| … | … | @@ -0,0 +1,41 @@ |
| 1 | #!/usr/bin/env python3 |
| 2 | # Score results-LVC/<stamp> against PREREG-LVC.md. Usage: summarize-LVC.py results-LVC/<stamp> results-LV/<stamp> results-U/<stamp> |
| 3 | import glob, math, os, sys |
| 4 | T975 = {9: 2.262} |
| 5 | def mean(x): return sum(x) / len(x) |
| 6 | def ci(d): |
| 7 | n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1)) |
| 8 | h = T975.get(n - 1, 1.96) * sd / math.sqrt(n) |
| 9 | return m, m - h, m + h |
| 10 | def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]' |
| 11 | def load (d): |
| 12 | Q, N, P, A = {}, {}, {}, {} |
| 13 | for f in glob.glob(os.path.join(d, 'seed-*.tsv')): |
| 14 | for line in open(f): |
| 15 | p = line.rstrip('\n').split('\t') |
| 16 | if p[0] == 'Q': Q[(int(p[2]), p[3], int(p[1]))] = tuple(map(float, p[4:10])) |
| 17 | elif p[0] == 'N': N[(int(p[2]), p[3], int(p[1]))] = int(p[4]) |
| 18 | elif p[0] == 'P' and p[4] == 'MST': P[(int(p[2]), p[3], int(p[1]))] = float(p[7]) |
| 19 | elif p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[6]), float(p[7])) |
| 20 | return Q, N, P, A |
| 21 | Q, N, P, A = load(sys.argv[1]); Q0, _, P0, _ = load(sys.argv[2]); _, _, Pu, _ = load(sys.argv[3]) |
| 22 | seeds = sorted({k[2] for k in Q}); tiers = sorted({k[0] for k in Q}); top = max(tiers) |
| 23 | out = [] |
| 24 | def say(s=''): out.append(s); print(s) |
| 25 | def m(d, t, k, i=None): return [d[(t, k, s)] if i is None else d[(t, k, s)][i] for s in seeds] |
| 26 | say('## Held-out labels (mean): V-measure / many-to-one / labels / share in a class / sense / sense-COLL / merges (runs at the 2000 cap)') |
| 27 | for t in tiers: |
| 28 | for arm in ['LV-I', 'LV-A']: |
| 29 | say(f'{t} {arm}: ' + ' / '.join(f'{mean(m(Q, t, arm, i)):.3f}' for i in range(6)) + f' / {mean(m(N, t, arm)):.1f} ({sum(x >= 2000 for x in m(N, t, arm))})') |
| 30 | say('\n## Re-parse (mean): link F1 / ucomp type recall / ucomp precision; demo-setting LV-I F1; L-MST F1') |
| 31 | for t in tiers: |
| 32 | for arm in ['LV-I', 'LV-A']: |
| 33 | k = f'M-{arm}-ucomp-MST' |
| 34 | say(f'{t} {arm}: {mean(m(P, t, arm)):.3f} / {mean(m(A, t, k, 1)):.3f} / {mean(m(A, t, k, 2)):.3f} demo {mean(m(P0, t, arm)):.3f} L-MST {mean(m(Pu, t, "L")):.3f}') |
| 35 | say(f'\n## Predictions (at {top})') |
| 36 | def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}') |
| 37 | x = mean(m(Q, top, 'LV-I', 3)); verdict('C1', x >= 0.8, f'share of held-out tokens in a class {x:.3f} (bar 0.8)') |
| 38 | c = ci([a - b for a, b in zip(m(Q, top, 'LV-I', 4), m(Q, top, 'LV-I', 5))]); verdict('C2', c[1] > 0, f'sense LV-I − LV-I-COLL {fmt(c)}') |
| 39 | c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(P0, top, 'LV-I'))]); verdict('C3', c[1] > 0, f'link F1 LV-I converged − demo setting {fmt(c)}') |
| 40 | c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(Pu, top, 'L'))]); verdict('C4', c[1] > -0.02, f'link F1 LV-I − L-MST {fmt(c)} (bar: lower > -0.02)') |
| 41 | open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n') |