#!/usr/bin/env python3
# Score a results-v3/<stamp>/ directory against PREREG-v3.md. Usage: summarize-v3.py results-v3/<stamp>
import glob, math, os, sys
T975 = {4: 2.776}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
V, L, C, A, S = {}, {}, {}, [], {}
for f in sorted(glob.glob(os.path.join(sys.argv[1], '*.tsv'))):
arm = os.path.basename(f).split('-')[0]
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'V': V[(int(p[2]), p[4], int(p[1]))] = tuple(map(float, p[5:8]))
elif p[0] == 'L': L[(int(p[2]), p[4], int(p[1]))] = float(p[5])
elif p[0] == 'C': C[(int(p[2]), p[4], int(p[5]), int(p[1]))] = float(p[6])
elif p[0] == 'S': S[(int(p[2]), p[4], int(p[1]))] = tuple(map(int, p[5:9])) # elig, split, split&poly, poly&elig
elif p[0] == 'A': A.append((arm, p[4], p[5]))
seeds = sorted({k[2] for k in V}); tiers = sorted({k[0] for k in V}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def v(t, m, i): return [V[(t, m, s)][i] for s in seeds]
def d(t, a, b, i): return ci([x - y for x, y in zip(v(t, a, i), v(t, b, i))])
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
say('## Token classes (mean): V-measure / many-to-one / sense accuracy')
for t in tiers:
say(f'tier {t}')
for m in [f'JT-{a}{c}{s}' for s in ['', '-SHUF'] for a in ['V2D', 'V3'] for c in ['', '-COLL']] + ['MAJ']:
if (t, m, seeds[0]) in V: say(f' {m:16s} ' + ' / '.join(f'{mean(v(t, m, i)):.3f}' for i in range(3)))
say('\n## Split diagnostics (mean over seeds): eligible, split, split precision, split recall, base rate')
def sp(t, m):
r = [S[(t, m, s)] for s in seeds]
prec = [x[2] / x[1] if x[1] else 0.0 for x in r]; rec = [x[2] / x[3] if x[3] else 0.0 for x in r]; base = [x[3] / x[0] if x[0] else 0.0 for x in r]
return mean([x[0] for x in r]), mean([x[1] for x in r]), mean(prec), mean(rec), mean(base)
for t in tiers:
for m in ['JT-V3', 'JT-V3-SHUF']:
if (t, m, seeds[0]) in S: say(f' {t} {m:11s} ' + ' '.join(f'{x:.3f}' if i > 1 else f'{x:.1f}' for i, x in enumerate(sp(t, m))))
say('\n## Parse link F1 (mean)')
for t in tiers: say(f'{t}: ' + ' '.join(f'{m} {mean([L[(t, m, s)] for s in seeds]):.3f}' for m in ['ATTN-V2D', 'ATTN-V3'] if (t, m, seeds[0]) in L))
say(f'\n## Predictions (at {top})')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
e, ns, prec, rec, base = sp(top, 'JT-V3')
verdict('S1', ns > 0 and prec >= 2 * base and rec >= 0.3, f'split precision {prec:.3f} (bar {2 * base:.3f} = 2 x base), recall {rec:.3f} (bar 0.3), words split {ns:.1f} of {e:.1f}')
c = d(top, 'JT-V3-COLL', 'JT-V2D-COLL', 2); verdict('S2', c[1] > 0, f'collapsed sense V3 − V2D {fmt(c)}')
c = d(top, 'JT-V3', 'JT-V2D', 2); verdict('S3', c[1] > 0, f'sense V3 − V2D {fmt(c)}')
nshuf = sp(top, 'JT-V3-SHUF')[1]; c = d(top, 'JT-V3-SHUF', 'JT-V2D-SHUF', 2)
verdict('S4', nshuf <= 0.5 * ns and not c[1] > 0, f'words split shuffled {nshuf:.1f} vs ordered {ns:.1f} (bar <= 0.5x); shuffled sense V3 − V2D {fmt(c)}')
c = d(top, 'JT-V3', 'JT-V2D', 0); verdict('S5', c[1] > -0.02, f'V-measure V3 − V2D {fmt(c)}')
r = [x for x in A if x[0] == 'v3']; verdict('S6', len(r) > 0 and all(x[2] == 'PASS' for x in r), f'{sum(x[2] == "PASS" for x in r)}/{len(r)} audit checks PASS')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')