#!/usr/bin/env python3
# Score a results-v1/<stamp>/ directory against PREREG-v1.md. Usage: summarize-v1.py results-v1/<stamp>
import glob, math, os, sys
T975 = {4: 2.776}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
V, L, C, A = {}, {}, {}, []
for f in sorted(glob.glob(os.path.join(sys.argv[1], '*.tsv'))):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'V': V[(int(p[2]), p[4], int(p[1]))] = tuple(map(float, p[5:8]))
elif p[0] == 'L': L[(int(p[2]), p[4], int(p[1]))] = float(p[5])
elif p[0] == 'C': C[(int(p[2]), p[4], int(p[5]), int(p[1]))] = float(p[6])
elif p[0] == 'A': A.append((p[4], p[5]))
seeds = sorted({k[2] for k in V}); tiers = sorted({k[0] for k in V}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def v(t, m, i): return [V[(t, m, s)][i] for s in seeds] # i: 0 vmeasure, 1 m1, 2 sense
def l(t, m): return [L[(t, m, s)] for s in seeds]
say('## Token classes (mean): V-measure / many-to-one / sense accuracy')
for t in tiers:
say(f'tier {t}')
for m in ['JT-V1', 'TYPE-A', 'TYPE', 'MAJ', 'JT-V1-SHUF', 'TYPE-A-SHUF']:
if (t, m, seeds[0]) in V: say(f' {m:12s} ' + ' / '.join(f'{mean(v(t, m, i)):.3f}' for i in range(3)))
say('\n## Parse link F1 (mean)')
for t in tiers: say(f'{t}: ' + ' '.join(f'{m} {mean(l(t, m)):.3f}' for m in ['ATTN-V1', 'WORD-MI', 'GOLD-MI'] if (t, m, seeds[0]) in L))
say('\n## Continuous checkpoints, tier ' + str(top) + ' (mean V-measure)')
cks = sorted({k[2] for k in C if k[0] == top and k[1] == 'JT-V1'})
say(' ' + ' '.join(f'{c}:{mean([C[(top, "JT-V1", c, s)] for s in seeds]):.3f}' for c in cks))
say('\n## Predictions')
def verdict(n, ok, d): say(f'{n}: {"PASS" if ok else "FAIL"} — {d}')
c = ci([a - b for a, b in zip(v(top, 'JT-V1', 2), v(top, 'TYPE-A', 2))])
verdict('J1', c[1] > 0, f'sense JT-V1 − TYPE-A at {top}: {c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]')
c = ci([a - b for a, b in zip(v(top, 'JT-V1', 2), v(top, 'MAJ', 2))])
verdict('J1+', c[1] > 0, f'sense JT-V1 − MAJ at {top}: {c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]')
c = ci([a - b for a, b in zip(v(top, 'JT-V1', 0), v(top, 'TYPE', 0))]); m = mean(v(top, 'JT-V1', 0))
verdict('J2', m >= 0.7 and c[1] > 0, f'V-measure JT-V1 {m:.3f} (bar 0.7); − TYPE {c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]')
ok = True; cells = []
for t in tiers:
c = ci([a - b for a, b in zip(l(t, 'ATTN-V1'), l(t, 'WORD-MI'))]); ok &= c[1] > 0
cells.append(f'{t}: {c[0]:+.3f} [{c[1]:+.3f}]')
verdict('J3', ok, 'ATTN-V1 − WORD-MI ' + '; '.join(cells))
seq = [mean([C[(top, 'JT-V1', ck, s)] for s in seeds]) for ck in cks if ck >= 4000]
verdict("J4'", all(seq[i] <= seq[i + 1] for i in range(len(seq) - 1)), 'checkpoints from 4K: ' + ' '.join(f'{x:.3f}' for x in seq))
verdict('J5', all(r == 'PASS' for _, r in A) and len(A) > 0, f'{sum(r == "PASS" for _, r in A)}/{len(A)} audit checks PASS')
c = ci([a - b for a, b in zip(v(top, 'JT-V1-SHUF', 2), v(top, 'TYPE-A-SHUF', 2))])
verdict('J6', not c[1] > 0, f'shuffled sense JT-V1 − TYPE-A at {top}: {c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')