#!/usr/bin/env python3
# Score a results-v2/<stamp>/ directory against PREREG-v2.md. Usage: summarize-v2.py results-v2/<stamp>
import glob, math, os, sys
T975 = {4: 2.776}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
V, L, C, A = {}, {}, {}, []
for f in sorted(glob.glob(os.path.join(sys.argv[1], '*.tsv'))):
arm = os.path.basename(f).split('-')[0]
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'V': V[(int(p[2]), p[4], int(p[1]))] = tuple(map(float, p[5:8]))
elif p[0] == 'L': L[(int(p[2]), p[4], int(p[1]))] = float(p[5])
elif p[0] == 'C': C[(int(p[2]), p[4], int(p[5]), int(p[1]))] = float(p[6])
elif p[0] == 'A': A.append((arm, p[4], p[5]))
seeds = sorted({k[2] for k in V}); tiers = sorted({k[0] for k in V}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def v(t, m, i): return [V[(t, m, s)][i] for s in seeds] # i: 0 vmeasure, 1 m1, 2 sense
def l(t, m): return [L[(t, m, s)] for s in seeds]
def d(t, a, b, i): return ci([x - y for x, y in zip(v(t, a, i), v(t, b, i))])
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
ARMS = ['V1', 'V2W', 'V2D']
say('## Token classes (mean): V-measure / many-to-one / sense accuracy')
for t in tiers:
say(f'tier {t}')
for m in [f'JT-{a}{c}{s}' for s in ['', '-SHUF'] for a in ARMS for c in ['', '-COLL']] + ['TYPE-A', 'TYPE-A-SHUF', 'TYPE', 'MAJ']:
if (t, m, seeds[0]) in V: say(f' {m:16s} ' + ' / '.join(f'{mean(v(t, m, i)):.3f}' for i in range(3)))
say('\n## Parse link F1 (mean)')
for t in tiers: say(f'{t}: ' + ' '.join(f'{m} {mean(l(t, m)):.3f}' for m in ['ATTN-V1', 'ATTN-V2W', 'ATTN-V2D', 'WORD-MI', 'GOLD-MI'] if (t, m, seeds[0]) in L))
say('\n## Continuous checkpoints, tier ' + str(top) + ' (mean V-measure)')
for a in ARMS:
cks = sorted({k[2] for k in C if k[0] == top and k[1] == f'JT-{a}'})
say(f' {a:4s} ' + ' '.join(f'{c}:{mean([C[(top, f"JT-{a}", c, s)] for s in seeds]):.3f}' for c in cks))
say('\n## Predictions (sense = sense accuracy on polysemous occurrences, at ' + str(top) + ')')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
for a, arm in [('V2W', 'word'), ('V2D', 'dj')]:
c = d(top, f'JT-{a}', f'JT-{a}-COLL', 2); verdict(f'K1 {a}', c[1] > 0, f'sense JT-{a} − JT-{a}-COLL {fmt(c)}')
c = d(top, f'JT-{a}-SHUF', f'JT-{a}-COLL-SHUF', 2); verdict(f'K2 {a}', not c[1] > 0, f'shuffled sense JT-{a} − COLL {fmt(c)}')
c = d(top, f'JT-{a}', 'JT-V1', 2); verdict(f'K3 {a}', c[1] > 0, f'sense JT-{a} − JT-V1 {fmt(c)}')
c = d(top, f'JT-{a}', 'MAJ', 2); verdict(f'K4 {a}', c[1] > 0, f'sense JT-{a} − MAJ {fmt(c)}')
c = d(top, f'JT-{a}', 'JT-V1', 0); verdict(f'K5 {a}', c[1] > -0.02, f'V-measure JT-{a} − JT-V1 {fmt(c)}')
r = [x for x in A if x[0] == arm]; verdict(f'K6 {a}', len(r) > 0 and all(x[2] == 'PASS' for x in r), f'{sum(x[2] == "PASS" for x in r)}/{len(r)} audit checks PASS')
say('\n## Scored, no prediction')
say(f'V1 sense − V1-COLL: {fmt(d(top, "JT-V1", "JT-V1-COLL", 2))}')
say(f'V1 shuffled sense − COLL: {fmt(d(top, "JT-V1-SHUF", "JT-V1-COLL-SHUF", 2))}')
say(f'V2W sense − V2D: {fmt(d(top, "JT-V2W", "JT-V2D", 2))}')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')