#!/usr/bin/env python3
# Score results-LV/<stamp> against PREREG-LV.md. Usage: summarize-LV.py results-LV/<stamp> results-reparse/<stamp> results-U/<stamp>
import glob, math, os, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
def load (d):
Q, N, P, A, RP = {}, {}, {}, {}, {}
for f in glob.glob(os.path.join(d, 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'Q': Q[(int(p[2]), p[3], int(p[1]))] = tuple(map(float, p[4:10]))
elif p[0] == 'N': N[(int(p[2]), p[3], int(p[1]))] = int(p[4])
elif p[0] == 'P' and p[4] == 'MST': P[(int(p[2]), p[3], int(p[1]))] = float(p[7]) # MST rows only
elif p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[6]), float(p[7]))
elif p[0] == 'RP': RP[(int(p[2]), int(p[3]), int(p[1]))] = float(p[6])
return Q, N, P, A, RP
Q, N, P, A, _ = load(sys.argv[1]); _, _, _, Ar, RPr = load(sys.argv[2]); _, _, Pu, _, _ = load(sys.argv[3])
seeds = sorted({k[2] for k in Q}); tiers = sorted({k[0] for k in Q}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def m(d, t, k, i=None): return [d[(t, k, s)] if i is None else d[(t, k, s)][i] for s in seeds]
say('## Held-out labels (mean): V-measure / many-to-one / labels / share in a class / sense / sense-COLL / merges')
for t in tiers:
for arm in ['LV-I', 'LV-A']:
say(f'{t} {arm}: ' + ' / '.join(f'{mean(m(Q, t, arm, i)):.3f}' for i in range(6)) + f' / {mean(m(N, t, arm)):.1f}')
say('\n## Re-parse link F1 and level 2 (mean): link F1 / ucomp types / ucomp type recall / ucomp precision')
for t in tiers:
say(f'{t} words(round 0) F1 {mean([RPr[(t, 0, s)] for s in seeds]):.3f} ucomp recall {mean(m(Ar, t, "M-RP0-ucomp", 1)):.3f}; L-MST F1 {mean(m(Pu, t, "L")):.3f}')
for arm in ['LV-I', 'LV-A']:
k = f'M-{arm}-ucomp-MST'
say(f' {arm}: {mean(m(P, t, arm)):.3f} / {mean(m(A, t, k, 0)):.0f} / {mean(m(A, t, k, 1)):.3f} / {mean(m(A, t, k, 2)):.3f}')
say(f'\n## Predictions (at {top})')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
c = ci([a - b for a, b in zip(m(Q, top, 'LV-I', 4), m(Q, top, 'LV-I', 5))]); verdict('L1', c[1] > 0, f'sense LV-I − LV-I-COLL {fmt(c)}')
c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), [RPr[(top, 0, s)] for s in seeds])]); verdict('L2', c[1] > 0, f'link F1 LV-I − word MST {fmt(c)}')
c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(Pu, top, 'L'))]); verdict('L3', c[1] > -0.02, f'link F1 LV-I − L-MST {fmt(c)} (bar: lower > -0.02)')
c = ci([a - b for a, b in zip(m(A, top, 'M-LV-I-ucomp-MST', 1), m(Ar, top, 'M-RP0-ucomp', 1))]); verdict('L4', c[1] > 0, f'ucomp type recall LV-I − word MST {fmt(c)}')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')