#!/usr/bin/env python3
# Score results-LVC/<stamp> against PREREG-LVC.md. Usage: summarize-LVC.py results-LVC/<stamp> results-LV/<stamp> results-U/<stamp>
import glob, math, os, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
def load (d):
Q, N, P, A = {}, {}, {}, {}
for f in glob.glob(os.path.join(d, 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'Q': Q[(int(p[2]), p[3], int(p[1]))] = tuple(map(float, p[4:10]))
elif p[0] == 'N': N[(int(p[2]), p[3], int(p[1]))] = int(p[4])
elif p[0] == 'P' and p[4] == 'MST': P[(int(p[2]), p[3], int(p[1]))] = float(p[7])
elif p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[6]), float(p[7]))
return Q, N, P, A
Q, N, P, A = load(sys.argv[1]); Q0, _, P0, _ = load(sys.argv[2]); _, _, Pu, _ = load(sys.argv[3])
seeds = sorted({k[2] for k in Q}); tiers = sorted({k[0] for k in Q}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def m(d, t, k, i=None): return [d[(t, k, s)] if i is None else d[(t, k, s)][i] for s in seeds]
say('## Held-out labels (mean): V-measure / many-to-one / labels / share in a class / sense / sense-COLL / merges (runs at the 2000 cap)')
for t in tiers:
for arm in ['LV-I', 'LV-A']:
say(f'{t} {arm}: ' + ' / '.join(f'{mean(m(Q, t, arm, i)):.3f}' for i in range(6)) + f' / {mean(m(N, t, arm)):.1f} ({sum(x >= 2000 for x in m(N, t, arm))})')
say('\n## Re-parse (mean): link F1 / ucomp type recall / ucomp precision; demo-setting LV-I F1; L-MST F1')
for t in tiers:
for arm in ['LV-I', 'LV-A']:
k = f'M-{arm}-ucomp-MST'
say(f'{t} {arm}: {mean(m(P, t, arm)):.3f} / {mean(m(A, t, k, 1)):.3f} / {mean(m(A, t, k, 2)):.3f} demo {mean(m(P0, t, arm)):.3f} L-MST {mean(m(Pu, t, "L")):.3f}')
say(f'\n## Predictions (at {top})')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
x = mean(m(Q, top, 'LV-I', 3)); verdict('C1', x >= 0.8, f'share of held-out tokens in a class {x:.3f} (bar 0.8)')
c = ci([a - b for a, b in zip(m(Q, top, 'LV-I', 4), m(Q, top, 'LV-I', 5))]); verdict('C2', c[1] > 0, f'sense LV-I − LV-I-COLL {fmt(c)}')
c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(P0, top, 'LV-I'))]); verdict('C3', c[1] > 0, f'link F1 LV-I converged − demo setting {fmt(c)}')
c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(Pu, top, 'L'))]); verdict('C4', c[1] > -0.02, f'link F1 LV-I − L-MST {fmt(c)} (bar: lower > -0.02)')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')