#!/usr/bin/env python3
# Score results-U/<stamp> against PREREG-U.md. Usage: summarize-U.py results-U/<stamp> results/<testA-stamp>
import glob, math, os, re, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
A, P = {}, {}
NEW = re.compile(r'uconn|ucomp|^P\t|C5-U')
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[5]), float(p[6]), float(p[7]))
elif p[0] == 'P': P[(int(p[2]), p[3], p[4], int(p[1]))] = tuple(map(float, p[5:8]))
seeds = sorted({k[2] for k in A}); tiers = sorted({k[0] for k in A}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
bad = 0; c5 = 0
for s in seeds:
new_all = list(open(os.path.join(sys.argv[1], f'seed-{s}.tsv')))
c5 += any(l.startswith('GUARD\tC5-U\tPASS') for l in new_all)
bad += list(open(os.path.join(sys.argv[2], f'seed-{s}.tsv'))) != [l for l in new_all if not NEW.search(l)]
say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A exactly')
say(f'C5-U: {"PASS" if c5 == len(seeds) else "FAIL"} — {c5}/{len(seeds)} seeds')
def a(t, m, i): return [A[(t, m, s)][i] for s in seeds]
def da(t, m1, m2, i): return ci([p - q for p, q in zip(a(t, m1, i), a(t, m2, i))])
def pf(t, x, p): return [P[(t, x, p, s)][2] for s in seeds]
say('\n## Link precision / recall / F1 (mean, held-out)')
for t in tiers:
say(f'{t}: ' + ' '.join(f'{x}-{p} ' + '/'.join(f'{mean([P[(t, x, p, s)][i] for s in seeds]):.3f}' for i in range(3)) for x in ['G', 'L', 'R'] for p in ['FOREST', 'MST', 'MPG']))
say('\n## Means: types / instance recall / type recall / candidate precision')
for t in tiers:
say(f'tier {t}')
for m in ['C-G', 'M-G-comp'] + [f'M-{x}-{r}-{p}' for x in ['G', 'L', 'R'] for p in ['FOREST', 'MST', 'MPG'] for r in ['uconn', 'ucomp']]:
say(f' {m:20s} {mean(a(t, m, 0)):8.0f} ' + ' '.join(f'{mean(a(t, m, i)):.3f}' for i in (1, 2, 3)))
say(f'\n## Predictions (at {top} unless stated)')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
c = da(top, 'M-G-ucomp-FOREST', 'M-G-comp', 2); verdict('U1', c[1] > 0, f'type recall M-G-ucomp-FOREST − M-G-comp {fmt(c)}')
for x in ['G', 'L']:
c = ci([p - q for p, q in zip(pf(top, x, 'MST'), pf(top, x, 'FOREST'))]); verdict(f'U2 {x}', c[1] > 0, f'link F1 MST − FOREST {fmt(c)}')
best = 'MST' if mean(pf(top, 'G', 'MST')) >= mean(pf(top, 'G', 'FOREST')) else 'FOREST'
m = f'M-G-ucomp-{best}'; r = mean(a(top, m, 2)) / mean(a(top, 'C-G', 2)); ty = mean(a(top, m, 0)) / mean(a(top, 'C-G', 0))
verdict('U3', r >= 0.7 and ty <= 0.10, f'{m}: type recall {r:.3f} x C-G (bar >= 0.7), types {ty:.3f} x C-G (bar <= 0.10)')
ok = True; cells = []
for t in tiers:
for x in ['G', 'L']: c = da(t, f'M-{x}-ucomp-MST', 'M-R-ucomp-MST', 2); ok &= c[1] > 0; cells.append(f'{t} {x}: {c[0]:+.3f} [{c[1]:+.3f}]')
verdict('U4', ok, 'type recall M-x-ucomp-MST − M-R-ucomp-MST ' + '; '.join(cells))
ok = True; cells = []
for t in tiers: c = da(t, 'M-G-ucomp-MST', 'C-G', 3); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
verdict('U5', ok, 'precision M-G-ucomp-MST − C-G ' + '; '.join(cells))
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')