#!/usr/bin/env python3
# Score results-reparse/<stamp> against PREREG-reparse.md. Usage: summarize-reparse.py results-reparse/<stamp> results-U/<stamp>
import glob, math, os, re, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
RP, A, P = {}, {}, {}
NEW = re.compile(r'^RP\t|RP[0-9]')
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'RP': RP[(int(p[2]), int(p[3]), int(p[1]))] = tuple(map(float, p[4:8]))
elif p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[5]), float(p[6]), float(p[7]))
elif p[0] == 'P': P[(int(p[2]), p[3], p[4], int(p[1]))] = tuple(map(float, p[5:8]))
seeds = sorted({k[2] for k in RP}); tiers = sorted({k[0] for k in RP}); top = max(tiers); rounds = sorted({k[1] for k in RP})
out = []
def say(s=''): out.append(s); print(s)
bad = sum(list(open(os.path.join(sys.argv[2], f'seed-{s}.tsv'))) != [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if not NEW.search(l)] for s in seeds)
say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A-U exactly')
def rp(t, r, i): return [RP[(t, r, s)][i] for s in seeds]
def at(t, m, i): return [A[(t, m, s)][i] for s in seeds]
def pf(t, x): return [P[(t, x, 'MST', s)][2] for s in seeds]
say('\n## Per round (mean): link F1 / many-to-one / ucomp type recall / ucomp precision')
for t in tiers:
say(f'tier {t}: G-MST F1 {mean(pf(t, "G")):.3f} L-MST F1 {mean(pf(t, "L")):.3f}, L ucomp recall {mean(at(t, "M-L-ucomp-MST", 2)):.3f}')
for r in rounds:
m = f'M-RP{r}-ucomp'
say(f' round {r}: {mean(rp(t, r, 2)):.3f} / {mean(rp(t, r, 3)):.3f} / {mean(at(t, m, 2)):.3f} / {mean(at(t, m, 3)):.3f}')
say('\n## Predictions')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
ok = True; cells = []
for t in tiers: c = ci([a - b for a, b in zip(rp(t, 1, 2), rp(t, 0, 2))]); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
verdict('P1', ok, 'link F1 round 1 − round 0 ' + '; '.join(cells))
last = max(rounds)
c = ci([a - b for a, b in zip(rp(top, last, 2), rp(top, 1, 2))]); verdict('P2', c[1] > 0, f'link F1 round {last} − round 1 at {top} {fmt(c)}')
c = ci([a - b for a, b in zip(rp(top, last, 2), pf(top, 'L'))]); verdict('P3', c[1] > 0, f'link F1 round {last} − L-MST at {top} {fmt(c)}')
c = ci([a - b for a, b in zip(at(top, f'M-RP{last}-ucomp', 2), at(top, 'M-L-ucomp-MST', 2))]); verdict('P4', c[1] > 0, f'ucomp type recall round {last} − L-MST at {top} {fmt(c)}')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')