#!/usr/bin/env python3
# Score results-B/<stamp> against PREREG-B.md. Usage: summarize-B.py results-B/<stamp> results-soft/<stamp>
import glob, math, os, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
B = {}
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'B': B[(int(p[2]), p[3], p[4], int(p[1]))] = tuple(map(float, p[5:9]))
seeds = sorted({k[3] for k in B}); tiers = sorted({k[0] for k in B}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
bad = 0
for s in seeds:
old = list(open(os.path.join(sys.argv[2], f'seed-{s}.tsv')))
new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if not l.startswith('B\t')]
bad += old != new
say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A-soft exactly')
def b(t, x, m, i): return [B[(t, x, m, s)][i] for s in seeds]
def d(t, x, m1, m2): return ci([p - q for p, q in zip(b(t, x, m1, 2), b(t, x, m2, 2))])
say('\n## Means: precision / recall / F1 / brackets chosen')
for t in tiers:
for x in ['G', 'L']:
say(f'{t} {x}: ' + ' '.join(f'{m} {mean(b(t, x, m, 0)):.3f}/{mean(b(t, x, m, 1)):.3f}/{mean(b(t, x, m, 2)):.3f}/{mean(b(t, x, m, 3)):.0f}' for m in ['CNT', 'HARD-R', 'SOFT-R', 'SOFT-RI']))
say('\n## Predictions')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
for x in ['G', 'L']:
ok = True; cells = []
for t in tiers: c = d(t, x, 'SOFT-R', 'CNT'); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
verdict(f'G1 {x}', ok, 'F1 SOFT-R − CNT ' + '; '.join(cells))
c = d(top, x, 'SOFT-R', 'HARD-R'); verdict(f'G2 {x}', c[1] > 0, f'F1 SOFT-R − HARD-R at {top} {fmt(c)}')
c = d(top, x, 'SOFT-RI', 'SOFT-R'); verdict(f'G3 {x}', c[1] > 0, f'F1 SOFT-RI − SOFT-R at {top} {fmt(c)}')
m = mean(b(top, 'G', 'SOFT-RI', 2)); verdict('G4', m >= 0.5, f'F1 SOFT-RI (G) at {top} = {m:.3f} (bar 0.5)')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')