#!/usr/bin/env python3
# Score a results/<stamp>/ directory against PREREG.md (run 1) or PREREG-2.md (run 2: pass BEST-G).
# Usage: summarize.py results/<stamp> [BACKOFF|BEST-G]
import glob, math, os, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
P, R, L = {}, {}, {}
for f in sorted(glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv'))):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'P': P[(int(p[2]), p[3], int(p[1]))] = float(p[4])
elif p[0] == 'R':
R[(int(p[2]), p[3], int(p[4]), int(p[1]))] = float(p[5]); L[(int(p[2]), p[3], int(p[1]))] = int(p[6])
seeds = sorted({k[2] for k in P}); tiers = sorted({k[0] for k in P})
COMB = sys.argv[2] if len(sys.argv) > 2 else 'BACKOFF' # the combined scorer under test
out = []
def say(s=''): out.append(s); print(s)
def pv(t, a): return [P[(t, a, s)] for s in seeds]
def rv(t, a, b): return [R[(t, a, b, s)] for s in seeds]
say('## Level-1 link F1 (mean)')
say('tier\t' + '\t'.join(['W', 'G', 'S1', 'S2', COMB]))
for t in tiers: say(f'{t}\t' + '\t'.join(f'{mean(pv(t, a)):.3f}' for a in ['W', 'G', 'S1', 'S2', COMB]))
say('\n## Level-2 type recall at budget (mean)')
arms = ['C-G', 'HARD', 'GRADED', 'GRADED-G', 'GRADED-R']
for t in tiers:
say(f'tier {t}: budget\t300\t1000\t3000\t10000\tall')
for a in arms:
say(f' {a}\t' + '\t'.join(f'{mean(rv(t, a, b)):.3f}' for b in [300, 1000, 3000, 10000, -1]) + f'\t(list {mean([L[(t, a, s)] for s in seeds]):.0f})')
say('\n## Predictions')
def verdict(n, ok, d): say(f'{n}: {"PASS" if ok else "FAIL"} — {d}')
cells = []; ok = True
for t in tiers:
best = max(['W', 'G', 'S1', 'S2'], key=lambda a: mean(pv(t, a)))
if t <= 10000:
for a in ['W', 'G', 'S1', 'S2']:
c = ci([x - y for x, y in zip(pv(t, COMB), pv(t, a))])
ok &= c[1] > 0
if a == best: cells.append(f'{t}: {COMB}-{a} {c[0]:+.3f}[{c[1]:+.3f}]')
else:
d = mean(pv(t, COMB)) - mean(pv(t, best)); ok &= d >= -0.01
cells.append(f'{t}: {COMB}-{best} {d:+.3f}')
verdict('F1', ok, '; '.join(cells))
g = [(t, mean(rv(t, 'GRADED', -1))) for t in tiers]
verdict('F2', all(x >= 0.95 for _, x in g), ' '.join(f'{t}:{x:.3f}' for t, x in g) + ' | HARD ' + ' '.join(f'{t}:{mean(rv(t, "HARD", -1)):.3f}' for t in tiers))
ok = True; cells = []
for t in tiers:
c = ci([x - y for x, y in zip(rv(t, 'GRADED', 1000), rv(t, 'C-G', 1000))]); ok &= c[1] > 0
cells.append(f'{t}: GRADED-C-G@1K {c[0]:+.3f}[{c[1]:+.3f}]')
# GRADED at B = |HARD list| vs HARD (interpolate GRADED at the nearest budget not exceeding the HARD list)
verdict('F3a', ok, '; '.join(cells))
say('F3b (GRADED >= HARD at B = |HARD list|): see note — compared at the largest common budget below each HARD list size:')
for t in tiers:
hl = mean([L[(t, 'HARD', s)] for s in seeds])
b = max([x for x in [300, 1000, 3000, 10000] if x <= hl] or [300])
say(f' {t}: |HARD| {hl:.0f}, budget {b}: GRADED {mean(rv(t, "GRADED", b)):.3f} vs HARD {mean(rv(t, "HARD", b)):.3f}')
ok = True; cells = []
for t in [x for x in tiers if x <= 10000]:
c = ci([x - y for x, y in zip(rv(t, 'GRADED', 1000), rv(t, 'GRADED-G', 1000))]); ok &= c[1] > 0
cells.append(f'{t}: {c[0]:+.3f}[{c[1]:+.3f}]')
verdict('F4', ok, 'GRADED-GRADED-G@1K ' + '; '.join(cells))
ok = True; cells = []
for t in tiers:
c = ci([x - y for x, y in zip(rv(t, 'GRADED-R', 1000), rv(t, 'C-G', 1000))]); ok &= not c[1] > 0
cells.append(f'{t}: {c[0]:+.3f}[{c[1]:+.3f}]')
verdict('F5', ok, 'GRADED-R-C-G@1K ' + '; '.join(cells))
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')