#!/usr/bin/env python3
# Summarize a results/<stamp>/ directory against PREREG.md. Usage: summarize.py results/<stamp>
import glob, math, os, sys
from collections import defaultdict
T975 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d)
if n < 2: return m, m, m
sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
A = {}; N1 = {}; B = {}; guards = []
for f in sorted(glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv'))):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (float(p[4]), float(p[5]))
elif p[0] == 'N1': N1[(int(p[2]), int(p[1]))] = float(p[4]) - float(p[5])
elif p[0] == 'B': B[(int(p[2]), p[3], int(p[4]), int(p[1]))] = dict(ev=int(p[5]), acc=int(p[6]), f=float(p[7]), p=float(p[8]), r=float(p[9]), ceil=float(p[10]))
elif p[0] == 'GUARD': guards.append(p[1:3])
seeds = sorted({k[2] for k in A})
tiers = sorted({k[0] for k in A})
def kept(t, arm, s):
hu, h = A[(t, arm, s)]; _, hg = A[(t, 'Gw', s)]
return (hu - h) / (hu - hg)
def F(t, arm, s, b=-1): return B[(t, arm, b, s)]['f']
def EV(t, arm, s, b=-1): return B[(t, arm, b, s)]['ev']
out = []
def say(s=''): out.append(s); print(s)
say(f'seeds={len(seeds)} tiers={tiers}')
say(f'guards: ' + ', '.join(f'{g[0]}={g[1]}' for g in guards[:1]) + f' ({sum(g[1]=="PASS" for g in guards)}/{len(guards)} pass)')
say()
say('## Part A — bits/word (held-out), kept = (H_uni - H_arm)/(H_uni - H_Gw)')
say('tier\tH_uni\tG\tGw\tL\tR\tU\tkept L\tkept R\tN1 gain(L)')
for t in tiers:
row = [t, mean([A[(t, 'G', s)][0] for s in seeds])] + [mean([A[(t, a, s)][1] for s in seeds]) for a in ('G', 'Gw', 'L', 'R', 'U')]
row += [mean([kept(t, 'L', s) for s in seeds]), mean([kept(t, 'R', s) for s in seeds])]
n1 = [N1[(t, s)] for s in seeds if (t, s) in N1]
row.append(mean(n1) if n1 else float('nan'))
say('\t'.join(f'{v:.3f}' if isinstance(v, float) else str(v) for v in row))
say()
say('## Part B — hard arms: link F1 (mean) / pair types evaluated (mean)')
say('tier\t' + '\t'.join(['G', 'Gw', 'L', 'R', 'U']) + '\tceiling')
for t in tiers:
say(f'{t}\t' + '\t'.join(f'{mean([F(t,a,s) for s in seeds]):.3f}/{mean([EV(t,a,s) for s in seeds]):.0f}' for a in ('G', 'Gw', 'L', 'R', 'U'))
+ f'\t{mean([B[(t,"G",-1,s)]["ceil"] for s in seeds]):.3f}')
say()
say('## Budgeted word-pair search: link F1 (mean) by budget')
budgets = sorted({k[2] for k in B if k[2] > 0})
for t in tiers:
say(f'tier {t}: budget\t' + '\t'.join(str(b) for b in budgets))
for arm in ('U-f', 'S-L', 'S-R'):
cells = []
for b in budgets:
v = [F(t, arm, s, b) for s in seeds if (t, arm, b, s) in B]
cells.append(f'{mean(v):.3f}' if len(v) == len(seeds) else '-')
say(f' {arm}\t' + '\t'.join(cells))
say()
say('## Predictions')
res = {}
def verdict(name, ok, detail): res[name] = ok; say(f'{name}: {"PASS" if ok else "FAIL"} — {detail}')
big = [t for t in tiers if t >= 10000]
v = [(t, mean([kept(t, 'L', s) for s in seeds])) for t in big]
r = [(t, max(kept(t, 'R', s) for s in seeds)) for t in tiers]
verdict('PA1', all(x >= 0.8 for _, x in v) and all(x <= 0.1 for _, x in r),
'kept(L) ' + ' '.join(f'{t}:{x:.2f}' for t, x in v) + ' | max kept(R) ' + ' '.join(f'{t}:{x:.2f}' for t, x in r))
mx = max(EV(t, a, s) for t in tiers for a in ('L', 'R') for s in seeds)
ratio = [(t, mean([EV(t, 'U', s) / EV(t, 'L', s) for s in seeds])) for t in big]
verdict('PA2', mx <= 900 and all(x >= 50 for _, x in ratio), f'max L/R types {mx}; U/L ratio ' + ' '.join(f'{t}:{x:.0f}x' for t, x in ratio))
d = [(t, ci([F(t, 'L', s) - F(t, 'R', s) for s in seeds])) for t in tiers]
order = all(mean([F(t, 'G', s) for s in seeds]) >= mean([F(t, 'Gw', s) for s in seeds]) >= mean([F(t, 'L', s) for s in seeds]) for t in tiers)
verdict('PB1', all(c[1] > 0 for _, c in d) and order,
'L-R ' + ' '.join(f'{t}:{c[0]:+.3f}[{c[1]:+.3f},{c[2]:+.3f}]' for t, c in d) + f' | G>=Gw>=L every tier: {order}')
small = [t for t in tiers if t <= 10000]
pb2a = [(t, mean([F(t, 'L', s) for s in seeds]) - mean([F(t, 'U', s) for s in seeds]), mean([EV(t, 'L', s) / EV(t, 'U', s) for s in seeds])) for t in small]
top = max(tiers)
gap = mean([F(top, 'L', s) for s in seeds]) - mean([F(top, 'U', s) for s in seeds])
verdict('PB2', all(x >= 0 and f <= 0.02 for _, x, f in pb2a) and gap >= -0.05,
'F(L)-F(U), types L/U: ' + ' '.join(f'{t}:{x:+.3f},{f*100:.2f}%' for t, x, f in pb2a) + f' | at {top}: {gap:+.3f}')
cells = []; ok3 = True
for t in tiers:
for b in (1000, 3000, 10000, 30000):
if not all((t, 'S-L', b, s) in B for s in seeds): continue
if all(EV(t, 'U-f', s, b) < b for s in seeds): continue # budget exceeds the space: all orders identical
c1 = ci([F(t, 'S-L', s, b) - F(t, 'U-f', s, b) for s in seeds])
c2 = ci([F(t, 'S-R', s, b) - F(t, 'U-f', s, b) for s in seeds])
ok3 = ok3 and c1[1] > 0 and not c2[1] > 0
cells.append(f'{t}/{b}: SL-Uf {c1[0]:+.3f}[{c1[1]:+.3f}] SR-Uf {c2[0]:+.3f}[{c2[1]:+.3f}]')
verdict('PB3', ok3, '; '.join(cells))
g1 = mean([F(min(tiers), 'Gw', s) - F(min(tiers), 'L', s) for s in seeds]); g2 = mean([F(top, 'Gw', s) - F(top, 'L', s) for s in seeds])
verdict('PB4', g1 > g2, f'Gw-L at {min(tiers)}: {g1:+.3f}, at {top}: {g2:+.3f}')
say()
say('## Controls')
verdict('C1', all(g[1] == 'PASS' for g in guards if g[0] == 'C1') and len(guards) == len(seeds), 'parser exact on gold edges, every seed')
n1 = [N1[k] for k in N1]
verdict('C3', all(x <= 0.02 for x in n1), f'N1 L gain over unigram: max {max(n1):+.3f} bits/word')
c4 = all(mean([F(t, 'G', s) for s in seeds]) >= max(mean([F(t, a, s) for s in seeds]) for a in ('Gw', 'L', 'R', 'U')) for t in tiers)
verdict('C4', c4, 'G highest mean F1 among hard arms at every tier')
say('C2: shown to fail under --mut-drop (see commit message); not re-run per seed.')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')