#!/usr/bin/env python3
# Score a Test A results/<stamp>/ directory against PREREG.md amendment 1. Usage: summarize.py results/<stamp>
import glob, math, os, sys
T975 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d)
sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
A = {}; X = {}; guards = []
for f in sorted(glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv'))):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = dict(types=int(p[4]), inst=float(p[5]), typ=float(p[6]), prec=float(p[7]))
elif p[0] == 'X': X[(int(p[2]), p[3], int(p[1]), int(p[4]))] = int(p[5])
elif p[0] == 'GUARD': guards.append((p[1], p[2]))
seeds = sorted({k[2] for k in A}); tiers = sorted({k[0] for k in A})
arms = []
for k in A:
if k[1] not in arms: arms.append(k[1])
top = max(tiers)
def v(t, a, f): return [A[(t, a, s)][f] for s in seeds]
def growth(t, a, s):
r = [X[(t, a, s, k + 1)] / X[(t, a, s, k)] for k in range(2, 6) if X[(t, a, s, k)] > 0]
return math.exp(mean([math.log(x) for x in r])) if r and all(x > 0 for x in r) else float('nan')
out = []
def say(s=''): out.append(s); print(s)
say(f'seeds={len(seeds)} tiers={tiers} guards {sum(g[1] == "PASS" for g in guards)}/{len(guards)} pass')
for t in tiers:
say(f'\n## tier {t}: arm\ttypes\tinst recall\ttype recall\tcand precision\tgrowth')
for a in arms:
say(f'{a}\t{mean(v(t,a,"types")):.0f}\t{mean(v(t,a,"inst")):.3f}\t{mean(v(t,a,"typ")):.3f}\t{mean(v(t,a,"prec")):.3f}\t{mean([growth(t,a,s) for s in seeds]):.2f}')
say('\n## Predictions')
def verdict(n, ok, d): say(f'{n}: {"PASS" if ok else "FAIL"} — {d}')
ratio = mean([A[(top, 'M-G-comp', s)]['types'] / A[(top, 'C-G', s)]['types'] for s in seeds])
rr = mean(v(top, 'M-G-comp', 'typ')) / mean(v(top, 'C-G', 'typ'))
verdict('A1', ratio <= 0.10 and rr >= 0.7, f'types M-G-comp/C-G = {ratio:.3f} (bar <= 0.10); type recall ratio = {rr:.3f} (bar >= 0.7)')
c = [(t, ci([a - b for a, b in zip(v(t, 'M-G-comp', 'typ'), v(t, 'M-R-comp', 'typ'))])) for t in tiers]
verdict('A2', all(x[1] > 0 for _, x in c), ' '.join(f'{t}:{x[0]:+.3f}[{x[1]:+.3f},{x[2]:+.3f}]' for t, x in c))
g = {a: mean([growth(top, a, s) for s in seeds]) for a in ('M-G-comp', 'C-G', 'F')}
verdict('A3', g['M-G-comp'] <= 0.5 * g['C-G'] and g['F'] >= g['C-G'], f'growth M-G-comp {g["M-G-comp"]:.2f}, C-G {g["C-G"]:.2f}, F {g["F"]:.2f}')
order = [mean(v(top, f'M-{x}-comp', 'typ')) for x in ('G', 'p10', 'p30', 'p50')]
s3, s2 = mean(v(top, 'M-S3-comp', 'typ')), mean(v(top, 'M-S2-comp', 'typ'))
verdict('A4', all(order[i] >= order[i + 1] for i in range(3)) and s3 < s2,
'G,p10,p30,p50 = ' + ', '.join(f'{x:.3f}' for x in order) + f' | S2 {s2:.3f}, S3 {s3:.3f}')
c = [(t, ci([a - b for a, b in zip(v(t, 'M-G-comp', 'prec'), v(t, 'C-G', 'prec'))])) for t in tiers]
verdict('A5', all(x[1] > 0 for _, x in c), ' '.join(f'{t}:{x[0]:+.3f}[{x[1]:+.3f},{x[2]:+.3f}]' for t, x in c))
say('\n## Controls')
verdict('C1/C5', all(g[1] == 'PASS' for g in guards), 'gold-span selector and true-tree rule checks, every seed')
ok = all(mean(v(t, 'M-G-comp', 'typ')) >= max(mean(v(t, a, 'typ')) for a in arms if a.startswith('M-') and a.endswith('-comp')) for t in tiers)
verdict("C4'", ok, 'M-G-comp has the highest type recall among M-x-comp at every tier')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')