#!/usr/bin/env python3
# Score a results-soft/<stamp>/ directory against PREREG-soft.md. Usage: summarize-soft.py results-soft/<stamp> results/<testA-stamp>
import glob, math, os, sys
T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
R = {}
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'R': R[(int(p[2]), p[3], p[4], int(p[5]), int(p[1]))] = (float(p[6]), float(p[7]))
seeds = sorted({k[4] for k in R}); tiers = sorted({k[0] for k in R}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
bad = 0
for s in seeds:
old = [l for l in open(os.path.join(sys.argv[2], f'seed-{s}.tsv'))]
new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if not l.startswith('R\t')]
bad += old != new
say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A exactly')
def r(t, x, m, b, i): return [R[(t, x, m, b, s)][i] for s in seeds]
def d(t, x, m1, m2, b, i): return ci([p - q for p, q in zip(r(t, x, m1, b, i), r(t, x, m2, b, i))])
say('\n## Means: R@B / P@B')
for t in tiers:
for x in ['G', 'L']:
say(f'{t} {x}: ' + ' '.join(f'{m} ' + ' '.join(f'{mean(r(t, x, m, b, 0)):.3f}/{mean(r(t, x, m, b, 1)):.3f}' for b in (1000, 3000, 10000)) for m in ['CNT', 'HARD', 'SOFT']))
say('\n## Predictions')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
for x in ['G', 'L']:
ok = True; cells = []
for t in tiers: c = d(t, x, 'SOFT', 'CNT', 3000, 0); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
verdict(f'E1 {x}', ok, 'R@3K SOFT − CNT ' + '; '.join(cells))
c = d(top, x, 'SOFT', 'HARD', 3000, 0); verdict(f'E2 {x}', c[1] > 0, f'R@3K SOFT − HARD at {top} {fmt(c)}')
c = d(top, x, 'SOFT', 'CNT', 3000, 1); verdict(f'E3 {x}', c[1] > 0, f'P@3K SOFT − CNT at {top} {fmt(c)}')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')