atomspace-layer-discovery / experiments / level2 / summarize-jt.py

#!/usr/bin/env python3
# Score a results-jt/<stamp>/ directory against PREREG-JT.md. Usage: summarize-jt.py results-jt/<stamp> results/<testA-stamp>
import glob, math, os, sys

T975 = {9: 2.262}
def mean(x): return sum(x) / len(x)
def ci(d):
    n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
    h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
    return m, m - h, m + h
def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'

A = {}  # (tier, arm, seed) -> (types, inst recall, type recall, precision)
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
    for line in open(f):
        p = line.rstrip('\n').split('\t')
        if p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[5]), float(p[6]), float(p[7]))
seeds = sorted({k[2] for k in A}); tiers = sorted({k[0] for k in A}); top = max(tiers)
out = []
def say(s=''): out.append(s); print(s)
def a(t, m, i): return [A[(t, m, s)][i] for s in seeds]
def d(t, x, y, i): return ci([p - q for p, q in zip(a(t, x, i), a(t, y, i))])

# R0: every non-JT line identical to Test A.
bad = 0
for s in seeds:
    old = [l for l in open(os.path.join(sys.argv[2], f'seed-{s}.tsv')) if 'JT' not in l]
    new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if 'JT' not in l]
    bad += old != new
say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A exactly')

say('\n## Means: candidate types / instance recall / type recall / candidate precision')
for t in tiers:
    say(f'tier {t}')
    for m in ['C-G', 'M-G-comp', 'C-L', 'M-L-comp', 'M-R-comp', 'C-JT', 'M-JT-conn', 'M-JT-comp', 'M-JTA-conn', 'M-JTA-comp']:
        say(f'  {m:11s} {mean(a(t, m, 0)):9.0f}  ' + '  '.join(f'{mean(a(t, m, i)):.3f}' for i in (1, 2, 3)))

say(f'\n## Predictions (at {top} unless stated)')
def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
c = d(top, 'M-JTA-comp', 'M-JT-comp', 2); verdict('B1', c[1] > 0, f'type recall M-JTA-comp − M-JT-comp {fmt(c)}')
c = d(top, 'M-JT-comp', 'M-L-comp', 2); verdict('B2', c[1] > 0, f'type recall M-JT-comp − M-L-comp {fmt(c)}')
c = d(top, 'M-JTA-comp', 'M-G-comp', 2); verdict('B3', c[1] > 0, f'type recall M-JTA-comp − M-G-comp {fmt(c)}')
ok = True; cells = []
for t in tiers: c = d(t, 'M-JTA-comp', 'C-JT', 3); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
verdict('B4', ok, 'precision M-JTA-comp − C-JT ' + '; '.join(cells))
ok = True; cells = []
for t in tiers:
    for m in ['M-JT-comp', 'M-JTA-comp']: c = d(t, m, 'M-R-comp', 2); ok &= c[1] > 0; cells.append(f'{t} {m}: {c[0]:+.3f} [{c[1]:+.3f}]')
verdict('B5', ok, 'type recall − M-R-comp ' + '; '.join(cells))
x, y = mean(a(top, 'M-JTA-comp', 0)), mean(a(top, 'C-JT', 0)); verdict('B6', x <= y, f'types M-JTA-comp {x:.0f} vs C-JT {y:.0f}')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')