#!/usr/bin/env python3
# Score a results-v3diag/<stamp>/ directory against PREREG-v3diag.md.
import glob, os, sys
def mean(x): return sum(x) / len(x) if x else 0.0
D = {}
for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
for line in open(f):
p = line.rstrip('\n').split('\t')
if p[0] == 'D': D[(p[4], int(p[1]))] = (int(p[5]), int(p[6]), int(p[7]), int(p[8]), float(p[9]), float(p[10]), int(p[11]))
seeds = sorted({k[1] for k in D}); out = []
def say(s=''): out.append(s); print(s)
R = {}
say('cell eligible split precision recall base minor-prec minor-rec (words)')
for c in ['GG', 'GJ', 'JG', 'JJ']:
r = [D[(c, s)] for s in seeds]
prec = mean([x[2] / x[1] if x[1] else 0.0 for x in r]); rec = mean([x[2] / x[3] if x[3] else 0.0 for x in r]); base = mean([x[3] / x[0] for x in r])
mp = mean([x[4] for x in r if x[6] > 0]); mr = mean([x[5] for x in r if x[6] > 0])
R[c] = (prec, rec, base, mp, mr)
say(f'{c} {mean([x[0] for x in r]):7.1f} {mean([x[1] for x in r]):5.1f} {prec:9.3f} {rec:6.3f} {base:.3f} {mp:10.3f} {mr:9.3f} ({mean([x[6] for x in r]):.1f})')
prec, rec, base, mp, mr = R['GG']
d1 = prec >= 2 * base and rec >= 0.3
say(f'\nD1: {"PASS" if d1 else "FAIL"} — GG split precision {prec:.3f} (bar {2 * base:.3f}), recall {rec:.3f} (bar 0.3)')
say(f'D2: {"PASS" if mp >= 0.5 and mr >= 0.5 else "FAIL"} — GG minor-component precision {mp:.3f}, recall {mr:.3f} (bars 0.5)')
if d1:
dj, dg = rec - R['GJ'][1], rec - R['JG'][1]
say(f'D3: main failing input = {"the connector classes (GJ)" if dj > dg else "the parse (JG)"} — recall drop GJ {dj:.3f}, JG {dg:.3f}')
else:
say('D3: the method fails (D1 FAIL with gold inputs)')
open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')