OverviewFilesCommitsIssuesDocsReleasesPeers

atomspace-layer-discovery commits / 1bef2b4

experiments: pooled-parse sense split (jt --splitdiag2) and Test B unit choice (l2 --soft B rows), run + scorers
--splitdiag output unchanged after factoring split_stats (seed 1, 3K); l2 Test A and R rows unchanged (seed 1, 3K). Test B smoke: SOFT-R keeps verb phrases and clause fragments (complete in the parse, not NP/PP gold); checked with L2_SOFT_DEBUG, not a bug. No parameters changed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1bef2b47ceca23dc278900789b428458ded8b312
by plankatron <fleeky@gmail.com> · 2026-10-18T22:30:13.000Z

Changed files (6) +233 -22

experiments/jigsaw-transformer/jt.c +107 -21
……@@ -5,6 +5,7 @@
55// [--v2 word|dj] PREREG-v2 arms (implies the v1 pipeline); adds JT-<arm>-COLL rows
66// [--v3] PREREG-v3: V2-DJ + sense units split by disjuncts; adds S rows (eligible, split, split∩poly, poly∩eligible)
77// [--splitdiag] PREREG-v3diag: D rows (split test on gold/JT parse x gold/JT classes); skips audit and baselines
8// [--splitdiag2] PREREG-v3pool: D rows for GG, JG1, JG6, PG1, PG6 (pooled-parse connectors, gold classes)
89// [--dump prefix] write <prefix>-<tier>.bin (final labels + attention parse) for ../level2/l2 --jt
910// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
1011// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
……@@ -708,6 +709,31 @@ static void train_v3 (const corpus_t *tr, params_t *p, int **cls_out, double per
708709 free(tcon); free(tnc); free(start); free(occ); free(fill); free(minor); free(tru.w);
709710}
710711
712// Split test over all eligible words with the bags in tcon/tnc; prints one D row.
713static void split_stats (const corpus_t *tr, const int *start, const int *occ, const int *tcon, const int *tnc, int *minor,
714 int seed, int tier, const char *cell) {
715 long elig = 0, split = 0, split_poly = 0, poly_elig = 0, nmp = 0;
716 double mp = 0, mr = 0;
717 for (int w = 0; w < V; w++) {
718 int m = start[w + 1] - start[w];
719 const int *ow = occ + start[w];
720 if (m < SPLIT_MIN) continue;
721 elig++; poly_elig += poly[w];
722 if (!split_word(ow, m, tcon, tnc, minor)) continue;
723 split++;
724 if (!poly[w]) continue;
725 split_poly++;
726 int cnt[64] = { 0 }, c1 = -1, c2 = -1;
727 for (int i = 0; i < m; i++) cnt[tr->c[ow[i]]]++;
728 for (int g = 0; g < NCLS; g++) { if (c1 < 0 || cnt[g] > cnt[c1]) { c2 = c1; c1 = g; } else if (c2 < 0 || cnt[g] > cnt[c2]) c2 = g; }
729 long inm = 0, ins = 0, both = 0;
730 for (int i = 0; i < m; i++) { int a = minor[i], b = tr->c[ow[i]] == c2; inm += a; ins += b; both += a && b; }
731 mp += inm ? (double) both / inm : 0; mr += ins ? (double) both / ins : 0; nmp++;
732 }
733 printf("D\t%d\t%d\t%d\t%s\t%ld\t%ld\t%ld\t%ld\t%.5f\t%.5f\t%ld\n", seed, tier, KJ, cell, elig, split, split_poly, poly_elig,
734 nmp ? mp / nmp : 0, nmp ? mr / nmp : 0, nmp);
735}
736
711737// PREREG-v3diag.md: the v3 split test on links from parse {gold tree, JT attention parse} x connector classes
712738// {gold, JT label}. D rows: cell, eligible, split, split&poly, poly&eligible, mean minor-component precision and
713739// recall over split polysemous words, number of such words.
……@@ -748,30 +774,88 @@ static void split_diag (const corpus_t *tr, const params_t *p, int seed, int tie
748774 tnc[o + t] = nc;
749775 }
750776 }
751 long elig = 0, split = 0, split_poly = 0, poly_elig = 0, nmp = 0;
752 double mp = 0, mr = 0;
753 for (int w = 0; w < V; w++) {
754 int m = start[w + 1] - start[w];
755 const int *ow = occ + start[w];
756 if (m < SPLIT_MIN) continue;
757 elig++; poly_elig += poly[w];
758 if (!split_word(ow, m, tcon, tnc, minor)) continue;
759 split++;
760 if (!poly[w]) continue;
761 split_poly++;
762 int cnt[64] = { 0 }, c1 = -1, c2 = -1;
763 for (int i = 0; i < m; i++) cnt[tr->c[ow[i]]]++;
764 for (int g = 0; g < NCLS; g++) { if (c1 < 0 || cnt[g] > cnt[c1]) { c2 = c1; c1 = g; } else if (c2 < 0 || cnt[g] > cnt[c2]) c2 = g; }
765 long inm = 0, ins = 0, both = 0;
766 for (int i = 0; i < m; i++) { int a = minor[i], b = tr->c[ow[i]] == c2; inm += a; ins += b; both += a && b; }
767 mp += inm ? (double) both / inm : 0; mr += ins ? (double) both / ins : 0; nmp++;
768 }
769 printf("D\t%d\t%d\t%d\t%s\t%ld\t%ld\t%ld\t%ld\t%.5f\t%.5f\t%ld\n", seed, tier, KJ, cell[c], elig, split, split_poly, poly_elig,
770 nmp ? mp / nmp : 0, nmp ? mr / nmp : 0, nmp);
777 split_stats(tr, start, occ, tcon, tnc, minor, seed, tier, cell[c]);
771778 }
772779 free(jl); free(jh); free(start); free(occ); free(fill); free(tcon); free(tnc); free(minor);
773780}
774781
782// PREREG-v3pool.md: gold connector classes; links from the gold tree, the JT attention parse or the window-PMI
783// parse over JT labels, each as the max parse or the consensus (>= 4 of 6) of Gumbel-perturbed parses.
784#define NPOOL 6
785static void split_diag2 (const corpus_t *tr, const params_t *p, int seed, int tier) {
786 int N = tr->ntok, hd[MAXN + 1];
787 int *jl = malloc(sizeof(int) * N);
788 double *ae = malloc(sizeof(double) * N * W); // ae[t*W + d-1] = α_{t,t+d} + α_{t+d,t}
789 static sent_state_t S;
790 for (int s = 0; s < tr->ns; s++) {
791 int o = tr->off[s], n = tr->off[s + 1] - o;
792 estep(tr, o, n, p, &S);
793 for (int t = 0; t < n; t++) {
794 int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; jl[o + t] = b;
795 for (int d = 1; d <= W; d++) ae[(size_t) (o + t) * W + d - 1] = t + d < n ? S.alpha[t][t + d] + S.alpha[t + d][t] : 0;
796 }
797 }
798 corpus_t Lc = *tr; Lc.c = jl;
799 arm_t a = { "l1", KJ, NULL, 1 };
800 pairs_t P = count_pairs(&a, &Lc);
801 long ev, acc;
802 double *Sc = hard_scores(&P, &ev, &acc);
803 free_pairs(&P);
804 double mJ = 0, mP = 0; long nJ = 0, nP = 0;
805 for (int s = 0; s < tr->ns; s++) {
806 int o = tr->off[s], n = tr->off[s + 1] - o;
807 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
808 double v = ae[(size_t) (o + i) * W + j - i - 1], u = Sc[(size_t) jl[o + i] * KJ + jl[o + j]];
809 if (v > 0) { mJ += v; nJ++; }
810 if (u > NEG && u > 0) { mP += u; nP++; }
811 }
812 }
813 mJ = nJ ? mJ / nJ : 0; mP = nP ? mP / nP : 0;
814 printf("#\tsplitdiag2\tmean_pos_edge_JT=%.5f\tmean_pos_edge_PMI=%.5f\n", mJ, mP);
815 int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * N), *fill = malloc(sizeof(int) * V);
816 for (int t = 0; t < N; t++) start[tr->w[t] + 1]++;
817 for (int w = 0; w < V; w++) start[w + 1] += start[w];
818 memcpy(fill, start, sizeof(int) * V);
819 for (int t = 0; t < N; t++) occ[fill[tr->w[t]]++] = t;
820 int *tcon = malloc(sizeof(int) * N * MAXC), *tnc = malloc(sizeof(int) * N), *minor = malloc(sizeof(int) * N);
821 static const char *cell[5] = { "GG", "JG1", "JG6", "PG1", "PG6" };
822 static int lc[MAXN][MAXN];
823 for (int c = 0; c < 5; c++) {
824 int pmi = c >= 3, KP = c == 2 || c == 4 ? NPOOL : 1, need = KP == 1 ? 1 : 4;
825 double scale = 0.3 * (pmi ? mP : mJ);
826 rng_t r = { 7777ULL * seed + 31ULL * tier + c };
827 for (int s = 0; s < tr->ns; s++) {
828 int o = tr->off[s], n = tr->off[s + 1] - o, m = n + 1;
829 for (int i = 0; i < n; i++) for (int j = 0; j < n; j++) lc[i][j] = 0;
830 if (c == 0) {
831 for (int t = 0; t < n; t++) { int h = tr->h[o + t]; if (h >= 0) { lc[t][h]++; lc[h][t]++; } }
832 } else for (int k = 0; k < KP; k++) {
833 for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
834 for (int d = 1; d < m; d++) sc[0][d] = 0;
835 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
836 double v;
837 if (pmi) v = Sc[(size_t) jl[o + i] * KJ + jl[o + j]];
838 else { v = ae[(size_t) (o + i) * W + j - i - 1]; if (!(v > 0)) v = NEG; }
839 if (KP > 1 && v > NEG) { double u = runif(&r); if (u <= 0) u = 1e-300; v += scale * -log(-log(u)); }
840 sc[i + 1][j + 1] = sc[j + 1][i + 1] = v;
841 }
842 eisner(m, hd);
843 for (int d = 1; d < m; d++) { int h = hd[d] - 1; if (h >= 0) { lc[d - 1][h]++; lc[h][d - 1]++; } }
844 }
845 for (int t = 0; t < n; t++) {
846 int nc = 0;
847 for (int j = 0; j < n; j++) {
848 if (j == t || lc[t][j] < need) continue;
849 if (nc < MAXC) tcon[(size_t) (o + t) * MAXC + nc++] = (j < t ? 0 : 64) + tr->c[o + j];
850 }
851 tnc[o + t] = nc;
852 }
853 }
854 split_stats(tr, start, occ, tcon, tnc, minor, seed, tier, cell[c]);
855 }
856 free(jl); free(ae); free(Sc); free(start); free(occ); free(fill); free(tcon); free(tnc); free(minor);
857}
858
775859static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
776860 if (V3) train_v3(tr, p, cls_out, perturb);
777861 else train_core(tr, tr, V, p, cls_out, perturb);
……@@ -844,7 +928,7 @@ static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
844928
845929int main (int argc, char **argv) {
846930 if (argc < 2) { fprintf(stderr, "usage: jt <seed> [--tiers a,b] [--heldout N] [--K k] [--shuffle] [--explain f] [--log f]\n"); return 1; }
847 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0, splitdiag = 0;
931 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0, splitdiag = 0, splitdiag2 = 0;
848932 const char *explain_path = NULL, *log_path = NULL, *dump_prefix = NULL;
849933 for (int i = 2; i < argc; i++) {
850934 if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
……@@ -852,6 +936,7 @@ int main (int argc, char **argv) {
852936 else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
853937 else if (!strcmp(argv[i], "--v1")) v1 = 1;
854938 else if (!strcmp(argv[i], "--splitdiag")) splitdiag = 1;
939 else if (!strcmp(argv[i], "--splitdiag2")) splitdiag2 = 1;
855940 else if (!strcmp(argv[i], "--v3")) { v1 = 1; MODE = 2; V3 = 1; }
856941 else if (!strcmp(argv[i], "--v2") && i + 1 < argc) { v1 = 1; i++; MODE = !strcmp(argv[i], "word") ? 1 : !strcmp(argv[i], "dj") ? 2 : -1; if (MODE < 0) { fprintf(stderr, "--v2 word|dj\n"); return 1; } }
857942 else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
……@@ -907,6 +992,7 @@ int main (int argc, char **argv) {
907992 printf("V\t%d\t%d\t%d\tJT-%s-COLL%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
908993 printf("L\t%d\t%d\t%d\tATTN-%s%s\t%.5f\n", seed, tier, KJ, nm, tag, attention_parse_f1(&ho, &p));
909994 if (splitdiag) { split_diag(&tr, &p, seed, tier); free(cls); continue; }
995 if (splitdiag2) { split_diag2(&tr, &p, seed, tier); free(cls); continue; }
910996 if (V3) printf("S\t%d\t%d\t%d\tJT-%s%s\t%ld\t%ld\t%ld\t%ld\n", seed, tier, KJ, nm, tag, st_elig, st_split, st_split_poly, st_poly_elig);
911997 if (dump_prefix) { char dp[512]; snprintf(dp, sizeof dp, "%s-%d.bin", dump_prefix, tier); dump_layer1(dp, &tr, &ho, &p); }
912998 for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
experiments/jigsaw-transformer/run-v3pool.sh +12 -0
……@@ -0,0 +1,12 @@
1#!/bin/sh
2# PREREG-v3pool: split test with pooled-parse connectors, 5 seeds, training text at 30K.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o jt jt.c -lm 2>/dev/null
6out=results-v3pool/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum jt.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5; do ./jt $s --v2 dj --tiers 30000 --splitdiag2 > "$out/seed-$s.tsv" & done
11wait
12echo "$out"
experiments/jigsaw-transformer/summarize-v3pool.py +29 -0
……@@ -0,0 +1,29 @@
1#!/usr/bin/env python3
2# Score results-v3pool/<stamp> against PREREG-v3pool.md. Usage: summarize-v3pool.py results-v3pool/<stamp> results-v3diag/<stamp>
3import glob, os, sys
4def mean(x): return sum(x) / len(x) if x else 0.0
5def load(d):
6 D = {}
7 for f in glob.glob(os.path.join(d, 'seed-*.tsv')):
8 for line in open(f):
9 p = line.rstrip('\n').split('\t')
10 if p[0] == 'D': D[(p[4], int(p[1]))] = p[5:]
11 return D
12D, O = load(sys.argv[1]), load(sys.argv[2])
13seeds = sorted({k[1] for k in D}); out = []
14def say(s=''): out.append(s); print(s)
15ok = all(D[('GG', s)] == O[('GG', s)] and D[('JG1', s)] == O[('JG', s)] for s in seeds)
16say(f'F0: {"PASS" if ok else "FAIL"} — GG and JG1 reproduce results-v3diag GG and JG')
17R = {}
18say('\ncell split precision recall base minor-prec minor-rec')
19for c in ['GG', 'JG1', 'JG6', 'PG1', 'PG6']:
20 r = [list(map(float, D[(c, s)])) for s in seeds]
21 prec = mean([x[2] / x[1] if x[1] else 0.0 for x in r]); rec = mean([x[2] / x[3] if x[3] else 0.0 for x in r]); base = mean([x[3] / x[0] for x in r])
22 R[c] = (prec, rec, base)
23 say(f'{c:4s} {mean([x[1] for x in r]):5.1f} {prec:9.3f} {rec:6.3f} {base:.3f} {mean([x[4] for x in r if x[6] > 0]):10.3f} {mean([x[5] for x in r if x[6] > 0]):9.3f}')
24say('')
25for n, c in [('F1', 'JG6'), ('F2', 'PG6')]:
26 say(f'{n}: {"PASS" if R[c][0] >= 2 * R[c][2] else "FAIL"} — {c} split precision {R[c][0]:.3f} (bar {2 * R[c][2]:.3f})')
27ok = R['JG6'][0] > R['JG1'][0] and R['PG6'][0] > R['PG1'][0]
28say(f'F3: {"PASS" if ok else "FAIL"} — precision JG6 {R["JG6"][0]:.3f} vs JG1 {R["JG1"][0]:.3f}; PG6 {R["PG6"][0]:.3f} vs PG1 {R["PG1"][0]:.3f}')
29open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')
experiments/level2/l2.c +33 -1
……@@ -274,6 +274,11 @@ static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, c
274274 e->soft += (float) c / KPARSE;
275275 }
276276 }
277 if (getenv("L2_SOFT_DEBUG")) for (size_t i = 0; i <= tmask; i++) if (tab[i].used && tab[i].cnum >= 3 && tab[i].soft / tab[i].cnum > 0.5) {
278 fprintf(stderr, "%s n=%u h=%u a=%.1f :", src, tab[i].cnum, tab[i].hard, tab[i].soft);
279 for (int k = 0; k < tab[i].len; k++) fprintf(stderr, " %d", tab[i].s[k]);
280 fprintf(stderr, "\n");
281 }
277282 size_t nt = 0, *idx = malloc(sizeof(size_t) * nfirst);
278283 for (size_t i = 0; i <= tmask; i++) if (tab[i].used) idx[nt++] = i;
279284 uint32_t *rank[3];
……@@ -285,10 +290,32 @@ static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, c
285290 for (size_t k = 0; k < nt; k++) rank[mth][idx[k]] = (uint32_t) k;
286291 }
287292 long gold = 0, hit[3][3] = { { 0 } }, cand[3][3] = { { 0 } }, cgold[3][3] = { { 0 } };
293 long bpred[4] = { 0 }, btp[4] = { 0 }; // Test B (PREREG-B.md): CNT, HARD-R, SOFT-R, SOFT-RI
294 static br_t cb[4][MAXN * LMAX];
295 rng_t r2 = { 2000003ULL * seed + 7919ULL * tier + (uint64_t) src[0] };
288296 br_t gb[MAXN];
289297 for (int s = 0; s < ho->ns; s++) {
290 int o = ho->off[s], n = ho->off[s + 1] - o, le6, ng = gold_spans(gold_ho, s, gb, &le6);
298 int o = ho->off[s], n = ho->off[s + 1] - o, le6, ng = gold_spans(gold_ho, s, gb, &le6), m = n + 1, nc[4] = { 0 };
291299 for (int b = 0; b < ng; b++) if (gb[b].j - gb[b].i + 1 <= LMAX) gold++;
300 for (int k = 0; k < KPARSE; k++) { fill_scores(ho, o, n, Sc, A, scale, &r2); eisner(m, hd); for (int d = 1; d < m; d++) hp[k][d - 1] = hd[d] - 1; }
301 for (int i = 0; i < n; i++) for (int len = 2; len <= LMAX && i + len <= n; len++) {
302 int j = i + len - 1, c = 0;
303 for (int k = 0; k < len; k++) x[k] = ho->c[o + i + k];
304 span_t *e = span_get(x, len, 0);
305 if (!e || e->cnum < 3) continue;
306 for (int k = 0; k < KPARSE; k++) c += complete(hp[k], 0, n, i, j);
307 double rh = (double) e->hard / e->cnum, rs = e->soft / e->cnum, ii = (double) c / KPARSE;
308 cb[0][nc[0]++] = (br_t) { i, j, e->cnum };
309 if (rh > 0.5) cb[1][nc[1]++] = (br_t) { i, j, rh };
310 if (rs > 0.5) cb[2][nc[2]++] = (br_t) { i, j, rs };
311 if (rs > 0.5 && ii > 0.5) cb[3][nc[3]++] = (br_t) { i, j, rs * ii };
312 }
313 for (int arm = 0; arm < 4; arm++) {
314 int k = select_brackets(cb[arm], nc[arm]);
315 bpred[arm] += k;
316 for (int a = 0; a < k; a++) for (int b = 0; b < ng; b++)
317 if (gb[b].j - gb[b].i + 1 <= LMAX && cb[arm][a].i == gb[b].i && cb[arm][a].j == gb[b].j) { btp[arm]++; break; }
318 }
292319 for (int i = 0; i < n; i++) for (int len = 2; len <= LMAX && i + len <= n; len++) {
293320 int j = i + len - 1, isg = 0;
294321 for (int k = 0; k < len; k++) x[k] = ho->c[o + i + k];
……@@ -305,6 +332,11 @@ static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, c
305332 for (int mth = 0; mth < 3; mth++) for (int bi = 0; bi < 3; bi++)
306333 printf("R\t%d\t%d\t%s\t%s\t%d\t%.5f\t%.5f\t%zu\n", seed, tier, src, mname[mth], budgets_soft[bi],
307334 gold ? (double) hit[mth][bi] / gold : 0, cand[mth][bi] ? (double) cgold[mth][bi] / cand[mth][bi] : 0, nt);
335 static const char *bname[4] = { "CNT", "HARD-R", "SOFT-R", "SOFT-RI" };
336 for (int arm = 0; arm < 4; arm++) {
337 double pr = bpred[arm] ? (double) btp[arm] / bpred[arm] : 0, rc = gold ? (double) btp[arm] / gold : 0;
338 printf("B\t%d\t%d\t%s\t%s\t%.5f\t%.5f\t%.5f\t%ld\n", seed, tier, src, bname[arm], pr, rc, pr + rc > 0 ? 2 * pr * rc / (pr + rc) : 0, bpred[arm]);
339 }
308340 for (int mth = 0; mth < 3; mth++) free(rank[mth]);
309341 free(idx);
310342}
experiments/level2/run-B.sh +12 -0
……@@ -0,0 +1,12 @@
1#!/bin/sh
2# PREREG-B full run: Test A + soft rankings + unit choice, 10 seeds x tiers 3K,10K,30K, in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -o l2 l2.c -lm 2>/dev/null
6out=results-B/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum l2.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5 6 7 8 9 10; do ./l2 $s --soft > "$out/seed-$s.tsv" & done
11wait
12echo "$out"
experiments/level2/summarize-B.py +40 -0
……@@ -0,0 +1,40 @@
1#!/usr/bin/env python3
2# Score results-B/<stamp> against PREREG-B.md. Usage: summarize-B.py results-B/<stamp> results-soft/<stamp>
3import glob, math, os, sys
4T975 = {9: 2.262}
5def mean(x): return sum(x) / len(x)
6def ci(d):
7 n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
8 h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
9 return m, m - h, m + h
10def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
11B = {}
12for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
13 for line in open(f):
14 p = line.rstrip('\n').split('\t')
15 if p[0] == 'B': B[(int(p[2]), p[3], p[4], int(p[1]))] = tuple(map(float, p[5:9]))
16seeds = sorted({k[3] for k in B}); tiers = sorted({k[0] for k in B}); top = max(tiers)
17out = []
18def say(s=''): out.append(s); print(s)
19bad = 0
20for s in seeds:
21 old = list(open(os.path.join(sys.argv[2], f'seed-{s}.tsv')))
22 new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if not l.startswith('B\t')]
23 bad += old != new
24say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A-soft exactly')
25def b(t, x, m, i): return [B[(t, x, m, s)][i] for s in seeds]
26def d(t, x, m1, m2): return ci([p - q for p, q in zip(b(t, x, m1, 2), b(t, x, m2, 2))])
27say('\n## Means: precision / recall / F1 / brackets chosen')
28for t in tiers:
29 for x in ['G', 'L']:
30 say(f'{t} {x}: ' + ' '.join(f'{m} {mean(b(t, x, m, 0)):.3f}/{mean(b(t, x, m, 1)):.3f}/{mean(b(t, x, m, 2)):.3f}/{mean(b(t, x, m, 3)):.0f}' for m in ['CNT', 'HARD-R', 'SOFT-R', 'SOFT-RI']))
31say('\n## Predictions')
32def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
33for x in ['G', 'L']:
34 ok = True; cells = []
35 for t in tiers: c = d(t, x, 'SOFT-R', 'CNT'); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
36 verdict(f'G1 {x}', ok, 'F1 SOFT-R − CNT ' + '; '.join(cells))
37 c = d(top, x, 'SOFT-R', 'HARD-R'); verdict(f'G2 {x}', c[1] > 0, f'F1 SOFT-R − HARD-R at {top} {fmt(c)}')
38 c = d(top, x, 'SOFT-RI', 'SOFT-R'); verdict(f'G3 {x}', c[1] > 0, f'F1 SOFT-RI − SOFT-R at {top} {fmt(c)}')
39m = mean(b(top, 'G', 'SOFT-RI', 2)); verdict('G4', m >= 0.5, f'F1 SOFT-RI (G) at {top} = {m:.3f} (bar 0.5)')
40open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')