OverviewFilesCommitsIssuesDocsReleasesPeers

atomspace-layer-discovery commits / 7918707

experiments: v3 split diagnosis (jt --splitdiag) and level2 soft ranking (l2 --soft), run + scorers
Control Z (noise 0: SOFT ranking == HARD) PASS and R0 PASS on seed 1, 3K. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79187077d309de236308e0c1f12a4128550ae0ef
by plankatron <fleeky@gmail.com> · 2026-10-18T22:19:44.000Z

Changed files (6) +265 -4

experiments/jigsaw-transformer/jt.c +68 -1
……@@ -4,6 +4,7 @@
44// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file] [--v1]
55// [--v2 word|dj] PREREG-v2 arms (implies the v1 pipeline); adds JT-<arm>-COLL rows
66// [--v3] PREREG-v3: V2-DJ + sense units split by disjuncts; adds S rows (eligible, split, split∩poly, poly∩eligible)
7// [--splitdiag] PREREG-v3diag: D rows (split test on gold/JT parse x gold/JT classes); skips audit and baselines
78// [--dump prefix] write <prefix>-<tier>.bin (final labels + attention parse) for ../level2/l2 --jt
89// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
910// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
……@@ -707,6 +708,70 @@ static void train_v3 (const corpus_t *tr, params_t *p, int **cls_out, double per
707708 free(tcon); free(tnc); free(start); free(occ); free(fill); free(minor); free(tru.w);
708709}
709710
711// PREREG-v3diag.md: the v3 split test on links from parse {gold tree, JT attention parse} x connector classes
712// {gold, JT label}. D rows: cell, eligible, split, split&poly, poly&eligible, mean minor-component precision and
713// recall over split polysemous words, number of such words.
714static void split_diag (const corpus_t *tr, const params_t *p, int seed, int tier) {
715 int N = tr->ntok, hd[MAXN + 1];
716 int *jl = malloc(sizeof(int) * N), *jh = malloc(sizeof(int) * N);
717 static sent_state_t S;
718 for (int s = 0; s < tr->ns; s++) {
719 int o = tr->off[s], n = tr->off[s + 1] - o, m = n + 1;
720 estep(tr, o, n, p, &S);
721 for (int t = 0; t < n; t++) { int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; jl[o + t] = b; }
722 for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
723 for (int d = 1; d < m; d++) sc[0][d] = 0;
724 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
725 double v = S.alpha[i][j] + S.alpha[j][i];
726 sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
727 }
728 eisner(m, hd);
729 for (int d = 1; d < m; d++) jh[o + d - 1] = hd[d] - 1;
730 }
731 int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * N), *fill = malloc(sizeof(int) * V);
732 for (int t = 0; t < N; t++) start[tr->w[t] + 1]++;
733 for (int w = 0; w < V; w++) start[w + 1] += start[w];
734 memcpy(fill, start, sizeof(int) * V);
735 for (int t = 0; t < N; t++) occ[fill[tr->w[t]]++] = t;
736 int *tcon = malloc(sizeof(int) * N * MAXC), *tnc = malloc(sizeof(int) * N), *minor = malloc(sizeof(int) * N);
737 static const char *cell[4] = { "GG", "GJ", "JG", "JJ" };
738 for (int c = 0; c < 4; c++) {
739 const int *head = c < 2 ? tr->h : jh, *cl = c % 2 == 0 ? tr->c : jl;
740 for (int s = 0; s < tr->ns; s++) {
741 int o = tr->off[s], n = tr->off[s + 1] - o;
742 for (int t = 0; t < n; t++) {
743 int nc = 0;
744 for (int j = 0; j < n; j++) {
745 if (j == t || (head[o + t] != j && head[o + j] != t)) continue;
746 if (nc < MAXC) tcon[(size_t) (o + t) * MAXC + nc++] = (j < t ? 0 : 64) + cl[o + j];
747 }
748 tnc[o + t] = nc;
749 }
750 }
751 long elig = 0, split = 0, split_poly = 0, poly_elig = 0, nmp = 0;
752 double mp = 0, mr = 0;
753 for (int w = 0; w < V; w++) {
754 int m = start[w + 1] - start[w];
755 const int *ow = occ + start[w];
756 if (m < SPLIT_MIN) continue;
757 elig++; poly_elig += poly[w];
758 if (!split_word(ow, m, tcon, tnc, minor)) continue;
759 split++;
760 if (!poly[w]) continue;
761 split_poly++;
762 int cnt[64] = { 0 }, c1 = -1, c2 = -1;
763 for (int i = 0; i < m; i++) cnt[tr->c[ow[i]]]++;
764 for (int g = 0; g < NCLS; g++) { if (c1 < 0 || cnt[g] > cnt[c1]) { c2 = c1; c1 = g; } else if (c2 < 0 || cnt[g] > cnt[c2]) c2 = g; }
765 long inm = 0, ins = 0, both = 0;
766 for (int i = 0; i < m; i++) { int a = minor[i], b = tr->c[ow[i]] == c2; inm += a; ins += b; both += a && b; }
767 mp += inm ? (double) both / inm : 0; mr += ins ? (double) both / ins : 0; nmp++;
768 }
769 printf("D\t%d\t%d\t%d\t%s\t%ld\t%ld\t%ld\t%ld\t%.5f\t%.5f\t%ld\n", seed, tier, KJ, cell[c], elig, split, split_poly, poly_elig,
770 nmp ? mp / nmp : 0, nmp ? mr / nmp : 0, nmp);
771 }
772 free(jl); free(jh); free(start); free(occ); free(fill); free(tcon); free(tnc); free(minor);
773}
774
710775static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
711776 if (V3) train_v3(tr, p, cls_out, perturb);
712777 else train_core(tr, tr, V, p, cls_out, perturb);
……@@ -779,13 +844,14 @@ static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
779844
780845int main (int argc, char **argv) {
781846 if (argc < 2) { fprintf(stderr, "usage: jt <seed> [--tiers a,b] [--heldout N] [--K k] [--shuffle] [--explain f] [--log f]\n"); return 1; }
782 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0;
847 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0, splitdiag = 0;
783848 const char *explain_path = NULL, *log_path = NULL, *dump_prefix = NULL;
784849 for (int i = 2; i < argc; i++) {
785850 if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
786851 else if (!strcmp(argv[i], "--K") && i + 1 < argc) KJ = atoi(argv[++i]);
787852 else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
788853 else if (!strcmp(argv[i], "--v1")) v1 = 1;
854 else if (!strcmp(argv[i], "--splitdiag")) splitdiag = 1;
789855 else if (!strcmp(argv[i], "--v3")) { v1 = 1; MODE = 2; V3 = 1; }
790856 else if (!strcmp(argv[i], "--v2") && i + 1 < argc) { v1 = 1; i++; MODE = !strcmp(argv[i], "word") ? 1 : !strcmp(argv[i], "dj") ? 2 : -1; if (MODE < 0) { fprintf(stderr, "--v2 word|dj\n"); return 1; } }
791857 else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
……@@ -840,6 +906,7 @@ int main (int argc, char **argv) {
840906 collapse(&ho, lab, KJ, lc); score_labels(&ho, lc, KJ, &vm, &m1, &se); free(lc);
841907 printf("V\t%d\t%d\t%d\tJT-%s-COLL%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
842908 printf("L\t%d\t%d\t%d\tATTN-%s%s\t%.5f\n", seed, tier, KJ, nm, tag, attention_parse_f1(&ho, &p));
909 if (splitdiag) { split_diag(&tr, &p, seed, tier); free(cls); continue; }
843910 if (V3) printf("S\t%d\t%d\t%d\tJT-%s%s\t%ld\t%ld\t%ld\t%ld\n", seed, tier, KJ, nm, tag, st_elig, st_split, st_split_poly, st_poly_elig);
844911 if (dump_prefix) { char dp[512]; snprintf(dp, sizeof dp, "%s-%d.bin", dump_prefix, tier); dump_layer1(dp, &tr, &ho, &p); }
845912 for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
experiments/jigsaw-transformer/run-v3diag.sh +12 -0
……@@ -0,0 +1,12 @@
1#!/bin/sh
2# PREREG-v3diag: split test on gold/JT parse x gold/JT classes, 5 seeds, training text at 30K.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o jt jt.c -lm 2>/dev/null
6out=results-v3diag/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum jt.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5; do ./jt $s --v2 dj --tiers 30000 --splitdiag > "$out/seed-$s.tsv" & done
11wait
12echo "$out"
experiments/jigsaw-transformer/summarize-v3diag.py +29 -0
……@@ -0,0 +1,29 @@
1#!/usr/bin/env python3
2# Score a results-v3diag/<stamp>/ directory against PREREG-v3diag.md.
3import glob, os, sys
4def mean(x): return sum(x) / len(x) if x else 0.0
5D = {}
6for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
7 for line in open(f):
8 p = line.rstrip('\n').split('\t')
9 if p[0] == 'D': D[(p[4], int(p[1]))] = (int(p[5]), int(p[6]), int(p[7]), int(p[8]), float(p[9]), float(p[10]), int(p[11]))
10seeds = sorted({k[1] for k in D}); out = []
11def say(s=''): out.append(s); print(s)
12R = {}
13say('cell eligible split precision recall base minor-prec minor-rec (words)')
14for c in ['GG', 'GJ', 'JG', 'JJ']:
15 r = [D[(c, s)] for s in seeds]
16 prec = mean([x[2] / x[1] if x[1] else 0.0 for x in r]); rec = mean([x[2] / x[3] if x[3] else 0.0 for x in r]); base = mean([x[3] / x[0] for x in r])
17 mp = mean([x[4] for x in r if x[6] > 0]); mr = mean([x[5] for x in r if x[6] > 0])
18 R[c] = (prec, rec, base, mp, mr)
19 say(f'{c} {mean([x[0] for x in r]):7.1f} {mean([x[1] for x in r]):5.1f} {prec:9.3f} {rec:6.3f} {base:.3f} {mp:10.3f} {mr:9.3f} ({mean([x[6] for x in r]):.1f})')
20prec, rec, base, mp, mr = R['GG']
21d1 = prec >= 2 * base and rec >= 0.3
22say(f'\nD1: {"PASS" if d1 else "FAIL"} — GG split precision {prec:.3f} (bar {2 * base:.3f}), recall {rec:.3f} (bar 0.3)')
23say(f'D2: {"PASS" if mp >= 0.5 and mr >= 0.5 else "FAIL"} — GG minor-component precision {mp:.3f}, recall {mr:.3f} (bars 0.5)')
24if d1:
25 dj, dg = rec - R['GJ'][1], rec - R['JG'][1]
26 say(f'D3: main failing input = {"the connector classes (GJ)" if dj > dg else "the parse (JG)"} — recall drop GJ {dj:.3f}, JG {dg:.3f}')
27else:
28 say('D3: the method fails (D1 FAIL with gold inputs)')
29open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')
experiments/level2/l2.c +105 -3
……@@ -106,10 +106,11 @@ static int complete (const int *head, int o, int n, int i, int j) {
106106
107107// ---------------------------------------------------------------- span-type table
108108
109typedef struct { uint16_t s[LMAX]; uint8_t len, used; uint32_t cnum; } span_t;
109typedef struct { uint16_t s[LMAX]; uint8_t len, used; uint32_t cnum, hard, first; float soft; } span_t; // hard/first/soft: Test A-soft
110110
111111static span_t *tab;
112112static size_t tmask;
113static uint32_t nfirst; // creation order (Test A-soft tie-break)
113114
114115static uint64_t span_hash (const int *x, int len) {
115116 uint64_t h = 1469598103934665603ULL ^ (uint64_t) len;
……@@ -123,7 +124,7 @@ static span_t *span_get (const int *x, int len, int create) {
123124 span_t *e = &tab[i];
124125 if (!e->used) {
125126 if (!create) return NULL;
126 e->used = 1; e->len = len;
127 e->used = 1; e->len = len; e->first = nfirst++;
127128 for (int k = 0; k < len; k++) e->s[k] = x[k];
128129 return e;
129130 }
……@@ -225,17 +226,103 @@ static void run_arm (int seed, int tier, const char *name, int rule, const corpu
225226 gold ? (double) inst / gold : 0, gold ? (double) typed / gold : 0, cand ? (double) cand_gold / cand : 0, gold, cand);
226227}
227228
229// ---------------------------------------------------------------- Test A-soft (PREREG-soft.md)
230
231#define KPARSE 6
232static const int budgets_soft[3] = { 1000, 3000, 10000 };
233static const span_t *rank_key_tab;
234static int rank_method; // 0 CNT, 1 HARD, 2 SOFT
235static int rank_cmp (const void *a, const void *b) {
236 const span_t *x = &rank_key_tab[*(const size_t *) a], *y = &rank_key_tab[*(const size_t *) b];
237 double kx = rank_method == 0 ? x->cnum : rank_method == 1 ? x->hard : x->soft;
238 double ky = rank_method == 0 ? y->cnum : rank_method == 1 ? y->hard : y->soft;
239 if (kx != ky) return kx > ky ? -1 : 1;
240 if (x->cnum != y->cnum) return x->cnum > y->cnum ? -1 : 1;
241 return x->first < y->first ? -1 : x->first > y->first;
242}
243
244// Fills ac.c's sc[][] for sentence o..o+n of S from gated scores Sc; Gumbel noise (scale) on gated edges.
245static void fill_scores (const corpus_t *S, int o, int n, const double *Sc, int A, double scale, rng_t *r) {
246 int m = n + 1;
247 for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
248 for (int d = 1; d < m; d++) sc[0][d] = 0;
249 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
250 double v = Sc[(size_t) S->c[o + i] * A + S->c[o + j]];
251 if (v > NEG && scale > 0) { double u = runif(r); if (u <= 0) u = 1e-300; v += scale * -log(-log(u)); }
252 sc[i + 1][j + 1] = sc[j + 1][i + 1] = v;
253 }
254}
255
256static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, const corpus_t *ho, const corpus_t *gold_ho,
257 const double *Sc, int A, double scale) {
258 memset(tab, 0, sizeof(span_t) * (tmask + 1));
259 nfirst = 0;
260 rng_t r = { 1000003ULL * seed + 7919ULL * tier + (uint64_t) src[0] };
261 int hd[MAXN + 1], hmax[MAXN], hp[KPARSE][MAXN], x[LMAX];
262 for (int s = 0; s < tr->ns; s++) {
263 int o = tr->off[s], n = tr->off[s + 1] - o, m = n + 1;
264 fill_scores(tr, o, n, Sc, A, 0, &r); eisner(m, hd);
265 for (int d = 1; d < m; d++) hmax[d - 1] = hd[d] - 1;
266 for (int k = 0; k < KPARSE; k++) { fill_scores(tr, o, n, Sc, A, scale, &r); eisner(m, hd); for (int d = 1; d < m; d++) hp[k][d - 1] = hd[d] - 1; }
267 for (int i = 0; i < n; i++) for (int len = 2; len <= LMAX && i + len <= n; len++) {
268 int j = i + len - 1, c = 0;
269 for (int k = 0; k < len; k++) x[k] = tr->c[o + i + k];
270 span_t *e = span_get(x, len, 1);
271 e->cnum++;
272 e->hard += complete(hmax, 0, n, i, j);
273 for (int k = 0; k < KPARSE; k++) c += complete(hp[k], 0, n, i, j);
274 e->soft += (float) c / KPARSE;
275 }
276 }
277 size_t nt = 0, *idx = malloc(sizeof(size_t) * nfirst);
278 for (size_t i = 0; i <= tmask; i++) if (tab[i].used) idx[nt++] = i;
279 uint32_t *rank[3];
280 rank_key_tab = tab;
281 for (int mth = 0; mth < 3; mth++) {
282 rank_method = mth;
283 qsort(idx, nt, sizeof(size_t), rank_cmp);
284 rank[mth] = malloc(sizeof(uint32_t) * (tmask + 1));
285 for (size_t k = 0; k < nt; k++) rank[mth][idx[k]] = (uint32_t) k;
286 }
287 long gold = 0, hit[3][3] = { { 0 } }, cand[3][3] = { { 0 } }, cgold[3][3] = { { 0 } };
288 br_t gb[MAXN];
289 for (int s = 0; s < ho->ns; s++) {
290 int o = ho->off[s], n = ho->off[s + 1] - o, le6, ng = gold_spans(gold_ho, s, gb, &le6);
291 for (int b = 0; b < ng; b++) if (gb[b].j - gb[b].i + 1 <= LMAX) gold++;
292 for (int i = 0; i < n; i++) for (int len = 2; len <= LMAX && i + len <= n; len++) {
293 int j = i + len - 1, isg = 0;
294 for (int k = 0; k < len; k++) x[k] = ho->c[o + i + k];
295 span_t *e = span_get(x, len, 0);
296 if (!e) continue;
297 for (int b = 0; b < ng; b++) if (gb[b].i == i && gb[b].j == j) isg = 1;
298 size_t slot = (size_t) (e - tab);
299 for (int mth = 0; mth < 3; mth++) for (int bi = 0; bi < 3; bi++) if (rank[mth][slot] < (uint32_t) budgets_soft[bi]) {
300 cand[mth][bi]++; if (isg) { cgold[mth][bi]++; hit[mth][bi]++; }
301 }
302 }
303 }
304 static const char *mname[3] = { "CNT", "HARD", "SOFT" };
305 for (int mth = 0; mth < 3; mth++) for (int bi = 0; bi < 3; bi++)
306 printf("R\t%d\t%d\t%s\t%s\t%d\t%.5f\t%.5f\t%zu\n", seed, tier, src, mname[mth], budgets_soft[bi],
307 gold ? (double) hit[mth][bi] / gold : 0, cand[mth][bi] ? (double) cgold[mth][bi] / cand[mth][bi] : 0, nt);
308 for (int mth = 0; mth < 3; mth++) free(rank[mth]);
309 free(idx);
310}
311
228312// ---------------------------------------------------------------- main
229313
230314int main (int argc, char **argv) {
231 if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix]\n"); return 1; }
315 if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix] [--soft [--soft-scale s]]\n"); return 1; }
232316 int seed = atoi(argv[1]);
233317 int tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000;
234318 const char *jt_prefix = NULL;
319 int soft = 0; double soft_scale = 1.0;
235320 for (int i = 2; i < argc; i++) {
236321 if (!strcmp(argv[i], "--mut-drop")) mut_drop = 1;
237322 else if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
238323 else if (!strcmp(argv[i], "--jt") && i + 1 < argc) jt_prefix = argv[++i];
324 else if (!strcmp(argv[i], "--soft")) soft = 1;
325 else if (!strcmp(argv[i], "--soft-scale") && i + 1 < argc) soft_scale = atof(argv[++i]);
239326 else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
240327 ntier = 0;
241328 for (char *p = strtok(argv[++i], ","); p && ntier < 16; p = strtok(NULL, ",")) tiers[ntier++] = atoi(p);
……@@ -317,6 +404,21 @@ int main (int argc, char **argv) {
317404 }
318405 free(Lmap);
319406
407 // PREREG-soft.md: rank span types by assembly-ness over KPARSE perturbed parses (sources G and L).
408 if (soft) for (int xi = 0; xi < 2; xi++) {
409 int x = xi ? SRC_L : SRC_G;
410 int *Lm = exchange(&tr, &passes);
411 int A = layer1_symbols(x, &tr, seed, Lm, Rmap, str);
412 layer1_symbols(x, &ho, seed, Lm, Rmap, sho);
413 corpus_t trs = with_symbols(&tr, str), hos = with_symbols(&ho, sho);
414 arm_t a = { "l1", A, NULL, 1 };
415 pairs_t P = count_pairs(&a, &trs);
416 long ev, acc;
417 double *Sc = hard_scores(&P, &ev, &acc);
418 run_soft(seed, tier, src_name[x], &trs, &hos, &ho, Sc, A, soft_scale);
419 free(Sc); free_pairs(&P); free(Lm);
420 }
421
320422 // PREREG-JT.md: layer 1 from the jigsaw transformer (labels + its attention parse), from jt --dump.
321423 if (jt_prefix) {
322424 char path[512];
experiments/level2/run-soft.sh +12 -0
……@@ -0,0 +1,12 @@
1#!/bin/sh
2# PREREG-soft full run: Test A + soft rankings, 10 seeds x tiers 3K,10K,30K, in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -o l2 l2.c -lm 2>/dev/null
6out=results-soft/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum l2.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5 6 7 8 9 10; do ./l2 $s --soft > "$out/seed-$s.tsv" & done
11wait
12echo "$out"
experiments/level2/summarize-soft.py +39 -0
……@@ -0,0 +1,39 @@
1#!/usr/bin/env python3
2# Score a results-soft/<stamp>/ directory against PREREG-soft.md. Usage: summarize-soft.py results-soft/<stamp> results/<testA-stamp>
3import glob, math, os, sys
4T975 = {9: 2.262}
5def mean(x): return sum(x) / len(x)
6def ci(d):
7 n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
8 h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
9 return m, m - h, m + h
10def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
11R = {}
12for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
13 for line in open(f):
14 p = line.rstrip('\n').split('\t')
15 if p[0] == 'R': R[(int(p[2]), p[3], p[4], int(p[5]), int(p[1]))] = (float(p[6]), float(p[7]))
16seeds = sorted({k[4] for k in R}); tiers = sorted({k[0] for k in R}); top = max(tiers)
17out = []
18def say(s=''): out.append(s); print(s)
19bad = 0
20for s in seeds:
21 old = [l for l in open(os.path.join(sys.argv[2], f'seed-{s}.tsv'))]
22 new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if not l.startswith('R\t')]
23 bad += old != new
24say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A exactly')
25def r(t, x, m, b, i): return [R[(t, x, m, b, s)][i] for s in seeds]
26def d(t, x, m1, m2, b, i): return ci([p - q for p, q in zip(r(t, x, m1, b, i), r(t, x, m2, b, i))])
27say('\n## Means: R@B / P@B')
28for t in tiers:
29 for x in ['G', 'L']:
30 say(f'{t} {x}: ' + ' '.join(f'{m} ' + ' '.join(f'{mean(r(t, x, m, b, 0)):.3f}/{mean(r(t, x, m, b, 1)):.3f}' for b in (1000, 3000, 10000)) for m in ['CNT', 'HARD', 'SOFT']))
31say('\n## Predictions')
32def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
33for x in ['G', 'L']:
34 ok = True; cells = []
35 for t in tiers: c = d(t, x, 'SOFT', 'CNT', 3000, 0); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
36 verdict(f'E1 {x}', ok, 'R@3K SOFT − CNT ' + '; '.join(cells))
37 c = d(top, x, 'SOFT', 'HARD', 3000, 0); verdict(f'E2 {x}', c[1] > 0, f'R@3K SOFT − HARD at {top} {fmt(c)}')
38 c = d(top, x, 'SOFT', 'CNT', 3000, 1); verdict(f'E3 {x}', c[1] > 0, f'P@3K SOFT − CNT at {top} {fmt(c)}')
39open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')