OverviewFilesCommitsIssuesDocsReleasesPeers

atomspace-layer-discovery commits / 5bf8c7a

experiments: v3 sense split (jt --v3), JT layer-1 dump (jt --dump, l2 --jt), run + scorers
v1 and V2-DJ outputs byte-identical after the refactor (seed 1, 3K); Test A arms byte-identical with --jt (R0 smoke). v3 smoke (seed 1, 3K): runs, audit PASS, 0 of 156 eligible words split (mixture G below the chi2 gate; checked with JT_SPLIT_DEBUG, not a bug). No parameters changed after the smoke. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5bf8c7ad9d052fbeb205bcaafdb535adf0af093d
by plankatron <fleeky@gmail.com> · 2026-10-18T21:52:30.000Z

Changed files (7) +349 -19

experiments/jigsaw-transformer/jt.c +186 -18
……@@ -3,6 +3,8 @@
33//
44// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file] [--v1]
55// [--v2 word|dj] PREREG-v2 arms (implies the v1 pipeline); adds JT-<arm>-COLL rows
6// [--v3] PREREG-v3: V2-DJ + sense units split by disjuncts; adds S rows (eligible, split, split∩poly, poly∩eligible)
7// [--dump prefix] write <prefix>-<tier>.bin (final labels + attention parse) for ../level2/l2 --jt
68// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
79// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
810// Output rows (stdout):
……@@ -23,6 +25,10 @@
2325
2426static int KJ = 30;
2527static int MODE = 0; // 0 v1 (class profiles), 1 V2-WORD (neighbour words), 2 V2-DJ (disjuncts); PREREG-v2.md
28static int V3 = 0; // PREREG-v3.md: V2-DJ, then split words into sense units by their disjuncts, re-learn
29static int VU; // size of the layer-0 alphabet (words, or sense units in v3)
30static uint64_t unit_hash; // v3: hash of the training unit assignment (audit)
31static long st_elig, st_split, st_split_poly, st_poly_elig; // v3 split diagnostics, last training call
2632
2733static uint64_t fnv_bytes (uint64_t h, const void *p, size_t n);
2834
……@@ -466,13 +472,13 @@ static double *word_scores (const corpus_t *tr) {
466472}
467473
468474// Layer-0 attention of occurrence t over its window, from word-pair PMI. Returns 0 if no neighbour passes.
469static int word_attention (const corpus_t *C, int o, int n, int t, const double *SW, double *alpha) {
475static int word_attention (const corpus_t *C, int o, int n, int t, const double *SW, int A, double *alpha) {
470476 double tot = 0;
471477 for (int j = 0; j < n; j++) {
472478 alpha[j] = 0;
473479 if (j == t || abs(j - t) > W) continue;
474480 int a = t < j ? C->w[o + t] : C->w[o + j], b = t < j ? C->w[o + j] : C->w[o + t];
475 double v = SW[(size_t) a * V + b];
481 double v = SW[(size_t) a * A + b];
476482 if (v > NEG) { alpha[j] = v; tot += v; }
477483 }
478484 if (tot <= 0) return 0;
……@@ -534,35 +540,39 @@ static int *exchange_weighted (const double *Wp, const corpus_t *tr) {
534540}
535541
536542// Full v1 learner on a training prefix. Leaves params in p and the layer-0 classes in *cls_out.
537static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
538 double *SW = word_scores(tr);
539 double *Wp = calloc((size_t) V * V, sizeof(double)), alpha[MAXN];
540 for (int s = 0; s < tr->ns; s++) {
541 int o = tr->off[s], n = tr->off[s + 1] - o;
543// tru: the same text with layer-0 symbols (words, or v3 sense units, alphabet vu); EM always runs over words (tr).
544static void train_core (const corpus_t *tr, const corpus_t *tru, int vu, params_t *p, int **cls_out, double perturb) {
545 int V0 = V;
546 V = VU = vu; // layer 0 and exchange over the unit alphabet
547 double *SW = word_scores(tru);
548 double *Wp = calloc((size_t) vu * vu, sizeof(double)), alpha[MAXN];
549 for (int s = 0; s < tru->ns; s++) {
550 int o = tru->off[s], n = tru->off[s + 1] - o;
542551 for (int t = 0; t < n; t++) {
543 if (!word_attention(tr, o, n, t, SW, alpha)) continue;
552 if (!word_attention(tru, o, n, t, SW, vu, alpha)) continue;
544553 for (int j = 0; j < n; j++) if (alpha[j] > 0) {
545 int a = t < j ? tr->w[o + t] : tr->w[o + j], b = t < j ? tr->w[o + j] : tr->w[o + t];
546 Wp[(size_t) a * V + b] += alpha[j];
554 int a = t < j ? tru->w[o + t] : tru->w[o + j], b = t < j ? tru->w[o + j] : tru->w[o + t];
555 Wp[(size_t) a * vu + b] += alpha[j];
547556 }
548557 }
549558 }
550 int *cls = exchange_weighted(Wp, tr);
559 int *cls = exchange_weighted(Wp, tru);
560 V = V0;
551561 // Initial counts: one-hot classes, word-level attention as messages.
552562 counts_t c; counts_alloc(&c);
553563 static sent_state_t S;
554564 for (int s = 0; s < tr->ns; s++) {
555565 int o = tr->off[s], n = tr->off[s + 1] - o;
556566 S.n = n;
557 for (int t = 0; t < n; t++) { for (int z = 0; z < KJ; z++) S.q[t][z] = 0; S.q[t][cls[tr->w[o + t]]] = 1; }
567 for (int t = 0; t < n; t++) { for (int z = 0; z < KJ; z++) S.q[t][z] = 0; S.q[t][cls[tru->w[o + t]]] = 1; }
558568 for (int t = 0; t < n; t++) {
559569 for (int y = 0; y < KJ; y++) { S.mL[t][y] = 0; S.mR[t][y] = 0; }
560 int ok = word_attention(tr, o, n, t, SW, alpha);
570 int ok = word_attention(tru, o, n, t, SW, vu, alpha);
561571 memcpy(S.alpha[t], alpha, sizeof(double) * n);
562572 if (!ok) continue;
563 for (int j = 0; j < n; j++) if (alpha[j] > 0) (j < t ? S.mL[t] : S.mR[t])[cls[tr->w[o + j]]] += alpha[j];
573 for (int j = 0; j < n; j++) if (alpha[j] > 0) (j < t ? S.mL[t] : S.mR[t])[cls[tru->w[o + j]]] += alpha[j];
564574 }
565 if (MODE == 2) { int cl[MAXN]; for (int j = 0; j < n; j++) cl[j] = cls[tr->w[o + j]]; disjuncts(&S, cl); }
575 if (MODE == 2) { int cl[MAXN]; for (int j = 0; j < n; j++) cl[j] = cls[tru->w[o + j]]; disjuncts(&S, cl); }
566576 accumulate(tr, o, &S, &c);
567577 }
568578 c.N[0] += perturb; // audit mutation test only
……@@ -576,6 +586,132 @@ static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double per
576586 *cls_out = cls;
577587}
578588
589// ---------------------------------------------------------------- v3 (PREREG-v3.md): sense units from disjuncts
590
591#define NCODE 128 // connector code = side * 64 + class
592#define SPLIT_MIN 20
593#define SPLIT_ADD 0.1
594
595// Log-likelihood of bags b[0..m) under multinomial counts c (add-SPLIT_ADD over NCODE codes).
596static double bag_ll (const int *codes, int nc, const double *lth) { double s = 0; for (int k = 0; k < nc; k++) s += lth[codes[k]]; return s; }
597static void mult_log (const double *c, double *lth) {
598 double tot = 0;
599 for (int k = 0; k < NCODE; k++) tot += c[k];
600 for (int k = 0; k < NCODE; k++) lth[k] = log((c[k] + SPLIT_ADD) / (tot + SPLIT_ADD * NCODE));
601}
602static double chi2_999 (int df) { double a = 2.0 / (9 * df), x = 1 - a + 3.0902 * sqrt(a); return df * x * x * x; } // Wilson–Hilferty
603
604// Split test for one word over its occurrences occ[0..m) (indices into tcon/tnc). Sets minor[i] = 1 for the
605// smaller component's occurrences. Returns 1 if the split is accepted.
606static int split_word (const int *occ, int m, const int *tcon, const int *tnc, int *minor) {
607 int *O = malloc(sizeof(int) * m), no = 0, used[NCODE] = { 0 }, kc = 0;
608 for (int i = 0; i < m; i++) if (tnc[occ[i]] > 0) O[no++] = i;
609 double c1[NCODE] = { 0 }, l1[NCODE], cA[NCODE], cB[NCODE], lA[NCODE], lB[NCODE];
610 for (int i = 0; i < no; i++) for (int k = 0; k < tnc[occ[O[i]]]; k++) c1[tcon[occ[O[i]] * MAXC + k]]++;
611 for (int k = 0; k < NCODE; k++) if (c1[k] > 0) { used[k] = 1; kc++; }
612 mult_log(c1, l1);
613 double LL1 = 0;
614 for (int i = 0; i < no; i++) LL1 += bag_ll(tcon + occ[O[i]] * MAXC, tnc[occ[O[i]]], l1);
615 // Initial split: the connector whose presence/absence gives the largest G (hard two-multinomial fit).
616 int best = -1; double bestG = -1;
617 for (int code = 0; code < NCODE; code++) {
618 if (!used[code]) continue;
619 memset(cA, 0, sizeof cA); memset(cB, 0, sizeof cB);
620 int nB = 0;
621 for (int i = 0; i < no; i++) {
622 const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
623 for (int k = 0; k < nc; k++) has |= b[k] == code;
624 nB += has;
625 for (int k = 0; k < nc; k++) (has ? cB : cA)[b[k]]++;
626 }
627 if (nB == 0 || nB == no) continue;
628 mult_log(cA, lA); mult_log(cB, lB);
629 double ll = 0;
630 for (int i = 0; i < no; i++) {
631 const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
632 for (int k = 0; k < nc; k++) has |= b[k] == code;
633 ll += bag_ll(b, nc, has ? lB : lA);
634 }
635 if (2 * (ll - LL1) > bestG) { bestG = 2 * (ll - LL1); best = code; }
636 }
637 int accept = 0;
638 if (best >= 0) {
639 double *r = malloc(sizeof(double) * no), wB = 0, LL2 = 0;
640 for (int i = 0; i < no; i++) {
641 const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
642 for (int k = 0; k < nc; k++) has |= b[k] == best;
643 r[i] = has;
644 }
645 for (int it = 0; it <= 20; it++) { // 20 EM passes, then one more E-step for LL2 and the posteriors
646 memset(cA, 0, sizeof cA); memset(cB, 0, sizeof cB); wB = 0;
647 for (int i = 0; i < no; i++) {
648 const int *b = tcon + occ[O[i]] * MAXC;
649 for (int k = 0; k < tnc[occ[O[i]]]; k++) { cA[b[k]] += 1 - r[i]; cB[b[k]] += r[i]; }
650 wB += r[i];
651 }
652 wB /= no;
653 if (wB <= 0 || wB >= 1) break;
654 mult_log(cA, lA); mult_log(cB, lB);
655 LL2 = 0;
656 for (int i = 0; i < no; i++) {
657 const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]];
658 double a = log(1 - wB) + bag_ll(b, nc, lA), bb = log(wB) + bag_ll(b, nc, lB), mx = a > bb ? a : bb;
659 double lse = mx + log(exp(a - mx) + exp(bb - mx));
660 LL2 += lse; r[i] = exp(bb - lse);
661 }
662 }
663 double G = 2 * (LL2 - LL1), wmin = wB < 0.5 ? wB : 1 - wB;
664 accept = wB > 0 && wB < 1 && G > chi2_999(kc) && wmin >= 0.05;
665 if (getenv("JT_SPLIT_DEBUG")) fprintf(stderr, "split m=%d no=%d kc=%d initG=%.1f G=%.1f thr=%.1f wB=%.3f LL1=%.1f LL2=%.1f accept=%d\n", m, no, kc, bestG, G, chi2_999(kc), wB, LL1, LL2, accept);
666 for (int i = 0; i < m; i++) minor[i] = 0;
667 if (accept) for (int i = 0; i < no; i++) minor[O[i]] = wB < 0.5 ? r[i] > 0.5 : r[i] <= 0.5;
668 free(r);
669 }
670 free(O);
671 return accept;
672}
673
674static void train_v3 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
675 int *cls0;
676 train_core(tr, tr, V, p, &cls0, 0);
677 free(cls0);
678 int *tcon = malloc(sizeof(int) * tr->ntok * MAXC), *tnc = malloc(sizeof(int) * tr->ntok);
679 static sent_state_t S;
680 for (int s = 0; s < tr->ns; s++) {
681 int o = tr->off[s];
682 estep(tr, o, tr->off[s + 1] - o, p, &S);
683 for (int t = 0; t < S.n; t++) { tnc[o + t] = S.nc[t]; memcpy(tcon + (size_t) (o + t) * MAXC, S.con[t], sizeof(int) * MAXC); }
684 }
685 // Occurrences grouped by word (counting sort).
686 int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * tr->ntok), *fill = malloc(sizeof(int) * V);
687 for (int t = 0; t < tr->ntok; t++) start[tr->w[t] + 1]++;
688 for (int w = 0; w < V; w++) start[w + 1] += start[w];
689 memcpy(fill, start, sizeof(int) * V);
690 for (int t = 0; t < tr->ntok; t++) occ[fill[tr->w[t]]++] = t;
691 corpus_t tru = *tr;
692 tru.w = malloc(sizeof(int) * tr->ntok);
693 memcpy(tru.w, tr->w, sizeof(int) * tr->ntok);
694 int vu = V, *minor = malloc(sizeof(int) * tr->ntok);
695 st_elig = st_split = st_split_poly = st_poly_elig = 0;
696 for (int w = 0; w < V; w++) {
697 int m = start[w + 1] - start[w];
698 if (m < SPLIT_MIN) continue;
699 st_elig++; st_poly_elig += poly[w];
700 if (!split_word(occ + start[w], m, tcon, tnc, minor)) continue;
701 st_split++; st_split_poly += poly[w];
702 for (int i = 0; i < m; i++) if (minor[i]) tru.w[occ[start[w] + i]] = vu;
703 vu++;
704 }
705 unit_hash = fnv_bytes(1469598103934665603ULL, tru.w, sizeof(int) * tr->ntok);
706 train_core(tr, &tru, vu, p, cls_out, perturb);
707 free(tcon); free(tnc); free(start); free(occ); free(fill); free(minor); free(tru.w);
708}
709
710static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
711 if (V3) train_v3(tr, p, cls_out, perturb);
712 else train_core(tr, tr, V, p, cls_out, perturb);
713}
714
579715static uint64_t params_hash (const params_t *p, const int *cls) {
580716 uint64_t h = 1469598103934665603ULL;
581717 h = fnv_bytes(h, p->lpi, sizeof(double) * V * KJ);
……@@ -583,7 +719,35 @@ static uint64_t params_hash (const params_t *p, const int *cls) {
583719 h = fnv_bytes(h, p->pmi, sizeof(double) * KJ * KJ);
584720 if (MODE == 1) { h = fnv_bytes(h, p->lpsiL, sizeof(double) * KJ * V); h = fnv_bytes(h, p->lpsiR, sizeof(double) * KJ * V); }
585721 if (MODE == 2) { h = fnv_bytes(h, p->dj.key, sizeof(uint64_t) * p->dj.cap); h = fnv_bytes(h, p->dj.cnt, sizeof(double) * p->dj.cap * KJ); }
586 return fnv_bytes(h, cls, sizeof(int) * V);
722 if (V3) h = fnv_bytes(h, &unit_hash, sizeof unit_hash);
723 return fnv_bytes(h, cls, sizeof(int) * VU);
724}
725
726// Layer-1 dump for ../level2 (PREREG-JT.md): per corpus ntok, labels[ntok], heads[ntok] (in-sentence offset, -1 root).
727static void dump_layer1 (const char *path, const corpus_t *tr, const corpus_t *ho, const params_t *p) {
728 FILE *f = fopen(path, "wb");
729 if (!f) { fprintf(stderr, "cannot write %s\n", path); exit(1); }
730 static sent_state_t S;
731 for (int k = 0; k < 2; k++) {
732 const corpus_t *C = k ? ho : tr;
733 int *lab = malloc(sizeof(int) * C->ntok), *head = malloc(sizeof(int) * C->ntok), hd[MAXN + 1];
734 for (int s = 0; s < C->ns; s++) {
735 int o = C->off[s], n = C->off[s + 1] - o, m = n + 1;
736 estep(C, o, n, p, &S);
737 for (int t = 0; t < n; t++) { int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; lab[o + t] = b; }
738 for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
739 for (int d = 1; d < m; d++) sc[0][d] = 0;
740 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
741 double v = S.alpha[i][j] + S.alpha[j][i];
742 sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
743 }
744 eisner(m, hd);
745 for (int d = 1; d < m; d++) head[o + d - 1] = hd[d] - 1;
746 }
747 fwrite(&C->ntok, sizeof(int), 1, f); fwrite(lab, sizeof(int), C->ntok, f); fwrite(head, sizeof(int), C->ntok, f);
748 free(lab); free(head);
749 }
750 fclose(f);
587751}
588752
589753// X-COLL (PREREG-v2): every occurrence of a word gets the word's most frequent label in lab.
……@@ -616,15 +780,17 @@ static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
616780int main (int argc, char **argv) {
617781 if (argc < 2) { fprintf(stderr, "usage: jt <seed> [--tiers a,b] [--heldout N] [--K k] [--shuffle] [--explain f] [--log f]\n"); return 1; }
618782 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0;
619 const char *explain_path = NULL, *log_path = NULL;
783 const char *explain_path = NULL, *log_path = NULL, *dump_prefix = NULL;
620784 for (int i = 2; i < argc; i++) {
621785 if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
622786 else if (!strcmp(argv[i], "--K") && i + 1 < argc) KJ = atoi(argv[++i]);
623787 else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
624788 else if (!strcmp(argv[i], "--v1")) v1 = 1;
789 else if (!strcmp(argv[i], "--v3")) { v1 = 1; MODE = 2; V3 = 1; }
625790 else if (!strcmp(argv[i], "--v2") && i + 1 < argc) { v1 = 1; i++; MODE = !strcmp(argv[i], "word") ? 1 : !strcmp(argv[i], "dj") ? 2 : -1; if (MODE < 0) { fprintf(stderr, "--v2 word|dj\n"); return 1; } }
626791 else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
627792 else if (!strcmp(argv[i], "--log") && i + 1 < argc) log_path = argv[++i];
793 else if (!strcmp(argv[i], "--dump") && i + 1 < argc) dump_prefix = argv[++i];
628794 else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
629795 ntier = 0;
630796 for (char *p = strtok(argv[++i], ","); p && ntier < 16; p = strtok(NULL, ",")) tiers[ntier++] = atoi(p);
……@@ -638,7 +804,7 @@ int main (int argc, char **argv) {
638804 for (int i = 0; i < ntier; i++) if (tiers[i] > maxt) maxt = tiers[i];
639805 corpus_t full = gen_corpus(1000003ULL * seed + 1, maxt), ho = gen_corpus(1000003ULL * seed + 2, nho);
640806 if (shuffle) { full = shuffle_within(full, 99ULL * seed + 1); ho = shuffle_within(ho, 99ULL * seed + 2); }
641 const char *tag = shuffle ? "-SHUF" : "", *nm = MODE == 0 ? "V1" : MODE == 1 ? "V2W" : "V2D";
807 const char *tag = shuffle ? "-SHUF" : "", *nm = V3 ? "V3" : MODE == 0 ? "V1" : MODE == 1 ? "V2W" : "V2D";
642808 int *lab = malloc(sizeof(int) * ho.ntok);
643809 params_t p; params_alloc(&p);
644810
……@@ -674,6 +840,8 @@ int main (int argc, char **argv) {
674840 collapse(&ho, lab, KJ, lc); score_labels(&ho, lc, KJ, &vm, &m1, &se); free(lc);
675841 printf("V\t%d\t%d\t%d\tJT-%s-COLL%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
676842 printf("L\t%d\t%d\t%d\tATTN-%s%s\t%.5f\n", seed, tier, KJ, nm, tag, attention_parse_f1(&ho, &p));
843 if (V3) printf("S\t%d\t%d\t%d\tJT-%s%s\t%ld\t%ld\t%ld\t%ld\n", seed, tier, KJ, nm, tag, st_elig, st_split, st_split_poly, st_poly_elig);
844 if (dump_prefix) { char dp[512]; snprintf(dp, sizeof dp, "%s-%d.bin", dump_prefix, tier); dump_layer1(dp, &tr, &ho, &p); }
677845 for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
678846 score_labels(&ho, lab, KJ, &vm, &m1, &se);
679847 printf("V\t%d\t%d\t%d\tTYPE-A%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
experiments/jigsaw-transformer/run-v3.sh +17 -0
……@@ -0,0 +1,17 @@
1#!/bin/sh
2# PREREG-v3 full run: v3 and V2-DJ x 5 seeds x tiers 3K,10K,30K, normal and shuffled-within-sentence, in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o jt jt.c -lm 2>/dev/null
6out=results-v3/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum jt.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5; do
11 ./jt $s --v3 > "$out/v3-seed-$s.tsv" &
12 ./jt $s --v3 --shuffle > "$out/v3-shuf-$s.tsv" &
13 ./jt $s --v2 dj > "$out/dj-seed-$s.tsv" &
14 ./jt $s --v2 dj --shuffle > "$out/dj-shuf-$s.tsv" &
15done
16wait
17echo "$out"
experiments/jigsaw-transformer/summarize-v3.py +55 -0
……@@ -0,0 +1,55 @@
1#!/usr/bin/env python3
2# Score a results-v3/<stamp>/ directory against PREREG-v3.md. Usage: summarize-v3.py results-v3/<stamp>
3import glob, math, os, sys
4
5T975 = {4: 2.776}
6def mean(x): return sum(x) / len(x)
7def ci(d):
8 n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
9 h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
10 return m, m - h, m + h
11
12V, L, C, A, S = {}, {}, {}, [], {}
13for f in sorted(glob.glob(os.path.join(sys.argv[1], '*.tsv'))):
14 arm = os.path.basename(f).split('-')[0]
15 for line in open(f):
16 p = line.rstrip('\n').split('\t')
17 if p[0] == 'V': V[(int(p[2]), p[4], int(p[1]))] = tuple(map(float, p[5:8]))
18 elif p[0] == 'L': L[(int(p[2]), p[4], int(p[1]))] = float(p[5])
19 elif p[0] == 'C': C[(int(p[2]), p[4], int(p[5]), int(p[1]))] = float(p[6])
20 elif p[0] == 'S': S[(int(p[2]), p[4], int(p[1]))] = tuple(map(int, p[5:9])) # elig, split, split&poly, poly&elig
21 elif p[0] == 'A': A.append((arm, p[4], p[5]))
22seeds = sorted({k[2] for k in V}); tiers = sorted({k[0] for k in V}); top = max(tiers)
23out = []
24def say(s=''): out.append(s); print(s)
25def v(t, m, i): return [V[(t, m, s)][i] for s in seeds]
26def d(t, a, b, i): return ci([x - y for x, y in zip(v(t, a, i), v(t, b, i))])
27def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
28
29say('## Token classes (mean): V-measure / many-to-one / sense accuracy')
30for t in tiers:
31 say(f'tier {t}')
32 for m in [f'JT-{a}{c}{s}' for s in ['', '-SHUF'] for a in ['V2D', 'V3'] for c in ['', '-COLL']] + ['MAJ']:
33 if (t, m, seeds[0]) in V: say(f' {m:16s} ' + ' / '.join(f'{mean(v(t, m, i)):.3f}' for i in range(3)))
34say('\n## Split diagnostics (mean over seeds): eligible, split, split precision, split recall, base rate')
35def sp(t, m):
36 r = [S[(t, m, s)] for s in seeds]
37 prec = [x[2] / x[1] if x[1] else 0.0 for x in r]; rec = [x[2] / x[3] if x[3] else 0.0 for x in r]; base = [x[3] / x[0] if x[0] else 0.0 for x in r]
38 return mean([x[0] for x in r]), mean([x[1] for x in r]), mean(prec), mean(rec), mean(base)
39for t in tiers:
40 for m in ['JT-V3', 'JT-V3-SHUF']:
41 if (t, m, seeds[0]) in S: say(f' {t} {m:11s} ' + ' '.join(f'{x:.3f}' if i > 1 else f'{x:.1f}' for i, x in enumerate(sp(t, m))))
42say('\n## Parse link F1 (mean)')
43for t in tiers: say(f'{t}: ' + ' '.join(f'{m} {mean([L[(t, m, s)] for s in seeds]):.3f}' for m in ['ATTN-V2D', 'ATTN-V3'] if (t, m, seeds[0]) in L))
44
45say(f'\n## Predictions (at {top})')
46def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
47e, ns, prec, rec, base = sp(top, 'JT-V3')
48verdict('S1', ns > 0 and prec >= 2 * base and rec >= 0.3, f'split precision {prec:.3f} (bar {2 * base:.3f} = 2 x base), recall {rec:.3f} (bar 0.3), words split {ns:.1f} of {e:.1f}')
49c = d(top, 'JT-V3-COLL', 'JT-V2D-COLL', 2); verdict('S2', c[1] > 0, f'collapsed sense V3 − V2D {fmt(c)}')
50c = d(top, 'JT-V3', 'JT-V2D', 2); verdict('S3', c[1] > 0, f'sense V3 − V2D {fmt(c)}')
51nshuf = sp(top, 'JT-V3-SHUF')[1]; c = d(top, 'JT-V3-SHUF', 'JT-V2D-SHUF', 2)
52verdict('S4', nshuf <= 0.5 * ns and not c[1] > 0, f'words split shuffled {nshuf:.1f} vs ordered {ns:.1f} (bar <= 0.5x); shuffled sense V3 − V2D {fmt(c)}')
53c = d(top, 'JT-V3', 'JT-V2D', 0); verdict('S5', c[1] > -0.02, f'V-measure V3 − V2D {fmt(c)}')
54r = [x for x in A if x[0] == 'v3']; verdict('S6', len(r) > 0 and all(x[2] == 'PASS' for x in r), f'{sum(x[2] == "PASS" for x in r)}/{len(r)} audit checks PASS')
55open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')
experiments/level2/.gitignore +1 -0
……@@ -1 +1,2 @@
11l2
2results-jt/*/*.bin
experiments/level2/l2.c +24 -1
……@@ -228,12 +228,14 @@ static void run_arm (int seed, int tier, const char *name, int rule, const corpu
228228// ---------------------------------------------------------------- main
229229
230230int main (int argc, char **argv) {
231 if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop]\n"); return 1; }
231 if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix]\n"); return 1; }
232232 int seed = atoi(argv[1]);
233233 int tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000;
234 const char *jt_prefix = NULL;
234235 for (int i = 2; i < argc; i++) {
235236 if (!strcmp(argv[i], "--mut-drop")) mut_drop = 1;
236237 else if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
238 else if (!strcmp(argv[i], "--jt") && i + 1 < argc) jt_prefix = argv[++i];
237239 else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
238240 ntier = 0;
239241 for (char *p = strtok(argv[++i], ","); p && ntier < 16; p = strtok(NULL, ",")) tiers[ntier++] = atoi(p);
……@@ -314,6 +316,27 @@ int main (int argc, char **argv) {
314316 run_arm(seed, tier, nm, RULE_COMP, &trs, &hos, &ho, htr, hho);
315317 }
316318 free(Lmap);
319
320 // PREREG-JT.md: layer 1 from the jigsaw transformer (labels + its attention parse), from jt --dump.
321 if (jt_prefix) {
322 char path[512];
323 snprintf(path, sizeof path, "%s-%d.bin", jt_prefix, tier);
324 FILE *f = fopen(path, "rb");
325 if (!f) { fprintf(stderr, "cannot read %s\n", path); return 1; }
326 int *jtr = malloc(sizeof(int) * tr.ntok), *jho = malloc(sizeof(int) * ho.ntok), *atr = malloc(sizeof(int) * tr.ntok), *aho = malloc(sizeof(int) * ho.ntok), nt;
327 int ok = fread(&nt, sizeof(int), 1, f) == 1 && nt == tr.ntok && fread(jtr, sizeof(int), nt, f) == (size_t) nt && fread(atr, sizeof(int), nt, f) == (size_t) nt
328 && fread(&nt, sizeof(int), 1, f) == 1 && nt == ho.ntok && fread(jho, sizeof(int), nt, f) == (size_t) nt && fread(aho, sizeof(int), nt, f) == (size_t) nt;
329 fclose(f);
330 if (!ok) { fprintf(stderr, "%s does not match this corpus\n", path); return 1; }
331 corpus_t trs = with_symbols(&tr, jtr), hos = with_symbols(&ho, jho);
332 run_arm(seed, tier, "C-JT", RULE_ALL, &trs, &hos, &ho, NULL, NULL);
333 layer1_parse(&trs, &hos, NCLS, htr, hho);
334 run_arm(seed, tier, "M-JT-conn", RULE_CONN, &trs, &hos, &ho, htr, hho);
335 run_arm(seed, tier, "M-JT-comp", RULE_COMP, &trs, &hos, &ho, htr, hho);
336 run_arm(seed, tier, "M-JTA-conn", RULE_CONN, &trs, &hos, &ho, atr, aho);
337 run_arm(seed, tier, "M-JTA-comp", RULE_COMP, &trs, &hos, &ho, atr, aho);
338 free(jtr); free(jho); free(atr); free(aho);
339 }
317340 }
318341 return 0;
319342}
experiments/level2/run-jt.sh +15 -0
……@@ -0,0 +1,15 @@
1#!/bin/sh
2# PREREG-JT full run: per seed, jt V2-DJ dumps its layer 1, then l2 runs Test A plus the JT arms. 10 seeds in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o ../jigsaw-transformer/jt ../jigsaw-transformer/jt.c -lm 2>/dev/null
6gcc -O2 -Wall -o l2 l2.c -lm 2>/dev/null
7out=results-jt/$(date +%s)
8mkdir -p "$out"
9git rev-parse HEAD > "$out/commit.txt"
10sha256sum l2.c ../jigsaw-transformer/jt.c ../across-cluster/ac.c > "$out/src.sha256"
11for s in 1 2 3 4 5 6 7 8 9 10; do
12 (../jigsaw-transformer/jt $s --v2 dj --dump "$out/jt-$s" > "$out/jt-$s.log" && ./l2 $s --jt "$out/jt-$s" > "$out/seed-$s.tsv") &
13done
14wait
15echo "$out"
experiments/level2/summarize-jt.py +51 -0
……@@ -0,0 +1,51 @@
1#!/usr/bin/env python3
2# Score a results-jt/<stamp>/ directory against PREREG-JT.md. Usage: summarize-jt.py results-jt/<stamp> results/<testA-stamp>
3import glob, math, os, sys
4
5T975 = {9: 2.262}
6def mean(x): return sum(x) / len(x)
7def ci(d):
8 n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
9 h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
10 return m, m - h, m + h
11def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
12
13A = {} # (tier, arm, seed) -> (types, inst recall, type recall, precision)
14for f in glob.glob(os.path.join(sys.argv[1], 'seed-*.tsv')):
15 for line in open(f):
16 p = line.rstrip('\n').split('\t')
17 if p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[5]), float(p[6]), float(p[7]))
18seeds = sorted({k[2] for k in A}); tiers = sorted({k[0] for k in A}); top = max(tiers)
19out = []
20def say(s=''): out.append(s); print(s)
21def a(t, m, i): return [A[(t, m, s)][i] for s in seeds]
22def d(t, x, y, i): return ci([p - q for p, q in zip(a(t, x, i), a(t, y, i))])
23
24# R0: every non-JT line identical to Test A.
25bad = 0
26for s in seeds:
27 old = [l for l in open(os.path.join(sys.argv[2], f'seed-{s}.tsv')) if 'JT' not in l]
28 new = [l for l in open(os.path.join(sys.argv[1], f'seed-{s}.tsv')) if 'JT' not in l]
29 bad += old != new
30say(f'R0: {"PASS" if bad == 0 else "FAIL"} — {len(seeds) - bad}/{len(seeds)} seeds reproduce Test A exactly')
31
32say('\n## Means: candidate types / instance recall / type recall / candidate precision')
33for t in tiers:
34 say(f'tier {t}')
35 for m in ['C-G', 'M-G-comp', 'C-L', 'M-L-comp', 'M-R-comp', 'C-JT', 'M-JT-conn', 'M-JT-comp', 'M-JTA-conn', 'M-JTA-comp']:
36 say(f' {m:11s} {mean(a(t, m, 0)):9.0f} ' + ' '.join(f'{mean(a(t, m, i)):.3f}' for i in (1, 2, 3)))
37
38say(f'\n## Predictions (at {top} unless stated)')
39def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
40c = d(top, 'M-JTA-comp', 'M-JT-comp', 2); verdict('B1', c[1] > 0, f'type recall M-JTA-comp − M-JT-comp {fmt(c)}')
41c = d(top, 'M-JT-comp', 'M-L-comp', 2); verdict('B2', c[1] > 0, f'type recall M-JT-comp − M-L-comp {fmt(c)}')
42c = d(top, 'M-JTA-comp', 'M-G-comp', 2); verdict('B3', c[1] > 0, f'type recall M-JTA-comp − M-G-comp {fmt(c)}')
43ok = True; cells = []
44for t in tiers: c = d(t, 'M-JTA-comp', 'C-JT', 3); ok &= c[1] > 0; cells.append(f'{t}: {fmt(c)}')
45verdict('B4', ok, 'precision M-JTA-comp − C-JT ' + '; '.join(cells))
46ok = True; cells = []
47for t in tiers:
48 for m in ['M-JT-comp', 'M-JTA-comp']: c = d(t, m, 'M-R-comp', 2); ok &= c[1] > 0; cells.append(f'{t} {m}: {c[0]:+.3f} [{c[1]:+.3f}]')
49verdict('B5', ok, 'type recall − M-R-comp ' + '; '.join(cells))
50x, y = mean(a(top, 'M-JTA-comp', 0)), mean(a(top, 'C-JT', 0)); verdict('B6', x <= y, f'types M-JTA-comp {x:.0f} vs C-JT {y:.0f}')
51open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')