OverviewFilesCommitsIssuesDocsReleasesPeers

atomspace-layer-discovery commits / 964b7e1

experiments/jigsaw-transformer: v1 learner (--v1): layer 0 from word-pair PMI, exchange over attended pairs
Smoke (1 seed, 3K): V-measure 0.61 (v0 0.42; own start 0.51), attention parse 0.48 (word MI tree 0.15), audit replay + mutation PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
964b7e17152602fbf9669ccb26b4418cd886be8c
by plankatron <fleeky@gmail.com> · 2026-10-18T21:01:38.000Z

Changed files (2) +198 -2

experiments/jigsaw-transformer/jt.c +183 -2
……@@ -1,7 +1,9 @@
11// Jigsaw transformer v0: count-based attention over jigsaws. See PREREG.md.
22// Reuses the planted grammar, exchange clusterer and Eisner parser of ../across-cluster/ac.c unchanged.
33//
4// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file]
4// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file] [--v1]
5// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
6// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
57// Output rows (stdout):
68// V <seed> <tier> <K> <method> <vmeasure> <m1acc> <senseacc> methods: JT-BATCH JT-CONT TYPE MAJ GOLD
79// L <seed> <tier> <K> <parser> <F1> parsers: ATTN WORD-MI GOLD-MI
……@@ -319,6 +321,134 @@ static void explain (FILE *f, const corpus_t *C, const params_t *p, const counts
319321 }
320322}
321323
324// ---------------------------------------------------------------- v1 (PREREG-v1.md): classes emerge from word-level PMI
325
326// Word-level gated PMI (ac.c arm U): S[a*V+b] for left word a, right word b; NEG if the pair fails the gate.
327static double *word_scores (const corpus_t *tr) {
328 arm_t a = { "U", V, NULL, 0 };
329 pairs_t P = count_pairs(&a, tr);
330 long ev, acc;
331 double *S = hard_scores(&P, &ev, &acc);
332 free_pairs(&P);
333 return S;
334}
335
336// Layer-0 attention of occurrence t over its window, from word-pair PMI. Returns 0 if no neighbour passes.
337static int word_attention (const corpus_t *C, int o, int n, int t, const double *SW, double *alpha) {
338 double tot = 0;
339 for (int j = 0; j < n; j++) {
340 alpha[j] = 0;
341 if (j == t || abs(j - t) > W) continue;
342 int a = t < j ? C->w[o + t] : C->w[o + j], b = t < j ? C->w[o + j] : C->w[o + t];
343 double v = SW[(size_t) a * V + b];
344 if (v > NEG) { alpha[j] = v; tot += v; }
345 }
346 if (tot <= 0) return 0;
347 for (int j = 0; j < n; j++) alpha[j] /= tot;
348 return 1;
349}
350
351// Exchange algorithm (Martin/Liermann/Ney, as ac.c exchange) over weighted word pairs Wp (left a, right b).
352static int *exchange_weighted (const double *Wp, const corpus_t *tr) {
353 double *freq = calloc(V, sizeof(double));
354 for (int t = 0; t < tr->ntok; t++) freq[tr->w[t]] += 1;
355 int *order = malloc(sizeof(int) * V);
356 for (int x = 0; x < V; x++) order[x] = x;
357 for (int i = 1; i < V; i++) { int x = order[i], j = i; while (j > 0 && freq[order[j - 1]] < freq[x]) { order[j] = order[j - 1]; j--; } order[j] = x; }
358 int *cls = malloc(sizeof(int) * V);
359 for (int i = 0; i < V; i++) cls[order[i]] = i % KJ;
360 double *Nc = calloc((size_t) KJ * KJ, sizeof(double));
361 for (int a = 0; a < V; a++) for (int b = 0; b < V; b++) { double v = Wp[(size_t) a * V + b]; if (v > 0) Nc[cls[a] * KJ + cls[b]] += v; }
362 double sv[64], pv[64];
363 for (int pass = 0; pass < 30; pass++) {
364 int moves = 0;
365 for (int i = 0; i < V; i++) {
366 int x = order[i];
367 if (freq[x] == 0) continue;
368 int a = cls[x];
369 double self = Wp[(size_t) x * V + x];
370 memset(sv, 0, sizeof sv); memset(pv, 0, sizeof pv);
371 for (int y = 0; y < V; y++) {
372 if (y == x) continue;
373 double r = Wp[(size_t) x * V + y], l = Wp[(size_t) y * V + x];
374 if (r > 0) sv[cls[y]] += r;
375 if (l > 0) pv[cls[y]] += l;
376 }
377 for (int c = 0; c < KJ; c++) { Nc[a * KJ + c] -= sv[c]; Nc[c * KJ + a] -= pv[c]; }
378 Nc[a * KJ + a] -= self;
379 int best = a; double bestF = -1e300;
380 for (int b = 0; b < KJ; b++) {
381 for (int c = 0; c < KJ; c++) { Nc[b * KJ + c] += sv[c]; Nc[c * KJ + b] += pv[c]; }
382 Nc[b * KJ + b] += self;
383 double f = 0;
384 for (int r = 0; r < KJ; r++) {
385 double row = 0, col = 0;
386 for (int c = 0; c < KJ; c++) { double v = Nc[r * KJ + c]; if (v > 0) f += v * log(v); row += Nc[r * KJ + c]; col += Nc[c * KJ + r]; }
387 if (row > 0) f -= row * log(row);
388 if (col > 0) f -= col * log(col);
389 }
390 if (f > bestF + 1e-9 || (b == a && f >= bestF - 1e-9)) { bestF = f; best = b; }
391 for (int c = 0; c < KJ; c++) { Nc[b * KJ + c] -= sv[c]; Nc[c * KJ + b] -= pv[c]; }
392 Nc[b * KJ + b] -= self;
393 }
394 for (int c = 0; c < KJ; c++) { Nc[best * KJ + c] += sv[c]; Nc[c * KJ + best] += pv[c]; }
395 Nc[best * KJ + best] += self;
396 if (best != a) { cls[x] = best; moves++; }
397 }
398 if (!moves) break;
399 }
400 free(freq); free(order); free(Nc);
401 return cls;
402}
403
404// Full v1 learner on a training prefix. Leaves params in p and the layer-0 classes in *cls_out.
405static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
406 double *SW = word_scores(tr);
407 double *Wp = calloc((size_t) V * V, sizeof(double)), alpha[MAXN];
408 for (int s = 0; s < tr->ns; s++) {
409 int o = tr->off[s], n = tr->off[s + 1] - o;
410 for (int t = 0; t < n; t++) {
411 if (!word_attention(tr, o, n, t, SW, alpha)) continue;
412 for (int j = 0; j < n; j++) if (alpha[j] > 0) {
413 int a = t < j ? tr->w[o + t] : tr->w[o + j], b = t < j ? tr->w[o + j] : tr->w[o + t];
414 Wp[(size_t) a * V + b] += alpha[j];
415 }
416 }
417 }
418 int *cls = exchange_weighted(Wp, tr);
419 // Initial counts: one-hot classes, word-level attention as messages.
420 counts_t c; counts_alloc(&c);
421 static sent_state_t S;
422 for (int s = 0; s < tr->ns; s++) {
423 int o = tr->off[s], n = tr->off[s + 1] - o;
424 S.n = n;
425 for (int t = 0; t < n; t++) { for (int z = 0; z < KJ; z++) S.q[t][z] = 0; S.q[t][cls[tr->w[o + t]]] = 1; }
426 for (int t = 0; t < n; t++) {
427 for (int y = 0; y < KJ; y++) { S.mL[t][y] = 0; S.mR[t][y] = 0; }
428 if (!word_attention(tr, o, n, t, SW, alpha)) continue;
429 for (int j = 0; j < n; j++) if (alpha[j] > 0) (j < t ? S.mL[t] : S.mR[t])[cls[tr->w[o + j]]] += alpha[j];
430 }
431 accumulate(tr, o, &S, &c);
432 }
433 c.N[0] += perturb; // audit mutation test only
434 derive(&c, p);
435 for (int it = 0; it < ITERS; it++) {
436 counts_zero(&c);
437 for (int s = 0; s < tr->ns; s++) { estep(tr, tr->off[s], tr->off[s + 1] - tr->off[s], p, &S); accumulate(tr, tr->off[s], &S, &c); }
438 derive(&c, p);
439 }
440 counts_free(&c); free(Wp); free(SW);
441 *cls_out = cls;
442}
443
444static uint64_t params_hash (const params_t *p, const int *cls) {
445 uint64_t h = 1469598103934665603ULL;
446 h = fnv_bytes(h, p->lpi, sizeof(double) * V * KJ);
447 h = fnv_bytes(h, p->lphL, sizeof(double) * KJ * KJ); h = fnv_bytes(h, p->lphR, sizeof(double) * KJ * KJ);
448 h = fnv_bytes(h, p->pmi, sizeof(double) * KJ * KJ);
449 return fnv_bytes(h, cls, sizeof(int) * V);
450}
451
322452// ---------------------------------------------------------------- main
323453
324454static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
……@@ -339,12 +469,13 @@ static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
339469
340470int main (int argc, char **argv) {
341471 if (argc < 2) { fprintf(stderr, "usage: jt <seed> [--tiers a,b] [--heldout N] [--K k] [--shuffle] [--explain f] [--log f]\n"); return 1; }
342 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0;
472 int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0;
343473 const char *explain_path = NULL, *log_path = NULL;
344474 for (int i = 2; i < argc; i++) {
345475 if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
346476 else if (!strcmp(argv[i], "--K") && i + 1 < argc) KJ = atoi(argv[++i]);
347477 else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
478 else if (!strcmp(argv[i], "--v1")) v1 = 1;
348479 else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
349480 else if (!strcmp(argv[i], "--log") && i + 1 < argc) log_path = argv[++i];
350481 else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
……@@ -369,6 +500,55 @@ int main (int argc, char **argv) {
369500 corpus_t tr = prefix(full, tier);
370501 double vm, m1, se;
371502
503 if (v1) { // PREREG-v1: checkpoints at doubling sizes, each a full re-learn + a hash-chained event
504 int ck[16], nck = 0;
505 for (int sz = 1000; sz < tier && nck < 15; sz *= 2) ck[nck++] = sz;
506 ck[nck++] = tier;
507 uint64_t hs[16], prev = 0;
508 FILE *log = log_path && ti == ntier - 1 ? fopen(log_path, "w") : NULL;
509 int *cls = NULL;
510 for (int k = 0; k < nck; k++) {
511 corpus_t pre = prefix(full, ck[k]);
512 free(cls);
513 train_v1(&pre, &p, &cls, 0);
514 uint64_t sh = fnv_bytes(1469598103934665603ULL, pre.w + pre.off[k ? ck[k - 1] : 0], sizeof(int) * (pre.ntok - pre.off[k ? ck[k - 1] : 0]));
515 uint64_t h = fnv_bytes(prev, &sh, sizeof sh), ph = params_hash(&p, cls);
516 h = fnv_bytes(h, &ph, sizeof ph);
517 hs[k] = h;
518 if (log) fprintf(log, "{\"event\":%d,\"sentences\":[%d,%d],\"sentences_hash\":\"%016llx\",\"state_hash\":\"%016llx\",\"prev\":\"%016llx\",\"hash\":\"%016llx\"}\n",
519 k, k ? ck[k - 1] : 0, ck[k], (unsigned long long) sh, (unsigned long long) ph, (unsigned long long) prev, (unsigned long long) h);
520 prev = h;
521 jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
522 printf("C\t%d\t%d\t%d\tJT-V1%s\t%d\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, ck[k], vm, m1, se);
523 }
524 if (log) fclose(log);
525 printf("V\t%d\t%d\t%d\tJT-V1%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
526 printf("L\t%d\t%d\t%d\tATTN-V1%s\t%.5f\n", seed, tier, KJ, tag, attention_parse_f1(&ho, &p));
527 for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
528 score_labels(&ho, lab, KJ, &vm, &m1, &se);
529 printf("V\t%d\t%d\t%d\tTYPE-A%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
530 if (explain_path && ti == ntier - 1) { FILE *f = fopen(explain_path, "w"); if (f) { explain(f, &ho, &p, NULL); fclose(f); } }
531 // Audit: replay all checkpoints; then alter one checkpoint's counts and confirm detection there.
532 int bad = -1, at = nck / 2, badm = -1;
533 prev = 0;
534 for (int pass = 0; pass < 2; pass++) {
535 uint64_t pv = 0;
536 for (int k = 0; k < nck; k++) {
537 corpus_t pre = prefix(full, ck[k]);
538 int *c2 = NULL;
539 train_v1(&pre, &p, &c2, pass == 1 && k == at ? 1.0 : 0);
540 uint64_t sh = fnv_bytes(1469598103934665603ULL, pre.w + pre.off[k ? ck[k - 1] : 0], sizeof(int) * (pre.ntok - pre.off[k ? ck[k - 1] : 0]));
541 uint64_t h = fnv_bytes(pv, &sh, sizeof sh), ph = params_hash(&p, c2);
542 h = fnv_bytes(h, &ph, sizeof ph);
543 if (h != hs[k]) { if (pass == 0 && bad < 0) bad = k; if (pass == 1 && badm < 0) badm = k; }
544 pv = h;
545 free(c2);
546 }
547 }
548 printf("A\t%d\t%d\t%d\treplay\t%s\tevents=%d first_mismatch=%d\n", seed, tier, KJ, bad < 0 ? "PASS" : "FAIL", nck, bad);
549 printf("A\t%d\t%d\t%d\tmutation\t%s\taltered=%d detected_at=%d\n", seed, tier, KJ, badm == at ? "PASS" : "FAIL", at, badm);
550 free(cls);
551 } else {
372552 train_batch(&tr, 1000ULL * seed + 7, &p);
373553 jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
374554 printf("V\t%d\t%d\t%d\tJT-BATCH%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
……@@ -389,6 +569,7 @@ int main (int argc, char **argv) {
389569 bad = train_cont(&tr, 1000ULL * seed + 7, &p, hs2, NULL, hs, at, NULL);
390570 printf("A\t%d\t%d\t%d\tmutation\t%s\taltered=%d detected_at=%d\n", seed, tier, KJ, bad == at ? "PASS" : "FAIL", at, bad);
391571 free(hs); free(hs2);
572 }
392573
393574 if (KJ == 30 && !shuffle) { // baselines (KJ-independent)
394575 int passes, *L = exchange(&tr, &passes);
experiments/jigsaw-transformer/run-v1.sh +15 -0
……@@ -0,0 +1,15 @@
1#!/bin/sh
2# PREREG-v1 full run: 5 seeds x tiers 3K,10K,30K, normal and shuffled-within-sentence, in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o jt jt.c -lm 2>/dev/null
6out=results-v1/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum jt.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5; do
11 ./jt $s --v1 > "$out/seed-$s.tsv" &
12 ./jt $s --v1 --shuffle > "$out/shuf-$s.tsv" &
13done
14wait
15echo "$out"