OverviewFilesCommitsIssuesDocsReleasesPeers

atomspace-layer-discovery commits / a2f3695

experiments/jigsaw-transformer: v2 arms (--v2 word|dj), collapse rows, run + scorer
v1 path unchanged: seed 1 tier 3K output byte-identical to the committed v1 results. Smoke (seed 1, 3K) runs both arms, audits PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
a2f36959ec58aae0a30c2848821550d37a381532
by plankatron <fleeky@gmail.com> · 2026-10-18T21:26:17.000Z

Changed files (3) +234 -10

experiments/jigsaw-transformer/jt.c +160 -10
……@@ -2,6 +2,7 @@
22// Reuses the planted grammar, exchange clusterer and Eisner parser of ../across-cluster/ac.c unchanged.
33//
44// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file] [--v1]
5// [--v2 word|dj] PREREG-v2 arms (implies the v1 pipeline); adds JT-<arm>-COLL rows
56// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
67// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
78// Output rows (stdout):
……@@ -18,7 +19,41 @@
1819#define ITERS 10
1920#define BATCH 1000
2021
22#define BETA 10.0
23
2124static int KJ = 30;
25static int MODE = 0; // 0 v1 (class profiles), 1 V2-WORD (neighbour words), 2 V2-DJ (disjuncts); PREREG-v2.md
26
27static uint64_t fnv_bytes (uint64_t h, const void *p, size_t n);
28
29// Disjunct table: key = disjunct hash (never 0), row = KJ soft counts.
30typedef struct { size_t cap, n; uint64_t *key; double *cnt; } djtab_t;
31
32static void dj_init (djtab_t *t, size_t cap) { t->cap = cap; t->n = 0; t->key = calloc(cap, sizeof(uint64_t)); t->cnt = calloc(cap * KJ, sizeof(double)); }
33static void dj_free (djtab_t *t) { free(t->key); free(t->cnt); }
34static double *dj_row (const djtab_t *t, uint64_t k) {
35 for (size_t i = k & (t->cap - 1);; i = (i + 1) & (t->cap - 1)) {
36 if (t->key[i] == k) return t->cnt + i * KJ;
37 if (!t->key[i]) return NULL;
38 }
39}
40static double *dj_add (djtab_t *t, uint64_t k) {
41 double *r = dj_row(t, k);
42 if (r) return r;
43 if (2 * (t->n + 1) > t->cap) {
44 djtab_t g; dj_init(&g, 2 * t->cap);
45 for (size_t i = 0; i < t->cap; i++) if (t->key[i]) {
46 size_t j = t->key[i] & (g.cap - 1);
47 while (g.key[j]) j = (j + 1) & (g.cap - 1);
48 g.key[j] = t->key[i]; memcpy(g.cnt + j * KJ, t->cnt + i * KJ, sizeof(double) * KJ);
49 }
50 g.n = t->n; dj_free(t); *t = g;
51 }
52 size_t i = k & (t->cap - 1);
53 while (t->key[i]) i = (i + 1) & (t->cap - 1);
54 t->key[i] = k; t->n++;
55 return t->cnt + i * KJ;
56}
2257
2358// ---------------------------------------------------------------- parameters (all counts)
2459
……@@ -26,26 +61,38 @@ typedef struct {
2661 double *pi; // V x KJ : soft count of class z on word w
2762 double *phL, *phR; // KJ x KJ : soft count of left/right neighbour class y around class z
2863 double *N; // KJ x KJ : soft count of class pair (left x, right y) within the window
64 double *psiL, *psiR; // KJ x V : soft count of attended left/right neighbour word around class z (V2-WORD)
65 djtab_t dj; // disjunct x KJ soft counts (V2-DJ)
2966} counts_t;
3067
31typedef struct { double *lpi, *lphL, *lphR, *pmi; } params_t; // logs / PMI+ derived from counts
68typedef struct { // logs / PMI+ derived from counts
69 double *lpi, *lphL, *lphR, *pmi;
70 double *lpsiL, *lpsiR; int *wseen; // V2-WORD
71 djtab_t dj; double djn[64]; // V2-DJ: copy of the counts and per-class totals
72} params_t;
3273
3374static void counts_alloc (counts_t *c) {
3475 c->pi = calloc((size_t) V * KJ, sizeof(double));
3576 c->phL = calloc((size_t) KJ * KJ, sizeof(double)); c->phR = calloc((size_t) KJ * KJ, sizeof(double));
3677 c->N = calloc((size_t) KJ * KJ, sizeof(double));
78 c->psiL = calloc((size_t) KJ * V, sizeof(double)); c->psiR = calloc((size_t) KJ * V, sizeof(double));
79 dj_init(&c->dj, 1 << 12);
3780}
3881static void counts_zero (counts_t *c) {
3982 memset(c->pi, 0, sizeof(double) * V * KJ); memset(c->phL, 0, sizeof(double) * KJ * KJ);
4083 memset(c->phR, 0, sizeof(double) * KJ * KJ); memset(c->N, 0, sizeof(double) * KJ * KJ);
84 memset(c->psiL, 0, sizeof(double) * KJ * V); memset(c->psiR, 0, sizeof(double) * KJ * V);
85 memset(c->dj.key, 0, sizeof(uint64_t) * c->dj.cap); memset(c->dj.cnt, 0, sizeof(double) * c->dj.cap * KJ); c->dj.n = 0;
4186}
42static void counts_free (counts_t *c) { free(c->pi); free(c->phL); free(c->phR); free(c->N); }
87static void counts_free (counts_t *c) { free(c->pi); free(c->phL); free(c->phR); free(c->N); free(c->psiL); free(c->psiR); dj_free(&c->dj); }
4388
4489static void params_alloc (params_t *p) {
4590 p->lpi = malloc(sizeof(double) * V * KJ);
4691 p->lphL = malloc(sizeof(double) * KJ * KJ); p->lphR = malloc(sizeof(double) * KJ * KJ); p->pmi = malloc(sizeof(double) * KJ * KJ);
92 p->lpsiL = calloc((size_t) KJ * V, sizeof(double)); p->lpsiR = calloc((size_t) KJ * V, sizeof(double)); p->wseen = calloc(V, sizeof(int));
93 dj_init(&p->dj, 1 << 12);
4794}
48static void params_free (params_t *p) { free(p->lpi); free(p->lphL); free(p->lphR); free(p->pmi); }
95static void params_free (params_t *p) { free(p->lpi); free(p->lphL); free(p->lphR); free(p->pmi); free(p->lpsiL); free(p->lpsiR); free(p->wseen); dj_free(&p->dj); }
4996
5097// M-step: add-λ smoothing, renormalise; PMI+ over class pairs.
5198static void derive (const counts_t *c, params_t *p) {
……@@ -69,11 +116,78 @@ static void derive (const counts_t *c, params_t *p) {
69116 p->pmi[x * KJ + y] = v > 0 ? v : 0;
70117 }
71118 free(rs); free(cs);
119 if (MODE == 1) { // Ψ(v | z) = (n(z, v) + β Σ_y Φ(y | z) P(v | y)) / (n(z) + β)
120 double *py = calloc(KJ, sizeof(double)), ph[64];
121 for (int w = 0; w < V; w++) {
122 double s = 0;
123 for (int y = 0; y < KJ; y++) { py[y] += c->pi[(size_t) w * KJ + y]; s += c->pi[(size_t) w * KJ + y]; }
124 p->wseen[w] = s > 0;
125 }
126 for (int side = 0; side < 2; side++) {
127 const double *n = side ? c->psiR : c->psiL, *lph = side ? p->lphR : p->lphL;
128 double *l = side ? p->lpsiR : p->lpsiL;
129 for (int z = 0; z < KJ; z++) {
130 double nz = 0;
131 for (int w = 0; w < V; w++) nz += n[(size_t) z * V + w];
132 for (int y = 0; y < KJ; y++) ph[y] = exp(lph[z * KJ + y]);
133 for (int w = 0; w < V; w++) {
134 if (!p->wseen[w]) { l[(size_t) z * V + w] = 0; continue; }
135 double b = 0;
136 for (int y = 0; y < KJ; y++) if (py[y] > 0) b += ph[y] * c->pi[(size_t) w * KJ + y] / py[y];
137 l[(size_t) z * V + w] = log((n[(size_t) z * V + w] + BETA * b) / (nz + BETA));
138 }
139 }
140 }
141 free(py);
142 }
143 if (MODE == 2) {
144 if (p->dj.cap != c->dj.cap) { dj_free(&p->dj); dj_init(&p->dj, c->dj.cap); }
145 memcpy(p->dj.key, c->dj.key, sizeof(uint64_t) * c->dj.cap); memcpy(p->dj.cnt, c->dj.cnt, sizeof(double) * c->dj.cap * KJ);
146 p->dj.n = c->dj.n;
147 for (int z = 0; z < KJ; z++) p->djn[z] = 0;
148 for (size_t i = 0; i < c->dj.cap; i++) if (c->dj.key[i]) for (int z = 0; z < KJ; z++) p->djn[z] += c->dj.cnt[i * KJ + z];
149 }
72150}
73151
74152// ---------------------------------------------------------------- one sentence: layers of attention
75153
76typedef struct { double q[MAXN][64], u[MAXN][64], mL[MAXN][64], mR[MAXN][64], alpha[MAXN][MAXN]; int n; } sent_state_t;
154#define MAXC 16
155typedef struct {
156 double q[MAXN][64], u[MAXN][64], mL[MAXN][64], mR[MAXN][64], alpha[MAXN][MAXN]; int n;
157 uint64_t dj[MAXN]; int nc[MAXN], con[MAXN][MAXC]; // V2-DJ: disjunct of each token; connector = side*64 + class
158} sent_state_t;
159
160// V2-DJ: Eisner over α_tj + α_jt; each token's disjunct = its links in position order, connector
161// (side, cl[j]) with side 0 = left, 1 = right.
162static void disjuncts (sent_state_t *S, const int *cl) {
163 int n = S->n, m = n + 1, head[MAXN + 1];
164 for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
165 for (int d = 1; d < m; d++) sc[0][d] = 0;
166 for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
167 double v = S->alpha[i][j] + S->alpha[j][i];
168 sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
169 }
170 eisner(m, head);
171 for (int t = 0; t < n; t++) {
172 uint64_t h = 1469598103934665603ULL;
173 S->nc[t] = 0;
174 for (int j = 0; j < n; j++) {
175 if (j == t || (head[t + 1] != j + 1 && head[j + 1] != t + 1)) continue;
176 int code = (j < t ? 0 : 64) + cl[j];
177 h = fnv_bytes(h, &code, sizeof code);
178 if (S->nc[t] < MAXC) S->con[t][S->nc[t]++] = code;
179 }
180 S->dj[t] = h | 1;
181 }
182}
183
184// V2-DJ: log P(d | z) = log (n(z, d) + β Π Φ_side(y | z)) / (n(z) + β).
185static double dj_logp (const params_t *p, const sent_state_t *S, int t, int z) {
186 const double *row = dj_row(&p->dj, S->dj[t]);
187 double b = 0;
188 for (int k = 0; k < S->nc[t]; k++) { int code = S->con[t][k]; b += (code < 64 ? p->lphL : p->lphR)[z * KJ + code % 64]; }
189 return log(((row ? row[z] : 0) + BETA * exp(b)) / (p->djn[z] + BETA));
190}
77191
78192// E-step on tokens [o, o+n) of C. Leaves q (final), alpha and messages (final layer) in S.
79193static void estep (const corpus_t *C, int o, int n, const params_t *p, sent_state_t *S) {
……@@ -115,11 +229,24 @@ static void estep (const corpus_t *C, int o, int n, const params_t *p, sent_stat
115229 double *m = j < t ? S->mL[t] : S->mR[t];
116230 for (int y = 0; y < KJ; y++) m[y] += S->alpha[t][j] * S->q[j][y];
117231 }
232 }
233 if (MODE == 2) {
234 int cl[MAXN];
235 for (int j = 0; j < n; j++) { int b = 0; for (int z = 1; z < KJ; z++) if (S->q[j][z] > S->q[j][b]) b = z; cl[j] = b; }
236 disjuncts(S, cl);
237 }
238 for (int t = 0; t < n; t++) {
118239 double mx = -1e300, sum = 0, e[64];
119240 const double *lp = p->lpi + (size_t) C->w[o + t] * KJ;
120241 for (int z = 0; z < KJ; z++) {
121242 double v = lp[z];
122 for (int y = 0; y < KJ; y++) v += S->mL[t][y] * p->lphL[z * KJ + y] + S->mR[t][y] * p->lphR[z * KJ + y];
243 if (MODE == 0) for (int y = 0; y < KJ; y++) v += S->mL[t][y] * p->lphL[z * KJ + y] + S->mR[t][y] * p->lphR[z * KJ + y];
244 else if (MODE == 1) {
245 for (int j = 0; j < n; j++) {
246 int wj = C->w[o + j];
247 if (S->alpha[t][j] > 0 && p->wseen[wj]) v += S->alpha[t][j] * (j < t ? p->lpsiL : p->lpsiR)[(size_t) z * V + wj];
248 }
249 } else v += dj_logp(p, S, t, z);
123250 e[z] = v;
124251 if (v > mx) mx = v;
125252 }
……@@ -142,6 +269,11 @@ static void accumulate (const corpus_t *C, int o, const sent_state_t *S, counts_
142269 c->phR[z * KJ + y] += S->q[t][z] * S->mR[t][y];
143270 }
144271 }
272 if (MODE == 1) for (int j = 0; j < n; j++) if (S->alpha[t][j] > 0) {
273 double *ps = (j < t ? c->psiL : c->psiR) + C->w[o + j];
274 for (int z = 0; z < KJ; z++) ps[(size_t) z * V] += S->q[t][z] * S->alpha[t][j];
275 }
276 if (MODE == 2) { double *r = dj_add(&c->dj, S->dj[t]); for (int z = 0; z < KJ; z++) r[z] += S->q[t][z]; }
145277 double win[64] = { 0 };
146278 for (int j = t + 1; j < n && j - t <= W; j++) for (int y = 0; y < KJ; y++) win[y] += S->q[j][y];
147279 for (int x = 0; x < KJ; x++) for (int y = 0; y < KJ; y++) c->N[x * KJ + y] += S->q[t][x] * win[y];
……@@ -425,9 +557,12 @@ static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double per
425557 for (int t = 0; t < n; t++) { for (int z = 0; z < KJ; z++) S.q[t][z] = 0; S.q[t][cls[tr->w[o + t]]] = 1; }
426558 for (int t = 0; t < n; t++) {
427559 for (int y = 0; y < KJ; y++) { S.mL[t][y] = 0; S.mR[t][y] = 0; }
428 if (!word_attention(tr, o, n, t, SW, alpha)) continue;
560 int ok = word_attention(tr, o, n, t, SW, alpha);
561 memcpy(S.alpha[t], alpha, sizeof(double) * n);
562 if (!ok) continue;
429563 for (int j = 0; j < n; j++) if (alpha[j] > 0) (j < t ? S.mL[t] : S.mR[t])[cls[tr->w[o + j]]] += alpha[j];
430564 }
565 if (MODE == 2) { int cl[MAXN]; for (int j = 0; j < n; j++) cl[j] = cls[tr->w[o + j]]; disjuncts(&S, cl); }
431566 accumulate(tr, o, &S, &c);
432567 }
433568 c.N[0] += perturb; // audit mutation test only
……@@ -446,9 +581,20 @@ static uint64_t params_hash (const params_t *p, const int *cls) {
446581 h = fnv_bytes(h, p->lpi, sizeof(double) * V * KJ);
447582 h = fnv_bytes(h, p->lphL, sizeof(double) * KJ * KJ); h = fnv_bytes(h, p->lphR, sizeof(double) * KJ * KJ);
448583 h = fnv_bytes(h, p->pmi, sizeof(double) * KJ * KJ);
584 if (MODE == 1) { h = fnv_bytes(h, p->lpsiL, sizeof(double) * KJ * V); h = fnv_bytes(h, p->lpsiR, sizeof(double) * KJ * V); }
585 if (MODE == 2) { h = fnv_bytes(h, p->dj.key, sizeof(uint64_t) * p->dj.cap); h = fnv_bytes(h, p->dj.cnt, sizeof(double) * p->dj.cap * KJ); }
449586 return fnv_bytes(h, cls, sizeof(int) * V);
450587}
451588
589// X-COLL (PREREG-v2): every occurrence of a word gets the word's most frequent label in lab.
590static void collapse (const corpus_t *C, const int *lab, int nlab, int *out) {
591 int *cnt = calloc((size_t) V * nlab, sizeof(int)), *maj = malloc(sizeof(int) * V);
592 for (int t = 0; t < C->ntok; t++) cnt[(size_t) C->w[t] * nlab + lab[t]]++;
593 for (int w = 0; w < V; w++) { int b = 0; for (int l = 1; l < nlab; l++) if (cnt[(size_t) w * nlab + l] > cnt[(size_t) w * nlab + b]) b = l; maj[w] = b; }
594 for (int t = 0; t < C->ntok; t++) out[t] = maj[C->w[t]];
595 free(cnt); free(maj);
596}
597
452598// ---------------------------------------------------------------- main
453599
454600static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
……@@ -476,6 +622,7 @@ int main (int argc, char **argv) {
476622 else if (!strcmp(argv[i], "--K") && i + 1 < argc) KJ = atoi(argv[++i]);
477623 else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
478624 else if (!strcmp(argv[i], "--v1")) v1 = 1;
625 else if (!strcmp(argv[i], "--v2") && i + 1 < argc) { v1 = 1; i++; MODE = !strcmp(argv[i], "word") ? 1 : !strcmp(argv[i], "dj") ? 2 : -1; if (MODE < 0) { fprintf(stderr, "--v2 word|dj\n"); return 1; } }
479626 else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
480627 else if (!strcmp(argv[i], "--log") && i + 1 < argc) log_path = argv[++i];
481628 else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
……@@ -491,7 +638,7 @@ int main (int argc, char **argv) {
491638 for (int i = 0; i < ntier; i++) if (tiers[i] > maxt) maxt = tiers[i];
492639 corpus_t full = gen_corpus(1000003ULL * seed + 1, maxt), ho = gen_corpus(1000003ULL * seed + 2, nho);
493640 if (shuffle) { full = shuffle_within(full, 99ULL * seed + 1); ho = shuffle_within(ho, 99ULL * seed + 2); }
494 const char *tag = shuffle ? "-SHUF" : "";
641 const char *tag = shuffle ? "-SHUF" : "", *nm = MODE == 0 ? "V1" : MODE == 1 ? "V2W" : "V2D";
495642 int *lab = malloc(sizeof(int) * ho.ntok);
496643 params_t p; params_alloc(&p);
497644
……@@ -519,11 +666,14 @@ int main (int argc, char **argv) {
519666 k, k ? ck[k - 1] : 0, ck[k], (unsigned long long) sh, (unsigned long long) ph, (unsigned long long) prev, (unsigned long long) h);
520667 prev = h;
521668 jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
522 printf("C\t%d\t%d\t%d\tJT-V1%s\t%d\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, ck[k], vm, m1, se);
669 printf("C\t%d\t%d\t%d\tJT-%s%s\t%d\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, ck[k], vm, m1, se);
523670 }
524671 if (log) fclose(log);
525 printf("V\t%d\t%d\t%d\tJT-V1%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
526 printf("L\t%d\t%d\t%d\tATTN-V1%s\t%.5f\n", seed, tier, KJ, tag, attention_parse_f1(&ho, &p));
672 printf("V\t%d\t%d\t%d\tJT-%s%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
673 int *lc = malloc(sizeof(int) * ho.ntok);
674 collapse(&ho, lab, KJ, lc); score_labels(&ho, lc, KJ, &vm, &m1, &se); free(lc);
675 printf("V\t%d\t%d\t%d\tJT-%s-COLL%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
676 printf("L\t%d\t%d\t%d\tATTN-%s%s\t%.5f\n", seed, tier, KJ, nm, tag, attention_parse_f1(&ho, &p));
527677 for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
528678 score_labels(&ho, lab, KJ, &vm, &m1, &se);
529679 printf("V\t%d\t%d\t%d\tTYPE-A%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
experiments/jigsaw-transformer/run-v2.sh +19 -0
……@@ -0,0 +1,19 @@
1#!/bin/sh
2# PREREG-v2 full run: arms v1, word, dj x 5 seeds x tiers 3K,10K,30K, normal and shuffled-within-sentence, in parallel.
3set -e
4cd "$(dirname "$0")"
5gcc -O2 -Wall -Wno-unused-function -o jt jt.c -lm 2>/dev/null
6out=results-v2/$(date +%s)
7mkdir -p "$out"
8git rev-parse HEAD > "$out/commit.txt"
9sha256sum jt.c ../across-cluster/ac.c > "$out/src.sha256"
10for s in 1 2 3 4 5; do
11 ./jt $s --v1 > "$out/v1-seed-$s.tsv" &
12 ./jt $s --v1 --shuffle > "$out/v1-shuf-$s.tsv" &
13 for a in word dj; do
14 ./jt $s --v2 $a > "$out/$a-seed-$s.tsv" &
15 ./jt $s --v2 $a --shuffle > "$out/$a-shuf-$s.tsv" &
16 done
17done
18wait
19echo "$out"
experiments/jigsaw-transformer/summarize-v2.py +55 -0
……@@ -0,0 +1,55 @@
1#!/usr/bin/env python3
2# Score a results-v2/<stamp>/ directory against PREREG-v2.md. Usage: summarize-v2.py results-v2/<stamp>
3import glob, math, os, sys
4
5T975 = {4: 2.776}
6def mean(x): return sum(x) / len(x)
7def ci(d):
8 n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1))
9 h = T975.get(n - 1, 1.96) * sd / math.sqrt(n)
10 return m, m - h, m + h
11
12V, L, C, A = {}, {}, {}, []
13for f in sorted(glob.glob(os.path.join(sys.argv[1], '*.tsv'))):
14 arm = os.path.basename(f).split('-')[0]
15 for line in open(f):
16 p = line.rstrip('\n').split('\t')
17 if p[0] == 'V': V[(int(p[2]), p[4], int(p[1]))] = tuple(map(float, p[5:8]))
18 elif p[0] == 'L': L[(int(p[2]), p[4], int(p[1]))] = float(p[5])
19 elif p[0] == 'C': C[(int(p[2]), p[4], int(p[5]), int(p[1]))] = float(p[6])
20 elif p[0] == 'A': A.append((arm, p[4], p[5]))
21seeds = sorted({k[2] for k in V}); tiers = sorted({k[0] for k in V}); top = max(tiers)
22out = []
23def say(s=''): out.append(s); print(s)
24def v(t, m, i): return [V[(t, m, s)][i] for s in seeds] # i: 0 vmeasure, 1 m1, 2 sense
25def l(t, m): return [L[(t, m, s)] for s in seeds]
26def d(t, a, b, i): return ci([x - y for x, y in zip(v(t, a, i), v(t, b, i))])
27def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]'
28
29ARMS = ['V1', 'V2W', 'V2D']
30say('## Token classes (mean): V-measure / many-to-one / sense accuracy')
31for t in tiers:
32 say(f'tier {t}')
33 for m in [f'JT-{a}{c}{s}' for s in ['', '-SHUF'] for a in ARMS for c in ['', '-COLL']] + ['TYPE-A', 'TYPE-A-SHUF', 'TYPE', 'MAJ']:
34 if (t, m, seeds[0]) in V: say(f' {m:16s} ' + ' / '.join(f'{mean(v(t, m, i)):.3f}' for i in range(3)))
35say('\n## Parse link F1 (mean)')
36for t in tiers: say(f'{t}: ' + ' '.join(f'{m} {mean(l(t, m)):.3f}' for m in ['ATTN-V1', 'ATTN-V2W', 'ATTN-V2D', 'WORD-MI', 'GOLD-MI'] if (t, m, seeds[0]) in L))
37say('\n## Continuous checkpoints, tier ' + str(top) + ' (mean V-measure)')
38for a in ARMS:
39 cks = sorted({k[2] for k in C if k[0] == top and k[1] == f'JT-{a}'})
40 say(f' {a:4s} ' + ' '.join(f'{c}:{mean([C[(top, f"JT-{a}", c, s)] for s in seeds]):.3f}' for c in cks))
41
42say('\n## Predictions (sense = sense accuracy on polysemous occurrences, at ' + str(top) + ')')
43def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}')
44for a, arm in [('V2W', 'word'), ('V2D', 'dj')]:
45 c = d(top, f'JT-{a}', f'JT-{a}-COLL', 2); verdict(f'K1 {a}', c[1] > 0, f'sense JT-{a} − JT-{a}-COLL {fmt(c)}')
46 c = d(top, f'JT-{a}-SHUF', f'JT-{a}-COLL-SHUF', 2); verdict(f'K2 {a}', not c[1] > 0, f'shuffled sense JT-{a} − COLL {fmt(c)}')
47 c = d(top, f'JT-{a}', 'JT-V1', 2); verdict(f'K3 {a}', c[1] > 0, f'sense JT-{a} − JT-V1 {fmt(c)}')
48 c = d(top, f'JT-{a}', 'MAJ', 2); verdict(f'K4 {a}', c[1] > 0, f'sense JT-{a} − MAJ {fmt(c)}')
49 c = d(top, f'JT-{a}', 'JT-V1', 0); verdict(f'K5 {a}', c[1] > -0.02, f'V-measure JT-{a} − JT-V1 {fmt(c)}')
50 r = [x for x in A if x[0] == arm]; verdict(f'K6 {a}', len(r) > 0 and all(x[2] == 'PASS' for x in r), f'{sum(x[2] == "PASS" for x in r)}/{len(r)} audit checks PASS')
51say('\n## Scored, no prediction')
52say(f'V1 sense − V1-COLL: {fmt(d(top, "JT-V1", "JT-V1-COLL", 2))}')
53say(f'V1 shuffled sense − COLL: {fmt(d(top, "JT-V1-SHUF", "JT-V1-COLL-SHUF", 2))}')
54say(f'V2W sense − V2D: {fmt(d(top, "JT-V2W", "JT-V2D", 2))}')
55open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n')