// Jigsaw transformer v0: count-based attention over jigsaws. See PREREG.md.
// Reuses the planted grammar, exchange clusterer and Eisner parser of ../across-cluster/ac.c unchanged.
//
// jt <seed> [--tiers 3000,10000,30000] [--heldout N] [--K 30] [--shuffle] [--explain file] [--log file] [--v1]
// [--v2 word|dj] PREREG-v2 arms (implies the v1 pipeline); adds JT-<arm>-COLL rows
// [--v3] PREREG-v3: V2-DJ + sense units split by disjuncts; adds S rows (eligible, split, split∩poly, poly∩eligible)
// [--splitdiag] PREREG-v3diag: D rows (split test on gold/JT parse x gold/JT classes); skips audit and baselines
// [--splitdiag2] PREREG-v3pool: D rows for GG, JG1, JG6, PG1, PG6 (pooled-parse connectors, gold classes)
// [--dump prefix] write <prefix>-<tier>.bin (final labels + attention parse) for ../level2/l2 --jt
// --v1 runs the PREREG-v1 learner (layer 0 from word-pair PMI, classes by exchange over attended pairs).
// C <seed> <tier> <K> <method> <checkpoint> <vmeasure> <m1acc> <senseacc> v1 continuous checkpoints
// Output rows (stdout):
// V <seed> <tier> <K> <method> <vmeasure> <m1acc> <senseacc> methods: JT-BATCH JT-CONT TYPE MAJ GOLD
// L <seed> <tier> <K> <parser> <F1> parsers: ATTN WORD-MI GOLD-MI
// A <seed> <tier> <K> <check> <PASS|FAIL> <detail> audit checks: replay, mutation
#define main ac_main
#include "../across-cluster/ac.c"
#undef main
#define LAMBDA 0.1
#define LAYERS 2
#define ITERS 10
#define BATCH 1000
#define BETA 10.0
static int KJ = 30;
static int MODE = 0; // 0 v1 (class profiles), 1 V2-WORD (neighbour words), 2 V2-DJ (disjuncts); PREREG-v2.md
static int V3 = 0; // PREREG-v3.md: V2-DJ, then split words into sense units by their disjuncts, re-learn
static int VU; // size of the layer-0 alphabet (words, or sense units in v3)
static uint64_t unit_hash; // v3: hash of the training unit assignment (audit)
static long st_elig, st_split, st_split_poly, st_poly_elig; // v3 split diagnostics, last training call
static uint64_t fnv_bytes (uint64_t h, const void *p, size_t n);
// Disjunct table: key = disjunct hash (never 0), row = KJ soft counts.
typedef struct { size_t cap, n; uint64_t *key; double *cnt; } djtab_t;
static void dj_init (djtab_t *t, size_t cap) { t->cap = cap; t->n = 0; t->key = calloc(cap, sizeof(uint64_t)); t->cnt = calloc(cap * KJ, sizeof(double)); }
static void dj_free (djtab_t *t) { free(t->key); free(t->cnt); }
static double *dj_row (const djtab_t *t, uint64_t k) {
for (size_t i = k & (t->cap - 1);; i = (i + 1) & (t->cap - 1)) {
if (t->key[i] == k) return t->cnt + i * KJ;
if (!t->key[i]) return NULL;
}
}
static double *dj_add (djtab_t *t, uint64_t k) {
double *r = dj_row(t, k);
if (r) return r;
if (2 * (t->n + 1) > t->cap) {
djtab_t g; dj_init(&g, 2 * t->cap);
for (size_t i = 0; i < t->cap; i++) if (t->key[i]) {
size_t j = t->key[i] & (g.cap - 1);
while (g.key[j]) j = (j + 1) & (g.cap - 1);
g.key[j] = t->key[i]; memcpy(g.cnt + j * KJ, t->cnt + i * KJ, sizeof(double) * KJ);
}
g.n = t->n; dj_free(t); *t = g;
}
size_t i = k & (t->cap - 1);
while (t->key[i]) i = (i + 1) & (t->cap - 1);
t->key[i] = k; t->n++;
return t->cnt + i * KJ;
}
// ---------------------------------------------------------------- parameters (all counts)
typedef struct {
double *pi; // V x KJ : soft count of class z on word w
double *phL, *phR; // KJ x KJ : soft count of left/right neighbour class y around class z
double *N; // KJ x KJ : soft count of class pair (left x, right y) within the window
double *psiL, *psiR; // KJ x V : soft count of attended left/right neighbour word around class z (V2-WORD)
djtab_t dj; // disjunct x KJ soft counts (V2-DJ)
} counts_t;
typedef struct { // logs / PMI+ derived from counts
double *lpi, *lphL, *lphR, *pmi;
double *lpsiL, *lpsiR; int *wseen; // V2-WORD
djtab_t dj; double djn[64]; // V2-DJ: copy of the counts and per-class totals
} params_t;
static void counts_alloc (counts_t *c) {
c->pi = calloc((size_t) V * KJ, sizeof(double));
c->phL = calloc((size_t) KJ * KJ, sizeof(double)); c->phR = calloc((size_t) KJ * KJ, sizeof(double));
c->N = calloc((size_t) KJ * KJ, sizeof(double));
c->psiL = calloc((size_t) KJ * V, sizeof(double)); c->psiR = calloc((size_t) KJ * V, sizeof(double));
dj_init(&c->dj, 1 << 12);
}
static void counts_zero (counts_t *c) {
memset(c->pi, 0, sizeof(double) * V * KJ); memset(c->phL, 0, sizeof(double) * KJ * KJ);
memset(c->phR, 0, sizeof(double) * KJ * KJ); memset(c->N, 0, sizeof(double) * KJ * KJ);
memset(c->psiL, 0, sizeof(double) * KJ * V); memset(c->psiR, 0, sizeof(double) * KJ * V);
memset(c->dj.key, 0, sizeof(uint64_t) * c->dj.cap); memset(c->dj.cnt, 0, sizeof(double) * c->dj.cap * KJ); c->dj.n = 0;
}
static void counts_free (counts_t *c) { free(c->pi); free(c->phL); free(c->phR); free(c->N); free(c->psiL); free(c->psiR); dj_free(&c->dj); }
static void params_alloc (params_t *p) {
p->lpi = malloc(sizeof(double) * V * KJ);
p->lphL = malloc(sizeof(double) * KJ * KJ); p->lphR = malloc(sizeof(double) * KJ * KJ); p->pmi = malloc(sizeof(double) * KJ * KJ);
p->lpsiL = calloc((size_t) KJ * V, sizeof(double)); p->lpsiR = calloc((size_t) KJ * V, sizeof(double)); p->wseen = calloc(V, sizeof(int));
dj_init(&p->dj, 1 << 12);
}
static void params_free (params_t *p) { free(p->lpi); free(p->lphL); free(p->lphR); free(p->pmi); free(p->lpsiL); free(p->lpsiR); free(p->wseen); dj_free(&p->dj); }
// M-step: add-λ smoothing, renormalise; PMI+ over class pairs.
static void derive (const counts_t *c, params_t *p) {
for (int w = 0; w < V; w++) {
double s = 0;
for (int z = 0; z < KJ; z++) s += c->pi[(size_t) w * KJ + z];
for (int z = 0; z < KJ; z++) p->lpi[(size_t) w * KJ + z] = log((c->pi[(size_t) w * KJ + z] + LAMBDA) / (s + KJ * LAMBDA));
}
for (int z = 0; z < KJ; z++) {
double sl = 0, sr = 0;
for (int y = 0; y < KJ; y++) { sl += c->phL[z * KJ + y]; sr += c->phR[z * KJ + y]; }
for (int y = 0; y < KJ; y++) {
p->lphL[z * KJ + y] = log((c->phL[z * KJ + y] + LAMBDA) / (sl + KJ * LAMBDA));
p->lphR[z * KJ + y] = log((c->phR[z * KJ + y] + LAMBDA) / (sr + KJ * LAMBDA));
}
}
double T = 0, *rs = calloc(KJ, sizeof(double)), *cs = calloc(KJ, sizeof(double));
for (int x = 0; x < KJ; x++) for (int y = 0; y < KJ; y++) { double v = c->N[x * KJ + y] + LAMBDA; T += v; rs[x] += v; cs[y] += v; }
for (int x = 0; x < KJ; x++) for (int y = 0; y < KJ; y++) {
double v = log2((c->N[x * KJ + y] + LAMBDA) * T / (rs[x] * cs[y]));
p->pmi[x * KJ + y] = v > 0 ? v : 0;
}
free(rs); free(cs);
if (MODE == 1) { // Ψ(v | z) = (n(z, v) + β Σ_y Φ(y | z) P(v | y)) / (n(z) + β)
double *py = calloc(KJ, sizeof(double)), ph[64];
for (int w = 0; w < V; w++) {
double s = 0;
for (int y = 0; y < KJ; y++) { py[y] += c->pi[(size_t) w * KJ + y]; s += c->pi[(size_t) w * KJ + y]; }
p->wseen[w] = s > 0;
}
for (int side = 0; side < 2; side++) {
const double *n = side ? c->psiR : c->psiL, *lph = side ? p->lphR : p->lphL;
double *l = side ? p->lpsiR : p->lpsiL;
for (int z = 0; z < KJ; z++) {
double nz = 0;
for (int w = 0; w < V; w++) nz += n[(size_t) z * V + w];
for (int y = 0; y < KJ; y++) ph[y] = exp(lph[z * KJ + y]);
for (int w = 0; w < V; w++) {
if (!p->wseen[w]) { l[(size_t) z * V + w] = 0; continue; }
double b = 0;
for (int y = 0; y < KJ; y++) if (py[y] > 0) b += ph[y] * c->pi[(size_t) w * KJ + y] / py[y];
l[(size_t) z * V + w] = log((n[(size_t) z * V + w] + BETA * b) / (nz + BETA));
}
}
}
free(py);
}
if (MODE == 2) {
if (p->dj.cap != c->dj.cap) { dj_free(&p->dj); dj_init(&p->dj, c->dj.cap); }
memcpy(p->dj.key, c->dj.key, sizeof(uint64_t) * c->dj.cap); memcpy(p->dj.cnt, c->dj.cnt, sizeof(double) * c->dj.cap * KJ);
p->dj.n = c->dj.n;
for (int z = 0; z < KJ; z++) p->djn[z] = 0;
for (size_t i = 0; i < c->dj.cap; i++) if (c->dj.key[i]) for (int z = 0; z < KJ; z++) p->djn[z] += c->dj.cnt[i * KJ + z];
}
}
// ---------------------------------------------------------------- one sentence: layers of attention
#define MAXC 16
typedef struct {
double q[MAXN][64], u[MAXN][64], mL[MAXN][64], mR[MAXN][64], alpha[MAXN][MAXN]; int n;
uint64_t dj[MAXN]; int nc[MAXN], con[MAXN][MAXC]; // V2-DJ: disjunct of each token; connector = side*64 + class
} sent_state_t;
// V2-DJ: Eisner over α_tj + α_jt; each token's disjunct = its links in position order, connector
// (side, cl[j]) with side 0 = left, 1 = right.
static void disjuncts (sent_state_t *S, const int *cl) {
int n = S->n, m = n + 1, head[MAXN + 1];
for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
for (int d = 1; d < m; d++) sc[0][d] = 0;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v = S->alpha[i][j] + S->alpha[j][i];
sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
}
eisner(m, head);
for (int t = 0; t < n; t++) {
uint64_t h = 1469598103934665603ULL;
S->nc[t] = 0;
for (int j = 0; j < n; j++) {
if (j == t || (head[t + 1] != j + 1 && head[j + 1] != t + 1)) continue;
int code = (j < t ? 0 : 64) + cl[j];
h = fnv_bytes(h, &code, sizeof code);
if (S->nc[t] < MAXC) S->con[t][S->nc[t]++] = code;
}
S->dj[t] = h | 1;
}
}
// V2-DJ: log P(d | z) = log (n(z, d) + β Π Φ_side(y | z)) / (n(z) + β).
static double dj_logp (const params_t *p, const sent_state_t *S, int t, int z) {
const double *row = dj_row(&p->dj, S->dj[t]);
double b = 0;
for (int k = 0; k < S->nc[t]; k++) { int code = S->con[t][k]; b += (code < 64 ? p->lphL : p->lphR)[z * KJ + code % 64]; }
return log(((row ? row[z] : 0) + BETA * exp(b)) / (p->djn[z] + BETA));
}
// E-step on tokens [o, o+n) of C. Leaves q (final), alpha and messages (final layer) in S.
static void estep (const corpus_t *C, int o, int n, const params_t *p, sent_state_t *S) {
S->n = n;
for (int t = 0; t < n; t++) { // layer 0: q from the word alone
double mx = -1e300, sum = 0;
const double *lp = p->lpi + (size_t) C->w[o + t] * KJ;
for (int z = 0; z < KJ; z++) if (lp[z] > mx) mx = lp[z];
for (int z = 0; z < KJ; z++) sum += (S->q[t][z] = exp(lp[z] - mx));
for (int z = 0; z < KJ; z++) S->q[t][z] /= sum;
}
static double nq[MAXN][64];
for (int layer = 0; layer < LAYERS; layer++) {
for (int t = 0; t < n; t++) for (int x = 0; x < KJ; x++) { // u_t = PMI+ q_t (t in the right role)
double s = 0;
for (int y = 0; y < KJ; y++) s += p->pmi[x * KJ + y] * S->q[t][y];
S->u[t][x] = s;
}
for (int t = 0; t < n; t++) {
double tot = 0;
for (int j = 0; j < n; j++) {
S->alpha[t][j] = 0;
if (j == t || abs(j - t) > W) continue;
int a = t < j ? t : j, b = t < j ? j : t;
double s = 0;
for (int x = 0; x < KJ; x++) s += S->q[a][x] * S->u[b][x];
S->alpha[t][j] = s > 0 ? s : 0;
tot += S->alpha[t][j];
}
int cnt = 0;
for (int j = 0; j < n; j++) if (j != t && abs(j - t) <= W) cnt++;
for (int j = 0; j < n; j++) {
if (j == t || abs(j - t) > W) continue;
S->alpha[t][j] = tot > 0 ? S->alpha[t][j] / tot : 1.0 / cnt;
}
for (int y = 0; y < KJ; y++) { S->mL[t][y] = 0; S->mR[t][y] = 0; }
for (int j = 0; j < n; j++) {
if (S->alpha[t][j] == 0) continue;
double *m = j < t ? S->mL[t] : S->mR[t];
for (int y = 0; y < KJ; y++) m[y] += S->alpha[t][j] * S->q[j][y];
}
}
if (MODE == 2) {
int cl[MAXN];
for (int j = 0; j < n; j++) { int b = 0; for (int z = 1; z < KJ; z++) if (S->q[j][z] > S->q[j][b]) b = z; cl[j] = b; }
disjuncts(S, cl);
}
for (int t = 0; t < n; t++) {
double mx = -1e300, sum = 0, e[64];
const double *lp = p->lpi + (size_t) C->w[o + t] * KJ;
for (int z = 0; z < KJ; z++) {
double v = lp[z];
if (MODE == 0) for (int y = 0; y < KJ; y++) v += S->mL[t][y] * p->lphL[z * KJ + y] + S->mR[t][y] * p->lphR[z * KJ + y];
else if (MODE == 1) {
for (int j = 0; j < n; j++) {
int wj = C->w[o + j];
if (S->alpha[t][j] > 0 && p->wseen[wj]) v += S->alpha[t][j] * (j < t ? p->lpsiL : p->lpsiR)[(size_t) z * V + wj];
}
} else v += dj_logp(p, S, t, z);
e[z] = v;
if (v > mx) mx = v;
}
for (int z = 0; z < KJ; z++) sum += (nq[t][z] = exp(e[z] - mx));
for (int z = 0; z < KJ; z++) nq[t][z] /= sum;
}
for (int t = 0; t < n; t++) memcpy(S->q[t], nq[t], sizeof(double) * KJ);
}
}
// Add one sentence's soft counts.
static void accumulate (const corpus_t *C, int o, const sent_state_t *S, counts_t *c) {
int n = S->n;
for (int t = 0; t < n; t++) {
double *pi = c->pi + (size_t) C->w[o + t] * KJ;
for (int z = 0; z < KJ; z++) {
pi[z] += S->q[t][z];
for (int y = 0; y < KJ; y++) {
c->phL[z * KJ + y] += S->q[t][z] * S->mL[t][y];
c->phR[z * KJ + y] += S->q[t][z] * S->mR[t][y];
}
}
if (MODE == 1) for (int j = 0; j < n; j++) if (S->alpha[t][j] > 0) {
double *ps = (j < t ? c->psiL : c->psiR) + C->w[o + j];
for (int z = 0; z < KJ; z++) ps[(size_t) z * V] += S->q[t][z] * S->alpha[t][j];
}
if (MODE == 2) { double *r = dj_add(&c->dj, S->dj[t]); for (int z = 0; z < KJ; z++) r[z] += S->q[t][z]; }
double win[64] = { 0 };
for (int j = t + 1; j < n && j - t <= W; j++) for (int y = 0; y < KJ; y++) win[y] += S->q[j][y];
for (int x = 0; x < KJ; x++) for (int y = 0; y < KJ; y++) c->N[x * KJ + y] += S->q[t][x] * win[y];
}
}
// ---------------------------------------------------------------- event log (audit)
static uint64_t fnv_bytes (uint64_t h, const void *p, size_t n) {
const unsigned char *b = p;
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 1099511628211ULL;
return h;
}
static uint64_t counts_hash (const counts_t *c) {
uint64_t h = 1469598103934665603ULL;
h = fnv_bytes(h, c->pi, sizeof(double) * V * KJ);
h = fnv_bytes(h, c->phL, sizeof(double) * KJ * KJ); h = fnv_bytes(h, c->phR, sizeof(double) * KJ * KJ);
return fnv_bytes(h, c->N, sizeof(double) * KJ * KJ);
}
static double counts_mass (const double *a, size_t n) { double s = 0; for (size_t i = 0; i < n; i++) s += a[i]; return s; }
// ---------------------------------------------------------------- training
static void init_counts (counts_t *c, uint64_t seed) {
rng_t r = { seed };
counts_zero(c);
for (int w = 0; w < V; w++) c->pi[(size_t) w * KJ + rpick(&r, KJ)] = 1.0; // random hard start: one class per word
}
static void train_batch (const corpus_t *tr, uint64_t seed, params_t *p) {
counts_t c; counts_alloc(&c); init_counts(&c, seed);
derive(&c, p);
static sent_state_t S;
for (int it = 0; it < ITERS; it++) {
counts_zero(&c);
for (int s = 0; s < tr->ns; s++) { estep(tr, tr->off[s], tr->off[s + 1] - tr->off[s], p, &S); accumulate(tr, tr->off[s], &S, &c); }
derive(&c, p);
}
counts_free(&c);
}
// Continuous: batches seen once, counts never deleted. Each M-step is an event (hash-chained).
// mutate_at >= 0 perturbs that batch's counts (audit mutation test). expect != NULL: verify hashes.
static int train_cont (const corpus_t *tr, uint64_t seed, params_t *p, uint64_t *hashes, int *nev, const uint64_t *expect, int mutate_at, FILE *log) {
counts_t c; counts_alloc(&c); init_counts(&c, seed);
derive(&c, p);
static sent_state_t S;
uint64_t prev = 0;
int ev = 0, first_bad = -1;
for (int b0 = 0; b0 < tr->ns; b0 += BATCH) {
int b1 = b0 + BATCH < tr->ns ? b0 + BATCH : tr->ns;
double m0 = counts_mass(c.N, (size_t) KJ * KJ);
uint64_t sh = fnv_bytes(1469598103934665603ULL, tr->w + tr->off[b0], sizeof(int) * (tr->off[b1] - tr->off[b0]));
for (int s = b0; s < b1; s++) { estep(tr, tr->off[s], tr->off[s + 1] - tr->off[s], p, &S); accumulate(tr, tr->off[s], &S, &c); }
if (ev == mutate_at) c.N[0] += 1.0;
derive(&c, p);
uint64_t h = fnv_bytes(prev, &sh, sizeof sh);
h = fnv_bytes(h, (uint64_t[]) { counts_hash(&c) }, sizeof(uint64_t));
if (log) fprintf(log, "{\"event\":%d,\"batch\":[%d,%d],\"sentences_hash\":\"%016llx\",\"pair_mass_added\":%.6f,\"prev\":\"%016llx\",\"hash\":\"%016llx\"}\n",
ev, b0, b1, (unsigned long long) sh, counts_mass(c.N, (size_t) KJ * KJ) - m0, (unsigned long long) prev, (unsigned long long) h);
if (expect && first_bad < 0 && expect[ev] != h) first_bad = ev;
if (hashes) hashes[ev] = h;
prev = h;
ev++;
}
if (nev) *nev = ev;
counts_free(&c);
return first_bad;
}
// ---------------------------------------------------------------- evaluation
static int *poly; // word has >= 2 gold classes
static void mark_polysemous (void) {
poly = calloc(V, sizeof(int));
int *ncls = calloc(V, sizeof(int));
for (int c = 0; c < NCLS; c++) for (int k = 0; k < ncw[c]; k++) ncls[cw[c][k]]++;
for (int w = 0; w < V; w++) poly[w] = ncls[w] >= 2;
free(ncls);
}
// V-measure, many-to-one accuracy, and sense accuracy (polysemous occurrences) of labels lab[] vs gold.
static void score_labels (const corpus_t *C, const int *lab, int nlab, double *vm, double *m1, double *sense) {
double *ct = calloc((size_t) nlab * NCLS, sizeof(double)), *rl = calloc(nlab, sizeof(double)), *rg = calloc(NCLS, sizeof(double)), Nt = C->ntok;
for (int t = 0; t < C->ntok; t++) { ct[(size_t) lab[t] * NCLS + C->c[t]]++; rl[lab[t]]++; rg[C->c[t]]++; }
double Hc = 0, Hk = 0, Hck = 0, Hkc = 0;
for (int g = 0; g < NCLS; g++) if (rg[g] > 0) Hc -= rg[g] / Nt * log(rg[g] / Nt);
for (int l = 0; l < nlab; l++) if (rl[l] > 0) Hk -= rl[l] / Nt * log(rl[l] / Nt);
for (int l = 0; l < nlab; l++) for (int g = 0; g < NCLS; g++) {
double v = ct[(size_t) l * NCLS + g];
if (v <= 0) continue;
Hck -= v / Nt * log(v / rl[l]);
Hkc -= v / Nt * log(v / rg[g]);
}
double hom = Hc > 0 ? 1 - Hck / Hc : 1, com = Hk > 0 ? 1 - Hkc / Hk : 1;
*vm = hom + com > 0 ? 2 * hom * com / (hom + com) : 0;
int *map = malloc(sizeof(int) * nlab);
for (int l = 0; l < nlab; l++) { int b = 0; for (int g = 1; g < NCLS; g++) if (ct[(size_t) l * NCLS + g] > ct[(size_t) l * NCLS + b]) b = g; map[l] = b; }
double ok = 0, pt = 0, pok = 0;
for (int t = 0; t < C->ntok; t++) {
int r = map[lab[t]] == C->c[t];
ok += r;
if (poly[C->w[t]]) { pt++; pok += r; }
}
*m1 = ok / Nt; *sense = pt > 0 ? pok / pt : 0;
free(ct); free(rl); free(rg); free(map);
}
static void jt_labels (const corpus_t *C, const params_t *p, int *lab) {
static sent_state_t S;
for (int s = 0; s < C->ns; s++) {
int o = C->off[s];
estep(C, o, C->off[s + 1] - o, p, &S);
for (int t = 0; t < S.n; t++) { int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; lab[o + t] = b; }
}
}
static double attention_parse_f1 (const corpus_t *C, const params_t *p) {
static sent_state_t S;
int head[MAXN + 1];
long tp = 0, pred = 0, gold = 0;
for (int s = 0; s < C->ns; s++) {
int o = C->off[s], n = C->off[s + 1] - o, m = n + 1;
estep(C, o, n, p, &S);
for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
for (int d = 1; d < m; d++) sc[0][d] = 0;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v = S.alpha[i][j] + S.alpha[j][i];
sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
}
eisner(m, head);
for (int d = 1; d < m; d++) {
if (C->h[o + d - 1] >= 0) gold++;
if (head[d] > 0) {
pred++;
int a = head[d] - 1, b = d - 1;
if (C->h[o + b] == a || C->h[o + a] == b) tp++;
}
}
}
double pr = pred ? (double) tp / pred : 0, rc = gold ? (double) tp / gold : 0;
return pr + rc > 0 ? 2 * pr * rc / (pr + rc) : 0;
}
static double mi_parse_f1 (const corpus_t *tr, const corpus_t *ho, int gold_classes) {
arm_t a = { gold_classes ? "G" : "U", gold_classes ? NCLS : V, NULL, gold_classes };
pairs_t P = count_pairs(&a, tr);
long ev, acc;
double *S = hard_scores(&P, &ev, &acc);
arm_edge_t e = { &a, S };
link_t L = parse_eval(ho, edge_arm, &e, 1);
free(S); free_pairs(&P);
double pr = L.pred ? (double) L.tp / L.pred : 0, rc = L.gold ? (double) L.tp / L.gold : 0;
return pr + rc > 0 ? 2 * pr * rc / (pr + rc) : 0;
}
static void explain (FILE *f, const corpus_t *C, const params_t *p, const counts_t *unused) {
(void) unused;
static sent_state_t S;
for (int s = 0; s < C->ns; s++) { // first polysemous occurrence
int o = C->off[s], n = C->off[s + 1] - o, t = -1;
for (int k = 0; k < n && t < 0; k++) if (poly[C->w[o + k]]) t = k;
if (t < 0) continue;
estep(C, o, n, p, &S);
fprintf(f, "sentence %d, token %d (word %d, gold class %d)\n", s, t, C->w[o + t], C->c[o + t]);
for (int z = 0; z < KJ; z++) if (S.q[t][z] > 0.05) fprintf(f, " q[class %d] = %.3f (log pi %.3f)\n", z, S.q[t][z], p->lpi[(size_t) C->w[o + t] * KJ + z]);
for (int j = 0; j < n; j++) if (S.alpha[t][j] > 0.05) {
int bt = 0, bj = 0;
for (int z = 1; z < KJ; z++) { if (S.q[t][z] > S.q[t][bt]) bt = z; if (S.q[j][z] > S.q[j][bj]) bj = z; }
int a = t < j ? bt : bj, b = t < j ? bj : bt;
fprintf(f, " attends token %d (word %d) alpha %.3f ; class pair (%d,%d) PMI+ %.3f\n", j, C->w[o + j], S.alpha[t][j], a, b, p->pmi[a * KJ + b]);
}
return;
}
}
// ---------------------------------------------------------------- v1 (PREREG-v1.md): classes emerge from word-level PMI
// Word-level gated PMI (ac.c arm U): S[a*V+b] for left word a, right word b; NEG if the pair fails the gate.
static double *word_scores (const corpus_t *tr) {
arm_t a = { "U", V, NULL, 0 };
pairs_t P = count_pairs(&a, tr);
long ev, acc;
double *S = hard_scores(&P, &ev, &acc);
free_pairs(&P);
return S;
}
// Layer-0 attention of occurrence t over its window, from word-pair PMI. Returns 0 if no neighbour passes.
static int word_attention (const corpus_t *C, int o, int n, int t, const double *SW, int A, double *alpha) {
double tot = 0;
for (int j = 0; j < n; j++) {
alpha[j] = 0;
if (j == t || abs(j - t) > W) continue;
int a = t < j ? C->w[o + t] : C->w[o + j], b = t < j ? C->w[o + j] : C->w[o + t];
double v = SW[(size_t) a * A + b];
if (v > NEG) { alpha[j] = v; tot += v; }
}
if (tot <= 0) return 0;
for (int j = 0; j < n; j++) alpha[j] /= tot;
return 1;
}
// Exchange algorithm (Martin/Liermann/Ney, as ac.c exchange) over weighted word pairs Wp (left a, right b).
static int *exchange_weighted (const double *Wp, const corpus_t *tr) {
double *freq = calloc(V, sizeof(double));
for (int t = 0; t < tr->ntok; t++) freq[tr->w[t]] += 1;
int *order = malloc(sizeof(int) * V);
for (int x = 0; x < V; x++) order[x] = x;
for (int i = 1; i < V; i++) { int x = order[i], j = i; while (j > 0 && freq[order[j - 1]] < freq[x]) { order[j] = order[j - 1]; j--; } order[j] = x; }
int *cls = malloc(sizeof(int) * V);
for (int i = 0; i < V; i++) cls[order[i]] = i % KJ;
double *Nc = calloc((size_t) KJ * KJ, sizeof(double));
for (int a = 0; a < V; a++) for (int b = 0; b < V; b++) { double v = Wp[(size_t) a * V + b]; if (v > 0) Nc[cls[a] * KJ + cls[b]] += v; }
double sv[64], pv[64];
for (int pass = 0; pass < 30; pass++) {
int moves = 0;
for (int i = 0; i < V; i++) {
int x = order[i];
if (freq[x] == 0) continue;
int a = cls[x];
double self = Wp[(size_t) x * V + x];
memset(sv, 0, sizeof sv); memset(pv, 0, sizeof pv);
for (int y = 0; y < V; y++) {
if (y == x) continue;
double r = Wp[(size_t) x * V + y], l = Wp[(size_t) y * V + x];
if (r > 0) sv[cls[y]] += r;
if (l > 0) pv[cls[y]] += l;
}
for (int c = 0; c < KJ; c++) { Nc[a * KJ + c] -= sv[c]; Nc[c * KJ + a] -= pv[c]; }
Nc[a * KJ + a] -= self;
int best = a; double bestF = -1e300;
for (int b = 0; b < KJ; b++) {
for (int c = 0; c < KJ; c++) { Nc[b * KJ + c] += sv[c]; Nc[c * KJ + b] += pv[c]; }
Nc[b * KJ + b] += self;
double f = 0;
for (int r = 0; r < KJ; r++) {
double row = 0, col = 0;
for (int c = 0; c < KJ; c++) { double v = Nc[r * KJ + c]; if (v > 0) f += v * log(v); row += Nc[r * KJ + c]; col += Nc[c * KJ + r]; }
if (row > 0) f -= row * log(row);
if (col > 0) f -= col * log(col);
}
if (f > bestF + 1e-9 || (b == a && f >= bestF - 1e-9)) { bestF = f; best = b; }
for (int c = 0; c < KJ; c++) { Nc[b * KJ + c] -= sv[c]; Nc[c * KJ + b] -= pv[c]; }
Nc[b * KJ + b] -= self;
}
for (int c = 0; c < KJ; c++) { Nc[best * KJ + c] += sv[c]; Nc[c * KJ + best] += pv[c]; }
Nc[best * KJ + best] += self;
if (best != a) { cls[x] = best; moves++; }
}
if (!moves) break;
}
free(freq); free(order); free(Nc);
return cls;
}
// Full v1 learner on a training prefix. Leaves params in p and the layer-0 classes in *cls_out.
// tru: the same text with layer-0 symbols (words, or v3 sense units, alphabet vu); EM always runs over words (tr).
static void train_core (const corpus_t *tr, const corpus_t *tru, int vu, params_t *p, int **cls_out, double perturb) {
int V0 = V;
V = VU = vu; // layer 0 and exchange over the unit alphabet
double *SW = word_scores(tru);
double *Wp = calloc((size_t) vu * vu, sizeof(double)), alpha[MAXN];
for (int s = 0; s < tru->ns; s++) {
int o = tru->off[s], n = tru->off[s + 1] - o;
for (int t = 0; t < n; t++) {
if (!word_attention(tru, o, n, t, SW, vu, alpha)) continue;
for (int j = 0; j < n; j++) if (alpha[j] > 0) {
int a = t < j ? tru->w[o + t] : tru->w[o + j], b = t < j ? tru->w[o + j] : tru->w[o + t];
Wp[(size_t) a * vu + b] += alpha[j];
}
}
}
int *cls = exchange_weighted(Wp, tru);
V = V0;
// Initial counts: one-hot classes, word-level attention as messages.
counts_t c; counts_alloc(&c);
static sent_state_t S;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o;
S.n = n;
for (int t = 0; t < n; t++) { for (int z = 0; z < KJ; z++) S.q[t][z] = 0; S.q[t][cls[tru->w[o + t]]] = 1; }
for (int t = 0; t < n; t++) {
for (int y = 0; y < KJ; y++) { S.mL[t][y] = 0; S.mR[t][y] = 0; }
int ok = word_attention(tru, o, n, t, SW, vu, alpha);
memcpy(S.alpha[t], alpha, sizeof(double) * n);
if (!ok) continue;
for (int j = 0; j < n; j++) if (alpha[j] > 0) (j < t ? S.mL[t] : S.mR[t])[cls[tru->w[o + j]]] += alpha[j];
}
if (MODE == 2) { int cl[MAXN]; for (int j = 0; j < n; j++) cl[j] = cls[tru->w[o + j]]; disjuncts(&S, cl); }
accumulate(tr, o, &S, &c);
}
c.N[0] += perturb; // audit mutation test only
derive(&c, p);
for (int it = 0; it < ITERS; it++) {
counts_zero(&c);
for (int s = 0; s < tr->ns; s++) { estep(tr, tr->off[s], tr->off[s + 1] - tr->off[s], p, &S); accumulate(tr, tr->off[s], &S, &c); }
derive(&c, p);
}
counts_free(&c); free(Wp); free(SW);
*cls_out = cls;
}
// ---------------------------------------------------------------- v3 (PREREG-v3.md): sense units from disjuncts
#define NCODE 128 // connector code = side * 64 + class
#define SPLIT_MIN 20
#define SPLIT_ADD 0.1
// Log-likelihood of bags b[0..m) under multinomial counts c (add-SPLIT_ADD over NCODE codes).
static double bag_ll (const int *codes, int nc, const double *lth) { double s = 0; for (int k = 0; k < nc; k++) s += lth[codes[k]]; return s; }
static void mult_log (const double *c, double *lth) {
double tot = 0;
for (int k = 0; k < NCODE; k++) tot += c[k];
for (int k = 0; k < NCODE; k++) lth[k] = log((c[k] + SPLIT_ADD) / (tot + SPLIT_ADD * NCODE));
}
static double chi2_999 (int df) { double a = 2.0 / (9 * df), x = 1 - a + 3.0902 * sqrt(a); return df * x * x * x; } // Wilson–Hilferty
// Split test for one word over its occurrences occ[0..m) (indices into tcon/tnc). Sets minor[i] = 1 for the
// smaller component's occurrences. Returns 1 if the split is accepted.
static int split_word (const int *occ, int m, const int *tcon, const int *tnc, int *minor) {
int *O = malloc(sizeof(int) * m), no = 0, used[NCODE] = { 0 }, kc = 0;
for (int i = 0; i < m; i++) if (tnc[occ[i]] > 0) O[no++] = i;
double c1[NCODE] = { 0 }, l1[NCODE], cA[NCODE], cB[NCODE], lA[NCODE], lB[NCODE];
for (int i = 0; i < no; i++) for (int k = 0; k < tnc[occ[O[i]]]; k++) c1[tcon[occ[O[i]] * MAXC + k]]++;
for (int k = 0; k < NCODE; k++) if (c1[k] > 0) { used[k] = 1; kc++; }
mult_log(c1, l1);
double LL1 = 0;
for (int i = 0; i < no; i++) LL1 += bag_ll(tcon + occ[O[i]] * MAXC, tnc[occ[O[i]]], l1);
// Initial split: the connector whose presence/absence gives the largest G (hard two-multinomial fit).
int best = -1; double bestG = -1;
for (int code = 0; code < NCODE; code++) {
if (!used[code]) continue;
memset(cA, 0, sizeof cA); memset(cB, 0, sizeof cB);
int nB = 0;
for (int i = 0; i < no; i++) {
const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
for (int k = 0; k < nc; k++) has |= b[k] == code;
nB += has;
for (int k = 0; k < nc; k++) (has ? cB : cA)[b[k]]++;
}
if (nB == 0 || nB == no) continue;
mult_log(cA, lA); mult_log(cB, lB);
double ll = 0;
for (int i = 0; i < no; i++) {
const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
for (int k = 0; k < nc; k++) has |= b[k] == code;
ll += bag_ll(b, nc, has ? lB : lA);
}
if (2 * (ll - LL1) > bestG) { bestG = 2 * (ll - LL1); best = code; }
}
int accept = 0;
if (best >= 0) {
double *r = malloc(sizeof(double) * no), wB = 0, LL2 = 0;
for (int i = 0; i < no; i++) {
const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]], has = 0;
for (int k = 0; k < nc; k++) has |= b[k] == best;
r[i] = has;
}
for (int it = 0; it <= 20; it++) { // 20 EM passes, then one more E-step for LL2 and the posteriors
memset(cA, 0, sizeof cA); memset(cB, 0, sizeof cB); wB = 0;
for (int i = 0; i < no; i++) {
const int *b = tcon + occ[O[i]] * MAXC;
for (int k = 0; k < tnc[occ[O[i]]]; k++) { cA[b[k]] += 1 - r[i]; cB[b[k]] += r[i]; }
wB += r[i];
}
wB /= no;
if (wB <= 0 || wB >= 1) break;
mult_log(cA, lA); mult_log(cB, lB);
LL2 = 0;
for (int i = 0; i < no; i++) {
const int *b = tcon + occ[O[i]] * MAXC; int nc = tnc[occ[O[i]]];
double a = log(1 - wB) + bag_ll(b, nc, lA), bb = log(wB) + bag_ll(b, nc, lB), mx = a > bb ? a : bb;
double lse = mx + log(exp(a - mx) + exp(bb - mx));
LL2 += lse; r[i] = exp(bb - lse);
}
}
double G = 2 * (LL2 - LL1), wmin = wB < 0.5 ? wB : 1 - wB;
accept = wB > 0 && wB < 1 && G > chi2_999(kc) && wmin >= 0.05;
if (getenv("JT_SPLIT_DEBUG")) fprintf(stderr, "split m=%d no=%d kc=%d initG=%.1f G=%.1f thr=%.1f wB=%.3f LL1=%.1f LL2=%.1f accept=%d\n", m, no, kc, bestG, G, chi2_999(kc), wB, LL1, LL2, accept);
for (int i = 0; i < m; i++) minor[i] = 0;
if (accept) for (int i = 0; i < no; i++) minor[O[i]] = wB < 0.5 ? r[i] > 0.5 : r[i] <= 0.5;
free(r);
}
free(O);
return accept;
}
static void train_v3 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
int *cls0;
train_core(tr, tr, V, p, &cls0, 0);
free(cls0);
int *tcon = malloc(sizeof(int) * tr->ntok * MAXC), *tnc = malloc(sizeof(int) * tr->ntok);
static sent_state_t S;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s];
estep(tr, o, tr->off[s + 1] - o, p, &S);
for (int t = 0; t < S.n; t++) { tnc[o + t] = S.nc[t]; memcpy(tcon + (size_t) (o + t) * MAXC, S.con[t], sizeof(int) * MAXC); }
}
// Occurrences grouped by word (counting sort).
int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * tr->ntok), *fill = malloc(sizeof(int) * V);
for (int t = 0; t < tr->ntok; t++) start[tr->w[t] + 1]++;
for (int w = 0; w < V; w++) start[w + 1] += start[w];
memcpy(fill, start, sizeof(int) * V);
for (int t = 0; t < tr->ntok; t++) occ[fill[tr->w[t]]++] = t;
corpus_t tru = *tr;
tru.w = malloc(sizeof(int) * tr->ntok);
memcpy(tru.w, tr->w, sizeof(int) * tr->ntok);
int vu = V, *minor = malloc(sizeof(int) * tr->ntok);
st_elig = st_split = st_split_poly = st_poly_elig = 0;
for (int w = 0; w < V; w++) {
int m = start[w + 1] - start[w];
if (m < SPLIT_MIN) continue;
st_elig++; st_poly_elig += poly[w];
if (!split_word(occ + start[w], m, tcon, tnc, minor)) continue;
st_split++; st_split_poly += poly[w];
for (int i = 0; i < m; i++) if (minor[i]) tru.w[occ[start[w] + i]] = vu;
vu++;
}
unit_hash = fnv_bytes(1469598103934665603ULL, tru.w, sizeof(int) * tr->ntok);
train_core(tr, &tru, vu, p, cls_out, perturb);
free(tcon); free(tnc); free(start); free(occ); free(fill); free(minor); free(tru.w);
}
// Split test over all eligible words with the bags in tcon/tnc; prints one D row.
static void split_stats (const corpus_t *tr, const int *start, const int *occ, const int *tcon, const int *tnc, int *minor,
int seed, int tier, const char *cell) {
long elig = 0, split = 0, split_poly = 0, poly_elig = 0, nmp = 0;
double mp = 0, mr = 0;
for (int w = 0; w < V; w++) {
int m = start[w + 1] - start[w];
const int *ow = occ + start[w];
if (m < SPLIT_MIN) continue;
elig++; poly_elig += poly[w];
if (!split_word(ow, m, tcon, tnc, minor)) continue;
split++;
if (!poly[w]) continue;
split_poly++;
int cnt[64] = { 0 }, c1 = -1, c2 = -1;
for (int i = 0; i < m; i++) cnt[tr->c[ow[i]]]++;
for (int g = 0; g < NCLS; g++) { if (c1 < 0 || cnt[g] > cnt[c1]) { c2 = c1; c1 = g; } else if (c2 < 0 || cnt[g] > cnt[c2]) c2 = g; }
long inm = 0, ins = 0, both = 0;
for (int i = 0; i < m; i++) { int a = minor[i], b = tr->c[ow[i]] == c2; inm += a; ins += b; both += a && b; }
mp += inm ? (double) both / inm : 0; mr += ins ? (double) both / ins : 0; nmp++;
}
printf("D\t%d\t%d\t%d\t%s\t%ld\t%ld\t%ld\t%ld\t%.5f\t%.5f\t%ld\n", seed, tier, KJ, cell, elig, split, split_poly, poly_elig,
nmp ? mp / nmp : 0, nmp ? mr / nmp : 0, nmp);
}
// PREREG-v3diag.md: the v3 split test on links from parse {gold tree, JT attention parse} x connector classes
// {gold, JT label}. D rows: cell, eligible, split, split&poly, poly&eligible, mean minor-component precision and
// recall over split polysemous words, number of such words.
static void split_diag (const corpus_t *tr, const params_t *p, int seed, int tier) {
int N = tr->ntok, hd[MAXN + 1];
int *jl = malloc(sizeof(int) * N), *jh = malloc(sizeof(int) * N);
static sent_state_t S;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o, m = n + 1;
estep(tr, o, n, p, &S);
for (int t = 0; t < n; t++) { int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; jl[o + t] = b; }
for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
for (int d = 1; d < m; d++) sc[0][d] = 0;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v = S.alpha[i][j] + S.alpha[j][i];
sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
}
eisner(m, hd);
for (int d = 1; d < m; d++) jh[o + d - 1] = hd[d] - 1;
}
int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * N), *fill = malloc(sizeof(int) * V);
for (int t = 0; t < N; t++) start[tr->w[t] + 1]++;
for (int w = 0; w < V; w++) start[w + 1] += start[w];
memcpy(fill, start, sizeof(int) * V);
for (int t = 0; t < N; t++) occ[fill[tr->w[t]]++] = t;
int *tcon = malloc(sizeof(int) * N * MAXC), *tnc = malloc(sizeof(int) * N), *minor = malloc(sizeof(int) * N);
static const char *cell[4] = { "GG", "GJ", "JG", "JJ" };
for (int c = 0; c < 4; c++) {
const int *head = c < 2 ? tr->h : jh, *cl = c % 2 == 0 ? tr->c : jl;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o;
for (int t = 0; t < n; t++) {
int nc = 0;
for (int j = 0; j < n; j++) {
if (j == t || (head[o + t] != j && head[o + j] != t)) continue;
if (nc < MAXC) tcon[(size_t) (o + t) * MAXC + nc++] = (j < t ? 0 : 64) + cl[o + j];
}
tnc[o + t] = nc;
}
}
split_stats(tr, start, occ, tcon, tnc, minor, seed, tier, cell[c]);
}
free(jl); free(jh); free(start); free(occ); free(fill); free(tcon); free(tnc); free(minor);
}
// PREREG-v3pool.md: gold connector classes; links from the gold tree, the JT attention parse or the window-PMI
// parse over JT labels, each as the max parse or the consensus (>= 4 of 6) of Gumbel-perturbed parses.
#define NPOOL 6
static void split_diag2 (const corpus_t *tr, const params_t *p, int seed, int tier) {
int N = tr->ntok, hd[MAXN + 1];
int *jl = malloc(sizeof(int) * N);
double *ae = malloc(sizeof(double) * N * W); // ae[t*W + d-1] = α_{t,t+d} + α_{t+d,t}
static sent_state_t S;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o;
estep(tr, o, n, p, &S);
for (int t = 0; t < n; t++) {
int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; jl[o + t] = b;
for (int d = 1; d <= W; d++) ae[(size_t) (o + t) * W + d - 1] = t + d < n ? S.alpha[t][t + d] + S.alpha[t + d][t] : 0;
}
}
corpus_t Lc = *tr; Lc.c = jl;
arm_t a = { "l1", KJ, NULL, 1 };
pairs_t P = count_pairs(&a, &Lc);
long ev, acc;
double *Sc = hard_scores(&P, &ev, &acc);
free_pairs(&P);
double mJ = 0, mP = 0; long nJ = 0, nP = 0;
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v = ae[(size_t) (o + i) * W + j - i - 1], u = Sc[(size_t) jl[o + i] * KJ + jl[o + j]];
if (v > 0) { mJ += v; nJ++; }
if (u > NEG && u > 0) { mP += u; nP++; }
}
}
mJ = nJ ? mJ / nJ : 0; mP = nP ? mP / nP : 0;
printf("#\tsplitdiag2\tmean_pos_edge_JT=%.5f\tmean_pos_edge_PMI=%.5f\n", mJ, mP);
int *start = calloc(V + 1, sizeof(int)), *occ = malloc(sizeof(int) * N), *fill = malloc(sizeof(int) * V);
for (int t = 0; t < N; t++) start[tr->w[t] + 1]++;
for (int w = 0; w < V; w++) start[w + 1] += start[w];
memcpy(fill, start, sizeof(int) * V);
for (int t = 0; t < N; t++) occ[fill[tr->w[t]]++] = t;
int *tcon = malloc(sizeof(int) * N * MAXC), *tnc = malloc(sizeof(int) * N), *minor = malloc(sizeof(int) * N);
static const char *cell[5] = { "GG", "JG1", "JG6", "PG1", "PG6" };
static int lc[MAXN][MAXN];
for (int c = 0; c < 5; c++) {
int pmi = c >= 3, KP = c == 2 || c == 4 ? NPOOL : 1, need = KP == 1 ? 1 : 4;
double scale = 0.3 * (pmi ? mP : mJ);
rng_t r = { 7777ULL * seed + 31ULL * tier + c };
for (int s = 0; s < tr->ns; s++) {
int o = tr->off[s], n = tr->off[s + 1] - o, m = n + 1;
for (int i = 0; i < n; i++) for (int j = 0; j < n; j++) lc[i][j] = 0;
if (c == 0) {
for (int t = 0; t < n; t++) { int h = tr->h[o + t]; if (h >= 0) { lc[t][h]++; lc[h][t]++; } }
} else for (int k = 0; k < KP; k++) {
for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
for (int d = 1; d < m; d++) sc[0][d] = 0;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v;
if (pmi) v = Sc[(size_t) jl[o + i] * KJ + jl[o + j]];
else { v = ae[(size_t) (o + i) * W + j - i - 1]; if (!(v > 0)) v = NEG; }
if (KP > 1 && v > NEG) { double u = runif(&r); if (u <= 0) u = 1e-300; v += scale * -log(-log(u)); }
sc[i + 1][j + 1] = sc[j + 1][i + 1] = v;
}
eisner(m, hd);
for (int d = 1; d < m; d++) { int h = hd[d] - 1; if (h >= 0) { lc[d - 1][h]++; lc[h][d - 1]++; } }
}
for (int t = 0; t < n; t++) {
int nc = 0;
for (int j = 0; j < n; j++) {
if (j == t || lc[t][j] < need) continue;
if (nc < MAXC) tcon[(size_t) (o + t) * MAXC + nc++] = (j < t ? 0 : 64) + tr->c[o + j];
}
tnc[o + t] = nc;
}
}
split_stats(tr, start, occ, tcon, tnc, minor, seed, tier, cell[c]);
}
free(jl); free(ae); free(Sc); free(start); free(occ); free(fill); free(tcon); free(tnc); free(minor);
}
static void train_v1 (const corpus_t *tr, params_t *p, int **cls_out, double perturb) {
if (V3) train_v3(tr, p, cls_out, perturb);
else train_core(tr, tr, V, p, cls_out, perturb);
}
static uint64_t params_hash (const params_t *p, const int *cls) {
uint64_t h = 1469598103934665603ULL;
h = fnv_bytes(h, p->lpi, sizeof(double) * V * KJ);
h = fnv_bytes(h, p->lphL, sizeof(double) * KJ * KJ); h = fnv_bytes(h, p->lphR, sizeof(double) * KJ * KJ);
h = fnv_bytes(h, p->pmi, sizeof(double) * KJ * KJ);
if (MODE == 1) { h = fnv_bytes(h, p->lpsiL, sizeof(double) * KJ * V); h = fnv_bytes(h, p->lpsiR, sizeof(double) * KJ * V); }
if (MODE == 2) { h = fnv_bytes(h, p->dj.key, sizeof(uint64_t) * p->dj.cap); h = fnv_bytes(h, p->dj.cnt, sizeof(double) * p->dj.cap * KJ); }
if (V3) h = fnv_bytes(h, &unit_hash, sizeof unit_hash);
return fnv_bytes(h, cls, sizeof(int) * VU);
}
// Layer-1 dump for ../level2 (PREREG-JT.md): per corpus ntok, labels[ntok], heads[ntok] (in-sentence offset, -1 root).
static void dump_layer1 (const char *path, const corpus_t *tr, const corpus_t *ho, const params_t *p) {
FILE *f = fopen(path, "wb");
if (!f) { fprintf(stderr, "cannot write %s\n", path); exit(1); }
static sent_state_t S;
for (int k = 0; k < 2; k++) {
const corpus_t *C = k ? ho : tr;
int *lab = malloc(sizeof(int) * C->ntok), *head = malloc(sizeof(int) * C->ntok), hd[MAXN + 1];
for (int s = 0; s < C->ns; s++) {
int o = C->off[s], n = C->off[s + 1] - o, m = n + 1;
estep(C, o, n, p, &S);
for (int t = 0; t < n; t++) { int b = 0; for (int z = 1; z < KJ; z++) if (S.q[t][z] > S.q[t][b]) b = z; lab[o + t] = b; }
for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG;
for (int d = 1; d < m; d++) sc[0][d] = 0;
for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) {
double v = S.alpha[i][j] + S.alpha[j][i];
sc[i + 1][j + 1] = sc[j + 1][i + 1] = v > 0 ? v : NEG;
}
eisner(m, hd);
for (int d = 1; d < m; d++) head[o + d - 1] = hd[d] - 1;
}
fwrite(&C->ntok, sizeof(int), 1, f); fwrite(lab, sizeof(int), C->ntok, f); fwrite(head, sizeof(int), C->ntok, f);
free(lab); free(head);
}
fclose(f);
}
// X-COLL (PREREG-v2): every occurrence of a word gets the word's most frequent label in lab.
static void collapse (const corpus_t *C, const int *lab, int nlab, int *out) {
int *cnt = calloc((size_t) V * nlab, sizeof(int)), *maj = malloc(sizeof(int) * V);
for (int t = 0; t < C->ntok; t++) cnt[(size_t) C->w[t] * nlab + lab[t]]++;
for (int w = 0; w < V; w++) { int b = 0; for (int l = 1; l < nlab; l++) if (cnt[(size_t) w * nlab + l] > cnt[(size_t) w * nlab + b]) b = l; maj[w] = b; }
for (int t = 0; t < C->ntok; t++) out[t] = maj[C->w[t]];
free(cnt); free(maj);
}
// ---------------------------------------------------------------- main
static corpus_t shuffle_within (corpus_t C, uint64_t seed) {
rng_t r = { seed };
corpus_t S = C;
S.w = malloc(sizeof(int) * C.ntok); S.c = malloc(sizeof(int) * C.ntok);
memcpy(S.w, C.w, sizeof(int) * C.ntok); memcpy(S.c, C.c, sizeof(int) * C.ntok);
for (int s = 0; s < C.ns; s++) {
int o = C.off[s], n = C.off[s + 1] - o;
for (int i = n - 1; i > 0; i--) {
int j = rpick(&r, i + 1);
int t = S.w[o + i]; S.w[o + i] = S.w[o + j]; S.w[o + j] = t;
t = S.c[o + i]; S.c[o + i] = S.c[o + j]; S.c[o + j] = t;
}
}
return S;
}
int main (int argc, char **argv) {
if (argc < 2) { fprintf(stderr, "usage: jt <seed> [--tiers a,b] [--heldout N] [--K k] [--shuffle] [--explain f] [--log f]\n"); return 1; }
int seed = atoi(argv[1]), tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000, shuffle = 0, v1 = 0, splitdiag = 0, splitdiag2 = 0;
const char *explain_path = NULL, *log_path = NULL, *dump_prefix = NULL;
for (int i = 2; i < argc; i++) {
if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]);
else if (!strcmp(argv[i], "--K") && i + 1 < argc) KJ = atoi(argv[++i]);
else if (!strcmp(argv[i], "--shuffle")) shuffle = 1;
else if (!strcmp(argv[i], "--v1")) v1 = 1;
else if (!strcmp(argv[i], "--splitdiag")) splitdiag = 1;
else if (!strcmp(argv[i], "--splitdiag2")) splitdiag2 = 1;
else if (!strcmp(argv[i], "--v3")) { v1 = 1; MODE = 2; V3 = 1; }
else if (!strcmp(argv[i], "--v2") && i + 1 < argc) { v1 = 1; i++; MODE = !strcmp(argv[i], "word") ? 1 : !strcmp(argv[i], "dj") ? 2 : -1; if (MODE < 0) { fprintf(stderr, "--v2 word|dj\n"); return 1; } }
else if (!strcmp(argv[i], "--explain") && i + 1 < argc) explain_path = argv[++i];
else if (!strcmp(argv[i], "--log") && i + 1 < argc) log_path = argv[++i];
else if (!strcmp(argv[i], "--dump") && i + 1 < argc) dump_prefix = argv[++i];
else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) {
ntier = 0;
for (char *p = strtok(argv[++i], ","); p && ntier < 16; p = strtok(NULL, ",")) tiers[ntier++] = atoi(p);
}
}
if (KJ > 64) { fprintf(stderr, "K <= 64\n"); return 1; }
setvbuf(stdout, NULL, _IOLBF, 0);
build_world();
mark_polysemous();
int maxt = 0;
for (int i = 0; i < ntier; i++) if (tiers[i] > maxt) maxt = tiers[i];
corpus_t full = gen_corpus(1000003ULL * seed + 1, maxt), ho = gen_corpus(1000003ULL * seed + 2, nho);
if (shuffle) { full = shuffle_within(full, 99ULL * seed + 1); ho = shuffle_within(ho, 99ULL * seed + 2); }
const char *tag = shuffle ? "-SHUF" : "", *nm = V3 ? "V3" : MODE == 0 ? "V1" : MODE == 1 ? "V2W" : "V2D";
int *lab = malloc(sizeof(int) * ho.ntok);
params_t p; params_alloc(&p);
for (int ti = 0; ti < ntier; ti++) {
int tier = tiers[ti];
corpus_t tr = prefix(full, tier);
double vm, m1, se;
if (v1) { // PREREG-v1: checkpoints at doubling sizes, each a full re-learn + a hash-chained event
int ck[16], nck = 0;
for (int sz = 1000; sz < tier && nck < 15; sz *= 2) ck[nck++] = sz;
ck[nck++] = tier;
uint64_t hs[16], prev = 0;
FILE *log = log_path && ti == ntier - 1 ? fopen(log_path, "w") : NULL;
int *cls = NULL;
for (int k = 0; k < nck; k++) {
corpus_t pre = prefix(full, ck[k]);
free(cls);
train_v1(&pre, &p, &cls, 0);
uint64_t sh = fnv_bytes(1469598103934665603ULL, pre.w + pre.off[k ? ck[k - 1] : 0], sizeof(int) * (pre.ntok - pre.off[k ? ck[k - 1] : 0]));
uint64_t h = fnv_bytes(prev, &sh, sizeof sh), ph = params_hash(&p, cls);
h = fnv_bytes(h, &ph, sizeof ph);
hs[k] = h;
if (log) fprintf(log, "{\"event\":%d,\"sentences\":[%d,%d],\"sentences_hash\":\"%016llx\",\"state_hash\":\"%016llx\",\"prev\":\"%016llx\",\"hash\":\"%016llx\"}\n",
k, k ? ck[k - 1] : 0, ck[k], (unsigned long long) sh, (unsigned long long) ph, (unsigned long long) prev, (unsigned long long) h);
prev = h;
jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
printf("C\t%d\t%d\t%d\tJT-%s%s\t%d\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, ck[k], vm, m1, se);
}
if (log) fclose(log);
printf("V\t%d\t%d\t%d\tJT-%s%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
int *lc = malloc(sizeof(int) * ho.ntok);
collapse(&ho, lab, KJ, lc); score_labels(&ho, lc, KJ, &vm, &m1, &se); free(lc);
printf("V\t%d\t%d\t%d\tJT-%s-COLL%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, nm, tag, vm, m1, se);
printf("L\t%d\t%d\t%d\tATTN-%s%s\t%.5f\n", seed, tier, KJ, nm, tag, attention_parse_f1(&ho, &p));
if (splitdiag) { split_diag(&tr, &p, seed, tier); free(cls); continue; }
if (splitdiag2) { split_diag2(&tr, &p, seed, tier); free(cls); continue; }
if (V3) printf("S\t%d\t%d\t%d\tJT-%s%s\t%ld\t%ld\t%ld\t%ld\n", seed, tier, KJ, nm, tag, st_elig, st_split, st_split_poly, st_poly_elig);
if (dump_prefix) { char dp[512]; snprintf(dp, sizeof dp, "%s-%d.bin", dump_prefix, tier); dump_layer1(dp, &tr, &ho, &p); }
for (int t = 0; t < ho.ntok; t++) lab[t] = cls[ho.w[t]];
score_labels(&ho, lab, KJ, &vm, &m1, &se);
printf("V\t%d\t%d\t%d\tTYPE-A%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
if (explain_path && ti == ntier - 1) { FILE *f = fopen(explain_path, "w"); if (f) { explain(f, &ho, &p, NULL); fclose(f); } }
// Audit: replay all checkpoints; then alter one checkpoint's counts and confirm detection there.
int bad = -1, at = nck / 2, badm = -1;
prev = 0;
for (int pass = 0; pass < 2; pass++) {
uint64_t pv = 0;
for (int k = 0; k < nck; k++) {
corpus_t pre = prefix(full, ck[k]);
int *c2 = NULL;
train_v1(&pre, &p, &c2, pass == 1 && k == at ? 1.0 : 0);
uint64_t sh = fnv_bytes(1469598103934665603ULL, pre.w + pre.off[k ? ck[k - 1] : 0], sizeof(int) * (pre.ntok - pre.off[k ? ck[k - 1] : 0]));
uint64_t h = fnv_bytes(pv, &sh, sizeof sh), ph = params_hash(&p, c2);
h = fnv_bytes(h, &ph, sizeof ph);
if (h != hs[k]) { if (pass == 0 && bad < 0) bad = k; if (pass == 1 && badm < 0) badm = k; }
pv = h;
free(c2);
}
}
printf("A\t%d\t%d\t%d\treplay\t%s\tevents=%d first_mismatch=%d\n", seed, tier, KJ, bad < 0 ? "PASS" : "FAIL", nck, bad);
printf("A\t%d\t%d\t%d\tmutation\t%s\taltered=%d detected_at=%d\n", seed, tier, KJ, badm == at ? "PASS" : "FAIL", at, badm);
free(cls);
} else {
train_batch(&tr, 1000ULL * seed + 7, &p);
jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
printf("V\t%d\t%d\t%d\tJT-BATCH%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
printf("L\t%d\t%d\t%d\tATTN%s\t%.5f\n", seed, tier, KJ, tag, attention_parse_f1(&ho, &p));
if (explain_path && ti == ntier - 1) { FILE *f = fopen(explain_path, "w"); if (f) { explain(f, &ho, &p, NULL); fclose(f); } }
// Continuous mode + audit (event log, replay, mutation).
int nb = (tier + BATCH - 1) / BATCH, nev = 0;
uint64_t *hs = malloc(sizeof(uint64_t) * nb), *hs2 = malloc(sizeof(uint64_t) * nb);
FILE *log = log_path && ti == ntier - 1 ? fopen(log_path, "w") : NULL;
train_cont(&tr, 1000ULL * seed + 7, &p, hs, &nev, NULL, -1, log);
if (log) fclose(log);
jt_labels(&ho, &p, lab); score_labels(&ho, lab, KJ, &vm, &m1, &se);
printf("V\t%d\t%d\t%d\tJT-CONT%s\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, tag, vm, m1, se);
int bad = train_cont(&tr, 1000ULL * seed + 7, &p, hs2, NULL, hs, -1, NULL);
printf("A\t%d\t%d\t%d\treplay\t%s\tevents=%d first_mismatch=%d\n", seed, tier, KJ, bad < 0 ? "PASS" : "FAIL", nev, bad);
int at = nev / 2;
bad = train_cont(&tr, 1000ULL * seed + 7, &p, hs2, NULL, hs, at, NULL);
printf("A\t%d\t%d\t%d\tmutation\t%s\taltered=%d detected_at=%d\n", seed, tier, KJ, bad == at ? "PASS" : "FAIL", at, bad);
free(hs); free(hs2);
}
if (KJ == 30 && !shuffle) { // baselines (KJ-independent)
int passes, *L = exchange(&tr, &passes);
for (int t = 0; t < ho.ntok; t++) lab[t] = L[ho.w[t]];
score_labels(&ho, lab, NCLS, &vm, &m1, &se); // exchange labels are < ac.c K = 30
printf("V\t%d\t%d\t%d\tTYPE\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, vm, m1, se);
free(L);
int *maj = malloc(sizeof(int) * V), *cnt = calloc((size_t) V * NCLS, sizeof(int));
for (int t = 0; t < tr.ntok; t++) cnt[(size_t) tr.w[t] * NCLS + tr.c[t]]++;
for (int w = 0; w < V; w++) { int b = primary[w]; for (int g = 0; g < NCLS; g++) if (cnt[(size_t) w * NCLS + g] > cnt[(size_t) w * NCLS + b]) b = g; maj[w] = b; }
for (int t = 0; t < ho.ntok; t++) lab[t] = maj[ho.w[t]];
score_labels(&ho, lab, NCLS, &vm, &m1, &se);
printf("V\t%d\t%d\t%d\tMAJ\t%.5f\t%.5f\t%.5f\n", seed, tier, KJ, vm, m1, se);
free(maj); free(cnt);
printf("L\t%d\t%d\t%d\tWORD-MI\t%.5f\n", seed, tier, KJ, mi_parse_f1(&tr, &ho, 0));
printf("L\t%d\t%d\t%d\tGOLD-MI\t%.5f\n", seed, tier, KJ, mi_parse_f1(&tr, &ho, 1));
}
}
params_free(&p);
return 0;
}