| … | … | @@ -285,6 +285,132 @@ static double m1_acc (const corpus_t *C, const int *lab, int nlab) { |
| 285 | 285 | return (double) ok / C->ntok; |
| 286 | 286 | } |
| 287 | 287 | |
| 288 | // ---------------------------------------------------------------- Test DICT (PREREG-dict.md) |
| 289 | |
| 290 | #define NCAND 31 // the MST + 30 perturbed MSTs |
| 291 | #define DBETA 10.0 |
| 292 | #define NCODE 128 // connector code = side * 64 + class |
| 293 | |
| 294 | typedef struct { size_t cap, n; uint64_t *key; double *cnt; } dtab_t; |
| 295 | typedef struct { dtab_t t; double nz[64], phi[64][NCODE], phin[64]; } dict_t; |
| 296 | |
| 297 | static void dt_init (dtab_t *t, size_t cap) { t->cap = cap; t->n = 0; t->key = calloc(cap, sizeof(uint64_t)); t->cnt = calloc(cap, sizeof(double)); } |
| 298 | static double *dt_slot (dtab_t *t, uint64_t k, int create) { |
| 299 | if (create && 2 * (t->n + 1) > t->cap) { |
| 300 | dtab_t g; dt_init(&g, 2 * t->cap); |
| 301 | for (size_t i = 0; i < t->cap; i++) if (t->key[i]) { size_t j = t->key[i] & (g.cap - 1); while (g.key[j]) j = (j + 1) & (g.cap - 1); g.key[j] = t->key[i]; g.cnt[j] = t->cnt[i]; } |
| 302 | g.n = t->n; free(t->key); free(t->cnt); *t = g; |
| 303 | } |
| 304 | size_t i = k & (t->cap - 1); |
| 305 | while (t->key[i] && t->key[i] != k) i = (i + 1) & (t->cap - 1); |
| 306 | if (!t->key[i]) { if (!create) return NULL; t->key[i] = k; t->n++; } |
| 307 | return &t->cnt[i]; |
| 308 | } |
| 309 | static void dict_init (dict_t *D) { dt_init(&D->t, 1 << 16); memset(D->nz, 0, sizeof D->nz); memset(D->phi, 0, sizeof D->phi); memset(D->phin, 0, sizeof D->phin); } |
| 310 | static void dict_free (dict_t *D) { free(D->t.key); free(D->t.cnt); } |
| 311 | |
| 312 | // Sentence-local undirected parse: nl[t] links of token t, lk[t][k] their in-sentence offsets. |
| 313 | typedef struct { int nl[MAXN], lk[MAXN][MAXL]; } sparse_t; |
| 314 | |
| 315 | // MST of one sentence over MI (root edges -1e6), with Gumbel noise of the given scale on every window edge. |
| 316 | static void parse_one (const corpus_t *S, int o, int n, const double *MI, int A, double scale, rng_t *r, sparse_t *T) { |
| 317 | int m = n + 1, hd[MAXN + 1]; |
| 318 | for (int h = 0; h < m; h++) for (int d = 0; d < m; d++) sc[h][d] = NEG; |
| 319 | for (int d = 1; d < m; d++) sc[0][d] = -1e6; |
| 320 | for (int i = 0; i < n; i++) for (int j = i + 1; j < n && j - i <= W; j++) { |
| 321 | double v = MI[(size_t) S->c[o + i] * A + S->c[o + j]]; |
| 322 | if (scale > 0) { double u = runif(r); if (u <= 0) u = 1e-300; v += scale * -log(-log(u)); } |
| 323 | sc[i + 1][j + 1] = sc[j + 1][i + 1] = v; |
| 324 | } |
| 325 | eisner(m, hd); |
| 326 | for (int t = 0; t < n; t++) T->nl[t] = 0; |
| 327 | for (int d = 1; d < m; d++) if (hd[d] > 0) { int a = d - 1, b = hd[d] - 1; T->lk[a][T->nl[a]++] = b; T->lk[b][T->nl[b]++] = a; } |
| 328 | } |
| 329 | |
| 330 | // Token t's disjunct: its links in position order as codes (side, class); returns the count. |
| 331 | static int disj (const corpus_t *S, int o, const sparse_t *T, int t, int *codes) { |
| 332 | int nb[MAXL], m = T->nl[t]; |
| 333 | memcpy(nb, T->lk[t], sizeof(int) * m); |
| 334 | for (int x = 1; x < m; x++) { int v = nb[x], y = x; while (y > 0 && nb[y - 1] > v) { nb[y] = nb[y - 1]; y--; } nb[y] = v; } |
| 335 | for (int x = 0; x < m; x++) codes[x] = (nb[x] < t ? 0 : 64) + S->c[o + nb[x]]; |
| 336 | return m; |
| 337 | } |
| 338 | static uint64_t disj_key (int z, const int *codes, int nc) { |
| 339 | uint64_t h = 1469598103934665603ULL ^ (uint64_t) z; |
| 340 | for (int k = 0; k < nc; k++) h = (h ^ (uint64_t) (codes[k] + 1)) * 1099511628211ULL; |
| 341 | h = (h ^ (uint64_t) nc) * 1099511628211ULL; |
| 342 | return mix(h) | 1; |
| 343 | } |
| 344 | static void dict_add (dict_t *D, const corpus_t *S, int o, int n, const sparse_t *T) { |
| 345 | int codes[MAXL]; |
| 346 | for (int t = 0; t < n; t++) { |
| 347 | int z = S->c[o + t], nc = disj(S, o, T, t, codes); |
| 348 | *dt_slot(&D->t, disj_key(z, codes, nc), 1) += 1; |
| 349 | D->nz[z] += 1; |
| 350 | for (int k = 0; k < nc; k++) { D->phi[z][codes[k]] += 1; D->phin[z] += 1; } |
| 351 | } |
| 352 | } |
| 353 | static double dict_score (const dict_t *D, const corpus_t *S, int o, int n, const sparse_t *T) { |
| 354 | int codes[MAXL]; |
| 355 | double tot = 0; |
| 356 | for (int t = 0; t < n; t++) { |
| 357 | int z = S->c[o + t], nc = disj(S, o, T, t, codes); |
| 358 | double lb = 0; |
| 359 | for (int k = 0; k < nc; k++) lb += log((D->phi[z][codes[k]] + 0.1) / (D->phin[z] + 0.1 * NCODE)); |
| 360 | const double *c = dt_slot((dtab_t *) &D->t, disj_key(z, codes, nc), 0); |
| 361 | tot += log(((c ? *c : 0) + DBETA * exp(lb)) / (D->nz[z] + DBETA)); |
| 362 | } |
| 363 | return tot; |
| 364 | } |
| 365 | static long gold_tp (const corpus_t *G, int o, int n, const sparse_t *T) { |
| 366 | long tp = 0; |
| 367 | for (int t = 0; t < n; t++) { int h = G->h[o + t]; if (h < 0) continue; for (int k = 0; k < T->nl[t]; k++) if (T->lk[t][k] == h) { tp++; break; } } |
| 368 | return tp; |
| 369 | } |
| 370 | |
| 371 | // One source: dictionary 0 from the training MST, then 3 iterations of parse-with-dictionary + re-estimation. |
| 372 | static void run_dict (int seed, int tier, const char *name, const corpus_t *tr, const corpus_t *ho, const double *MI, int A) { |
| 373 | static sparse_t T, best; |
| 374 | dict_t D, Dn; |
| 375 | dict_init(&D); |
| 376 | long gold = 0, tp = 0, pred = 0; |
| 377 | for (int s = 0; s < tr->ns; s++) { int o = tr->off[s], n = tr->off[s + 1] - o; parse_one(tr, o, n, MI, A, 0, NULL, &T); dict_add(&D, tr, o, n, &T); } |
| 378 | for (int s = 0; s < ho->ns; s++) { |
| 379 | int o = ho->off[s], n = ho->off[s + 1] - o; |
| 380 | parse_one(ho, o, n, MI, A, 0, NULL, &T); |
| 381 | tp += gold_tp(ho, o, n, &T); |
| 382 | for (int t = 0; t < n; t++) { gold += ho->h[o + t] >= 0; pred += T.nl[t]; } |
| 383 | } |
| 384 | pred /= 2; |
| 385 | printf("DP\t%d\t%d\t%s\t0\t%.5f\t%.5f\n", seed, tier, name, 2.0 * tp / (pred + gold), 2.0 * tp / (pred + gold)); |
| 386 | for (int it = 1; it <= 3; it++) { |
| 387 | rng_t r = { 1000003ULL * seed + 7919ULL * tier + 131ULL * it + (uint64_t) name[0] }; |
| 388 | dict_init(&Dn); |
| 389 | for (int k = 0; k < 2; k++) { |
| 390 | const corpus_t *C = k ? ho : tr; |
| 391 | long ctp = 0, cor = 0, cpred = 0, cgold = 0; |
| 392 | for (int s = 0; s < C->ns; s++) { |
| 393 | int o = C->off[s], n = C->off[s + 1] - o; |
| 394 | double bs = -1e300; long otp = -1; |
| 395 | for (int c = 0; c < NCAND; c++) { |
| 396 | parse_one(C, o, n, MI, A, c ? 1.0 : 0, &r, &T); |
| 397 | double v = dict_score(&D, C, o, n, &T); |
| 398 | if (v > bs) { bs = v; best = T; } |
| 399 | if (k) { long g = gold_tp(C, o, n, &T); if (g > otp) otp = g; } |
| 400 | } |
| 401 | if (!k) dict_add(&Dn, C, o, n, &best); |
| 402 | else { |
| 403 | ctp += gold_tp(C, o, n, &best); cor += otp; |
| 404 | for (int t = 0; t < n; t++) { cgold += C->h[o + t] >= 0; cpred += best.nl[t]; } |
| 405 | } |
| 406 | } |
| 407 | if (k) { cpred /= 2; printf("DP\t%d\t%d\t%s\t%d\t%.5f\t%.5f\n", seed, tier, name, it, 2.0 * ctp / (cpred + cgold), 2.0 * cor / (cpred + cgold)); } |
| 408 | } |
| 409 | dict_free(&D); D = Dn; |
| 410 | } |
| 411 | dict_free(&D); |
| 412 | } |
| 413 | |
| 288 | 414 | // ---------------------------------------------------------------- span-type table |
| 289 | 415 | |
| 290 | 416 | typedef struct { uint16_t s[LMAX]; uint8_t len, used; uint32_t cnum, hard, first; float soft; } span_t; // hard/first/soft: Test A-soft |
| … | … | @@ -530,11 +656,11 @@ static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, c |
| 530 | 656 | // ---------------------------------------------------------------- main |
| 531 | 657 | |
| 532 | 658 | int main (int argc, char **argv) { |
| 533 | | if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix] [--soft [--soft-scale s]] [--undirected] [--reparse] [--sections prefix] [--labels prefix [--labels-name n]]\n"); return 1; } |
| 659 | if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix] [--soft [--soft-scale s]] [--undirected] [--reparse] [--sections prefix] [--labels prefix [--labels-name n]] [--dict]\n"); return 1; } |
| 534 | 660 | int seed = atoi(argv[1]); |
| 535 | 661 | int tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000; |
| 536 | 662 | const char *jt_prefix = NULL; |
| 537 | | int soft = 0, undirected = 0, reparse = 0; double soft_scale = 1.0; |
| 663 | int soft = 0, undirected = 0, reparse = 0, dict = 0; double soft_scale = 1.0; |
| 538 | 664 | const char *sect_prefix = NULL, *lab_prefix = NULL, *lab_name = "LV"; |
| 539 | 665 | for (int i = 2; i < argc; i++) { |
| 540 | 666 | if (!strcmp(argv[i], "--mut-drop")) mut_drop = 1; |
| … | … | @@ -543,6 +669,7 @@ int main (int argc, char **argv) { |
| 543 | 669 | else if (!strcmp(argv[i], "--soft")) soft = 1; |
| 544 | 670 | else if (!strcmp(argv[i], "--undirected")) undirected = 1; |
| 545 | 671 | else if (!strcmp(argv[i], "--reparse")) reparse = 1; |
| 672 | else if (!strcmp(argv[i], "--dict")) dict = 1; |
| 546 | 673 | else if (!strcmp(argv[i], "--sections") && i + 1 < argc) sect_prefix = argv[++i]; |
| 547 | 674 | else if (!strcmp(argv[i], "--labels") && i + 1 < argc) lab_prefix = argv[++i]; |
| 548 | 675 | else if (!strcmp(argv[i], "--labels-name") && i + 1 < argc) lab_name = argv[++i]; |
| … | … | @@ -622,6 +749,41 @@ int main (int argc, char **argv) { |
| 622 | 749 | for (int w = V - 1; w > 0; w--) { int j = rpick(&rr, w + 1), t = perm[w]; perm[w] = perm[j]; perm[j] = t; } |
| 623 | 750 | for (int w = 0; w < V; w++) Rmap[w] = primary[perm[w]]; |
| 624 | 751 | |
| 752 | // PREREG-dict.md: parse with the learned dictionary, sources G and RP1. Skips every other arm. |
| 753 | if (dict) { |
| 754 | if (!Ptr.lnk) { Ptr = parse_alloc(full.ntok); Pho = parse_alloc(ho.ntok); } |
| 755 | for (int xi = 0; xi < 2; xi++) { |
| 756 | if (xi == 0) { layer1_symbols(SRC_G, &tr, seed, Lmap, Rmap, str); layer1_symbols(SRC_G, &ho, seed, Lmap, Rmap, sho); } |
| 757 | else { // RP1: exchange over the word-MST links (PREREG-reparse round 1) |
| 758 | corpus_t trw = with_symbols(&tr, tr.w); |
| 759 | arm_t a = { "l1", V, NULL, 1 }; |
| 760 | pairs_t P = count_pairs(&a, &trw); |
| 761 | double *MI = pair_mi(&P); |
| 762 | free_pairs(&P); |
| 763 | parse_u(&trw, MI, V, 0, &Ptr); |
| 764 | free(MI); |
| 765 | double *Wp = calloc((size_t) V * V, sizeof(double)); |
| 766 | for (int s = 0; s < tr.ns; s++) { |
| 767 | int o = tr.off[s], n = tr.off[s + 1] - o; |
| 768 | for (int t = 0; t < n; t++) for (int k = 0; k < Ptr.nl[o + t]; k++) { int b = Ptr.lnk[(o + t) * MAXL + k]; if (b > t) Wp[(size_t) tr.w[o + t] * V + tr.w[o + b]] += 1; } |
| 769 | } |
| 770 | int *cls = exchange_pairs(Wp, &tr); |
| 771 | for (int t = 0; t < tr.ntok; t++) str[t] = cls[tr.w[t]]; |
| 772 | for (int t = 0; t < ho.ntok; t++) sho[t] = cls[ho.w[t]]; |
| 773 | free(cls); free(Wp); |
| 774 | } |
| 775 | corpus_t trs = with_symbols(&tr, str), hos = with_symbols(&ho, sho); |
| 776 | arm_t a = { "l1", NCLS, NULL, 1 }; |
| 777 | pairs_t P = count_pairs(&a, &trs); |
| 778 | double *MI = pair_mi(&P); |
| 779 | free_pairs(&P); |
| 780 | run_dict(seed, tier, xi ? "RP1" : "G", &trs, &hos, MI, NCLS); |
| 781 | free(MI); |
| 782 | } |
| 783 | free(Lmap); |
| 784 | continue; |
| 785 | } |
| 786 | |
| 625 | 787 | // PREREG-LV.md: external per-token labels (<prefix>-<tier>-{tr,ho}.lab, one int per token) as layer 1: |
| 626 | 788 | // MST over the labels, link F1, C-x and M-x-ucomp-MST. Skips every other arm. |
| 627 | 789 | if (lab_prefix) { |