| … | … | @@ -425,13 +425,305 @@ static void explode_sections (void) { |
| 425 | 425 | } |
| 426 | 426 | } |
| 427 | 427 | |
| 428 | // ---------------------------------------------------------------- membership (gram-class-api.scm) |
| 429 | |
| 430 | static int is_class (long a) { return atoms.keys[a][0] == 'C'; } |
| 431 | |
| 432 | typedef struct { long w, cls; double n; } memb_t; |
| 433 | static memb_t *membs; |
| 434 | static size_t nmembs, capmembs; |
| 435 | |
| 436 | static long find_memb (long w, long cls) { |
| 437 | for (size_t k = 0; k < nmembs; k++) if (membs[k].w == w && membs[k].cls == cls) return k; |
| 438 | return -1; |
| 439 | } |
| 440 | |
| 441 | static long make_memb (long w, long cls) { |
| 442 | long k = find_memb(w, cls); |
| 443 | if (k >= 0) return k; |
| 444 | if (nmembs == capmembs) { capmembs = capmembs ? capmembs * 2 : 256; membs = realloc(membs, sizeof(memb_t) * capmembs); } |
| 445 | membs[nmembs] = (memb_t) { w, cls, 0 }; |
| 446 | return nmembs++; |
| 447 | } |
| 448 | |
| 449 | static int is_member (long a, long cls) { return a >= 0 && find_memb(a, cls) >= 0; } |
| 450 | |
| 451 | // make-class-node (gram-majority.scm:86-109): members' names joined by spaces, ".i" until unique. |
| 452 | static long make_class_node (const long *g, int n) { |
| 453 | static char key[1 << 16]; |
| 454 | int k = snprintf(key, sizeof key, "C "); |
| 455 | for (int i = 0; i < n; i++) k += snprintf(key + k, sizeof key - k, i ? " %s" : "%s", aname(g[i])); |
| 456 | while (lookup(&atoms, key) >= 0) k += snprintf(key + k, sizeof key - k, ".i"); |
| 457 | return intern(&atoms, key); |
| 458 | } |
| 459 | |
| 460 | // ---------------------------------------------------------------- sections and cross-sections (shape-vec.scm) |
| 461 | |
| 462 | static int is_section (long p) { return !cols[pairs[p].c].shape; } |
| 463 | |
| 464 | // make-cross-sections (shape-vec.scm:501-521): create (if needed) and return the crosses of a Section. |
| 465 | static int make_cross_sections (long sect, long *out) { |
| 466 | col_t s = cols[pairs[sect].c]; |
| 467 | long ca[64]; char cd[64]; |
| 468 | memcpy(ca, s.ca, sizeof(long) * s.nc); memcpy(cd, s.cd, s.nc); |
| 469 | long germ = pairs[sect].r; |
| 470 | for (int i = 0; i < s.nc; i++) { |
| 471 | long c = make_col(1, germ, s.nc, ca, cd, i); |
| 472 | out[i] = make_pair(ca[i], c); |
| 473 | } |
| 474 | return s.nc; |
| 475 | } |
| 476 | |
| 477 | // get-cross-sections (shape-vec.scm:475-498): only those that exist. |
| 478 | static int get_cross_sections (long sect, long *out) { |
| 479 | col_t s = cols[pairs[sect].c]; |
| 480 | static char key[1 << 16]; |
| 481 | int n = 0; |
| 482 | for (int i = 0; i < s.nc; i++) { |
| 483 | int k = snprintf(key, sizeof key, "X%ld", pairs[sect].r); |
| 484 | for (int j = 0; j < s.nc; j++) k += snprintf(key + k, sizeof key - k, " %ld%c", j == i ? -1L : s.ca[j], s.cd[j]); |
| 485 | long c = lookup(&colkeys, key); |
| 486 | long x = c < 0 ? -1 : get_pair(s.ca[i], c); |
| 487 | if (x >= 0) out[n++] = x; |
| 488 | } |
| 489 | return n; |
| 490 | } |
| 491 | |
| 492 | // The Section a CrossSection covers: germ goes back into the hole. |
| 493 | static long section_col_of (long cross) { |
| 494 | col_t sh = cols[pairs[cross].c]; |
| 495 | long ca[64]; char cd[64]; |
| 496 | memcpy(ca, sh.ca, sizeof(long) * sh.nc); memcpy(cd, sh.cd, sh.nc); |
| 497 | ca[sh.hole] = pairs[cross].r; |
| 498 | return make_col(0, -1, sh.nc, ca, cd, -1); |
| 499 | } |
| 500 | |
| 501 | static long make_section (long cross) { long c = section_col_of(cross); return make_pair(cols[pairs[cross].c].point, c); } |
| 502 | static long get_section (long cross) { long c = section_col_of(cross); return get_pair(cols[pairs[cross].c].point, c); } |
| 503 | |
| 504 | // flatten-section / flatten-cross / make-flat (shape-vec.scm:310-399). |
| 505 | static long make_flat (long cls, long pnt) { |
| 506 | col_t c = cols[pairs[pnt].c]; |
| 507 | long ca[64]; char cd[64]; |
| 508 | memcpy(ca, c.ca, sizeof(long) * c.nc); memcpy(cd, c.cd, c.nc); |
| 509 | long germ = pairs[pnt].r, point = c.point; |
| 510 | for (int i = 0; i < c.nc; i++) if (i != c.hole && is_member(ca[i], cls)) ca[i] = cls; |
| 511 | if (c.shape) { if (is_member(point, cls)) point = cls; ca[c.hole] = -1; } |
| 512 | if (is_member(germ, cls)) germ = cls; |
| 513 | long col = make_col(c.shape, c.shape ? point : -1, c.nc, ca, cd, c.shape ? c.hole : -1); |
| 514 | return make_pair(germ, col); |
| 515 | } |
| 516 | |
| 517 | // is-nonflat? (shape-vec.scm:405-411): does a connector of the pair already name the class? |
| 518 | static int is_nonflat (long cls, long pnt) { |
| 519 | col_t c = cols[pairs[pnt].c]; |
| 520 | for (int i = 0; i < c.nc; i++) if (i != c.hole && c.ca[i] == cls) return 1; |
| 521 | return 0; |
| 522 | } |
| 523 | |
| 524 | // ---------------------------------------------------------------- counts (count-api.scm, shape-project.scm) |
| 525 | |
| 526 | static void rebalance_count (long sect, double cnt) { |
| 527 | long xs[64]; |
| 528 | pairs[sect].n = cnt; |
| 529 | int n = make_cross_sections(sect, xs); |
| 530 | for (int i = 0; i < n; i++) pairs[xs[i]].n = cnt; |
| 531 | } |
| 532 | |
| 533 | static void rebalance_merge (long mrg, long donor) { |
| 534 | int is_sect = is_section(donor); |
| 535 | long m = is_sect ? mrg : make_section(mrg); |
| 536 | long d = is_sect ? donor : make_section(donor); |
| 537 | long xs[64]; |
| 538 | make_cross_sections(m, xs); |
| 539 | rebalance_count(m, pairs[mrg].n); |
| 540 | rebalance_count(d, pairs[donor].n); |
| 541 | } |
| 542 | |
| 543 | static double accumulate_count (long acc, long donor, double frac) { |
| 544 | double moved = frac * pairs[donor].n; |
| 545 | if (!(moved < 1.0e-10)) { |
| 546 | pairs[acc].n += moved; |
| 547 | pairs[donor].n -= moved; |
| 548 | rebalance_merge(acc, donor); |
| 549 | } |
| 550 | return moved; |
| 551 | } |
| 552 | |
| 553 | // ---------------------------------------------------------------- merge-majority (gram-majority.scm:113-477) |
| 554 | |
| 555 | static long cur_cls; |
| 556 | static long *cur_g; |
| 557 | static int cur_n, cur_thresh; |
| 558 | |
| 559 | static int vote_to_accept (long col) { |
| 560 | int v = 0; |
| 561 | for (int i = 0; i < cur_n; i++) if (get_pair(cur_g[i], col) >= 0) v++; |
| 562 | return cur_thresh <= v; |
| 563 | } |
| 564 | |
| 565 | static double do_merge (long w, long col, int accept) { |
| 566 | long sect = get_pair(w, col); |
| 567 | if (sect < 0) return 0; |
| 568 | int full = accept || pairs[sect].n <= NOISE || is_nonflat(cur_cls, sect) || is_class(w); |
| 569 | if (!full) return 0; // FRAC = 0 |
| 570 | long flat = make_flat(cur_cls, sect); |
| 571 | double moved = accumulate_count(flat, sect, 1.0); |
| 572 | membs[make_memb(w, cur_cls)].n += moved; |
| 573 | return 1.0; |
| 574 | } |
| 575 | |
| 576 | static void rebalance_dj (long col) { |
| 577 | for (int i = 0; i < cur_n; i++) { |
| 578 | long sect = get_pair(cur_g[i], col); |
| 579 | if (sect >= 0) { long flat = make_flat(cur_cls, sect); rebalance_merge(flat, sect); } |
| 580 | } |
| 581 | } |
| 582 | |
| 583 | static char *done, *shape_done; |
| 584 | static size_t capdone; |
| 585 | |
| 586 | static void mark (char **set, long col) { |
| 587 | if ((size_t) col >= capdone) { |
| 588 | size_t nc = colkeys.n * 2 + 1024; |
| 589 | done = realloc(done, nc); shape_done = realloc(shape_done, nc); |
| 590 | memset(done + capdone, 0, nc - capdone); memset(shape_done + capdone, 0, nc - capdone); |
| 591 | capdone = nc; |
| 592 | } |
| 593 | (*set)[col] = 1; |
| 594 | } |
| 595 | |
| 596 | static int is_marked (char *set, long col) { return (size_t) col < capdone && set[col]; } |
| 597 | |
| 598 | // make-once-predicate: false the first time it is asked about an atom, true afterwards. |
| 599 | static int once_shape_done (long col) { if (is_marked(shape_done, col)) return 1; mark(&shape_done, col); return 0; } |
| 600 | |
| 601 | static void move_member_counts (long from) { |
| 602 | double old = 0, nw; |
| 603 | for (size_t k = 0; k < nmembs; k++) if (membs[k].cls == from) old += membs[k].n; |
| 604 | long d = find_memb(from, cur_cls); |
| 605 | nw = d < 0 ? 0 : membs[d].n; |
| 606 | double fract = nw / old; |
| 607 | for (size_t k = 0; k < nmembs; k++) { |
| 608 | if (membs[k].cls != from) continue; |
| 609 | double x = membs[k].n * fract; |
| 610 | long w = membs[k].w; |
| 611 | membs[make_memb(w, cur_cls)].n += x; |
| 612 | membs[find_memb(w, from)].n -= x; |
| 613 | } |
| 614 | if (d >= 0) { membs[d] = membs[--nmembs]; } |
| 615 | } |
| 616 | |
| 617 | static void merge_majority (long cls, long *g, int n) { |
| 618 | for (int i = 0; i < n; i++) make_memb(g[i], cls); |
| 619 | cur_cls = cls; cur_g = g; cur_n = n; |
| 620 | cur_thresh = n <= 2 ? 2 : round_even(QUORUM * n); |
| 621 | free(done); free(shape_done); done = shape_done = NULL; capdone = 0; |
| 622 | mark(&done, 0); done[0] = 0; |
| 623 | |
| 624 | // get-all-djs: the columns of all members' pairs, first-seen order (Linas: atom-set order). |
| 625 | size_t ndj = 0, capdj = 1024; |
| 626 | long *djs = malloc(sizeof(long) * capdj); |
| 627 | char *seen = calloc(colkeys.n + 1, 1); |
| 628 | for (int i = 0; i < n; i++) for (size_t k = 0; k < rowpn[g[i]]; k++) { |
| 629 | long c = pairs[rowp[g[i]][k]].c; |
| 630 | if (!pairs[rowp[g[i]][k]].alive || seen[c]) continue; |
| 631 | seen[c] = 1; |
| 632 | if (ndj == capdj) { capdj *= 2; djs = realloc(djs, sizeof(long) * capdj); } |
| 633 | djs[ndj++] = c; |
| 634 | } |
| 635 | free(seen); |
| 636 | |
| 637 | int scnt = 0, mscnt = 0; |
| 638 | for (size_t k = 0; k < ndj; k++) { |
| 639 | long dj = djs[k]; |
| 640 | if (cols[dj].shape) continue; |
| 641 | int have = vote_to_accept(dj); |
| 642 | double merged = 0; |
| 643 | for (int i = 0; i < n; i++) merged += do_merge(g[i], dj, have); |
| 644 | if (0 < merged) { |
| 645 | mark(&done, dj); // record-done: the DJ and the Shapes of its members' Sections |
| 646 | for (int i = 0; i < n; i++) { |
| 647 | long sect = get_pair(g[i], dj); |
| 648 | if (sect < 0) continue; |
| 649 | long xs[64]; |
| 650 | int nx = make_cross_sections(sect, xs); |
| 651 | for (int j = 0; j < nx; j++) mark(&done, pairs[xs[j]].c); |
| 652 | } |
| 653 | mscnt++; |
| 654 | } |
| 655 | rebalance_dj(dj); |
| 656 | mark(&done, dj); |
| 657 | scnt++; |
| 658 | } |
| 659 | printf("------ merge-majority: Merge %d of %d sections\n", mscnt, scnt); |
| 660 | |
| 661 | // Left-over CrossSections (gram-majority.scm:333-422). |
| 662 | int nleft = 0, mshcnt = 0; |
| 663 | long *left = malloc(sizeof(long) * (ndj + 1)); |
| 664 | for (size_t k = 0; k < ndj; k++) if (!is_marked(done, djs[k])) left[nleft++] = djs[k]; |
| 665 | for (int k = 0; k < nleft; k++) { |
| 666 | long shp = left[k]; |
| 667 | if (once_shape_done(shp)) continue; |
| 668 | long alt = -1; |
| 669 | if (vote_to_accept(shp)) alt = shp; |
| 670 | else { |
| 671 | // get-alt-shapes: Shapes of the other crosses of the members' Sections, not yet done (marks them). |
| 672 | long cand[4096]; int nc = 0; |
| 673 | for (int i = 0; i < n; i++) { |
| 674 | long x = get_pair(g[i], shp); |
| 675 | if (x < 0) continue; |
| 676 | long sect = get_section(x); |
| 677 | if (sect < 0) continue; |
| 678 | long xs[64]; |
| 679 | int nx = make_cross_sections(sect, xs); |
| 680 | for (int j = 0; j < nx; j++) { |
| 681 | long c = pairs[xs[j]].c; |
| 682 | int dup = 0; |
| 683 | for (int q = 0; q < nc; q++) if (cand[q] == c) dup = 1; |
| 684 | if (!dup && nc < 4096) cand[nc++] = c; |
| 685 | } |
| 686 | } |
| 687 | for (int q = 0; q < nc && alt < 0; q++) if (!once_shape_done(cand[q]) && vote_to_accept(cand[q])) alt = cand[q]; |
| 688 | } |
| 689 | int have = alt >= 0; |
| 690 | long mrg = have ? alt : shp; |
| 691 | double mcnt = 0; |
| 692 | for (int i = 0; i < n; i++) mcnt += do_merge(g[i], mrg, have); |
| 693 | rebalance_dj(mrg); |
| 694 | if (0 < mcnt) mshcnt++; |
| 695 | } |
| 696 | printf("------ merge-majority: Remaining %d of %d cross\n", mshcnt, nleft); |
| 697 | for (int i = 0; i < n; i++) if (is_class(g[i])) move_member_counts(g[i]); |
| 698 | free(left); free(djs); |
| 699 | } |
| 700 | |
| 701 | // ---------------------------------------------------------------- cleanup (shape-project.scm:310-387) |
| 702 | |
| 703 | static void del_sect (long sect) { |
| 704 | long xs[64]; |
| 705 | int nx = get_cross_sections(sect, xs); |
| 706 | for (int i = 0; i < nx; i++) pairs[xs[i]].alive = 0; |
| 707 | pairs[sect].alive = 0; |
| 708 | } |
| 709 | |
| 710 | static void remove_empty (long row) { |
| 711 | size_t n0 = npairs; |
| 712 | for (size_t k = 0; k < n0; k++) { |
| 713 | if (!pairs[k].alive || pairs[k].r != row || !(pairs[k].n < 1.0e-10)) continue; |
| 714 | if (is_section(k)) del_sect(k); |
| 715 | else { long s = get_section(k); if (s >= 0) del_sect(s); } |
| 716 | } |
| 717 | } |
| 718 | |
| 428 | 719 | // ---------------------------------------------------------------- main |
| 429 | 720 | |
| 430 | 721 | int main (int argc, char **argv) { |
| 431 | | if (argc < 2) { fprintf(stderr, "usage: cluster <counts.tsv> [--nrank 50] [--intended]\n"); return 1; } |
| 432 | | int nrank = 50; |
| 722 | if (argc < 2) { fprintf(stderr, "usage: cluster <counts.tsv> [--nrank 50] [--loops 5] [--intended]\n"); return 1; } |
| 723 | int nrank = 50, loops = 5; |
| 433 | 724 | for (int i = 2; i < argc; i++) { |
| 434 | 725 | if (!strcmp(argv[i], "--nrank") && i + 1 < argc) nrank = atoi(argv[++i]); |
| 726 | else if (!strcmp(argv[i], "--loops") && i + 1 < argc) loops = atoi(argv[++i]); |
| 435 | 727 | else if (!strcmp(argv[i], "--intended")) as_run = 0; |
| 436 | 728 | } |
| 437 | 729 | setvbuf(stdout, NULL, _IOLBF, 0); |
| … | … | @@ -442,26 +734,67 @@ int main (int argc, char **argv) { |
| 442 | 734 | sims_upper(nrank); |
| 443 | 735 | printf("Done setting up similarity to %d\n", nrank); |
| 444 | 736 | |
| 445 | | // Round 1 (main-loop iteration 1): seed, in-group, trim. |
| 446 | | int iter = 1; |
| 447 | | printf("------ Round %d Next in line:\n", iter); |
| 448 | | long best = -1; |
| 449 | | for (size_t k = 0; k < nsims; k++) { |
| 450 | | if (sims[k].a == sims[k].b || !(MI_CUTOFF < sims[k].fmi)) continue; |
| 451 | | if (best < 0 || sims[k].rmi > sims[best].rmi) best = k; |
| 737 | for (int iter = 1; iter <= loops; iter++) { |
| 738 | // top-ranked-mi-pairs: fmi > 4, a != b, by ranked-MI; print the first 12. |
| 739 | printf("------ Round %d Next in line:\n", iter); |
| 740 | size_t nc = 0; |
| 741 | long *cand = malloc(sizeof(long) * (nsims + 1)); |
| 742 | for (size_t k = 0; k < nsims; k++) if (sims[k].a != sims[k].b && MI_CUTOFF < sims[k].fmi) cand[nc++] = k; |
| 743 | for (size_t i = 1; i < nc; i++) { // insertion sort, rmi descending (stable) |
| 744 | long x = cand[i]; size_t j = i; |
| 745 | while (j > 0 && sims[cand[j - 1]].rmi < sims[x].rmi) { cand[j] = cand[j - 1]; j--; } |
| 746 | cand[j] = x; |
| 747 | } |
| 748 | for (size_t i = 0; i < nc && i < 12; i++) |
| 749 | printf("ranked-MI = %.4f MI = %.4f (`%s`, `%s`)\n", sims[cand[i]].rmi, sims[cand[i]].fmi, aname(sims[cand[i]].a), aname(sims[cand[i]].b)); |
| 750 | if (!nc) { printf("no pair above the MI cutoff\n"); free(cand); break; } |
| 751 | long wa = sims[cand[0]].a, wb = sims[cand[0]].b; |
| 752 | free(cand); |
| 753 | printf("------ Start MI-based merge %d with seed pair `%s` and `%s`\n", iter, aname(wa), aname(wb)); |
| 754 | |
| 755 | rank_words(); |
| 756 | int ncand = nranked < (size_t) (nrank + 3 * iter) ? (int) nranked : nrank + 3 * iter; |
| 757 | long grp[4096]; |
| 758 | int n = optimal_in_group(wa, wb, ranked, ncand, grp); |
| 759 | printf("Initial in-group size=%d:", n); |
| 760 | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 761 | printf("\n"); |
| 762 | n = trim_greedy(grp, n); |
| 763 | printf("In-group size=%d:", n); |
| 764 | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 765 | printf("\n"); |
| 766 | |
| 767 | long cls = make_class_node(grp, n); |
| 768 | merge_majority(cls, grp, n); |
| 769 | printf("------ Merged into `%s`\n", aname(cls)); |
| 770 | |
| 771 | recompute(); // marginals of everything touched (full recompute is equivalent) |
| 772 | remove_empty(cls); |
| 773 | for (int i = 0; i < n; i++) remove_empty(grp[i]); |
| 774 | recompute(); |
| 775 | |
| 776 | // recomp-all-sim for the in-group (PRECISE-SIM #f): every stored sim that involves a member. |
| 777 | for (int i = 0; i < n; i++) { |
| 778 | long w = grp[i]; |
| 779 | int has = 0; |
| 780 | for (size_t k = 0; k < nsims && !has; k++) if (sims[k].a == w || sims[k].b == w) has = 1; |
| 781 | if (!has) continue; |
| 782 | simmer(w, w); |
| 783 | size_t n0 = nsims; |
| 784 | for (size_t k = 0; k < n0; k++) { |
| 785 | long o = sims[k].a == w ? sims[k].b : sims[k].b == w ? sims[k].a : -1; |
| 786 | if (o < 0 || o == w) continue; |
| 787 | int earlier = 0; |
| 788 | for (int j = 0; j < i; j++) if (grp[j] == o) earlier = 1; |
| 789 | if (!earlier) simmer(o, w); |
| 790 | } |
| 791 | } |
| 792 | simmer(cls, cls); |
| 793 | for (size_t r = 0; r < atoms.n; r++) if (is_class(r) && rowpn[r] > 0) simmer(cls, r); |
| 794 | |
| 795 | // expand-universe: sims for missing pairs within the top NRANK + 2*iter. |
| 796 | rank_words(); |
| 797 | sims_upper(nrank + 2 * iter); |
| 452 | 798 | } |
| 453 | | if (best < 0) { printf("no pair above the MI cutoff\n"); return 0; } |
| 454 | | long wa = sims[best].a, wb = sims[best].b; |
| 455 | | printf("ranked-MI = %.4f MI = %.4f (`%s`, `%s`)\n", sims[best].rmi, sims[best].fmi, aname(wa), aname(wb)); |
| 456 | | int ncand = nranked < (size_t) (nrank + 3 * iter) ? (int) nranked : nrank + 3 * iter; |
| 457 | | long grp[4096]; |
| 458 | | int n = optimal_in_group(wa, wb, ranked, ncand, grp); |
| 459 | | printf("Initial in-group size=%d:", n); |
| 460 | | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 461 | | printf("\n"); |
| 462 | | n = trim_greedy(grp, n); |
| 463 | | printf("In-group size=%d:", n); |
| 464 | | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 465 | | printf("\n"); |
| 466 | 799 | return 0; |
| 467 | 800 | } |