| … | … | @@ -0,0 +1,467 @@ |
| 1 | // Stage E: port of Linas's in-group-mi-cluster (learn scm/gram-class/agglo-mi-rank.scm), as it runs today |
| 2 | // (STAGE_E_SPEC.md; similarities over Shape columns only, §5). Built in steps; this file currently does |
| 3 | // step E2a: the first round's seed pair, in-group (optimal-mi-in-group) and jaccard trim, printed in |
| 4 | // Linas's log format so the two logs can be diffed. |
| 5 | // |
| 6 | // Matrix model (shape-vec.scm, direct-sum.scm): rows are words or word-classes; a pair is either a Section |
| 7 | // (row = germ, column = ConnectorSeq) or a CrossSection (row = connector word, column = Shape). Pairs are |
| 8 | // explicit, with their own counts, so later steps can merge them the way gram-majority.scm does. |
| 9 | // |
| 10 | // cluster <counts.tsv> [--nrank 50] [--intended] |
| 11 | |
| 12 | #include <math.h> |
| 13 | #include <stdint.h> |
| 14 | #include <stdio.h> |
| 15 | #include <stdlib.h> |
| 16 | #include <string.h> |
| 17 | |
| 18 | #define QUORUM 0.7 |
| 19 | #define COMMONALITY 0.2 |
| 20 | #define NOISE 4.0 |
| 21 | #define MI_CUTOFF 4.0 |
| 22 | |
| 23 | // ---------------------------------------------------------------- interning |
| 24 | |
| 25 | typedef struct { char **keys; size_t n, cap, mask; long *tab; } intern_t; |
| 26 | |
| 27 | static uint64_t fnv (const char *s) { |
| 28 | uint64_t h = 1469598103934665603ULL; |
| 29 | while (*s) h = (h ^ (unsigned char) *s++) * 1099511628211ULL; |
| 30 | return h; |
| 31 | } |
| 32 | |
| 33 | static void rehash (intern_t *t) { |
| 34 | t->mask = t->mask ? t->mask * 2 + 1 : 4095; |
| 35 | free(t->tab); |
| 36 | t->tab = malloc(sizeof(long) * (t->mask + 1)); |
| 37 | for (size_t i = 0; i <= t->mask; i++) t->tab[i] = -1; |
| 38 | for (size_t k = 0; k < t->n; k++) { |
| 39 | size_t i = fnv(t->keys[k]) & t->mask; |
| 40 | while (t->tab[i] >= 0) i = (i + 1) & t->mask; |
| 41 | t->tab[i] = k; |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | static long lookup (intern_t *t, const char *s) { |
| 46 | if (!t->mask) return -1; |
| 47 | size_t i = fnv(s) & t->mask; |
| 48 | while (t->tab[i] >= 0) { |
| 49 | if (!strcmp(t->keys[t->tab[i]], s)) return t->tab[i]; |
| 50 | i = (i + 1) & t->mask; |
| 51 | } |
| 52 | return -1; |
| 53 | } |
| 54 | |
| 55 | static long intern (intern_t *t, const char *s) { |
| 56 | long k = lookup(t, s); |
| 57 | if (k >= 0) return k; |
| 58 | if (!t->mask) rehash(t); |
| 59 | if (t->n == t->cap) { t->cap = t->cap ? t->cap * 2 : 4096; t->keys = realloc(t->keys, sizeof(char *) * t->cap); } |
| 60 | t->keys[t->n] = strdup(s); |
| 61 | size_t i = fnv(s) & t->mask; |
| 62 | while (t->tab[i] >= 0) i = (i + 1) & t->mask; |
| 63 | t->tab[i] = t->n++; |
| 64 | if (t->n * 2 > t->mask) rehash(t); |
| 65 | return t->n - 1; |
| 66 | } |
| 67 | |
| 68 | // ---------------------------------------------------------------- atoms and columns |
| 69 | |
| 70 | // Rows: words and word-classes, keyed "W <name>" / "C <name>". |
| 71 | static intern_t atoms; |
| 72 | static const char *aname (long a) { return atoms.keys[a] + 2; } |
| 73 | |
| 74 | // A column is a ConnectorSeq (shape = 0) or a Shape (shape = 1: point + connectors, one of them the hole). |
| 75 | typedef struct { int shape, nc, hole; long point; long *ca; char *cd; } col_t; |
| 76 | static intern_t colkeys; |
| 77 | static col_t *cols; |
| 78 | static size_t capcols; |
| 79 | |
| 80 | static long make_col (int shape, long point, int nc, const long *ca, const char *cd, int hole) { |
| 81 | static char key[1 << 16]; |
| 82 | int k = snprintf(key, sizeof key, "%c%ld", shape ? 'X' : 'S', shape ? point : -1L); |
| 83 | for (int i = 0; i < nc; i++) k += snprintf(key + k, sizeof key - k, " %ld%c", i == hole ? -1L : ca[i], cd[i]); |
| 84 | size_t before = colkeys.n; |
| 85 | long c = intern(&colkeys, key); |
| 86 | if ((size_t) c >= capcols) { capcols = capcols ? capcols * 2 : 1 << 16; cols = realloc(cols, sizeof(col_t) * capcols); } |
| 87 | if (colkeys.n > before) { |
| 88 | cols[c] = (col_t) { shape, nc, hole, point, malloc(sizeof(long) * nc), malloc(nc) }; |
| 89 | memcpy(cols[c].ca, ca, sizeof(long) * nc); |
| 90 | memcpy(cols[c].cd, cd, nc); |
| 91 | } |
| 92 | return c; |
| 93 | } |
| 94 | |
| 95 | // ---------------------------------------------------------------- pairs |
| 96 | |
| 97 | typedef struct { long r, c; double n; int alive; } pair_t; |
| 98 | static pair_t *pairs; |
| 99 | static size_t npairs, cappairs, pmask; |
| 100 | static long *ptab; |
| 101 | |
| 102 | static uint64_t phash (long r, long c) { return ((uint64_t) r * 0x9e3779b97f4a7c15ULL) ^ ((uint64_t) c * 0xbf58476d1ce4e5b9ULL); } |
| 103 | |
| 104 | static void prehash (void) { |
| 105 | pmask = pmask ? pmask * 2 + 1 : (1 << 20) - 1; |
| 106 | free(ptab); |
| 107 | ptab = malloc(sizeof(long) * (pmask + 1)); |
| 108 | for (size_t i = 0; i <= pmask; i++) ptab[i] = -1; |
| 109 | for (size_t k = 0; k < npairs; k++) { |
| 110 | size_t i = phash(pairs[k].r, pairs[k].c) & pmask; |
| 111 | while (ptab[i] >= 0) i = (i + 1) & pmask; |
| 112 | ptab[i] = k; |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | static long get_pair (long r, long c) { |
| 117 | if (!pmask) return -1; |
| 118 | size_t i = phash(r, c) & pmask; |
| 119 | while (ptab[i] >= 0) { |
| 120 | pair_t *p = &pairs[ptab[i]]; |
| 121 | if (p->r == r && p->c == c) return p->alive ? ptab[i] : -1; |
| 122 | i = (i + 1) & pmask; |
| 123 | } |
| 124 | return -1; |
| 125 | } |
| 126 | |
| 127 | static long make_pair (long r, long c) { |
| 128 | if (!pmask) prehash(); |
| 129 | size_t i = phash(r, c) & pmask; |
| 130 | while (ptab[i] >= 0) { |
| 131 | pair_t *p = &pairs[ptab[i]]; |
| 132 | if (p->r == r && p->c == c) { p->alive = 1; return ptab[i]; } |
| 133 | i = (i + 1) & pmask; |
| 134 | } |
| 135 | if (npairs == cappairs) { cappairs = cappairs ? cappairs * 2 : 1 << 18; pairs = realloc(pairs, sizeof(pair_t) * cappairs); } |
| 136 | pairs[npairs] = (pair_t) { r, c, 0, 1 }; |
| 137 | ptab[i] = npairs++; |
| 138 | if (npairs * 2 > pmask) prehash(); |
| 139 | return npairs - 1; |
| 140 | } |
| 141 | |
| 142 | static double pair_count (long r, long c) { long p = get_pair(r, c); return p < 0 ? 0 : pairs[p].n; } |
| 143 | |
| 144 | // ---------------------------------------------------------------- marginals (support.scm, transpose.scm) |
| 145 | |
| 146 | typedef struct { double l0, l1, l2, lq, su; } norms_t; |
| 147 | static norms_t *rown, *coln; |
| 148 | static double *mmt, T_mmt, S_sum; |
| 149 | static long **rowp; // per row: its live pairs (right-stars), rebuilt by recompute() |
| 150 | static size_t *rowpn; |
| 151 | |
| 152 | static void norm_add (norms_t *m, double n) { m->l0 += 1; m->l1 += fabs(n); m->l2 += n * n; m->lq += sqrt(fabs(n)); m->su += n; } |
| 153 | |
| 154 | static void recompute (void) { |
| 155 | free(rown); free(coln); free(mmt); |
| 156 | rown = calloc(atoms.n, sizeof(norms_t)); coln = calloc(colkeys.n, sizeof(norms_t)); mmt = calloc(atoms.n, sizeof(double)); |
| 157 | if (rowp) { for (size_t r = 0; r < atoms.n; r++) free(rowp[r]); free(rowp); free(rowpn); } |
| 158 | rowp = calloc(atoms.n, sizeof(long *)); rowpn = calloc(atoms.n, sizeof(size_t)); |
| 159 | size_t *cap = calloc(atoms.n, sizeof(size_t)); |
| 160 | for (size_t k = 0; k < npairs; k++) { |
| 161 | if (!pairs[k].alive) continue; |
| 162 | long r = pairs[k].r; |
| 163 | norm_add(&rown[r], pairs[k].n); norm_add(&coln[pairs[k].c], pairs[k].n); |
| 164 | if (rowpn[r] == cap[r]) { cap[r] = cap[r] ? cap[r] * 2 : 8; rowp[r] = realloc(rowp[r], sizeof(long) * cap[r]); } |
| 165 | rowp[r][rowpn[r]++] = k; |
| 166 | } |
| 167 | free(cap); |
| 168 | S_sum = 0; T_mmt = 0; |
| 169 | for (size_t r = 0; r < atoms.n; r++) S_sum += rown[r].su; |
| 170 | for (size_t k = 0; k < npairs; k++) if (pairs[k].alive) mmt[pairs[k].r] += coln[pairs[k].c].l1 * fabs(pairs[k].n); |
| 171 | for (size_t r = 0; r < atoms.n; r++) T_mmt += mmt[r]; |
| 172 | } |
| 173 | |
| 174 | // ---------------------------------------------------------------- similarities (symmetric-mi.scm, mi-similarity.scm) |
| 175 | |
| 176 | static int as_run = 1; |
| 177 | |
| 178 | typedef struct { long a, b; double fmi, rmi; int used; } sim_t; |
| 179 | static sim_t *sims; |
| 180 | static size_t nsims, capsims, smask; |
| 181 | static long *stab; |
| 182 | |
| 183 | static uint64_t shash (long a, long b) { if (a > b) { long t = a; a = b; b = t; } return phash(a, b); } |
| 184 | |
| 185 | static long find_sim (long a, long b) { |
| 186 | if (!smask) return -1; |
| 187 | size_t i = shash(a, b) & smask; |
| 188 | while (stab[i] >= 0) { |
| 189 | sim_t *s = &sims[stab[i]]; |
| 190 | if ((s->a == a && s->b == b) || (s->a == b && s->b == a)) return stab[i]; |
| 191 | i = (i + 1) & smask; |
| 192 | } |
| 193 | return -1; |
| 194 | } |
| 195 | |
| 196 | static void srehash (void) { |
| 197 | smask = smask ? smask * 2 + 1 : 4095; |
| 198 | free(stab); |
| 199 | stab = malloc(sizeof(long) * (smask + 1)); |
| 200 | for (size_t i = 0; i <= smask; i++) stab[i] = -1; |
| 201 | for (size_t k = 0; k < nsims; k++) { |
| 202 | size_t i = shash(sims[k].a, sims[k].b) & smask; |
| 203 | while (stab[i] >= 0) i = (i + 1) & smask; |
| 204 | stab[i] = k; |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // make-gram-mi-simmer: compute and store (overwrite) the similarity of a and b. |
| 209 | static void simmer (long a, long b) { |
| 210 | double dot = 0; |
| 211 | for (size_t i = 0; i < rowpn[a]; i++) { |
| 212 | pair_t *p = &pairs[rowp[a][i]]; |
| 213 | if (as_run && !cols[p->c].shape) continue; // §5: ConnectorSeq columns fail in-right-basis? |
| 214 | long q = get_pair(b, p->c); |
| 215 | if (q >= 0) dot += p->n * pairs[q].n; |
| 216 | } |
| 217 | double fmi = dot > 0 ? log2(dot * T_mmt / (mmt[a] * mmt[b])) : -INFINITY; |
| 218 | double rmi = fmi + 0.5 * log2((mmt[a] / T_mmt) * (mmt[b] / T_mmt)) + (-log2(T_mmt / (S_sum * S_sum))); |
| 219 | long k = find_sim(a, b); |
| 220 | if (k < 0) { |
| 221 | if (!smask) srehash(); |
| 222 | if (nsims == capsims) { capsims = capsims ? capsims * 2 : 4096; sims = realloc(sims, sizeof(sim_t) * capsims); } |
| 223 | sims[nsims] = (sim_t) { a, b, fmi, rmi, 1 }; |
| 224 | size_t i = shash(a, b) & smask; |
| 225 | while (stab[i] >= 0) i = (i + 1) & smask; |
| 226 | stab[i] = nsims++; |
| 227 | if (nsims * 2 > smask) srehash(); |
| 228 | } else { |
| 229 | sims[k].fmi = fmi; sims[k].rmi = rmi; |
| 230 | } |
| 231 | if (fmi > 6) printf("\tMI(`%s`, `%s`) = %.6f rank-MI = %.6f\n", aname(a), aname(b), fmi, rmi); |
| 232 | } |
| 233 | |
| 234 | static double mi_sim (long a, long b) { long k = find_sim(a, b); return k < 0 ? -INFINITY : sims[k].fmi; } |
| 235 | |
| 236 | // ---------------------------------------------------------------- ranking (similarity.scm rank-words) |
| 237 | |
| 238 | static long *ranked; |
| 239 | static size_t nranked; |
| 240 | |
| 241 | static int by_rank (const void *x, const void *y) { |
| 242 | long a = *(const long *) x, b = *(const long *) y; |
| 243 | if (rown[a].l1 != rown[b].l1) return rown[a].l1 > rown[b].l1 ? -1 : 1; |
| 244 | if (rown[a].l0 != rown[b].l0) return rown[a].l0 > rown[b].l0 ? -1 : 1; |
| 245 | return (a > b) - (a < b); |
| 246 | } |
| 247 | |
| 248 | static void rank_words (void) { |
| 249 | free(ranked); |
| 250 | ranked = malloc(sizeof(long) * atoms.n); |
| 251 | nranked = 0; |
| 252 | for (size_t r = 0; r < atoms.n; r++) if (rowpn[r] > 0) ranked[nranked++] = r; |
| 253 | qsort(ranked, nranked, sizeof(long), by_rank); |
| 254 | } |
| 255 | |
| 256 | // loop-upper-diagonal 0..depth: compute sims not already stored. |
| 257 | static void sims_upper (size_t depth) { |
| 258 | if (depth > nranked) depth = nranked; |
| 259 | for (size_t i = 0; i < depth; i++) for (size_t j = i; j < depth; j++) |
| 260 | if (find_sim(ranked[i], ranked[j]) < 0) simmer(ranked[i], ranked[j]); |
| 261 | } |
| 262 | |
| 263 | // ---------------------------------------------------------------- in-group (in-group.scm) |
| 264 | |
| 265 | static int round_even (double x) { return (int) rint(x); } // Guile `round` = round half to even |
| 266 | |
| 267 | // find-in-group: start with (WA WB), fold over candidates, cons accepted ones on the front. |
| 268 | static int find_in_group (long wa, long wb, double lower, double eps, const long *cand, int ncand, long *grp) { |
| 269 | double minscore = mi_sim(wa, wb) - eps; |
| 270 | int n = 2; |
| 271 | grp[0] = wa; grp[1] = wb; // list order: grp[0] is the head |
| 272 | for (int c = 0; c < ncand; c++) { |
| 273 | long w = cand[c]; |
| 274 | if (w == wa || w == wb) continue; |
| 275 | int tight = n == 2 ? 2 : round_even(QUORUM * n); |
| 276 | int maxfail = n - tight, fail = 0, ok = 1; |
| 277 | for (int m = 0; m < n && ok; m++) { |
| 278 | double s = mi_sim(w, grp[m]); |
| 279 | if (s < lower) fail += maxfail + 999; |
| 280 | if (s < minscore) fail += 1; |
| 281 | ok = fail <= maxfail; |
| 282 | } |
| 283 | if (ok) { memmove(grp + 1, grp, sizeof(long) * n); grp[0] = w; n++; } |
| 284 | } |
| 285 | return n; |
| 286 | } |
| 287 | |
| 288 | // optimal-mi-in-group (tightness 0.7, step 0.1, max 8.5, lower 1.0, max-jump 2.5, window 1.0, max 12). |
| 289 | static int optimal_in_group (long wa, long wb, const long *cand, int ncand, long *out) { |
| 290 | long grp[4096]; |
| 291 | int nsteps = (int) rint(8.5 / 0.1), slots = (int) rint(1.0 / 0.1); |
| 292 | int window[64]; |
| 293 | for (int i = 0; i < slots; i++) window[i] = 2; |
| 294 | double eps = 0; |
| 295 | for (int N = 1; N <= nsteps; N++) { |
| 296 | eps = N * 0.1; |
| 297 | int sz = find_in_group(wb, wa, 1.0, eps, cand, ncand, grp); |
| 298 | int prev = window[0]; |
| 299 | memmove(window, window + 1, sizeof(int) * (slots - 1)); |
| 300 | window[slots - 1] = sz; |
| 301 | if (!(sz - prev < 2.5 && sz < 12)) break; |
| 302 | } |
| 303 | int n = find_in_group(wa, wb, 1.0, eps - 0.1, cand, ncand, grp); |
| 304 | for (int i = 0; i < n; i++) out[i] = grp[n - 1 - i]; // reverse |
| 305 | return n; |
| 306 | } |
| 307 | |
| 308 | // ---------------------------------------------------------------- jaccard trim (jaccard.scm, gram-majority.scm) |
| 309 | |
| 310 | // count-shared-conseq: (shared, total) over columns with count > NOISE in some member. |
| 311 | static void shared_conseq (const long *g, int n, double *shared, double *total) { |
| 312 | int thresh = n <= 2 ? 2 : round_even(QUORUM * n), low = thresh - 1; |
| 313 | static long *seen; static size_t capseen; |
| 314 | static int *votes; static long *mark; static size_t capmark; static long epoch; |
| 315 | if (capmark < colkeys.n) { capmark = colkeys.n * 2; mark = realloc(mark, sizeof(long) * capmark); votes = realloc(votes, sizeof(int) * capmark); memset(mark, 0, sizeof(long) * capmark); } |
| 316 | epoch++; |
| 317 | size_t ns = 0; |
| 318 | for (int m = 0; m < n; m++) for (size_t i = 0; i < rowpn[g[m]]; i++) { |
| 319 | pair_t *p = &pairs[rowp[g[m]][i]]; |
| 320 | if (!(p->n > NOISE)) continue; |
| 321 | if (mark[p->c] != epoch) { |
| 322 | mark[p->c] = epoch; votes[p->c] = 0; |
| 323 | if (ns == capseen) { capseen = capseen ? capseen * 2 : 1024; seen = realloc(seen, sizeof(long) * capseen); } |
| 324 | seen[ns++] = p->c; |
| 325 | } |
| 326 | } |
| 327 | int sh = 0; |
| 328 | for (size_t k = 0; k < ns; k++) { |
| 329 | int v = 0; |
| 330 | for (int m = 0; m < n; m++) if (pair_count(g[m], seen[k]) > NOISE) v++; |
| 331 | if (v > low) sh++; |
| 332 | } |
| 333 | *shared = sh; *total = ns; |
| 334 | } |
| 335 | |
| 336 | static int trim_greedy (long *g, int n) { |
| 337 | double sh, tot; |
| 338 | shared_conseq(g, n, &sh, &tot); |
| 339 | double cm = sh / tot; |
| 340 | printf("Start: size=%d overlap = %.0f of %.0f disjuncts, commonality= %4.2f%%\n", n, sh, tot, cm * 100); |
| 341 | for (;;) { |
| 342 | if (n == 2 || COMMONALITY <= cm) break; |
| 343 | double best = cm, bsh = sh, btot = tot; |
| 344 | long bg[4096]; int bn = n, changed = 0; |
| 345 | for (int N = 0; N < n; N++) { |
| 346 | long m[4096]; int k = 0; |
| 347 | for (int i = 0; i < n; i++) if (i != N) m[k++] = g[i]; |
| 348 | double s2, t2; |
| 349 | shared_conseq(m, k, &s2, &t2); |
| 350 | double c2 = s2 / t2; |
| 351 | if (best <= c2) { best = c2; bsh = s2; btot = t2; memcpy(bg, m, sizeof(long) * k); bn = k; changed = 1; } |
| 352 | if (COMMONALITY < c2) break; |
| 353 | } |
| 354 | if (!changed) break; |
| 355 | printf("Better: size=%d overlap = %.0f of %.0f disjuncts, commonality= %4.2f%%\n", bn, bsh, btot, best * 100); |
| 356 | memcpy(g, bg, sizeof(long) * bn); n = bn; cm = best; sh = bsh; tot = btot; |
| 357 | } |
| 358 | printf("Best: size=%d overlap = %.0f of %.0f disjuncts, commonality= %4.2f%%\n", n, sh, tot, cm * 100); |
| 359 | return n; |
| 360 | } |
| 361 | |
| 362 | // ---------------------------------------------------------------- input |
| 363 | |
| 364 | static int connectors (const char *seq, char words[][1024], char *dirs, int max) { |
| 365 | int n = 0; |
| 366 | const char *p = seq; |
| 367 | while ((p = strstr(p, "(Connector (WordNode \"")) && n < max) { |
| 368 | p += 22; |
| 369 | const char *e = strstr(p, "\") (SexNode \""); |
| 370 | if (!e || e - p >= 1024) return -1; |
| 371 | memcpy(words[n], p, e - p); |
| 372 | words[n][e - p] = 0; |
| 373 | dirs[n] = e[13]; |
| 374 | p = e + 13; |
| 375 | n++; |
| 376 | } |
| 377 | return n; |
| 378 | } |
| 379 | |
| 380 | static long word (const char *w) { |
| 381 | static char key[2048]; |
| 382 | snprintf(key, sizeof key, "W %s", w); |
| 383 | return intern(&atoms, key); |
| 384 | } |
| 385 | |
| 386 | static void load_sections (const char *path) { |
| 387 | FILE *f = fopen(path, "r"); |
| 388 | if (!f) { perror(path); exit(1); } |
| 389 | static char line[1 << 16]; |
| 390 | static char cw[64][1024]; |
| 391 | char cd[64]; |
| 392 | long ca[64]; |
| 393 | const char *pre = "(Section (WordNode \""; |
| 394 | while (fgets(line, sizeof line, f)) { |
| 395 | if (strncmp(line, pre, strlen(pre))) continue; |
| 396 | char *g0 = line + strlen(pre), *g1 = strstr(g0, "\") (ConnectorSeq"); |
| 397 | char *t1 = strchr(line, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL; |
| 398 | if (!g1 || !t1 || !t2) continue; |
| 399 | double a, b, n; |
| 400 | if (3 != sscanf(t2 + 1, "%lf %lf %lf", &a, &b, &n)) { fprintf(stderr, "bad count: %s", line); exit(1); } |
| 401 | *g1 = 0; |
| 402 | int nc = connectors(g1 + 3, cw, cd, 64); |
| 403 | if (nc <= 0) { fprintf(stderr, "bad section\n"); exit(1); } |
| 404 | long g = word(g0); |
| 405 | for (int i = 0; i < nc; i++) ca[i] = word(cw[i]); |
| 406 | long p = make_pair(g, make_col(0, -1, nc, ca, cd, -1)); // may realloc `pairs` |
| 407 | pairs[p].n = n; |
| 408 | } |
| 409 | fclose(f); |
| 410 | } |
| 411 | |
| 412 | // explode-sections (shape-vec.scm:547-568): a CrossSection per connector, count copied from the Section. |
| 413 | static void explode_sections (void) { |
| 414 | size_t n0 = npairs; |
| 415 | for (size_t k = 0; k < n0; k++) { |
| 416 | if (!pairs[k].alive || cols[pairs[k].c].shape) continue; |
| 417 | col_t *s = &cols[pairs[k].c]; |
| 418 | long ca[64]; char cd[64]; int nc = s->nc; |
| 419 | memcpy(ca, s->ca, sizeof(long) * nc); memcpy(cd, s->cd, nc); |
| 420 | long g = pairs[k].r; double n = pairs[k].n; |
| 421 | for (int i = 0; i < nc; i++) { |
| 422 | long x = make_pair(ca[i], make_col(1, g, nc, ca, cd, i)); |
| 423 | pairs[x].n = n; |
| 424 | } |
| 425 | } |
| 426 | } |
| 427 | |
| 428 | // ---------------------------------------------------------------- main |
| 429 | |
| 430 | int main (int argc, char **argv) { |
| 431 | if (argc < 2) { fprintf(stderr, "usage: cluster <counts.tsv> [--nrank 50] [--intended]\n"); return 1; } |
| 432 | int nrank = 50; |
| 433 | for (int i = 2; i < argc; i++) { |
| 434 | if (!strcmp(argv[i], "--nrank") && i + 1 < argc) nrank = atoi(argv[++i]); |
| 435 | else if (!strcmp(argv[i], "--intended")) as_run = 0; |
| 436 | } |
| 437 | setvbuf(stdout, NULL, _IOLBF, 0); |
| 438 | load_sections(argv[1]); |
| 439 | explode_sections(); |
| 440 | recompute(); |
| 441 | rank_words(); |
| 442 | sims_upper(nrank); |
| 443 | printf("Done setting up similarity to %d\n", nrank); |
| 444 | |
| 445 | // Round 1 (main-loop iteration 1): seed, in-group, trim. |
| 446 | int iter = 1; |
| 447 | printf("------ Round %d Next in line:\n", iter); |
| 448 | long best = -1; |
| 449 | for (size_t k = 0; k < nsims; k++) { |
| 450 | if (sims[k].a == sims[k].b || !(MI_CUTOFF < sims[k].fmi)) continue; |
| 451 | if (best < 0 || sims[k].rmi > sims[best].rmi) best = k; |
| 452 | } |
| 453 | if (best < 0) { printf("no pair above the MI cutoff\n"); return 0; } |
| 454 | long wa = sims[best].a, wb = sims[best].b; |
| 455 | printf("ranked-MI = %.4f MI = %.4f (`%s`, `%s`)\n", sims[best].rmi, sims[best].fmi, aname(wa), aname(wb)); |
| 456 | int ncand = nranked < (size_t) (nrank + 3 * iter) ? (int) nranked : nrank + 3 * iter; |
| 457 | long grp[4096]; |
| 458 | int n = optimal_in_group(wa, wb, ranked, ncand, grp); |
| 459 | printf("Initial in-group size=%d:", n); |
| 460 | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 461 | printf("\n"); |
| 462 | n = trim_greedy(grp, n); |
| 463 | printf("In-group size=%d:", n); |
| 464 | for (int i = 0; i < n; i++) printf(" `%s`", aname(grp[i])); |
| 465 | printf("\n"); |
| 466 | return 0; |
| 467 | } |