| … | … | @@ -147,15 +147,15 @@ typedef struct { double l0, l1, l2, lq, su; } norms_t; |
| 147 | 147 | static norms_t *rown, *coln; |
| 148 | 148 | static double *mmt, T_mmt, S_sum; |
| 149 | 149 | static long **rowp; // per row: its live pairs (right-stars), rebuilt by recompute() |
| 150 | | static size_t *rowpn; |
| 150 | static size_t *rowpn, rowp_n; // rowp_n: rows allocated (atoms can be added between recomputes) |
| 151 | 151 | |
| 152 | 152 | static void norm_add (norms_t *m, double n) { m->l0 += 1; m->l1 += fabs(n); m->l2 += n * n; m->lq += sqrt(fabs(n)); m->su += n; } |
| 153 | 153 | |
| 154 | 154 | static void recompute (void) { |
| 155 | 155 | free(rown); free(coln); free(mmt); |
| 156 | 156 | rown = calloc(atoms.n, sizeof(norms_t)); coln = calloc(colkeys.n, sizeof(norms_t)); mmt = calloc(atoms.n, sizeof(double)); |
| 157 | | if (rowp) { for (size_t r = 0; r < atoms.n; r++) free(rowp[r]); free(rowp); free(rowpn); } |
| 158 | | rowp = calloc(atoms.n, sizeof(long *)); rowpn = calloc(atoms.n, sizeof(size_t)); |
| 157 | if (rowp) { for (size_t r = 0; r < rowp_n; r++) free(rowp[r]); free(rowp); free(rowpn); } |
| 158 | rowp = calloc(atoms.n, sizeof(long *)); rowpn = calloc(atoms.n, sizeof(size_t)); rowp_n = atoms.n; |
| 159 | 159 | size_t *cap = calloc(atoms.n, sizeof(size_t)); |
| 160 | 160 | for (size_t k = 0; k < npairs; k++) { |
| 161 | 161 | if (!pairs[k].alive) continue; |
| … | … | @@ -238,14 +238,31 @@ static double mi_sim (long a, long b) { long k = find_sim(a, b); return k < 0 ? |
| 238 | 238 | static long *ranked; |
| 239 | 239 | static size_t nranked; |
| 240 | 240 | |
| 241 | // Diagnostic (CLUSTER_STALE_RANK): rank by the counts a row had when it was first ranked, as a test of |
| 242 | // whether Linas's rank-words reads pre-merge marginals. |
| 243 | static double *rank_l1, *rank_l0; |
| 244 | static size_t rank_n; |
| 245 | static int stale_rank; |
| 246 | |
| 247 | static double rk1 (long a) { return stale_rank && (size_t) a < rank_n && rank_l1[a] >= 0 ? rank_l1[a] : rown[a].l1; } |
| 248 | static double rk0 (long a) { return stale_rank && (size_t) a < rank_n && rank_l1[a] >= 0 ? rank_l0[a] : rown[a].l0; } |
| 249 | |
| 241 | 250 | static int by_rank (const void *x, const void *y) { |
| 242 | 251 | long a = *(const long *) x, b = *(const long *) y; |
| 243 | | if (rown[a].l1 != rown[b].l1) return rown[a].l1 > rown[b].l1 ? -1 : 1; |
| 244 | | if (rown[a].l0 != rown[b].l0) return rown[a].l0 > rown[b].l0 ? -1 : 1; |
| 252 | if (rk1(a) != rk1(b)) return rk1(a) > rk1(b) ? -1 : 1; |
| 253 | if (rk0(a) != rk0(b)) return rk0(a) > rk0(b) ? -1 : 1; |
| 245 | 254 | return (a > b) - (a < b); |
| 246 | 255 | } |
| 247 | 256 | |
| 248 | 257 | static void rank_words (void) { |
| 258 | if (stale_rank) { // remember each row's counts the first time it is ranked |
| 259 | if (rank_n < atoms.n) { |
| 260 | rank_l1 = realloc(rank_l1, sizeof(double) * atoms.n); rank_l0 = realloc(rank_l0, sizeof(double) * atoms.n); |
| 261 | for (size_t r = rank_n; r < atoms.n; r++) rank_l1[r] = -1; |
| 262 | rank_n = atoms.n; |
| 263 | } |
| 264 | for (size_t r = 0; r < atoms.n; r++) if (rank_l1[r] < 0 && rowpn[r] > 0) { rank_l1[r] = rown[r].l1; rank_l0[r] = rown[r].l0; } |
| 265 | } |
| 249 | 266 | free(ranked); |
| 250 | 267 | ranked = malloc(sizeof(long) * atoms.n); |
| 251 | 268 | nranked = 0; |
| … | … | @@ -735,6 +752,7 @@ int main (int argc, char **argv) { |
| 735 | 752 | else if (!strcmp(argv[i], "--intended")) as_run = 0; |
| 736 | 753 | } |
| 737 | 754 | setvbuf(stdout, NULL, _IOLBF, 0); |
| 755 | stale_rank = getenv("CLUSTER_STALE_RANK") != NULL; |
| 738 | 756 | load_sections(argv[1]); |
| 739 | 757 | explode_sections(); |
| 740 | 758 | recompute(); |
| … | … | @@ -809,6 +827,34 @@ int main (int argc, char **argv) { |
| 809 | 827 | // expand-universe: sims for missing pairs within the top NRANK + 2*iter. |
| 810 | 828 | rank_words(); |
| 811 | 829 | sims_upper(nrank + 2 * iter); |
| 830 | if (getenv("CLUSTER_DEBUG")) { // diagnostic: where does the class rank, and its sims with members |
| 831 | for (size_t i = 0; i < nranked; i++) if (ranked[i] == cls) printf("debug: class rank %zu of %zu (count %.0f)\n", i, nranked, rown[cls].l1); |
| 832 | for (int i = 0; i < n; i++) { long k = find_sim(cls, grp[i]); printf("debug: sim(class, %s) = %s\n", aname(grp[i]), k < 0 ? "none" : ""); if (k >= 0) printf("debug: fmi %.4f rmi %.4f\n", sims[k].fmi, sims[k].rmi); } |
| 833 | { long q = grp[0]; // the first member's remaining pairs: column, count, does the class have it? |
| 834 | for (size_t k = 0; k < rowpn[q] && k < 8; k++) { |
| 835 | pair_t *p = &pairs[rowp[q][k]]; |
| 836 | printf("debug: %s pair col %ld [%s] n=%.0f alive=%d; class has it: %d\n", aname(q), p->c, colkeys.keys[p->c], p->n, p->alive, get_pair(cls, p->c) >= 0); |
| 837 | } |
| 838 | size_t nzero = 0; for (size_t k = 0; k < rowpn[q]; k++) if (pairs[rowp[q][k]].n < 1e-10) nzero++; |
| 839 | printf("debug: %s zero-count pairs %zu of %zu\n", aname(q), nzero, rowpn[q]); |
| 840 | } |
| 841 | for (int i = 0; i < n; i++) { // shared columns between the class and each member, by kind |
| 842 | int sh_x = 0, sh_s = 0; double dot_x = 0; |
| 843 | for (size_t k = 0; k < rowpn[cls]; k++) { |
| 844 | pair_t *p = &pairs[rowp[cls][k]]; |
| 845 | long q = get_pair(grp[i], p->c); |
| 846 | if (q < 0) continue; |
| 847 | if (cols[p->c].shape) { sh_x++; dot_x += p->n * pairs[q].n; } else sh_s++; |
| 848 | } |
| 849 | printf("debug: class vs %s: shared shapes %d (dot %.0f), shared sections %d\n", aname(grp[i]), sh_x, dot_x, sh_s); |
| 850 | } |
| 851 | for (int i = 0; i < n; i++) { |
| 852 | size_t rk = nranked; for (size_t j = 0; j < nranked; j++) if (ranked[j] == grp[i]) rk = j; |
| 853 | double secs = 0, xs = 0; |
| 854 | for (size_t k = 0; k < rowpn[grp[i]]; k++) { pair_t *p = &pairs[rowp[grp[i]][k]]; if (cols[p->c].shape) xs += p->n; else secs += p->n; } |
| 855 | printf("debug: member %s rank %zu count %.0f (sections %.0f, crosses %.0f) pairs %zu\n", aname(grp[i]), rk, rown[grp[i]].l1, secs, xs, rowpn[grp[i]]); |
| 856 | } |
| 857 | } |
| 812 | 858 | } |
| 813 | 859 | return 0; |
| 814 | 860 | } |