// Port of Linas's stage D: marginals of the pseudo-cset matrix (rows = words, columns = disjuncts =
// ConnectorSeqs, entries = Section counts), as oracle/g0.scm runs it:
// ((add-support-compute psa) 'cache-all) -> matrix support.scm: Norm Key per row/column,
// Left/Right Total Key, Supp Dimension Key
// ((batch-transpose psa) 'mmt-marginals) -> matrix transpose.scm set-mmt-marginals / set-mmt-totals
// MM^T Product Key per word w (transpose.scm:546-574), sums over its disjuncts d, with the column
// marginals of d:
// l0 = sum supp(d) l1 = sum count(d)*N(w,d) l2 = sum (len(d)*N(w,d))^2 (not square-rooted)
// lq = sum sqrt(amp(d)*N(w,d)) (not squared) su = sum sum(d)*N(w,d)
// Totals (transpose.scm:598-604): l0, l1, su summed over words; l2 = sqrt(sum l2_w^2); lq = (sum sqrt(lq_w))^2.
//
// Input: Section counts, one line per Section, in g0 dump format (the output of `count`).
// csetmarg <counts.tsv> > state-D-ours.tsv
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct { char *s; size_t n; } str_t;
typedef struct { char **keys; size_t n, cap, mask; long *tab; } intern_t;
static uint64_t fnv (const char *s) {
uint64_t h = 1469598103934665603ULL;
while (*s) h = (h ^ (unsigned char) *s++) * 1099511628211ULL;
return h;
}
static void intern_rehash (intern_t *t) {
t->mask = t->mask ? t->mask * 2 + 1 : 4095;
free(t->tab);
t->tab = malloc(sizeof(long) * (t->mask + 1));
for (size_t i = 0; i <= t->mask; i++) t->tab[i] = -1;
for (size_t k = 0; k < t->n; k++) {
size_t i = fnv(t->keys[k]) & t->mask;
while (t->tab[i] >= 0) i = (i + 1) & t->mask;
t->tab[i] = k;
}
}
static long intern (intern_t *t, const char *s) {
if (!t->mask) intern_rehash(t);
size_t i = fnv(s) & t->mask;
while (t->tab[i] >= 0) {
if (!strcmp(t->keys[t->tab[i]], s)) return t->tab[i];
i = (i + 1) & t->mask;
}
if (t->n == t->cap) { t->cap = t->cap ? t->cap * 2 : 4096; t->keys = realloc(t->keys, sizeof(char *) * t->cap); }
t->keys[t->n] = strdup(s);
t->tab[i] = t->n++;
if (t->n * 2 > t->mask) intern_rehash(t);
return t->n - 1;
}
typedef struct { long w, d; double n; } entry_t;
typedef struct { double l0, l1, l2, lq, su; } norms_t;
static void norm_add (norms_t *m, double n) { m->l0 += 1; m->l1 += fabs(n); m->l2 += n * n; m->lq += sqrt(fabs(n)); m->su += n; }
static void put (const char *atom, const char *key, int nv, const double *v) {
printf("%s\t(PredicateNode \"%s\")\t", atom, key);
for (int k = 0; k < nv; k++) printf(k ? " %.17g" : "%.17g", v[k]);
printf("\n");
}
int main (int argc, char **argv) {
if (argc != 2) { fprintf(stderr, "usage: csetmarg <counts.tsv>\n"); return 1; }
FILE *f = fopen(argv[1], "r");
if (!f) { perror(argv[1]); return 1; }
intern_t words = { 0 }, djs = { 0 };
entry_t *ents = NULL;
size_t nents = 0, capents = 0;
static char line[1 << 16];
const char *pre = "(Section (WordNode \"";
while (fgets(line, sizeof line, f)) {
if (strncmp(line, pre, strlen(pre))) continue;
char *w0 = line + strlen(pre), *w1 = strstr(w0, "\") (ConnectorSeq");
char *t1 = strchr(line, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL;
if (!w1 || !t1 || !t2 || strncmp(t1 + 1, "(PredicateNode \"*-TruthValueKey-*\")", 35)) continue;
double a, b, n;
if (3 != sscanf(t2 + 1, "%lf %lf %lf", &a, &b, &n)) { fprintf(stderr, "bad count: %s", line); return 1; }
*w1 = 0;
char *d0 = w1 + 3; // "(ConnectorSeq ..." up to the Section's closing paren
t1[-1] = 0; // drop the Section's final ")"
if (nents == capents) { capents = capents ? capents * 2 : 65536; ents = realloc(ents, sizeof(entry_t) * capents); }
ents[nents++] = (entry_t) { intern(&words, w0), intern(&djs, d0), n };
}
fclose(f);
// support.scm: right marginals per word (row), left marginals per disjunct (column).
norms_t *row = calloc(words.n, sizeof(norms_t)), *col = calloc(djs.n, sizeof(norms_t));
for (size_t k = 0; k < nents; k++) { norm_add(&row[ents[k].w], ents[k].n); norm_add(&col[ents[k].d], ents[k].n); }
static char atom[1 << 16];
double v[5], tl[3] = { 0 }, tr[3] = { 0 };
for (size_t w = 0; w < words.n; w++) {
snprintf(atom, sizeof atom, "(ListLink (WordNode \"%s\") (AnyNode \"cset-disjunct\"))", words.keys[w]);
v[0] = row[w].l0; v[1] = row[w].l1; v[2] = sqrt(row[w].l2); v[3] = row[w].lq * row[w].lq; v[4] = row[w].su;
put(atom, "*-Norm Key-*", 5, v);
tl[0] += row[w].l0; tl[1] += row[w].l1; tl[2] += row[w].su;
}
double *clen = malloc(sizeof(double) * djs.n), *camp = malloc(sizeof(double) * djs.n);
for (size_t d = 0; d < djs.n; d++) {
snprintf(atom, sizeof atom, "(ListLink (AnyNode \"cset-word\") %s)", djs.keys[d]);
clen[d] = sqrt(col[d].l2); camp[d] = col[d].lq * col[d].lq;
v[0] = col[d].l0; v[1] = col[d].l1; v[2] = clen[d]; v[3] = camp[d]; v[4] = col[d].su;
put(atom, "*-Norm Key-*", 5, v);
tr[0] += col[d].l0; tr[1] += col[d].l1; tr[2] += col[d].su;
}
const char *ww = "(ListLink (AnyNode \"cset-word\") (AnyNode \"cset-disjunct\"))";
put(ww, "*-Left Total Key-*", 3, tl);
put(ww, "*-Right Total Key-*", 3, tr);
v[0] = words.n; v[1] = djs.n; v[2] = nents; put(ww, "*-Supp Dimension Key-*", 3, v);
// transpose.scm: MM^T per word, then totals.
norms_t *mmt = calloc(words.n, sizeof(norms_t));
for (size_t k = 0; k < nents; k++) {
norms_t *m = &mmt[ents[k].w];
long d = ents[k].d;
double cnt = ents[k].n, acnt = fabs(cnt), rms = clen[d] * cnt;
m->l0 += col[d].l0; m->su += col[d].su * cnt; m->l1 += col[d].l1 * acnt;
m->l2 += rms * rms; m->lq += sqrt(camp[d] * acnt);
}
double t0 = 0, t1 = 0, t2 = 0, tq = 0, ts = 0;
for (size_t w = 0; w < words.n; w++) {
snprintf(atom, sizeof atom, "(ListLink (WordNode \"%s\") (AnyNode \"cset-disjunct\"))", words.keys[w]);
v[0] = mmt[w].l0; v[1] = mmt[w].l1; v[2] = mmt[w].l2; v[3] = mmt[w].lq; v[4] = mmt[w].su;
put(atom, "*-MM^T Product Key-*", 5, v);
t0 += mmt[w].l0; t1 += mmt[w].l1; t2 += mmt[w].l2 * mmt[w].l2; tq += sqrt(mmt[w].lq); ts += mmt[w].su;
}
v[0] = t0; v[1] = t1; v[2] = sqrt(t2); v[3] = tq * tq; v[4] = ts;
put(ww, "*-MM^T Product Key-*", 5, v);
return 0;
}