atomspace-layer-discovery / experiments / ladder / port / count.c

// Port of Linas's counting pipe (learn scm/pipe-parse/pipe-count.scm make-parse-pipe): every atom the parser
// emits gets its count (TruthValueKey index 2) incremented by one. For LgParseBonds the atoms are the
// parse's words and its edges; for LgParseSections they are its Sections and its edges (words not counted).
//
// Input: one or more recorded-parse files (<window>\t<parse>\t<atom>, from oracle/g0.scm), applied in order,
// so `count parses-A.tsv parses-C.tsv` gives the counts after stage C (which adds onto stage A's).
//
//   count <parses.tsv>... > counts.tsv

#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

typedef struct { char *atom; double n; } ent_t;

static ent_t *ents;
static size_t nents, capents, tmask = 65535;
static long *tab;

static uint64_t fnv (const char *s) {
  uint64_t h = 1469598103934665603ULL;
  while (*s) h = (h ^ (unsigned char) *s++) * 1099511628211ULL;
  return h;
}

static void grow (void) {
  tmask = tmask * 2 + 1;
  free(tab);
  tab = malloc(sizeof(long) * (tmask + 1));
  for (size_t i = 0; i <= tmask; i++) tab[i] = -1;
  for (size_t k = 0; k < nents; k++) {
    size_t i = fnv(ents[k].atom) & tmask;
    while (tab[i] >= 0) i = (i + 1) & tmask;
    tab[i] = k;
  }
}

static void incr (const char *atom) {
  size_t i = fnv(atom) & tmask;
  while (tab[i] >= 0) {
    if (!strcmp(ents[tab[i]].atom, atom)) { ents[tab[i]].n += 1; return; }
    i = (i + 1) & tmask;
  }
  if (nents == capents) { capents = capents ? capents * 2 : 4096; ents = realloc(ents, sizeof(ent_t) * capents); }
  ents[nents] = (ent_t) { strdup(atom), 1 };
  tab[i] = nents++;
  if (nents * 2 > tmask) grow();
}

int main (int argc, char **argv) {
  if (argc < 2) { fprintf(stderr, "usage: count <parses.tsv>...\n"); return 1; }
  tab = malloc(sizeof(long) * (tmask + 1));
  for (size_t i = 0; i <= tmask; i++) tab[i] = -1;
  static char line[1 << 16];
  for (int a = 1; a < argc; a++) {
    FILE *f = fopen(argv[a], "r");
    if (!f) { perror(argv[a]); return 1; }
    long nline = 0;
    while (fgets(line, sizeof line, f)) {
      nline++;
      size_t len = strlen(line);
      if (len && line[len - 1] == '\n') line[--len] = 0;
      else if (len == sizeof line - 1) { fprintf(stderr, "%s:%ld: line too long\n", argv[a], nline); return 1; }
      char *atom = strchr(line, '\t');
      if (atom) atom = strchr(atom + 1, '\t');
      if (!atom) { fprintf(stderr, "%s:%ld: malformed\n", argv[a], nline); return 1; }
      incr(atom + 1);
    }
    fclose(f);
  }
  for (size_t k = 0; k < nents; k++) printf("%s\t(PredicateNode \"*-TruthValueKey-*\")\t0 0 %.17g\n", ents[k].atom, ents[k].n);
  return 0;
}