// Stage C (MPG sections) through Link Grammar itself, the way Linas's pipeline runs it, minus Guile:
// * pair MI is loaded into an in-RAM AtomSpace as (Edge (Bond "ANY") (List w1 w2)) with
// (Predicate "*-Mutual Info Key-*") = FloatValue(mi, fmi) -- what LG's dict-atomese reads
// (link-grammar dict-atomese/word-pairs.cc pair_mi, index = `pair-index` in storage.dict);
// * the dictionary is Linas's dict-pair, opened by path, attached with lg_config_atomspace(as, null);
// * text is cut into windows exactly as learn scm/parse/sliding-block.scm make-observe-block does;
// * each window is parsed with the options of lg-atomese LGParseLink.cc execute(), and the first
// NUML valid linkages are turned into Sections and bonds as in make_sects / make_bonds.
// Output: <window>\t<parse>\t<atom> lines, the format of oracle/g0.scm's parses-C.tsv. Per-window parser
// stats go to the file given with --stats: <window>\t<linkages found>\t<post_processed>\t<valid>\t<sampled>.
//
// lgsect <text> <mi.tsv> <dict-dir> [--win 12] [--numl 3] [--stats file] [--first N]
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <opencog/atomspace/AtomSpace.h>
#include <opencog/atoms/value/FloatValue.h>
#include <opencog/atoms/atom_types/atom_types.h>
#include <opencog/lg/types/atom_types.h>
#include <link-grammar/dict-atomese.h>
using namespace opencog;
// ---------------------------------------------------------------- windows (sliding-block.scm)
// Guile char-set:whitespace = Unicode White_Space.
static bool uws (unsigned cp) {
return (cp >= 9 && cp <= 13) || cp == 32 || cp == 0x85 || cp == 0xA0 || cp == 0x1680 ||
(cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 || cp == 0x2029 || cp == 0x202F ||
cp == 0x205F || cp == 0x3000;
}
// Decode the text into code points, remembering each one's byte offset.
static void decode (const std::string& s, std::vector<unsigned>& cps, std::vector<size_t>& off) {
for (size_t i = 0; i < s.size();) {
unsigned char c = s[i];
int n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4;
unsigned cp = n == 1 ? c : c & (0xFF >> (n + 1));
for (int k = 1; k < n && i + k < s.size(); k++) cp = (cp << 6) | (s[i + k] & 0x3F);
cps.push_back(cp); off.push_back(i);
i += n;
}
off.push_back(s.size());
}
// get-deltas: each chunk runs from the current position to the first non-white after the first white.
static std::vector<std::string> windows (const std::string& text, int win) {
std::vector<unsigned> cps; std::vector<size_t> off;
decode(text, cps, off);
std::vector<std::pair<size_t, size_t>> chunks; // code-point [start, end)
size_t pos = 0, n = cps.size();
for (;;) {
size_t white = pos;
while (white < n && !uws(cps[white])) white++;
if (white == n) { chunks.push_back({pos, n}); break; }
size_t nonwhite = white;
while (nonwhite < n && uws(cps[nonwhite])) nonwhite++;
chunks.push_back({pos, nonwhite});
if (nonwhite == n) break;
pos = nonwhite;
}
std::vector<std::string> out;
if ((int) chunks.size() >= win) {
for (size_t k = 0; k + win <= chunks.size(); k++)
out.push_back(text.substr(off[chunks[k].first], off[chunks[k + win - 1].second] - off[chunks[k].first]));
} else {
out.push_back(text.substr(0, off[chunks.back().second]));
}
return out;
}
// ---------------------------------------------------------------- LGParseLink.cc helpers
static std::string word_string (Linkage lkg, int w, const char* phr) {
size_t sb = linkage_get_word_byte_start(lkg, w), eb = linkage_get_word_byte_end(lkg, w);
if (eb != sb) return std::string(phr + sb, eb - sb);
const char* wrd = linkage_get_word(lkg, w);
if (0 == w && 0 == strcmp(wrd, "LEFT-WALL")) return "###LEFT-WALL###";
if (linkage_get_num_words(lkg) - 1 == w && 0 == strcmp(wrd, "RIGHT-WALL")) return "###RIGHT-WALL###";
return wrd;
}
static void emit_linkage (Linkage lkg, const char* phr, int win, int pi) {
int nwords = linkage_get_num_words(lkg), nlinks = linkage_get_num_links(lkg);
for (int w = 0; w < nwords; w++) { // make_sects / make_conseq
std::vector<int> lks;
for (int li = 0; li < nlinks; li++) {
int lw = linkage_get_link_lword(lkg, li), rw = linkage_get_link_rword(lkg, li);
if (lw == w) lks.push_back(rw); else if (rw == w) lks.push_back(lw);
}
if (lks.empty()) continue;
std::sort(lks.begin(), lks.end());
std::string s = "(Section (WordNode \"" + word_string(lkg, w, phr) + "\") (ConnectorSeq";
for (int c : lks) s += " (Connector (WordNode \"" + word_string(lkg, c, phr) + "\") (SexNode \"" + (c < w ? "-" : "+") + "\"))";
s += "))";
printf("%d\t%d\t%s\n", win, pi, s.c_str());
}
for (int li = 0; li < nlinks; li++) { // make_bonds
int lw = linkage_get_link_lword(lkg, li), rw = linkage_get_link_rword(lkg, li);
printf("%d\t%d\t(EdgeLink (BondNode \"%s\") (ListLink (WordNode \"%s\") (WordNode \"%s\")))\n", win, pi,
linkage_get_link_label(lkg, li), word_string(lkg, lw, phr).c_str(), word_string(lkg, rw, phr).c_str());
}
}
// ---------------------------------------------------------------- main
static void load_mi (AtomSpacePtr& as, const char* path) {
std::ifstream f(path);
if (!f) { perror(path); exit(1); }
Handle bany = as->add_node(BOND_NODE, "ANY"), key = as->add_node(PREDICATE_NODE, "*-Mutual Info Key-*");
const std::string pre = "(EdgeLink (BondNode \"ANY\") (ListLink (WordNode \"";
std::string line;
long n = 0;
while (std::getline(f, line)) {
if (line.compare(0, pre.size(), pre) || line.find("\t(PredicateNode \"*-Mutual Info Key-*\")\t") == std::string::npos) continue;
size_t a0 = pre.size(), a1 = line.find("\") (WordNode \"", a0);
if (a1 == std::string::npos) continue; // a marginal (AnyNode), not a pair
size_t b0 = a1 + 14, b1 = line.find("\")))\t", b0);
double mi, fmi;
if (b1 == std::string::npos || 2 != sscanf(line.c_str() + line.rfind('\t') + 1, "%lf %lf", &mi, &fmi)) {
fprintf(stderr, "%s: bad line: %s\n", path, line.c_str());
exit(1);
}
Handle lst = as->add_link(LIST_LINK, as->add_node(WORD_NODE, line.substr(a0, a1 - a0)),
as->add_node(WORD_NODE, line.substr(b0, b1 - b0)));
as->set_value(as->add_link(EDGE_LINK, bany, lst), key, createFloatValue(std::vector<double>{mi, fmi}));
n++;
}
fprintf(stderr, "loaded %ld pair MI values\n", n);
}
int main (int argc, char** argv) {
if (argc < 4) { fprintf(stderr, "usage: lgsect <text> <mi.tsv> <dict-dir> [--win 12] [--numl 3] [--stats f] [--first N]\n"); return 1; }
int win = 12, numl = 3, first = -1;
const char* statsp = nullptr;
for (int i = 4; i < argc; i++) {
if (!strcmp(argv[i], "--win") && i + 1 < argc) win = atoi(argv[++i]);
else if (!strcmp(argv[i], "--numl") && i + 1 < argc) numl = atoi(argv[++i]);
else if (!strcmp(argv[i], "--stats") && i + 1 < argc) statsp = argv[++i];
else if (!strcmp(argv[i], "--first") && i + 1 < argc) first = atoi(argv[++i]);
}
std::ifstream tf(argv[1]);
std::stringstream ss; ss << tf.rdbuf();
std::vector<std::string> wins = windows(ss.str(), win);
fprintf(stderr, "%zu windows of %d\n", wins.size(), win);
AtomSpacePtr as = createAtomSpace();
load_mi(as, argv[2]);
lg_config_atomspace(as, nullptr);
Dictionary dict = dictionary_create_lang(argv[3]);
if (!dict) { fprintf(stderr, "cannot open dictionary %s\n", argv[3]); return 1; }
FILE* stats = statsp ? fopen(statsp, "w") : nullptr;
for (size_t wi = 0; wi < wins.size() && (first < 0 || (int) wi < first); wi++) {
const char* phr = wins[wi].c_str();
Sentence sent = sentence_create(phr, dict);
Parse_Options opts = parse_options_create();
parse_options_set_verbosity(opts, 0);
parse_options_set_max_parse_time(opts, 150);
parse_options_set_linkage_limit(opts, 15000);
parse_options_set_repeatable_rand(opts, 0);
if (numl > 15000) parse_options_set_linkage_limit(opts, numl);
int found = sentence_parse(sent, opts);
if (found == 0 && !parse_options_resources_exhausted(opts)) {
parse_options_reset_resources(opts);
parse_options_set_min_null_count(opts, 1);
parse_options_set_max_null_count(opts, sentence_length(sent));
found = sentence_parse(sent, opts);
}
int valid = found > 0 ? sentence_num_valid_linkages(sent) : 0;
int avail = found > 0 ? sentence_num_linkages_post_processed(sent) : 0;
int total = found > 0 ? sentence_num_linkages_found(sent) : 0;
if (stats) fprintf(stats, "%zu\t%d\t%d\t%d\t%d\n", wi, total, avail, valid, total > 15000);
int take = valid < numl ? valid : numl, jct = 0;
for (int i = 0; jct < take && i < avail; i++) {
if (0 < sentence_num_violations(sent, i)) continue;
Linkage lkg = linkage_create(i, sent, opts);
emit_linkage(lkg, phr, (int) wi, jct);
linkage_delete(lkg);
jct++;
}
sentence_delete(sent);
parse_options_delete(opts);
}
if (stats) fclose(stats);
dictionary_delete(dict);
return 0;
}