| … | … | @@ -744,12 +744,14 @@ static void remove_empty (long row) { |
| 744 | 744 | // ---------------------------------------------------------------- main |
| 745 | 745 | |
| 746 | 746 | int main (int argc, char **argv) { |
| 747 | if (argc < 2) { fprintf(stderr, "usage: cluster <counts.tsv> [--nrank 50] [--loops 5] [--intended]\n"); return 1; } | |
| 747 | if (argc < 2) { fprintf(stderr, "usage: cluster <counts.tsv> [--nrank 50] [--loops 5] [--intended] [--dump state.tsv]\n"); return 1; } | |
| 748 | 748 | int nrank = 50, loops = 5; |
| 749 | const char *dump_path = NULL; | |
| 749 | 750 | for (int i = 2; i < argc; i++) { |
| 750 | 751 | if (!strcmp(argv[i], "--nrank") && i + 1 < argc) nrank = atoi(argv[++i]); |
| 751 | 752 | else if (!strcmp(argv[i], "--loops") && i + 1 < argc) loops = atoi(argv[++i]); |
| 752 | 753 | else if (!strcmp(argv[i], "--intended")) as_run = 0; |
| 754 | else if (!strcmp(argv[i], "--dump") && i + 1 < argc) dump_path = argv[++i]; | |
| 753 | 755 | } |
| 754 | 756 | setvbuf(stdout, NULL, _IOLBF, 0); |
| 755 | 757 | stale_rank = getenv("CLUSTER_STALE_RANK") != NULL; |
| … | … | @@ -856,5 +858,20 @@ int main (int argc, char **argv) { |
| 856 | 858 | } |
| 857 | 859 | } |
| 858 | 860 | } |
| 861 | // Final state (level2/PREREG-LV.md): atoms (id, is_class, name), memberships, live Section pairs. | |
| 862 | if (dump_path) { | |
| 863 | FILE *f = fopen(dump_path, "w"); | |
| 864 | if (!f) { perror(dump_path); return 1; } | |
| 865 | for (size_t a = 0; a < atoms.n; a++) fprintf(f, "A\t%zu\t%d\t%s\n", a, is_class(a), aname(a)); | |
| 866 | for (size_t k = 0; k < nmembs; k++) fprintf(f, "M\t%ld\t%ld\t%.6f\n", membs[k].w, membs[k].cls, membs[k].n); | |
| 867 | for (size_t k = 0; k < npairs; k++) { | |
| 868 | if (!pairs[k].alive || pairs[k].n <= 0 || cols[pairs[k].c].shape) continue; | |
| 869 | col_t c = cols[pairs[k].c]; | |
| 870 | fprintf(f, "S\t%ld\t%.6f\t", pairs[k].r, pairs[k].n); | |
| 871 | for (int i = 0; i < c.nc; i++) fprintf(f, i ? " %ld%c" : "%ld%c", c.ca[i], c.cd[i]); | |
| 872 | fprintf(f, "\n"); | |
| 873 | } | |
| 874 | fclose(f); | |
| 875 | } | |
| 859 | 876 | return 0; |
| 860 | 877 | } |
| … | … | @@ -1,2 +1,3 @@ |
| 1 | 1 | l2 |
| 2 | 2 | results-jt/*/*.bin |
| 3 | results-LV/*/work/ |
| … | … | @@ -530,11 +530,12 @@ static void run_soft (int seed, int tier, const char *src, const corpus_t *tr, c |
| 530 | 530 | // ---------------------------------------------------------------- main |
| 531 | 531 | |
| 532 | 532 | int main (int argc, char **argv) { |
| 533 | if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix] [--soft [--soft-scale s]] [--undirected] [--reparse]\n"); return 1; } | |
| 533 | if (argc < 2) { fprintf(stderr, "usage: l2 <seed> [--tiers a,b] [--heldout N] [--mut-drop] [--jt prefix] [--soft [--soft-scale s]] [--undirected] [--reparse] [--sections prefix] [--labels prefix [--labels-name n]]\n"); return 1; } | |
| 534 | 534 | int seed = atoi(argv[1]); |
| 535 | 535 | int tiers[16] = { 3000, 10000, 30000 }, ntier = 3, nho = 2000; |
| 536 | 536 | const char *jt_prefix = NULL; |
| 537 | 537 | int soft = 0, undirected = 0, reparse = 0; double soft_scale = 1.0; |
| 538 | const char *sect_prefix = NULL, *lab_prefix = NULL, *lab_name = "LV"; | |
| 538 | 539 | for (int i = 2; i < argc; i++) { |
| 539 | 540 | if (!strcmp(argv[i], "--mut-drop")) mut_drop = 1; |
| 540 | 541 | else if (!strcmp(argv[i], "--heldout") && i + 1 < argc) nho = atoi(argv[++i]); |
| … | … | @@ -542,6 +543,9 @@ int main (int argc, char **argv) { |
| 542 | 543 | else if (!strcmp(argv[i], "--soft")) soft = 1; |
| 543 | 544 | else if (!strcmp(argv[i], "--undirected")) undirected = 1; |
| 544 | 545 | else if (!strcmp(argv[i], "--reparse")) reparse = 1; |
| 546 | else if (!strcmp(argv[i], "--sections") && i + 1 < argc) sect_prefix = argv[++i]; | |
| 547 | else if (!strcmp(argv[i], "--labels") && i + 1 < argc) lab_prefix = argv[++i]; | |
| 548 | else if (!strcmp(argv[i], "--labels-name") && i + 1 < argc) lab_name = argv[++i]; | |
| 545 | 549 | else if (!strcmp(argv[i], "--soft-scale") && i + 1 < argc) soft_scale = atof(argv[++i]); |
| 546 | 550 | else if (!strcmp(argv[i], "--tiers") && i + 1 < argc) { |
| 547 | 551 | ntier = 0; |
| … | … | @@ -618,6 +622,39 @@ int main (int argc, char **argv) { |
| 618 | 622 | for (int w = V - 1; w > 0; w--) { int j = rpick(&rr, w + 1), t = perm[w]; perm[w] = perm[j]; perm[j] = t; } |
| 619 | 623 | for (int w = 0; w < V; w++) Rmap[w] = primary[perm[w]]; |
| 620 | 624 | |
| 625 | // PREREG-LV.md: external per-token labels (<prefix>-<tier>-{tr,ho}.lab, one int per token) as layer 1: | |
| 626 | // MST over the labels, link F1, C-x and M-x-ucomp-MST. Skips every other arm. | |
| 627 | if (lab_prefix) { | |
| 628 | if (!Ptr.lnk) { Ptr = parse_alloc(full.ntok); Pho = parse_alloc(ho.ntok); } | |
| 629 | u_tr = &Ptr; u_ho = &Pho; | |
| 630 | int A = 0; | |
| 631 | for (int k = 0; k < 2; k++) { | |
| 632 | char path[512]; | |
| 633 | snprintf(path, sizeof path, "%s-%d-%s.lab", lab_prefix, tier, k ? "ho" : "tr"); | |
| 634 | FILE *f = fopen(path, "r"); | |
| 635 | if (!f) { perror(path); return 1; } | |
| 636 | int *dst = k ? sho : str, nt = k ? ho.ntok : tr.ntok; | |
| 637 | for (int t = 0; t < nt; t++) { if (fscanf(f, "%d", &dst[t]) != 1) { fprintf(stderr, "%s: short\n", path); return 1; } if (dst[t] + 1 > A) A = dst[t] + 1; } | |
| 638 | fclose(f); | |
| 639 | } | |
| 640 | corpus_t trs = with_symbols(&tr, str), hos = with_symbols(&ho, sho); | |
| 641 | arm_t a = { "l1", A, NULL, 1 }; | |
| 642 | pairs_t P = count_pairs(&a, &trs); | |
| 643 | double *MI = pair_mi(&P), pr, rc, f1; | |
| 644 | free_pairs(&P); | |
| 645 | parse_u(&trs, MI, A, 0, &Ptr); parse_u(&hos, MI, A, 0, &Pho); | |
| 646 | free(MI); | |
| 647 | link_prf(&hos, &Pho, &pr, &rc, &f1); | |
| 648 | printf("P\t%d\t%d\t%s\tMST\t%.5f\t%.5f\t%.5f\n", seed, tier, lab_name, pr, rc, f1); | |
| 649 | char nm[48]; | |
| 650 | snprintf(nm, sizeof nm, "C-%s", lab_name); | |
| 651 | run_arm(seed, tier, nm, RULE_ALL, &trs, &hos, &ho, NULL, NULL); | |
| 652 | snprintf(nm, sizeof nm, "M-%s-ucomp-MST", lab_name); | |
| 653 | run_arm(seed, tier, nm, RULE_UCOMP, &trs, &hos, &ho, (const int *) u_tr, (const int *) u_ho); | |
| 654 | free(Lmap); | |
| 655 | continue; | |
| 656 | } | |
| 657 | ||
| 621 | 658 | // F and MW: words |
| 622 | 659 | corpus_t trw = with_symbols(&tr, tr.w), how = with_symbols(&ho, ho.w); |
| 623 | 660 | run_arm(seed, tier, "F", RULE_ALL, &trw, &how, &ho, NULL, NULL); |
| … | … | @@ -708,6 +745,37 @@ int main (int argc, char **argv) { |
| 708 | 745 | } |
| 709 | 746 | free(cls); free(Wp); |
| 710 | 747 | } |
| 748 | // Step 3 (#15): per-token Sections of the word-level MST (Linas: germ word + linked words, -/+ by position), | |
| 749 | // written as <prefix>-<tier>-{tr,ho}.tsv lines: token index, germ, ConnectorSeq. | |
| 750 | if (sect_prefix) { | |
| 751 | if (!Ptr.lnk) { Ptr = parse_alloc(full.ntok); Pho = parse_alloc(ho.ntok); } | |
| 752 | corpus_t trw = with_symbols(&tr, tr.w), how = with_symbols(&ho, ho.w); | |
| 753 | arm_t a = { "l1", V, NULL, 1 }; | |
| 754 | pairs_t P = count_pairs(&a, &trw); | |
| 755 | double *MI = pair_mi(&P); | |
| 756 | free_pairs(&P); | |
| 757 | parse_u(&trw, MI, V, 0, &Ptr); parse_u(&how, MI, V, 0, &Pho); | |
| 758 | free(MI); | |
| 759 | for (int k = 0; k < 2; k++) { | |
| 760 | const corpus_t *C = k ? &how : &trw; const parse_t *Pp = k ? &Pho : &Ptr; | |
| 761 | char path[512]; | |
| 762 | snprintf(path, sizeof path, "%s-%d-%s.tsv", sect_prefix, tier, k ? "ho" : "tr"); | |
| 763 | FILE *f = fopen(path, "w"); | |
| 764 | if (!f) { perror(path); return 1; } | |
| 765 | for (int s = 0; s < C->ns; s++) { | |
| 766 | int o = C->off[s], n = C->off[s + 1] - o; | |
| 767 | for (int t = 0; t < n; t++) { | |
| 768 | int nb[MAXL], m = 0; | |
| 769 | for (int q = 0; q < Pp->nl[o + t]; q++) nb[m++] = Pp->lnk[(o + t) * MAXL + q]; | |
| 770 | for (int x = 1; x < m; x++) { int v = nb[x], y = x; while (y > 0 && nb[y - 1] > v) { nb[y] = nb[y - 1]; y--; } nb[y] = v; } | |
| 771 | fprintf(f, "%d\tw%04d\t%d\t", o + t, C->w[o + t], (k ? &ho : &tr)->c[o + t]); | |
| 772 | for (int x = 0; x < m; x++) fprintf(f, "(Connector (WordNode \"w%04d\") (SexNode \"%c\")) ", C->w[o + nb[x]], nb[x] < t ? '-' : '+'); | |
| 773 | fprintf(f, "\n"); | |
| 774 | } | |
| 775 | } | |
| 776 | fclose(f); | |
| 777 | } | |
| 778 | } | |
| 711 | 779 | free(Lmap); |
| 712 | 780 | |
| 713 | 781 | // PREREG-soft.md: rank span types by assembly-ness over KPARSE perturbed parses (sources G and L). |
| … | … | @@ -0,0 +1,102 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | # PREREG-LV.md: Linas's clusterer (ladder/port/cluster.c) as layer 1. | |
| 3 | # lv.py agg <sections-tr.tsv> <out.tsv> Sections summed over tokens, cluster.c format | |
| 4 | # lv.py label <sections-tr.tsv> <sections-ho.tsv> <state.tsv> <out-prefix> | |
| 5 | # label writes <out-prefix>-{tr,ho}.lab (one int per token) and prints one Q row of held-out measures: | |
| 6 | # Q <vmeasure> <m1> <nlabels> <class-share> <sense> <sense-coll> | |
| 7 | import collections, itertools, math, sys | |
| 8 | ||
| 9 | def read_sections (path): | |
| 10 | toks = [] # (token index, germ, gold class, [(word, dir)]) | |
| 11 | for line in open(path): | |
| 12 | idx, g, gold, cs = line.rstrip('\n').split('\t') | |
| 13 | conns = [] | |
| 14 | for part in cs.split('(Connector (WordNode "')[1:]: | |
| 15 | conns.append((part[:part.index('"')], part[part.index('(SexNode "') + 10])) | |
| 16 | toks.append((int(idx), g, int(gold), conns)) | |
| 17 | return toks | |
| 18 | ||
| 19 | def agg (src, out): | |
| 20 | c = collections.Counter() | |
| 21 | for _, g, _, conns in read_sections(src): | |
| 22 | if conns: c[(g, ' '.join(f'(Connector (WordNode "{w}") (SexNode "{d}"))' for w, d in conns))] += 1 | |
| 23 | with open(out, 'w') as f: | |
| 24 | for (g, cs), n in sorted(c.items()): f.write(f'(Section (WordNode "{g}") (ConnectorSeq {cs}))\t0\t0 0 {n}\n') | |
| 25 | ||
| 26 | def vmeasure (lab, gold): | |
| 27 | N = len(lab); ct = collections.Counter(zip(lab, gold)); rl = collections.Counter(lab); rg = collections.Counter(gold) | |
| 28 | Hc = -sum(v / N * math.log(v / N) for v in rg.values()); Hk = -sum(v / N * math.log(v / N) for v in rl.values()) | |
| 29 | Hck = -sum(v / N * math.log(v / rl[l]) for (l, g), v in ct.items()); Hkc = -sum(v / N * math.log(v / rg[g]) for (l, g), v in ct.items()) | |
| 30 | h = 1 - Hck / Hc if Hc > 0 else 1; c = 1 - Hkc / Hk if Hk > 0 else 1 | |
| 31 | return 2 * h * c / (h + c) if h + c > 0 else 0 | |
| 32 | ||
| 33 | def m1_map (lab, gold): | |
| 34 | ct = collections.Counter(zip(lab, gold)); best = {} | |
| 35 | for (l, g), v in ct.items(): | |
| 36 | if v > best.get(l, (0, None))[0]: best[l] = (v, g) | |
| 37 | return {l: g for l, (v, g) in best.items()} | |
| 38 | ||
| 39 | def label (tr_path, ho_path, state, prefix): | |
| 40 | name, is_cls, parents, sect = {}, {}, collections.defaultdict(list), {} | |
| 41 | for line in open(state): | |
| 42 | p = line.rstrip('\n').split('\t') | |
| 43 | if p[0] == 'A': is_cls[int(p[1])] = p[2] == '1'; name[p[3]] = int(p[1]) if p[2] == '0' else name.get(p[3]) | |
| 44 | elif p[0] == 'M': parents[int(p[1])].append((int(p[2]), float(p[3]))) | |
| 45 | elif p[0] == 'S': | |
| 46 | key = tuple((int(x[:-1]), x[-1]) for x in p[3].split()) | |
| 47 | sect[(int(p[1]), key)] = float(p[2]) | |
| 48 | word_id = {n: i for n, i in name.items() if i is not None and not is_cls[i]} | |
| 49 | anc_cache = {} | |
| 50 | def ancestors (a): # all classes a belongs to, transitively | |
| 51 | if a not in anc_cache: | |
| 52 | out, todo = [], [a] | |
| 53 | while todo: | |
| 54 | x = todo.pop() | |
| 55 | for c, _ in parents.get(x, []): | |
| 56 | if c not in out: out.append(c); todo.append(c) | |
| 57 | anc_cache[a] = out | |
| 58 | return anc_cache[a] | |
| 59 | extra = {} | |
| 60 | def unseen (w): # held-out word never seen by the clusterer: its own new label | |
| 61 | if w not in extra: extra[w] = len(is_cls) + len(extra) | |
| 62 | return extra[w] | |
| 63 | def assign (g, conns): | |
| 64 | a = word_id.get(g) | |
| 65 | if a is None: return unseen(g), False | |
| 66 | rows = ancestors(a) | |
| 67 | if rows and conns: | |
| 68 | opts = [[word_id.get(w, -1)] + ancestors(word_id[w]) if w in word_id else [-1] for w, _ in conns] | |
| 69 | best, bn = None, 0 | |
| 70 | for combo in itertools.islice(itertools.product(*opts), 4096): | |
| 71 | key = tuple(zip(combo, (d for _, d in conns))) | |
| 72 | for r in rows: | |
| 73 | n = sect.get((r, key), 0) | |
| 74 | if n > bn: best, bn = r, n | |
| 75 | if best is not None: return best, True | |
| 76 | if parents.get(a): return max(parents[a], key=lambda x: x[1])[0], True | |
| 77 | return a, False | |
| 78 | out = {} | |
| 79 | for split, path in (('tr', tr_path), ('ho', ho_path)): | |
| 80 | toks = sorted(read_sections(path)) | |
| 81 | labs = [assign(g, conns) for _, g, _, conns in toks] | |
| 82 | with open(f'{prefix}-{split}.lab', 'w') as f: f.write('\n'.join(str(l) for l, _ in labs) + '\n') | |
| 83 | out[split] = (toks, labs) | |
| 84 | tr_toks, _ = out['tr'] | |
| 85 | senses = collections.defaultdict(set) | |
| 86 | for _, g, gold, _ in tr_toks: senses[g].add(gold) | |
| 87 | toks, labs = out['ho'] | |
| 88 | lab = [l for l, _ in labs]; gold = [t[2] for t in toks]; words = [t[1] for t in toks] | |
| 89 | mp = m1_map(lab, gold) | |
| 90 | m1 = sum(mp[l] == g for l, g in zip(lab, gold)) / len(lab) | |
| 91 | coll_of = {} | |
| 92 | for (w, l), _ in collections.Counter(zip(words, lab)).most_common(): | |
| 93 | coll_of.setdefault(w, l) | |
| 94 | coll = [coll_of[w] for w in words] | |
| 95 | mpc = m1_map(coll, gold) | |
| 96 | poly = [len(senses.get(w, ())) >= 2 for w in words] | |
| 97 | def sense (ls, m): n = sum(poly); return sum(m[l] == g for l, g, p in zip(ls, gold, poly) if p) / n if n else 0 | |
| 98 | print('Q\t%.5f\t%.5f\t%d\t%.5f\t%.5f\t%.5f' % (vmeasure(lab, gold), m1, len(set(lab)), sum(c for _, c in labs) / len(labs), sense(lab, mp), sense(coll, mpc))) | |
| 99 | ||
| 100 | if __name__ == '__main__': | |
| 101 | if sys.argv[1] == 'agg': agg(sys.argv[2], sys.argv[3]) | |
| 102 | else: label(*sys.argv[2:6]) |
| … | … | @@ -0,0 +1,29 @@ |
| 1 | #!/bin/sh | |
| 2 | # PREREG-LV full run: per seed, word-MST Sections -> Linas's clusterer (port; intended and as-run) -> token labels | |
| 3 | # -> re-parse and level 2. 10 seeds in parallel, tiers 3K,10K,30K. | |
| 4 | set -e | |
| 5 | cd "$(dirname "$0")" | |
| 6 | gcc -O2 -Wall -o l2 l2.c -lm 2>/dev/null | |
| 7 | gcc -O2 -Wall -o ../ladder/port/cluster ../ladder/port/cluster.c -lm 2>/dev/null | |
| 8 | out=results-LV/$(date +%s) | |
| 9 | mkdir -p "$out/work" | |
| 10 | git rev-parse HEAD > "$out/commit.txt" | |
| 11 | sha256sum l2.c lv.py ../ladder/port/cluster.c ../across-cluster/ac.c > "$out/src.sha256" | |
| 12 | for s in 1 2 3 4 5 6 7 8 9 10; do | |
| 13 | ( | |
| 14 | w="$out/work/s$s" | |
| 15 | ./l2 $s --sections "$w" > /dev/null | |
| 16 | for t in 3000 10000 30000; do | |
| 17 | python3 lv.py agg "$w-$t-tr.tsv" "$w-$t-agg.tsv" | |
| 18 | for arm in I A; do | |
| 19 | fl=""; [ $arm = I ] && fl="--intended" | |
| 20 | ../ladder/port/cluster "$w-$t-agg.tsv" --nrank 200 --loops 100 $fl --dump "$w-$arm-$t-state.tsv" > "$w-$arm-$t.log" | |
| 21 | python3 lv.py label "$w-$t-tr.tsv" "$w-$t-ho.tsv" "$w-$arm-$t-state.tsv" "$w-lab$arm-$t" | sed "s/^Q/Q\t$s\t$t\tLV-$arm/" >> "$out/seed-$s.tsv" | |
| 22 | printf 'N\t%s\t%s\tLV-%s\t%s\n' $s $t $arm "$(grep -c 'Merged into' "$w-$arm-$t.log")" >> "$out/seed-$s.tsv" | |
| 23 | done | |
| 24 | done | |
| 25 | for arm in I A; do ./l2 $s --labels "$w-lab$arm" --labels-name LV-$arm >> "$out/seed-$s.tsv"; done | |
| 26 | ) & | |
| 27 | done | |
| 28 | wait | |
| 29 | echo "$out" |
| … | … | @@ -0,0 +1,43 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | # Score results-LV/<stamp> against PREREG-LV.md. Usage: summarize-LV.py results-LV/<stamp> results-reparse/<stamp> results-U/<stamp> | |
| 3 | import glob, math, os, sys | |
| 4 | T975 = {9: 2.262} | |
| 5 | def mean(x): return sum(x) / len(x) | |
| 6 | def ci(d): | |
| 7 | n = len(d); m = mean(d); sd = math.sqrt(sum((v - m) ** 2 for v in d) / (n - 1)) | |
| 8 | h = T975.get(n - 1, 1.96) * sd / math.sqrt(n) | |
| 9 | return m, m - h, m + h | |
| 10 | def fmt(c): return f'{c[0]:+.3f} [{c[1]:+.3f}, {c[2]:+.3f}]' | |
| 11 | def load (d): | |
| 12 | Q, N, P, A, RP = {}, {}, {}, {}, {} | |
| 13 | for f in glob.glob(os.path.join(d, 'seed-*.tsv')): | |
| 14 | for line in open(f): | |
| 15 | p = line.rstrip('\n').split('\t') | |
| 16 | if p[0] == 'Q': Q[(int(p[2]), p[3], int(p[1]))] = tuple(map(float, p[4:10])) | |
| 17 | elif p[0] == 'N': N[(int(p[2]), p[3], int(p[1]))] = int(p[4]) | |
| 18 | elif p[0] == 'P': P[(int(p[2]), p[3], int(p[1]))] = float(p[7]) | |
| 19 | elif p[0] == 'A': A[(int(p[2]), p[3], int(p[1]))] = (int(p[4]), float(p[6]), float(p[7])) | |
| 20 | elif p[0] == 'RP': RP[(int(p[2]), int(p[3]), int(p[1]))] = float(p[6]) | |
| 21 | return Q, N, P, A, RP | |
| 22 | Q, N, P, A, _ = load(sys.argv[1]); _, _, _, Ar, RPr = load(sys.argv[2]); _, _, Pu, _, _ = load(sys.argv[3]) | |
| 23 | seeds = sorted({k[2] for k in Q}); tiers = sorted({k[0] for k in Q}); top = max(tiers) | |
| 24 | out = [] | |
| 25 | def say(s=''): out.append(s); print(s) | |
| 26 | def m(d, t, k, i=None): return [d[(t, k, s)] if i is None else d[(t, k, s)][i] for s in seeds] | |
| 27 | say('## Held-out labels (mean): V-measure / many-to-one / labels / share in a class / sense / sense-COLL / merges') | |
| 28 | for t in tiers: | |
| 29 | for arm in ['LV-I', 'LV-A']: | |
| 30 | say(f'{t} {arm}: ' + ' / '.join(f'{mean(m(Q, t, arm, i)):.3f}' for i in range(6)) + f' / {mean(m(N, t, arm)):.1f}') | |
| 31 | say('\n## Re-parse link F1 and level 2 (mean): link F1 / ucomp types / ucomp type recall / ucomp precision') | |
| 32 | for t in tiers: | |
| 33 | say(f'{t} words(round 0) F1 {mean([RPr[(t, 0, s)] for s in seeds]):.3f} ucomp recall {mean(m(Ar, t, "M-RP0-ucomp", 1)):.3f}; L-MST F1 {mean(m(Pu, t, "L")):.3f}') | |
| 34 | for arm in ['LV-I', 'LV-A']: | |
| 35 | k = f'M-{arm}-ucomp-MST' | |
| 36 | say(f' {arm}: {mean(m(P, t, arm)):.3f} / {mean(m(A, t, k, 0)):.0f} / {mean(m(A, t, k, 1)):.3f} / {mean(m(A, t, k, 2)):.3f}') | |
| 37 | say(f'\n## Predictions (at {top})') | |
| 38 | def verdict(n, ok, txt): say(f'{n}: {"PASS" if ok else "FAIL"} — {txt}') | |
| 39 | c = ci([a - b for a, b in zip(m(Q, top, 'LV-I', 4), m(Q, top, 'LV-I', 5))]); verdict('L1', c[1] > 0, f'sense LV-I − LV-I-COLL {fmt(c)}') | |
| 40 | c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), [RPr[(top, 0, s)] for s in seeds])]); verdict('L2', c[1] > 0, f'link F1 LV-I − word MST {fmt(c)}') | |
| 41 | c = ci([a - b for a, b in zip(m(P, top, 'LV-I'), m(Pu, top, 'L'))]); verdict('L3', c[1] > -0.02, f'link F1 LV-I − L-MST {fmt(c)} (bar: lower > -0.02)') | |
| 42 | c = ci([a - b for a, b in zip(m(A, top, 'M-LV-I-ucomp-MST', 1), m(Ar, top, 'M-RP0-ucomp', 1))]); verdict('L4', c[1] > 0, f'ucomp type recall LV-I − word MST {fmt(c)}') | |
| 43 | open(os.path.join(sys.argv[1], 'summary.txt'), 'w').write('\n'.join(out) + '\n') |