#! /usr/bin/env -S guile
!#
;
; trim-pair.scm
;
; After word-pair counting has been completed, this script is run to
; to remove word-pairs with low observation counts. This will also
; compute the marginals, so can be run instead of `marginals-pair.scm`
;
(load "cogserver.scm")
; Init the statistics objects.
(define ala (make-any-link-api))
(define asa (add-pair-stars ala))
(define trm (add-trimmer asa))
; Compute the statistics; these are needed for trimming.
(batch-pairs asa)
(barrier storage-node)
; Trim the dataset.
; XXX This includes some ad-hoc constants. See the Sept 2021 diary entry
; for why these are chosen, instead of something else. Basically,
; trimming this much raises the average MI of the dataset to its
; highest value. It also cuts the number of word pairs about half.
(trm 'subtotal-trim asa 10 10 4)
; Re-init the stats.
(set! ala (make-any-link-api))
(set! asa (add-pair-stars ala))
(batch-all-pair-mi asa)
(barrier storage-node)
; (cog-close storage-node)