;
; cogserver-start.scm -- Phase 3a: CogServer with RocksDB backend
;
; Starts CogServer with pre-loaded AtomSpace from Phase 1+2 data.
; After initial load, AtomSpace stays in memory for continuous learning.
;
; Usage:
; cogserver -p 17001 # Start server
; Then in telnet/rlwrap: (load "cogserver-start.scm")
;
; Or direct:
; guile -l cogserver-start.scm # For testing without network
;
(use-modules (ice-9 format))
(use-modules (srfi srfi-1)) ; for take, etc.
(use-modules (opencog) (opencog exec))
(use-modules (opencog persist) (opencog persist-rocks))
(use-modules (opencog lg))
(use-modules (opencog matrix))
(use-modules (opencog sensory) (opencog opencl))
; ═══════════════════════════════════════════════════════════════════
; TYPE ALIASES (same as Phase 2)
; ParseNode/SentenceNode were in old opencog-nlp, not installed.
; We inject + export them into (opencog lg) before loading learn module.
; ═══════════════════════════════════════════════════════════════════
(define (ParseNode name) (Anchor (string-append "parse:" name)))
(define (SentenceNode name) (Anchor (string-append "sentence:" name)))
;; Add to module AND export so other modules see them
(let ((lg-mod (resolve-module '(opencog lg))))
(module-define! lg-mod 'ParseNode ParseNode)
(module-define! lg-mod 'SentenceNode SentenceNode)
(module-export! lg-mod '(ParseNode SentenceNode)))
(use-modules (opencog learn))
; ═══════════════════════════════════════════════════════════════════
; CONFIGURATION
; ═══════════════════════════════════════════════════════════════════
(define *rocks-path* "rocks://data/pairs.rdb")
(define *max-memory-gb* 8) ; RAM budget for AtomSpace
(define *hot-pair-threshold* 5) ; Minimum count to keep in RAM
; ═══════════════════════════════════════════════════════════════════
; GLOBAL STATE
; ═══════════════════════════════════════════════════════════════════
(define *storage* #f)
(define *pair-stars* #f)
(define *section-stars* #f)
;; Dashboard tracking state
(define *learning-epoch* 0)
(define *total-sentences* 0)
(define *total-parses* 0)
(define *mi-cache-hits* 0)
(define *mi-compute-count* 0)
(define *activity-log* '())
(define *max-activity-log* 100)
(define *server-start-time* (current-time))
;; GPU state
(define *gpu-node* #f)
(define *gpu-kernel-path* "/storage/projects/code/hypercore-kb/opencog-kb/language-learning/mvp/mi-kernel.cl")
(define *gpu-enabled* #f)
;; ═══════════════════════════════════════════════════════════════════
;; FACTORED MI: N-independent PMI storage
;;
;; MI = PMI_raw + log₂[N]
;; where PMI_raw = log₂[count(x,y)] - log₂[count(x,*)] - log₂[count(*,y)]
;;
;; Key insight: N drift doesn't affect MST parsing (constant offset
;; preserves ranking). We only adjust MI for reporting.
;; ═══════════════════════════════════════════════════════════════════
(define *n-current* 0) ; Current total pair count
(define *n-at-batch* 0) ; N when MI was last batch-computed
(define *pmi-raw-key* (Predicate "*-PMI-raw-*"))
(define (safe-log2 x)
"Log base 2, returns -inf for 0."
(if (> x 0) (/ (log x) (log 2)) -inf.0))
(define (n-drift-ratio)
"Return N drift as ratio: (N_current - N_batch) / N_batch"
(if (> *n-at-batch* 0)
(/ (- *n-current* *n-at-batch*) *n-at-batch*)
0.0))
(define (n-drift-bits)
"Return MI error in bits due to N drift: log₂[N_current/N_batch]"
(if (> *n-at-batch* 0)
(safe-log2 (/ *n-current* *n-at-batch*))
0.0))
(define (get-pmi-raw pair)
"Get stored PMI_raw from pair, or compute from existing MI."
(let ((val (cog-value pair *pmi-raw-key*)))
(if val
(cog-value-ref val 0)
;; Fallback: convert stored MI to PMI_raw
(let ((mi (cog-value pair (Predicate "*-FMI Key-*"))))
(if mi
(- (cog-value-ref mi 0) (safe-log2 *n-at-batch*))
#f)))))
(define (get-mi-adjusted pair)
"Get MI adjusted for current N: MI = PMI_raw + log₂[N_current]"
(let ((pmi-raw (get-pmi-raw pair)))
(if pmi-raw
(+ pmi-raw (safe-log2 *n-current*))
#f)))
(define (update-n-current!)
"Update N_current from actual edge count."
(set! *n-current* (cog-count-atoms 'Edge)))
; ═══════════════════════════════════════════════════════════════════
; GPU-ACCELERATED MI COMPUTATION
; Uses atomese-simd OpenCL for 385K pairs/second throughput
; ═══════════════════════════════════════════════════════════════════
(define (init-gpu!)
"Initialize GPU/OpenCL for MI computation."
(display "Initializing GPU for MI computation...\n")
(catch #t
(lambda ()
;; Copy kernel to /tmp (required by OpenCL loader)
(copy-file *gpu-kernel-path* "/tmp/mi-kernel.cl")
;; Create OpenCL node
(set! *gpu-node* (OpenclNode "opencl://:/tmp/mi-kernel.cl"))
;; Open with FloatValue result type
(cog-execute!
(SetValue *gpu-node* (Predicate "*-open-*") (Type 'FloatValue)))
;; Get device info
(let ((desc (cog-value *gpu-node* (Predicate "*-description-*"))))
(format #t "GPU initialized: ~a\n" desc))
(set! *gpu-enabled* #t)
*gpu-node*)
(lambda (key . args)
(format #t "GPU initialization failed: ~a ~a\n" key args)
(set! *gpu-enabled* #f)
#f)))
(define (close-gpu!)
"Close GPU connection."
(when *gpu-node*
(catch #t
(lambda ()
(cog-execute!
(SetValue *gpu-node* (Predicate "*-close-*") (Number 0))))
(lambda (key . args) #f))
(set! *gpu-node* #f)
(set! *gpu-enabled* #f)))
(define (gpu-status)
"Print GPU status."
(format #t "GPU enabled: ~a\n" *gpu-enabled*)
(when *gpu-enabled*
(let ((desc (cog-value *gpu-node* (Predicate "*-description-*"))))
(format #t "GPU device: ~a\n" desc))))
(define (compute-mi-gpu-batch pairs n-total)
"Compute MI for a list of pairs using GPU.
Returns list of (pair . mi) pairs."
(when (not *gpu-enabled*)
(error "GPU not initialized. Call (init-gpu!) first."))
(let ((n-pairs (length pairs)))
(if (= n-pairs 0)
'() ; Early return for empty list
;; Extract data from pairs using fold to build lists
(let loop ((remaining pairs)
(counts '())
(left-margs '())
(right-margs '()))
(if (null? remaining)
;; All pairs processed, run GPU
(let* ((counts-rev (reverse counts))
(left-rev (reverse left-margs))
(right-rev (reverse right-margs)))
;; Run GPU kernel
(cog-execute!
(SetValue *gpu-node* (Predicate "*-write-*")
(Section
(Item "compute_mi")
(ConnectorSeq
(apply Number (make-list n-pairs 0.0))
(apply Number counts-rev)
(apply Number left-rev)
(apply Number right-rev)
(apply Number (make-list n-pairs n-total))))))
;; Get results
(let* ((result (cog-execute! (ValueOf *gpu-node* (Predicate "*-read-*"))))
(args (cog-value-ref result 1))
(out (cog-value-ref args 0))
(mi-values (cog-value->list out)))
(map cons pairs mi-values)))
;; Process next pair
(let* ((pair (car remaining))
(cnt-val (cog-value pair *count-key*))
(cnt (if cnt-val (cog-value-ref cnt-val 2) 1.0))
(left (if *pair-stars*
(or (*pair-stars* 'left-wild-count pair) 1000.0)
1000.0))
(right (if *pair-stars*
(or (*pair-stars* 'right-wild-count pair) 1000.0)
1000.0)))
(loop (cdr remaining)
(cons cnt counts)
(cons left left-margs)
(cons right right-margs))))))))
(define* (recompute-mi-gpu #:key (chunk-size 50000))
"Recompute MI for all pairs using GPU acceleration.
~825x faster than CPU (385K pairs/sec vs ~450 pairs/sec)."
(when (not *gpu-enabled*)
(display "Initializing GPU...\n")
(init-gpu!))
(if (not *gpu-enabled*)
;; Fallback to CPU if GPU not available
(begin
(display "GPU not available, falling back to CPU.\n")
(recompute-mi))
;; GPU path
(begin
(display "═══ GPU-Accelerated MI Computation ═══\n\n")
(let* ((start-time (current-time))
(all-pairs (*pair-stars* 'get-all-elts))
(n-pairs (length all-pairs))
(n-total (*pair-stars* 'wild-wild-count))
(n-chunks (ceiling (/ n-pairs chunk-size)))
(mi-key (Predicate "*-FMI Key-*")))
(format #t "Total pairs: ~a\n" n-pairs)
(format #t "Total N: ~a\n" n-total)
(format #t "Processing in ~a chunks of ~a\n\n" n-chunks chunk-size)
;; Process in chunks using named let
(let loop ((remaining all-pairs)
(processed 0)
(chunk-num 0))
(when (not (null? remaining))
(let* ((chunk-len (min chunk-size (length remaining)))
(chunk (take remaining chunk-len))
(rest (drop remaining chunk-len))
(new-chunk-num (+ chunk-num 1)))
(format #t "Chunk ~a/~a: ~a pairs..." new-chunk-num n-chunks (length chunk))
;; Compute MI on GPU
(let ((mi-results (compute-mi-gpu-batch chunk n-total)))
(for-each
(lambda (pair-mi)
(cog-set-value! (car pair-mi) mi-key (FloatValue (cdr pair-mi))))
mi-results))
(let ((new-processed (+ processed (length chunk))))
(format #t " done (~,1f%)\n" (* 100.0 (/ new-processed n-pairs)))
(loop rest new-processed new-chunk-num)))))
;; Update N tracking
(update-n-current!)
(set! *n-at-batch* *n-current*)
(set! *learning-epoch* (+ 1 *learning-epoch*))
(let ((elapsed (- (current-time) start-time)))
(format #t "\n═══ GPU MI Complete ═══\n")
(format #t "Total time: ~a seconds\n" elapsed)
(format #t "Throughput: ~,0f pairs/second\n" (/ n-pairs (max elapsed 1)))
(format #t "N_at_batch = ~a, epoch = ~a\n" *n-at-batch* *learning-epoch*))))))
; ═══════════════════════════════════════════════════════════════════
; MEMORY MONITORING
; ═══════════════════════════════════════════════════════════════════
(define (get-atomspace-stats)
"Return basic AtomSpace statistics."
;; Note: cog-count-atoms 'Atom returns 0 - count concrete types instead
(let* ((edges (cog-count-atoms 'Edge))
(words (cog-count-atoms 'WordNode))
(bonds (cog-count-atoms 'BondNode))
(sections (cog-count-atoms 'Section))
(predicates (cog-count-atoms 'PredicateNode))
(anchors (cog-count-atoms 'AnchorNode))
(num-nodes (+ words bonds predicates anchors))
(num-links (+ edges sections))
(num-atoms (+ num-nodes num-links)))
(list num-atoms num-nodes num-links)))
(define (estimate-memory-mb)
"Rough estimate of AtomSpace memory usage in MB."
;; ~500 bytes per atom average (nodes smaller, links larger)
(let* ((edges (cog-count-atoms 'Edge))
(words (cog-count-atoms 'WordNode))
(sections (cog-count-atoms 'Section))
(total (+ edges words sections)))
(/ (* total 500) (* 1024 1024))))
(define (memory-status)
"Print memory and AtomSpace status."
(let* ((stats (get-atomspace-stats))
(mem-mb (estimate-memory-mb)))
(format #t "AtomSpace: ~a atoms (~a nodes, ~a links)~%"
(car stats) (cadr stats) (caddr stats))
(format #t "Estimated RAM: ~,1f MB (~,2f GB)~%"
mem-mb (/ mem-mb 1024))
(format #t "Budget: ~a GB~%" *max-memory-gb*)))
; ═══════════════════════════════════════════════════════════════════
; SELECTIVE LOADING
; ═══════════════════════════════════════════════════════════════════
(define (load-all-pairs)
"Load ALL pairs from RocksDB (full cold start)."
(display "Loading all pairs from RocksDB...\n")
(let ((start-time (current-time)))
(*pair-stars* 'fetch-pairs)
(let ((elapsed (- (current-time) start-time)))
;; Use cog-count-atoms instead of iterating get-all-elts (much faster)
(format #t "Elapsed time to load pairs: ~a secs~%" elapsed)
(format #t "Edges in AtomSpace: ~a~%" (cog-count-atoms 'Edge)))))
(define (load-hot-pairs min-count)
"Load only pairs with count >= min-count (warm start)."
(display "Loading hot pairs (selective)...\n")
;; For now, load all then filter
;; TODO: Implement server-side filtering in RocksDB
(load-all-pairs)
(let ((cold-count 0))
;; Remove cold pairs from AtomSpace (keep in RocksDB)
(for-each
(lambda (pair)
(let ((cnt (cog-value-ref
(cog-value pair (Predicate "*-TruthValueKey-*")) 2)))
(when (< cnt min-count)
(cog-extract! pair)
(set! cold-count (+ cold-count 1)))))
(*pair-stars* 'get-all-elts))
(format #t "Evicted ~a cold pairs (count < ~a)~%"
cold-count min-count)))
; ═══════════════════════════════════════════════════════════════════
; STARTUP MODES
; ═══════════════════════════════════════════════════════════════════
(define* (startup #:key (mode 'warm) (min-count 5))
"Initialize CogServer with RocksDB backend.
mode: 'cold = load everything, 'warm = load hot pairs only"
(display "═══ Phase 3a: CogServer Startup ═══\n\n")
;; Open RocksDB
(display "Opening RocksDB...\n")
(set! *storage* (RocksStorageNode *rocks-path*))
(cog-open *storage*)
(display " Database connected.\n\n")
;; Create matrix APIs
(display "Setting up matrix APIs...\n")
(define ala (make-any-link-api))
(define aca (add-count-api ala))
(set! *pair-stars* (add-pair-stars aca))
(define pca (make-pseudo-cset-api))
(set! *section-stars* (add-pair-stars pca))
(display " APIs ready.\n\n")
;; Load data based on mode
(case mode
((cold)
(display "COLD START: Loading all data...\n")
(load-all-pairs))
((warm)
(format #t "WARM START: Loading pairs with count >= ~a...~%"
min-count)
(load-hot-pairs min-count))
(else
(display "HOT START: Assuming data already loaded.\n")))
;; Initialize N tracking for factored MI
(update-n-current!)
(set! *n-at-batch* *n-current*)
(format #t "\nN tracking initialized: N = ~a\n" *n-current*)
;; Report status
(newline)
(memory-status)
(newline)
(display "CogServer ready for continuous learning.\n\n")
(display "Commands:\n")
(display " (parse-text \"...\") ; MST parse sentence\n")
(display " (learn-text \"...\") ; Count pairs from text\n")
(display " (mi-status) ; Show N drift status\n")
(display " (maybe-recompute-mi!) ; Auto-recompute if drift > 5%\n")
(display " (recompute-mi) ; Force MI recomputation (CPU)\n")
(display " (init-gpu!) ; Initialize GPU for fast MI\n")
(display " (recompute-mi-gpu) ; Force MI recomputation (GPU, 825x faster)\n")
(display " (memory-status) ; RAM usage\n")
(display " (shutdown) ; Graceful exit\n"))
; ═══════════════════════════════════════════════════════════════════
; CONTINUOUS LEARNING API
; ═══════════════════════════════════════════════════════════════════
(define *count-key* (Predicate "*-TruthValueKey-*"))
(define *window-size* 6)
(define (tokenize-text text)
"Split text into lowercase words."
(filter
(lambda (w) (>= (string-length w) 1))
(map (lambda (w)
(string-downcase
(string-filter char-alphabetic? w)))
(string-split text #\space))))
(define (count-inc! atom cnt)
"Increment count on ATOM by CNT."
(if (not (cog-value atom *count-key*))
(fetch-atom atom))
(cog-inc-value! atom *count-key* cnt 2)
(store-atom atom))
(define (learn-text text)
"Process text, count pairs within sliding window, store to RocksDB."
(define pairs-before (cog-count-atoms 'Edge))
(define words (tokenize-text text))
(define n (length words))
(define new-pair-count 0)
;; Count word occurrences
(for-each
(lambda (w) (count-inc! (Word w) 1))
words)
;; Count pairs within sliding window
(do ((i 0 (+ i 1)))
((>= i n))
(do ((j (+ i 1) (+ j 1)))
((or (>= j n) (> (- j i) *window-size*)))
(let ((pair (Edge (Bond "ANY")
(List (Word (list-ref words i))
(Word (list-ref words j))))))
(count-inc! pair 1)
(set! new-pair-count (+ new-pair-count 1)))))
;; Flush to RocksDB
(barrier *storage*)
;; Update stats
(define pairs-after (cog-count-atoms 'Edge))
(define new-pairs (- pairs-after pairs-before))
(set! *total-sentences* (+ 1 *total-sentences*))
(log-activity 'learn (format #f "~a words, ~a new pairs" n new-pairs))
(format #t "Learned: ~a words, ~a new pairs\n" n new-pairs))
(define* (parse-text sentence #:key (dict "dict-pair") (num-parses 5))
"Parse a sentence using MST parsing with dict-pair dictionary.
Returns Sections (word + connector-sets) from the parse.
Usage:
(parse-text \"the cat sat on the mat\")
(parse-text \"hello world\" #:num-parses 3)"
(display "Parsing: \"") (display sentence) (display "\"\n")
;; Execute LgParseLink with atomese dictionary
(define result
(cog-execute!
(LgParseLink
(Phrase sentence)
(LgDict dict)
(Number num-parses)
(cog-atomspace)
*storage*)))
;; Count results
(define result-list (cog-value->list result))
(define num-results (length result-list))
(format #t " Got ~a parses\n" num-results)
;; Show linkages from first parse
(when (> num-results 0)
(let* ((first-parse (car result-list))
(parse-parts (cog-value->list first-parse)))
(when (>= (length parse-parts) 2)
(display " Linkage:\n")
(let ((linkage (list-ref parse-parts 1)))
(for-each
(lambda (edge)
(format #t " ~a\n" edge))
(cog-value->list linkage))))))
;; Count new Sections
(define sections (cog-get-atoms 'Section))
(format #t " Total Sections: ~a\n" (length sections))
;; Update dashboard stats
(set! *total-parses* (+ 1 *total-parses*))
(log-activity 'parse (format #f "~a parses, ~a sections"
num-results (length sections)))
;; Return the parse result
result)
(define (parse-corpus-line line)
"Parse a single line and extract Sections (for batch processing)."
(when (> (string-length (string-trim-both line)) 0)
(catch #t
(lambda ()
(parse-text line #:num-parses 1)
#t)
(lambda (key . args)
(format #t " Parse error: ~a\n" key)
#f))))
(define (recompute-mi)
"Recompute MI on all pairs (batch operation).
Also records N_at_batch for drift tracking."
(display "Recomputing MI...\n")
(batch-all-pair-mi *pair-stars*)
;; Record N at this batch computation
(update-n-current!)
(set! *n-at-batch* *n-current*)
(set! *learning-epoch* (+ 1 *learning-epoch*))
(format #t "MI computation complete. N_at_batch = ~a, epoch = ~a\n"
*n-at-batch* *learning-epoch*)
(print-matrix-summary-report *pair-stars*))
(define *auto-recompute-threshold* 0.05) ; 5% N drift triggers recompute
(define* (maybe-recompute-mi! #:key (use-gpu #t))
"Recompute MI if N drift exceeds threshold.
Uses GPU if available and use-gpu is #t.
Returns #t if recomputed, #f otherwise."
(let ((drift (n-drift-ratio)))
(if (> (abs drift) *auto-recompute-threshold*)
(begin
(format #t "N drift ~,2f% exceeds ~,1f% threshold, recomputing MI...\n"
(* 100 drift) (* 100 *auto-recompute-threshold*))
(if (and use-gpu *gpu-enabled*)
(recompute-mi-gpu)
(recompute-mi))
#t)
#f)))
(define (mi-status)
"Print MI and N drift status."
(update-n-current!)
(format #t "N_current: ~a\n" *n-current*)
(format #t "N_at_batch: ~a\n" *n-at-batch*)
(format #t "N drift: ~,2f% (~,4f bits MI error)\n"
(* 100 (n-drift-ratio)) (n-drift-bits))
(format #t "Epoch: ~a\n" *learning-epoch*)
(format #t "Auto-recompute threshold: ~,1f%\n"
(* 100 *auto-recompute-threshold*)))
; ═══════════════════════════════════════════════════════════════════
; MEMORY MANAGEMENT
; ═══════════════════════════════════════════════════════════════════
(define (evict-cold-data min-count)
"Remove low-count pairs from RAM (kept in RocksDB)."
(let ((evicted 0))
(for-each
(lambda (pair)
(let ((cnt (cog-value-ref
(cog-value pair (Predicate "*-TruthValueKey-*")) 2)))
(when (< cnt min-count)
(cog-extract! pair)
(set! evicted (+ evicted 1)))))
(*pair-stars* 'get-all-elts))
(format #t "Evicted ~a pairs with count < ~a~%" evicted min-count)
(memory-status)))
(define (fetch-word-pairs word)
"Load all pairs containing WORD from RocksDB into RAM."
(fetch-incoming-set (Word word) *storage*)
(format #t "Loaded pairs for word: ~a~%" word))
; ═══════════════════════════════════════════════════════════════════
; DASHBOARD API - Stats and monitoring for web dashboard
; ═══════════════════════════════════════════════════════════════════
(define (log-activity type data)
"Add an activity entry to the log."
(set! *activity-log*
(take (cons (list (current-time) type data) *activity-log*)
(min (+ 1 (length *activity-log*)) *max-activity-log*))))
(define (learning-stats)
"Return current learning statistics as an alist (for dashboard)."
(update-n-current!) ; Sync N with actual count
(let* ((edges *n-current*)
(words (cog-count-atoms 'WordNode))
(sections (cog-count-atoms 'Section))
(total-atoms (+ edges words sections))
(drift-ratio (n-drift-ratio))
(drift-bits (n-drift-bits)))
(list
(cons 'timestamp (current-time))
(cons 'uptime (- (current-time) *server-start-time*))
(cons 'pairs edges)
(cons 'words words)
(cons 'sections sections)
(cons 'epoch *learning-epoch*)
(cons 'sentences *total-sentences*)
(cons 'parses *total-parses*)
(cons 'n-current *n-current*)
(cons 'n-at-batch *n-at-batch*)
(cons 'n-drift-pct (* 100.0 drift-ratio)) ; % change in N
(cons 'n-drift-bits drift-bits) ; MI error in bits
(cons 'memory-mb (exact->inexact (/ (* total-atoms 500) (* 1024 1024))))
(cons 'atoms total-atoms))))
(define (learning-stats-json)
"Return learning stats as JSON string."
(define stats (learning-stats))
(string-append "{"
(string-join
(map (lambda (pair)
(format #f "\"~a\":~a" (car pair)
(if (string? (cdr pair))
(format #f "\"~a\"" (cdr pair))
(cdr pair))))
stats)
",")
"}"))
(define (recent-activity count)
"Return last COUNT activity entries."
(take *activity-log* (min count (length *activity-log*))))
(define (recent-activity-json count)
"Return recent activity as JSON string."
(define activities (recent-activity count))
(string-append "["
(string-join
(map (lambda (entry)
(format #f "{\"time\":~a,\"type\":\"~a\",\"data\":\"~a\"}"
(car entry) (cadr entry) (caddr entry)))
activities)
",")
"]"))
(define (increment-epoch!)
"Increment the learning epoch (called after batch learning)."
(set! *learning-epoch* (+ 1 *learning-epoch*))
(set! *n-at-batch* (cog-count-atoms 'Edge)))
; ═══════════════════════════════════════════════════════════════════
; SHUTDOWN
; ═══════════════════════════════════════════════════════════════════
(define (shutdown)
"Graceful shutdown - ensure all data is saved."
(display "Shutting down...\n")
(when *storage*
(barrier *storage*) ; Ensure all writes complete
(cog-close *storage*)
(set! *storage* #f))
(display "Shutdown complete.\n"))
; ═══════════════════════════════════════════════════════════════════
; AUTO-START (when loaded)
; ═══════════════════════════════════════════════════════════════════
;; Uncomment to auto-start on load:
;; (startup #:mode 'warm #:min-count 5)
(display "
CogServer startup script loaded.
To start:
(startup #:mode 'cold) ; Load everything (~90 sec)
(startup #:mode 'warm) ; Load hot pairs only
(startup #:mode 'hot) ; Assume already loaded
Continuous Learning:
(parse-text \"the cat sat\") ; MST parse, extract Sections
(learn-text \"your text here\") ; Count pairs from text
MI Management (Factored: MI = PMI_raw + log₂[N]):
(mi-status) ; Show N drift & MI error
(maybe-recompute-mi!) ; Auto-recompute if drift > 5%
(recompute-mi) ; Force batch MI recompute (CPU)
Key insight: N drift doesn't affect MST parsing (constant offset
preserves ranking). Only recompute when drift > 5%.
GPU-Accelerated MI (385K pairs/sec, ~825x faster):
(init-gpu!) ; Initialize GPU
(gpu-status) ; Check GPU status
(recompute-mi-gpu) ; Force batch MI recompute (GPU)
(close-gpu!) ; Release GPU resources
Dashboard API:
(learning-stats) ; Stats as alist
(learning-stats-json) ; Stats as JSON
Maintenance:
(memory-status) ; Check RAM usage
(evict-cold-data 10) ; Free RAM
(shutdown) ; Graceful exit
")