;
; gap-detector.scm - Identify knowledge gaps
;
; Types of gaps:
; 1. Unknown term: word seen few times, high vagueness
; 2. Weak association: words co-occur but PMI is low/uncertain
; 3. Undefined relation: "X is Y" but what does that MEAN?
; 4. Missing part-whole: X has parts, but what are they?
; 5. Missing operation: X can be done, but HOW?
;
(use-modules (opencog))
(use-modules (ice-9 format))
(use-modules (srfi srfi-1))
;; ═══════════════════════════════════════════════════════════════════
;; GAP TYPES
;; ═══════════════════════════════════════════════════════════════════
(define *gap-types*
'((unknown-term . "Word seen but meaning unclear")
(weak-association . "Words co-occur but relationship unclear")
(undefined-relation . "Relation stated but not explained")
(missing-parts . "Thing mentioned but parts unknown")
(missing-operation . "Action mentioned but method unknown")))
(define (make-gap type subject context confidence)
"Create a gap record"
(list type subject context confidence))
(define (gap-type g) (car g))
(define (gap-subject g) (cadr g))
(define (gap-context g) (caddr g))
(define (gap-confidence g) (cadddr g))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DETECTION: UNKNOWN TERMS
;; ═══════════════════════════════════════════════════════════════════
(define *min-known-count* 3) ; Need at least this many occurrences
(define *min-known-assocs* 2) ; Need at least this many associations
(define (detect-unknown-terms)
"Find words that appear but aren't well understood"
(let ((gaps '()))
(for-each
(lambda (word-entry)
(let* ((word (car word-entry))
(count (cdr word-entry))
(assocs (layer1-associations word 5))
(vagueness (word-vagueness word)))
;; Gap if: low count OR few associations OR high vagueness
(when (or (< count *min-known-count*)
(< (length assocs) *min-known-assocs*)
(> vagueness 0.5))
(set! gaps
(cons (make-gap 'unknown-term
word
(format #f "seen ~a times, ~a assocs, vagueness ~,2f"
count (length assocs) vagueness)
vagueness)
gaps)))))
(get-all-word-counts))
;; Sort by vagueness (highest first = most unknown)
(sort gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DETECTION: WEAK ASSOCIATIONS
;; ═══════════════════════════════════════════════════════════════════
(define *weak-pmi-threshold* 1.5) ; PMI below this is "weak"
(define (detect-weak-associations)
"Find word pairs that co-occur but relationship is unclear"
(let ((gaps '()))
(for-each
(lambda (word-entry)
(let* ((word (car word-entry))
(assocs (layer1-associations word 10)))
(for-each
(lambda (assoc)
(let ((other (cdr assoc))
(pmi (car assoc)))
;; Weak if PMI is low but words DO co-occur
(when (and (> pmi 0) (< pmi *weak-pmi-threshold*))
(set! gaps
(cons (make-gap 'weak-association
(cons word other)
(format #f "co-occur but PMI only ~,2f" pmi)
(/ 1 (+ 1 pmi))) ; Lower PMI = higher gap
gaps)))))
assocs)))
(get-all-word-counts))
(sort gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DETECTION: UNDEFINED RELATIONS
;; ═══════════════════════════════════════════════════════════════════
;; Relation indicators - words that suggest a definition
(define *relation-words*
'("is" "are" "means" "refers" "defines" "represents" "consists" "contains"))
(define (detect-undefined-relations)
"Find 'X is Y' patterns where the relation isn't grounded"
(let ((gaps '()))
(for-each
(lambda (rel-word)
(let ((assocs (layer1-associations rel-word 20)))
(for-each
(lambda (assoc)
(let* ((word (cdr assoc))
(pmi (car assoc))
;; Check if the related word is itself well-defined
(word-assocs (layer1-associations word 5))
(word-vagueness (word-vagueness word)))
;; Gap if: strong association with "is/are/means" but the term is vague
(when (and (> pmi 1.0) (> word-vagueness 0.3))
(set! gaps
(cons (make-gap 'undefined-relation
word
(format #f "related to '~a' but poorly defined" rel-word)
word-vagueness)
gaps)))))
assocs)))
*relation-words*)
(delete-duplicates
(sort gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))
(lambda (a b) (equal? (gap-subject a) (gap-subject b))))))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DETECTION: MISSING PARTS
;; ═══════════════════════════════════════════════════════════════════
;; Part-whole indicators
(define *part-words*
'("has" "have" "contains" "includes" "consists" "made" "composed" "parts"))
(define (detect-missing-parts)
"Find things mentioned with 'has/contains' but parts unknown"
(let ((gaps '()))
(for-each
(lambda (part-word)
(let ((assocs (layer1-associations part-word 20)))
(for-each
(lambda (assoc)
(let* ((word (cdr assoc))
(pmi (car assoc))
;; Look for what this thing "has"
(word-assocs (layer1-associations word 10))
;; Count how many concrete parts we know
(concrete-parts (filter
(lambda (a)
(and (> (car a) 2.0) ; Strong association
(not (member (cdr a) *part-words*))))
word-assocs)))
;; Gap if: associated with "has" but few concrete parts known
(when (and (> pmi 1.0) (< (length concrete-parts) 2))
(set! gaps
(cons (make-gap 'missing-parts
word
(format #f "has parts but only ~a known"
(length concrete-parts))
(/ 1 (+ 1 (length concrete-parts))))
gaps)))))
assocs)))
*part-words*)
(delete-duplicates
(sort gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))
(lambda (a b) (equal? (gap-subject a) (gap-subject b))))))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DETECTION: MISSING OPERATIONS
;; ═══════════════════════════════════════════════════════════════════
;; Operation indicators
(define *operation-words*
'("send" "create" "make" "build" "sign" "verify" "encode" "decode"
"encrypt" "decrypt" "connect" "open" "close" "read" "write"))
(define (detect-missing-operations)
"Find operations mentioned but method unknown"
(let ((gaps '()))
(for-each
(lambda (op-word)
(let* ((count (get-word-count-by-name op-word))
(assocs (layer1-associations op-word 10))
(vagueness (word-vagueness op-word)))
;; Gap if: operation word seen but few associations (don't know HOW)
(when (and (> count 0)
(or (< (length assocs) 3)
(> vagueness 0.4)))
(set! gaps
(cons (make-gap 'missing-operation
op-word
(format #f "operation mentioned but ~a associations, vagueness ~,2f"
(length assocs) vagueness)
vagueness)
gaps)))))
*operation-words*)
(sort gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))))
;; ═══════════════════════════════════════════════════════════════════
;; AGGREGATE GAP DETECTION
;; ═══════════════════════════════════════════════════════════════════
(define (detect-all-gaps)
"Detect all types of gaps, return prioritized list"
(let ((all-gaps '()))
;; Collect all gap types
(set! all-gaps (append all-gaps (detect-unknown-terms)))
(set! all-gaps (append all-gaps (detect-weak-associations)))
(set! all-gaps (append all-gaps (detect-undefined-relations)))
(set! all-gaps (append all-gaps (detect-missing-parts)))
(set! all-gaps (append all-gaps (detect-missing-operations)))
;; Sort by confidence (highest = most important gap)
(sort all-gaps (lambda (a b) (> (gap-confidence a) (gap-confidence b))))))
(define (top-gaps n)
"Get top N most important gaps"
(take (detect-all-gaps) (min n (length (detect-all-gaps)))))
;; ═══════════════════════════════════════════════════════════════════
;; GAP DISPLAY
;; ═══════════════════════════════════════════════════════════════════
(define (format-gap gap)
"Format a gap for display"
(let ((type (gap-type gap))
(subject (gap-subject gap))
(context (gap-context gap))
(conf (gap-confidence gap)))
(format #f "[~a] ~a\n ~a (priority: ~,2f)"
type
(if (pair? subject)
(format #f "~a <-> ~a" (car subject) (cdr subject))
subject)
context
conf)))
(define (show-gaps n)
"Display top N gaps"
(let ((gaps (top-gaps n)))
(if (null? gaps)
"No knowledge gaps detected"
(string-join
(cons (format #f "Top ~a knowledge gaps:" (min n (length gaps)))
(map format-gap gaps))
"\n\n"))))
(define (show-gaps-by-type type n)
"Show gaps of a specific type"
(let* ((all-gaps (detect-all-gaps))
(typed-gaps (filter (lambda (g) (eq? (gap-type g) type)) all-gaps))
(top-typed (take typed-gaps (min n (length typed-gaps)))))
(if (null? top-typed)
(format #f "No ~a gaps detected" type)
(string-join
(cons (format #f "~a gaps:" type)
(map format-gap top-typed))
"\n\n"))))
;; ═══════════════════════════════════════════════════════════════════
;; QUESTION GENERATION (Phase 2.2 preview)
;; ═══════════════════════════════════════════════════════════════════
(define (gap-to-question gap)
"Convert a gap into a natural language question"
(let ((type (gap-type gap))
(subject (gap-subject gap)))
(case type
((unknown-term)
(format #f "What is ~a?" subject))
((weak-association)
(format #f "How does ~a relate to ~a?" (car subject) (cdr subject)))
((undefined-relation)
(format #f "What does ~a mean?" subject))
((missing-parts)
(format #f "What are the parts/components of ~a?" subject))
((missing-operation)
(format #f "How do you ~a something?" subject))
(else
(format #f "What is ~a?" subject)))))
(define (generate-questions n)
"Generate top N questions from gaps"
(let ((gaps (top-gaps n)))
(map gap-to-question gaps)))
(define (show-questions n)
"Display top N questions"
(let ((questions (generate-questions n)))
(if (null? questions)
"No questions to ask"
(string-join
(cons "Questions to fill knowledge gaps:"
(map (lambda (i q)
(format #f "~a. ~a" (+ 1 i) q))
(iota (length questions))
questions))
"\n"))))