;
; multimodal-disambiguation.scm
;
; Demonstrate how multi-modal data and layered structure
; can solve disambiguation problems like:
;
; "I deposited money at the bank by the river"
;
; This shows the path from raw statistics to grounded meaning.
;
(use-modules (opencog) (opencog exec))
(use-modules (srfi srfi-1))
(display "═══════════════════════════════════════════════════════════════════\n")
(display " MULTI-MODAL DISAMBIGUATION: Bank by the River Problem \n")
(display "═══════════════════════════════════════════════════════════════════\n\n")
;; ═══════════════════════════════════════════════════════════════════
;; PART 1: THE KNOWLEDGE BASE (what the system has learned)
;; ═══════════════════════════════════════════════════════════════════
(display "LAYER 1: Lexical Statistics (from parsing millions of sentences)\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
; Word co-occurrence counts (simplified - real system has millions)
(define cooccur (make-hash-table))
; BANK-FINANCIAL associations
(hash-set! cooccur '("bank" "money") 50000)
(hash-set! cooccur '("bank" "deposit") 35000)
(hash-set! cooccur '("bank" "account") 42000)
(hash-set! cooccur '("bank" "loan") 28000)
(hash-set! cooccur '("bank" "withdraw") 22000)
(hash-set! cooccur '("bank" "teller") 15000)
(hash-set! cooccur '("bank" "ATM") 18000)
; BANK-GEOGRAPHIC associations
(hash-set! cooccur '("bank" "river") 12000)
(hash-set! cooccur '("bank" "shore") 8000)
(hash-set! cooccur '("bank" "water") 9000)
(hash-set! cooccur '("bank" "flood") 5000)
(hash-set! cooccur '("bank" "erosion") 3000)
; Cross-domain (rare - buildings can be near rivers)
(hash-set! cooccur '("building" "river") 4000)
(hash-set! cooccur '("located" "river") 6000)
(display " bank ── money: 50,000 co-occurrences\n")
(display " bank ── deposit: 35,000\n")
(display " bank ── river: 12,000\n")
(display " bank ── shore: 8,000\n\n")
;; ═══════════════════════════════════════════════════════════════════
(display "LAYER 2: Semantic Frames (clusters of co-occurring concepts)\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
; Frames are clusters that tend to appear together
(define frames (make-hash-table))
(hash-set! frames 'FINANCIAL-TRANSACTION
'(bank money deposit withdraw account transfer loan interest
teller ATM check savings balance credit))
(hash-set! frames 'GEOGRAPHIC-WATER
'(river bank shore water stream lake pond flood erosion
bridge boat swim fish current flow))
(hash-set! frames 'PHYSICAL-LOCATION
'(at by near next beside located building street address
river park downtown corner))
(display " FINANCIAL-TRANSACTION frame:\n")
(display " {bank, money, deposit, withdraw, account, loan, ...}\n\n")
(display " GEOGRAPHIC-WATER frame:\n")
(display " {river, bank, shore, water, flood, erosion, ...}\n\n")
(display " PHYSICAL-LOCATION frame:\n")
(display " {at, by, near, located, building, river, park, ...}\n\n")
;; ═══════════════════════════════════════════════════════════════════
(display "LAYER 3: Multi-Modal Grounding (linking words to perception)\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
; Visual features (from image recognition training)
(define visual-features (make-hash-table))
(hash-set! visual-features "bank-building"
'((shape . rectangular)
(material . brick/glass)
(has-sign . #t)
(has-ATM . #t)
(has-door . #t)
(typical-colors . (gray white blue))
(typical-text . ("Bank" "ATM" "24hr"))))
(hash-set! visual-features "river-bank"
'((shape . irregular)
(material . soil/sand/rock)
(has-vegetation . #t)
(adjacent-to . water)
(typical-colors . (brown green blue))
(texture . natural)))
(hash-set! visual-features "money"
'((shape . rectangular)
(material . paper/metal)
(has-numbers . #t)
(has-portraits . #t)
(typical-colors . (green gray gold))))
(hash-set! visual-features "river"
'((shape . linear-flowing)
(material . water)
(has-movement . #t)
(reflective . #t)
(typical-colors . (blue green brown))))
(display " 'bank' (building) visual signature:\n")
(display " shape: rectangular, material: brick/glass, has-ATM: yes\n\n")
(display " 'bank' (geographic) visual signature:\n")
(display " shape: irregular, material: soil/sand, adjacent-to: water\n\n")
(display " When you SEE a rectangular building with 'Bank' sign,\n")
(display " the visual features activate BANK-FINANCIAL sense.\n\n")
;; ═══════════════════════════════════════════════════════════════════
;; PART 2: PARSING THE SENTENCE
;; ═══════════════════════════════════════════════════════════════════
(display "═══════════════════════════════════════════════════════════════════\n")
(display "NOW: Parsing 'I deposited money at the bank by the river'\n")
(display "═══════════════════════════════════════════════════════════════════\n\n")
(define sentence '("I" "deposited" "money" "at" "the" "bank" "by" "the" "river"))
(display "STEP 1: Syntactic Parse (Link Grammar bonds)\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
(display " I ──Ss──→ deposited (subject → verb)\n")
(display " deposited ──Os──→ money (verb → object)\n")
(display " deposited ──MVp──→ at (verb → prep phrase)\n")
(display " at ──Js──→ bank (prep → object)\n")
(display " the ──Ds──→ bank (det → noun)\n")
(display " bank ──Mp──→ by (noun → prep phrase)\n")
(display " by ──Js──→ river (prep → object)\n")
(display " the ──Ds──→ river (det → noun)\n\n")
(display " Structure extracted:\n")
(display " ┌─────────────────────────────────────────────────┐\n")
(display " │ [I] ─SUBJ→ [deposited] ─OBJ→ [money] │\n")
(display " │ │ │\n")
(display " │ └─LOCATION→ [at [the bank]] │\n")
(display " │ │ │\n")
(display " │ └─LOC→ [by [the river]]\n")
(display " └─────────────────────────────────────────────────┘\n\n")
;; ═══════════════════════════════════════════════════════════════════
(display "STEP 2: Frame Activation (which contexts light up?)\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
; Function to compute frame activation
(define (frame-activation sentence frame-words)
(let ((matches (filter (lambda (w) (member w frame-words)) sentence)))
(length matches)))
(define fin-frame (hash-ref frames 'FINANCIAL-TRANSACTION))
(define geo-frame (hash-ref frames 'GEOGRAPHIC-WATER))
(define loc-frame (hash-ref frames 'PHYSICAL-LOCATION))
(define fin-score (frame-activation sentence fin-frame))
(define geo-score (frame-activation sentence geo-frame))
(define loc-score (frame-activation sentence loc-frame))
(format #t " FINANCIAL-TRANSACTION activation: ~a words match\n" fin-score)
(format #t " Matches: ~a\n"
(filter (lambda (w) (member w fin-frame)) sentence))
(format #t "\n GEOGRAPHIC-WATER activation: ~a words match\n" geo-score)
(format #t " Matches: ~a\n"
(filter (lambda (w) (member w geo-frame)) sentence))
(format #t "\n PHYSICAL-LOCATION activation: ~a words match\n" loc-score)
(format #t " Matches: ~a\n\n"
(filter (lambda (w) (member w loc-frame)) sentence))
;; ═══════════════════════════════════════════════════════════════════
(display "STEP 3: Disambiguation via Frame Competition\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
(display " The word 'bank' is AMBIGUOUS - appears in two frames.\n\n")
(display " But look at the STRUCTURAL CONTEXT:\n\n")
(display " 'deposited' ──OBJ──→ 'money'\n")
(display " 'deposited' ──LOCATION──→ 'at the bank'\n\n")
(display " The verb 'deposited' STRONGLY activates FINANCIAL-TRANSACTION:\n")
(display " - 'deposit' is a core member of that frame\n")
(display " - 'money' is a core member of that frame\n")
(display " - Together they have PMI >> with FINANCIAL-bank\n\n")
(display " Meanwhile, 'river' activates GEOGRAPHIC-WATER,\n")
(display " but it's attached via 'by' (PHYSICAL-LOCATION frame).\n\n")
(display " KEY INSIGHT: The prepositional structure matters!\n\n")
(display " 'at the bank' → GOAL/DESTINATION of action\n")
(display " 'by the river' → PHYSICAL LOCATION modifier\n\n")
(display " These are DIFFERENT STRUCTURAL ROLES:\n")
(display " - 'at the bank' answers: WHERE did you deposit?\n")
(display " - 'by the river' answers: WHERE is that bank located?\n\n")
;; ═══════════════════════════════════════════════════════════════════
(display "STEP 4: Resolution\n")
(display "─────────────────────────────────────────────────────────────────\n\n")
(display " FINANCIAL wins for 'bank' because:\n\n")
(display " 1. VERB SELECTIONAL RESTRICTION:\n")
(display " 'deposit' takes FINANCIAL-things as locations\n")
(display " You deposit AT financial institutions, not AT dirt\n\n")
(display " 2. OBJECT-VERB COHERENCE:\n")
(display " deposit(money) → FINANCIAL frame (PMI very high)\n")
(display " This frame propagates to the location argument\n\n")
(display " 3. PREPOSITIONAL SEMANTICS:\n")
(display " 'at X' for transactions → X is the institution\n")
(display " 'by Y' for location → Y is nearby landmark\n\n")
(display " 4. NO CONFLICT:\n")
(display " A financial bank CAN BE by a river!\n")
(display " 'river' doesn't compete - it just locates.\n\n")
(display " ┌─────────────────────────────────────────────────────────────┐\n")
(display " │ │\n")
(display " │ RESOLVED INTERPRETATION: │\n")
(display " │ │\n")
(display " │ 'I deposited money at the bank[FINANCIAL] by the river' │\n")
(display " │ │\n")
(display " │ Meaning: I performed a financial deposit │\n")
(display " │ at a banking institution │\n")
(display " │ which is located near a river │\n")
(display " │ │\n")
(display " └─────────────────────────────────────────────────────────────┘\n\n")
;; ═══════════════════════════════════════════════════════════════════
;; PART 3: THE MULTI-MODAL ENHANCEMENT
;; ═══════════════════════════════════════════════════════════════════
(display "═══════════════════════════════════════════════════════════════════\n")
(display "BONUS: How Multi-Modal Data Makes This Stronger\n")
(display "═══════════════════════════════════════════════════════════════════\n\n")
(display "Imagine the system also has VISUAL experience:\n\n")
(display " LEARNED ASSOCIATIONS (from captioned images):\n\n")
(display " [Image: person at ATM] + caption: 'depositing money'\n")
(display " [Image: bank building] + caption: 'the local bank'\n")
(display " [Image: riverside scene] + caption: 'the river bank'\n")
(display " [Image: person by water] + caption: 'standing by the river'\n\n")
(display " These create CROSS-MODAL LINKS:\n\n")
(display " 'deposit' + 'bank' → visual: [building, ATM, indoor]\n")
(display " 'river' + 'bank' → visual: [outdoor, water, soil]\n\n")
(display " When parsing 'deposited money at the bank':\n\n")
(display " The phrase 'deposited money' activates:\n")
(display " - linguistic: FINANCIAL-TRANSACTION frame\n")
(display " - visual: [indoor, ATM, counter, transaction]\n\n")
(display " This visual expectation is INCOMPATIBLE with:\n")
(display " - river-bank visual: [outdoor, water, soil]\n\n")
(display " So multi-modal grounding provides ADDITIONAL EVIDENCE\n")
(display " for the FINANCIAL interpretation.\n\n")
;; ═══════════════════════════════════════════════════════════════════
(display "═══════════════════════════════════════════════════════════════════\n")
(display "THE ALGORITHM (how to prove it computationally)\n")
(display "═══════════════════════════════════════════════════════════════════\n\n")
(display "To PROVE disambiguation works, we need:\n\n")
(display "1. FRAME INDUCTION (already done by Linas' clustering):\n")
(display " - Cluster words by co-occurrence patterns\n")
(display " - High-PMI clusters become 'frames'\n")
(display " - 'bank' appears in multiple clusters → polysemous\n\n")
(display "2. SELECTIONAL PREFERENCE LEARNING:\n")
(display " - Count: deposit(SUBJ, OBJ, at-LOC)\n")
(display " - Find: what fills the LOC slot when OBJ='money'?\n")
(display " - Result: LOC is usually FINANCIAL-institution\n")
(display " - This is just MORE COUNTING at the frame level\n\n")
(display "3. STRUCTURAL ROLE DISCRIMINATION:\n")
(display " - 'at X' vs 'by X' have different roles\n")
(display " - Count what fills each role\n")
(display " - 'at' + transaction-verb → GOAL\n")
(display " - 'by' + noun → LOCATION-modifier\n\n")
(display "4. FRAME COHERENCE SCORING:\n")
(display " - For each word-sense assignment, compute total PMI\n")
(display " - bank=FINANCIAL + deposit + money = high coherence\n")
(display " - bank=GEOGRAPHIC + deposit + money = low coherence\n")
(display " - Choose maximum coherence interpretation\n\n")
(display "5. MULTI-MODAL VERIFICATION (optional but powerful):\n")
(display " - Visual expectations from frame → predict scene\n")
(display " - If scene matches, boost confidence\n")
(display " - If scene conflicts, reduce confidence\n\n")
(display "═══════════════════════════════════════════════════════════════════\n")
(display " THE KEY INSIGHT \n")
(display "═══════════════════════════════════════════════════════════════════\n\n")
(display "The 'bank by the river' problem LOOKS hard because we assume:\n")
(display " 'bank' must be EITHER financial OR geographic\n\n")
(display "But the sentence actually contains NO CONFLICT:\n")
(display " - 'bank' (the destination) = financial institution\n")
(display " - 'river' (the location) = geographic feature\n")
(display " - A building CAN BE near a river\n\n")
(display "The disambiguation happens through STRUCTURAL ROLES:\n")
(display " - 'at the bank' = where the ACTION happens\n")
(display " - 'by the river' = where the PLACE is located\n\n")
(display "This is COMPOSITIONAL SEMANTICS:\n")
(display " deposit(agent=I, theme=money, goal=bank[FIN], location=by-river)\n\n")
(display "And it emerges from:\n")
(display " Level 1: Word statistics (PMI between words)\n")
(display " Level 2: Frame statistics (which words cluster)\n")
(display " Level 3: Role statistics (what fills what slots)\n")
(display " Level 4: Coherence scoring (which assignment fits best)\n\n")
(display "All of this is COUNTING at different levels of structure!\n")
(display "Your intuition was correct.\n\n")