knowledge-ratchet / atomspace / ingest.js

/**
 * ingest.js — Combined ingest entry point
 *
 * Loads codetree structure + ratchet correction data into AtomSpace.
 * Gracefully skips if CogServer is not running.
 *
 * Usage:
 *   node atomspace/ingest.js [codetree-scan-dir]
 */

const { isCogServerUp, evalScheme } = require('./cogserver')
const { ingestCodetree } = require('./ingest-codetree')
const { ingestRatchet } = require('./ingest-ratchet')
const { execSync } = require('child_process')
const path = require('path')

async function ingestAll (scanDir) {
  const up = await isCogServerUp()
  if (!up) {
    console.log('[atomspace-ingest] CogServer not running — skipping all ingest')
    return { ok: false, reason: 'cogserver-down' }
  }

  const start = Date.now()
  const results = {}

  // Ingest codetree first (creates file: and mod: nodes)
  results.codetree = await ingestCodetree(scanDir)

  // Then ratchet data (creates loop: and rule: nodes, links to file: nodes)
  results.ratchet = await ingestRatchet()

  // Classify all files with language/domain/role
  try {
    const classifyScript = path.join(__dirname, 'classify-files.js')
    execSync(`node ${classifyScript}`, { stdio: 'inherit', timeout: 60000 })
    results.classify = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Classify failed:', err.message)
    results.classify = { ok: false, error: err.message }
  }

  // Ingest session counts per module (unfamiliarity signal)
  try {
    results.sessions = await ingestSessionCounts()
  } catch (err) {
    results.sessions = { ok: false, error: err.message }
  }

  // Generate contrast-derived meta-cognition rules
  try {
    const contrastScript = path.join(__dirname, 'contrast-rules.js')
    execSync(`node ${contrastScript}`, { stdio: 'inherit', timeout: 60000 })
    results.contrasts = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Contrast rule generation failed:', err.message)
    results.contrasts = { ok: false, error: err.message }
  }

  // Compute wisdom scores (referent_health × utility × breadth)
  try {
    const wisdomScript = path.join(__dirname, 'wisdom-score.js')
    execSync(`node ${wisdomScript}`, { stdio: 'inherit', timeout: 180000 })
    results.wisdom = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Wisdom score generation failed:', err.message)
    results.wisdom = { ok: false, error: err.message }
  }

  // Categorize sessions (topics + continuation chains — long-term memory)
  try {
    const categorizeScript = path.join(__dirname, 'session-categorize.js')
    execSync(`node ${categorizeScript}`, { stdio: 'inherit', timeout: 60000 })
    results.categorize = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Session categorization failed:', err.message)
    results.categorize = { ok: false, error: err.message }
  }

  // Compute advanced metrics (time-to-correction, per-domain, saturation)
  try {
    const advancedScript = path.join(__dirname, 'advanced-metrics.js')
    execSync(`node ${advancedScript}`, { stdio: 'inherit', timeout: 60000 })
    results.advanced = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Advanced metrics failed:', err.message)
    results.advanced = { ok: false, error: err.message }
  }

  // Compute chain-level metrics (speed, complexity, productivity)
  try {
    const chainScript = path.join(__dirname, 'chain-analysis.js')
    execSync(`node ${chainScript}`, { stdio: 'inherit', timeout: 60000 })
    results.chains = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Chain analysis failed:', err.message)
    results.chains = { ok: false, error: err.message }
  }

  // Capture success patterns (approvals + context leading to them)
  try {
    const successScript = path.join(__dirname, 'success-memory.js')
    execSync(`node ${successScript}`, { stdio: 'inherit', timeout: 120000 })
    results.successes = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Success memory failed:', err.message)
    results.successes = { ok: false, error: err.message }
  }

  // Promote repeated successes to preferred-approach rules
  try {
    const promoteScript = path.join(__dirname, 'success-promote.js')
    execSync(`node ${promoteScript}`, { stdio: 'inherit', timeout: 60000 })
    results.promoted = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Success promotion failed:', err.message)
    results.promoted = { ok: false, error: err.message }
  }

  // Analyze rule-type meta-patterns (which types have highest utility)
  try {
    const metaScript = path.join(__dirname, 'meta-patterns.js')
    execSync(`node ${metaScript}`, { stdio: 'inherit', timeout: 30000 })
    results.metaPatterns = { ok: true }
  } catch (err) {
    console.error('[atomspace-ingest] Meta patterns failed:', err.message)
    results.metaPatterns = { ok: false, error: err.message }
  }

  const elapsed = Date.now() - start
  console.log(`[atomspace-ingest] Total ingest: ${elapsed}ms`)

  return {
    ok: results.codetree.ok || results.ratchet.ok,
    elapsed,
    ...results
  }
}

/**
 * Ingest session counts per module — unfamiliarity signal.
 * Reads fused-scores.json and counts distinct sessions per project cwd.
 */
async function ingestSessionCounts () {
  const fs = require('fs')
  const scoresFile = path.join(__dirname, '..', 'data', 'fused-scores.json')
  if (!fs.existsSync(scoresFile)) return { ok: false, reason: 'no-scores-file' }

  const data = JSON.parse(fs.readFileSync(scoresFile, 'utf8'))
  const sessions = data.sessions || []

  // Count sessions per module (from cwd path)
  const modSessions = {}
  for (const s of sessions) {
    if (!s.cwd) continue
    // Extract module name from cwd like /storage/projects/code/hypercore-kb/hyper-fleeky/MODULE/
    const match = s.cwd.match(/\/hyper-fleeky\/([^/]+)/)
    if (match) {
      const mod = match[1]
      modSessions[mod] = (modSessions[mod] || 0) + 1
    }
  }

  // Store as Values on mod: atoms
  const batch = []
  for (const [mod, count] of Object.entries(modSessions)) {
    const modNode = `(ConceptNode "mod:${mod.replace(/"/g, '\\"')}")`
    batch.push(`(cog-set-value! ${modNode} (Predicate "session-count") (FloatValue ${count}))`)
  }

  if (batch.length > 0) {
    await evalScheme('(begin ' + batch.join(' ') + ')')
  }

  return { ok: true, modules: Object.keys(modSessions).length }
}

if (require.main === module) {
  const scanDir = process.argv[2]
  ingestAll(scanDir).then(result => {
    console.log(JSON.stringify(result, null, 2))
    process.exit(result.ok ? 0 : 1)
  })
}

module.exports = { ingestAll }