knowledge-ratchet / atomspace / ingest-ratchet.js

/**
 * ingest-ratchet.js — Load ratchet correction data into AtomSpace
 *
 * Reads loops.json, learned-rules.json, project-maps.json and creates:
 *   - Loop nodes (loop:FINGERPRINT) with type, project, duration values
 *   - Rule nodes (rule:ID) with symptom, fix values
 *   - EdgeLinks: loop-in-file, resolved-by, rule-for-file
 *   - Danger level values on file nodes
 *
 * Usage:
 *   node ingest-ratchet.js
 */

const path = require('path')
const fs = require('fs')
const { evalScheme, isCogServerUp } = require('./cogserver')
const S = require('./schema')

const DATA_DIR = path.resolve(__dirname, '../data')

function readJSON (filename) {
  const filepath = path.join(DATA_DIR, filename)
  if (!fs.existsSync(filepath)) return null
  return JSON.parse(fs.readFileSync(filepath, 'utf8'))
}

/**
 * Convert loops.json to Scheme expressions.
 */
function loopsToScheme (loopsData) {
  const exprs = []
  if (!loopsData || !loopsData.loops) return exprs

  for (const loop of loopsData.loops) {
    const fp = loop.fingerprint || loop.id || ''
    if (!fp) continue

    const node = S.loopNode(fp)
    exprs.push(node)

    if (loop.type) exprs.push(S.setStringValue(node, 'type', loop.type))
    if (loop.project) exprs.push(S.setStringValue(node, 'project', loop.project))
    if (loop.duration) exprs.push(S.setFloatValue(node, 'duration', loop.duration))
    if (loop.date) exprs.push(S.setStringValue(node, 'date', loop.date))
    exprs.push(S.setFloatValue(node, 'resolved', loop.resolved ? 1 : 0))

    // Correction count
    if (loop.corrections) {
      exprs.push(S.setFloatValue(node, 'correction-count', loop.corrections.length))
    }

    // Files tried — create loop-in-file edges
    if (loop.filesTried) {
      for (const file of loop.filesTried) {
        const fNode = S.fileNode(normalizeRatchetPath(file, loop.project))
        exprs.push(fNode)
        exprs.push(S.edge('loop-in-file', node, fNode))
      }
    }

    // Resolution files — create resolved-by edges
    if (loop.resolutionFiles) {
      for (const file of loop.resolutionFiles) {
        const fNode = S.fileNode(normalizeRatchetPath(file, loop.project))
        exprs.push(fNode)
        exprs.push(S.edge('resolved-by', node, fNode))
      }
    }
  }

  return exprs
}

/**
 * Convert learned-rules.json to Scheme expressions.
 */
function rulesToScheme (rulesData) {
  const exprs = []
  if (!rulesData) return exprs

  const rules = Array.isArray(rulesData) ? rulesData : (rulesData.rules || [])

  for (const rule of rules) {
    const id = rule.id || ''
    if (!id) continue

    const node = S.ruleNode(id)
    exprs.push(node)

    if (rule.quality) exprs.push(S.setStringValue(node, 'quality', rule.quality))

    // Symptom text
    if (rule.trigger && rule.trigger.symptom) {
      exprs.push(S.setStringValue(node, 'symptom', rule.trigger.symptom.slice(0, 200)))
    }

    // Fix description
    if (rule.action && rule.action.description) {
      exprs.push(S.setStringValue(node, 'fix', rule.action.description.slice(0, 200)))
    }

    // Antipattern
    if (rule.antipattern && rule.antipattern.description) {
      exprs.push(S.setStringValue(node, 'antipattern', rule.antipattern.description.slice(0, 200)))
    }

    // Files the rule applies to
    const ruleFiles = []
    if (rule.trigger && rule.trigger.files) ruleFiles.push(...rule.trigger.files)
    if (rule.action && rule.action.fixFiles) ruleFiles.push(...rule.action.fixFiles)

    for (const file of [...new Set(ruleFiles)]) {
      const project = rule.trigger ? rule.trigger.project : ''
      const fNode = S.fileNode(normalizeRatchetPath(file, project))
      exprs.push(fNode)
      exprs.push(S.edge('rule-for-file', node, fNode))
    }
  }

  return exprs
}

/**
 * Convert project-maps.json to danger-level values on file nodes.
 */
function projectMapsToScheme (mapsData) {
  const exprs = []
  if (!mapsData || !mapsData.projects) return exprs

  for (const [projectName, project] of Object.entries(mapsData.projects)) {
    if (!project.subsystems) continue

    for (const [subsysName, subsys] of Object.entries(project.subsystems)) {
      if (!subsys.dangerLevel) continue

      // Create a node for the subsystem
      const subsysPath = projectName + '/' + subsysName
      const fNode = S.fileNode(subsysPath)
      exprs.push(fNode)
      exprs.push(S.setStringValue(fNode, 'danger-level', subsys.dangerLevel))
      exprs.push(S.setFloatValue(fNode, 'loop-count', subsys.loopCount || 0))
      exprs.push(S.setFloatValue(fNode, 'correction-count', subsys.correctionCount || 0))

      // Individual files in the subsystem
      if (subsys.files) {
        for (const file of subsys.files) {
          const fileFNode = S.fileNode(normalizeRatchetPath(file, projectName))
          exprs.push(fileFNode)
          exprs.push(S.setStringValue(fileFNode, 'danger-level', subsys.dangerLevel))
          // Subsystem contains file
          exprs.push(S.edge('contains', fNode, fileFNode))
        }
      }
    }
  }

  return exprs
}

/**
 * Normalize a ratchet file path to match codetree conventions.
 * Ratchet paths may be absolute or relative to project root.
 */
function normalizeRatchetPath (filePath, project) {
  if (!filePath) return ''
  // Strip absolute prefix if present
  let normalized = filePath
    .replace(/^\/storage\/projects\/code\/hypercore-kb\/hyper-fleeky\//, '')
    .replace(/^\/storage\/projects\/code\/hypercore-kb\//, '')
  // If project is known and path doesn't start with it, prepend
  if (project && !normalized.startsWith(project + '/')) {
    // Check if it's a bare filename
    if (!normalized.includes('/')) {
      normalized = project + '/' + normalized
    }
  }
  return normalized
}

/**
 * Ingest all ratchet data into AtomSpace.
 */
async function ingestRatchet () {
  const up = await isCogServerUp()
  if (!up) {
    console.log('[atomspace-ingest] CogServer not running, skipping ratchet ingest')
    return { ok: false, reason: 'cogserver-down' }
  }

  const allExprs = []

  // Loops
  const loopsData = readJSON('loops.json')
  if (loopsData) {
    const loopExprs = loopsToScheme(loopsData)
    allExprs.push(...loopExprs)
    console.log(`[atomspace-ingest] ${loopExprs.length} expressions from ${(loopsData.loops || []).length} loops`)
  }

  // Rules
  const rulesData = readJSON('learned-rules.json')
  if (rulesData) {
    const ruleExprs = rulesToScheme(rulesData)
    allExprs.push(...ruleExprs)
    const ruleCount = Array.isArray(rulesData) ? rulesData.length : (rulesData.rules || []).length
    console.log(`[atomspace-ingest] ${ruleExprs.length} expressions from ${ruleCount} rules`)
  }

  // Project maps
  const mapsData = readJSON('project-maps.json')
  if (mapsData) {
    const mapExprs = projectMapsToScheme(mapsData)
    allExprs.push(...mapExprs)
    console.log(`[atomspace-ingest] ${mapExprs.length} expressions from project maps`)
  }

  // Send in batches
  const BATCH_SIZE = 50
  let sent = 0
  for (let i = 0; i < allExprs.length; i += BATCH_SIZE) {
    const batch = allExprs.slice(i, i + BATCH_SIZE)
    const combined = '(begin ' + batch.join(' ') + ')'
    const result = await evalScheme(combined)
    if (result === null) {
      console.error(`[atomspace-ingest] Connection lost at batch ${i}`)
      return { ok: false, reason: 'connection-lost', sent }
    }
    sent += batch.length
  }

  console.log(`[atomspace-ingest] Ratchet data ingested: ${sent} atoms/edges`)
  return { ok: true, sent }
}

// CLI entry point
if (require.main === module) {
  ingestRatchet().then(result => {
    console.log(JSON.stringify(result, null, 2))
    process.exit(result.ok ? 0 : 1)
  })
}

module.exports = { ingestRatchet, loopsToScheme, rulesToScheme, projectMapsToScheme }