/**
* ingest-codetree.js — Load codetree scan output into AtomSpace
*
* Runs codetree scan (or reads cached JSON), converts to Atomese atoms:
* - Module nodes (mod:NAME)
* - File nodes (file:MODULE/PATH)
* - Function/class nodes (fn:NAME)
* - EdgeLinks: contains, imports, depends-on, exports
*
* Usage:
* node ingest-codetree.js [scan-dir]
* Default scan-dir: ../../hyper-fleeky
*/
const { execSync } = require('child_process')
const path = require('path')
const fs = require('fs')
const { evalScheme, evalSchemeVoid, isCogServerUp } = require('./cogserver')
const S = require('./schema')
const CODETREE_BIN = path.resolve(__dirname, '../../hyper-fleeky/codetree/index.js')
const CACHE_FILE = path.resolve(__dirname, '../data/codetree-cache.json')
const HASH_FILE = path.resolve(__dirname, '../data/codetree-hash.txt')
/**
* Run codetree scan and return parsed JSON.
* Uses cache if available and unchanged.
*/
function scanCodetree (scanDir) {
const dir = scanDir || path.resolve(__dirname, '../../hyper-fleeky')
try {
const output = execSync(`node ${CODETREE_BIN} scan ${dir}`, {
encoding: 'utf8',
timeout: 60000,
maxBuffer: 10 * 1024 * 1024
})
const tree = JSON.parse(output)
// Cache for next time
fs.mkdirSync(path.dirname(CACHE_FILE), { recursive: true })
fs.writeFileSync(CACHE_FILE, output)
return tree
} catch (err) {
// Fall back to cache
if (fs.existsSync(CACHE_FILE)) {
console.log('codetree scan failed, using cache')
return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'))
}
console.error('codetree scan failed and no cache:', err.message)
return null
}
}
/**
* Convert codetree JSON to Scheme expressions for AtomSpace.
* Returns array of Scheme expression strings.
*/
function treeToScheme (tree) {
const exprs = []
const modules = tree.modules || {}
const system = tree.system || {}
// Module nodes + their files
for (const [modName, mod] of Object.entries(modules)) {
// Create module node
exprs.push(S.modNode(modName))
if (mod.version) {
exprs.push(S.setStringValue(S.modNode(modName), 'version', mod.version))
}
if (mod.type) {
exprs.push(S.setStringValue(S.modNode(modName), 'module-type', mod.type))
}
// Create file nodes and contains edges
const files = mod.files || {}
for (const [filePath, fileData] of Object.entries(files)) {
const fullPath = modName + '/' + filePath
const fNode = S.fileNode(fullPath)
exprs.push(fNode)
if (fileData.role) {
exprs.push(S.setStringValue(fNode, 'role', fileData.role))
}
// Module contains file
exprs.push(S.edge('contains', S.modNode(modName), fNode))
// File exports
if (fileData.exports) {
for (const exp of fileData.exports) {
if (exp.name && exp.name !== 'default') {
const fn = S.fnNode(modName + ':' + exp.name)
exprs.push(fn)
if (exp.type) {
exprs.push(S.setStringValue(fn, 'export-type', exp.type))
}
exprs.push(S.edge('exports', fNode, fn))
}
}
}
// File imports (local imports within the module or cross-module)
if (fileData.localImports) {
for (const imp of fileData.localImports) {
const spec = typeof imp === 'string' ? imp : (imp.specifier || imp.resolved || '')
if (!spec) continue
// Resolve relative to module
const resolved = typeof imp === 'object' && imp.resolved
? imp.resolved
: spec
// Only track if it looks like a file path
if (resolved.startsWith('.') || resolved.startsWith('/')) {
// Normalize the import target
const targetPath = normalizeImportPath(modName, filePath, resolved)
if (targetPath) {
exprs.push(S.edge('imports', fNode, S.fileNode(targetPath)))
}
}
}
}
// External imports (npm packages AND cross-module deps)
if (fileData.imports) {
for (const imp of fileData.imports) {
const spec = imp.specifier || ''
// Skip relative paths and node builtins
if (spec.startsWith('.') || spec.startsWith('/') || spec.startsWith('node:')) continue
// Package name (handle @scope/package)
const pkgName = spec.split('/')[0].startsWith('@')
? spec.split('/').slice(0, 2).join('/')
: spec.split('/')[0]
if (!pkgName) continue
// If we scanned this package as a module, link to mod:
if (modules[pkgName]) {
exprs.push(S.edge('imports', fNode, S.modNode(pkgName)))
} else {
// External npm package — create pkg: node so we capture real boundaries
const pkgNode = `(ConceptNode "pkg:${pkgName.replace(/"/g, '\\"')}")`
exprs.push(pkgNode)
exprs.push(S.edge('imports', fNode, pkgNode))
}
}
}
}
}
// Module-level dependencies from system.dependencies
if (system.dependencies) {
for (const [mod, deps] of Object.entries(system.dependencies)) {
for (const dep of deps) {
if (modules[dep]) {
exprs.push(S.edge('depends-on', S.modNode(mod), S.modNode(dep)))
}
}
}
}
return exprs
}
/**
* Normalize an import path relative to a file in a module.
*/
function normalizeImportPath (modName, fromFile, importPath) {
if (!importPath.startsWith('.')) return null
const fromDir = path.dirname(fromFile)
let resolved = path.normalize(path.join(fromDir, importPath))
// Add .js if no extension
if (!path.extname(resolved)) resolved += '.js'
return modName + '/' + resolved
}
/**
* Ingest codetree into AtomSpace via CogServer.
*/
async function ingestCodetree (scanDir) {
const up = await isCogServerUp()
if (!up) {
console.log('[atomspace-ingest] CogServer not running, skipping codetree ingest')
return { ok: false, reason: 'cogserver-down' }
}
console.log('[atomspace-ingest] Scanning codetree...')
const tree = scanCodetree(scanDir)
if (!tree) {
return { ok: false, reason: 'scan-failed' }
}
const exprs = treeToScheme(tree)
console.log(`[atomspace-ingest] Generated ${exprs.length} Scheme expressions from codetree`)
// Send in batches to avoid overwhelming the TCP connection
const BATCH_SIZE = 50
let sent = 0
for (let i = 0; i < exprs.length; i += BATCH_SIZE) {
const batch = exprs.slice(i, i + BATCH_SIZE)
const combined = '(begin ' + batch.join(' ') + ')'
const result = await evalScheme(combined)
if (result === null) {
console.error(`[atomspace-ingest] CogServer connection lost at batch ${i}`)
return { ok: false, reason: 'connection-lost', sent }
}
sent += batch.length
}
console.log(`[atomspace-ingest] Codetree ingested: ${sent} atoms/edges`)
return { ok: true, sent, modules: Object.keys(tree.modules || {}).length }
}
// CLI entry point
if (require.main === module) {
const scanDir = process.argv[2]
ingestCodetree(scanDir).then(result => {
console.log(JSON.stringify(result, null, 2))
process.exit(result.ok ? 0 : 1)
})
}
module.exports = { ingestCodetree, scanCodetree, treeToScheme }