#!/usr/bin/env node
// Project Mapper — Autodiscover Subsystems from Code
// Knowledge Ratchet Phase 2
//
// Walks project directory trees, parses require/import for dependency graphs,
// clusters files into subsystems, and cross-references with loop history.
//
// Outputs data/project-maps.json — a machine-readable map of every known
// project's subsystem structure, used by preflight-review, loop-detector,
// loop-briefing instead of hardcoded maps.
//
// Usage:
// node project-mapper.js # scan all known projects
// node project-mapper.js --project=pear-git # scan one project
// node project-mapper.js --verbose # show dependency details
const fs = require('fs')
const path = require('path')
const DATA_DIR = path.join(__dirname, 'data')
const OUTPUT_FILE = path.join(DATA_DIR, 'project-maps.json')
const LOOPS_FILE = path.join(DATA_DIR, 'loops.json')
// Known project roots — autodiscovered from hyper-fleeky + godot dirs
const PROJECT_ROOTS = {
'pear-git': '/storage/projects/code/hypercore-kb/hyper-fleeky/pear-git',
'hypernote': '/storage/projects/code/hypercore-kb/hyper-fleeky/hypernote',
'hyper-overlay': '/storage/projects/code/hypercore-kb/hyper-fleeky/hyper-overlay',
'hyper-social': '/storage/projects/code/hypercore-kb/hyper-social/hyper-social-web',
'dc-terrain': '/storage/projects/code/godot/dc-terrain',
'hyper-tps': '/storage/projects/code/godot/hyper-tps',
'blender-bridge': '/storage/projects/code/godot/blender-bridge',
'proc-nodes': '/storage/projects/code/godot/proc-nodes',
}
const SKIP_DIRS = new Set(['node_modules', '.git', 'vendor', 'dist', 'build', '.pear', '.claude'])
const CODE_EXTS = new Set(['.js', '.mjs', '.cjs', '.cpp', '.h', '.gd', '.glsl', '.gdshader'])
// ─── File Tree Scanner ───
function scanDir (dir, prefix = '') {
const files = []
let entries
try { entries = fs.readdirSync(dir, { withFileTypes: true }) }
catch { return files }
for (const entry of entries) {
if (SKIP_DIRS.has(entry.name)) continue
if (entry.name.startsWith('.')) continue
const fullPath = path.join(dir, entry.name)
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name
if (entry.isDirectory()) {
files.push(...scanDir(fullPath, relPath))
} else if (CODE_EXTS.has(path.extname(entry.name))) {
files.push({ path: fullPath, rel: relPath, name: entry.name })
}
}
return files
}
// ─── Dependency Parser ───
// Extract require() and import calls from JS files
function parseDeps (filePath) {
const deps = []
let content
try { content = fs.readFileSync(filePath, 'utf8') }
catch { return deps }
// require('./foo') or require('../bar')
const requireRe = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g
// import ... from './foo'
const importRe = /import\s+.*?\s+from\s+['"]([^'"]+)['"]/g
for (const re of [requireRe, importRe]) {
let match
while ((match = re.exec(content)) !== null) {
const dep = match[1]
// Only track local deps (not npm packages)
if (dep.startsWith('.') || dep.startsWith('/')) {
deps.push(dep)
}
}
}
return deps
}
// ─── Subsystem Clustering ───
// Group files by their top-level directory under lib/src
function clusterSubsystems (files, projectRoot) {
const subsystems = {}
for (const file of files) {
const rel = file.rel
const parts = rel.split('/')
// Determine subsystem from directory structure
let subsystem = 'root'
if (parts.length === 1) {
// Top-level file (e.g., pear-git-server.js, index.js)
subsystem = 'server-core'
} else if (parts[0] === 'lib' || parts[0] === 'src') {
if (parts.length === 2) {
// lib/foo.js → classify by name
subsystem = classifyByName(parts[1])
} else {
// lib/subsystem/file.js → use directory name
subsystem = parts[1]
}
} else if (parts[0] === 'test' || parts[0] === 'tests') {
subsystem = 'tests'
} else if (parts[0] === 'bin' || parts[0] === 'scripts') {
subsystem = 'cli'
} else {
subsystem = parts[0]
}
if (!subsystems[subsystem]) {
subsystems[subsystem] = { files: [], deps: new Set(), dependedBy: new Set() }
}
subsystems[subsystem].files.push(file.rel)
}
return subsystems
}
function classifyByName (filename) {
const base = filename.replace(/\.(js|mjs|cjs)$/, '').toLowerCase()
if (base.includes('auth') || base.includes('identity')) return 'identity'
if (base.includes('repo') || base.includes('registry')) return 'repo-management'
if (base.includes('route') || base.includes('middleware')) return 'http-routes'
if (base.includes('sync') || base.includes('federation') || base.includes('discovery')) return 'p2p-sync'
if (base.includes('deploy') || base.includes('ci')) return 'ci-deploy'
if (base.includes('-ui') || base.includes('nav') || base.includes('layout')) return 'frontend-ui'
if (base.includes('git') || base.includes('diff')) return 'git-layer'
if (base.includes('issue')) return 'issues'
if (base.includes('namespace')) return 'namespaces'
if (base.includes('error') || base.includes('response')) return 'shared-utils'
if (base.includes('storage') || base.includes('backend')) return 'storage'
return 'server-core'
}
// ─── Dependency Graph Between Subsystems ───
function buildDependencyGraph (files, subsystems, projectRoot) {
// Map file paths to their subsystem
const fileToSubsystem = {}
for (const [sub, data] of Object.entries(subsystems)) {
for (const f of data.files) {
fileToSubsystem[f] = sub
}
}
// Parse deps and map to subsystem-level edges
for (const file of files) {
if (!file.path.endsWith('.js')) continue
const fromSub = fileToSubsystem[file.rel]
if (!fromSub) continue
const deps = parseDeps(file.path)
for (const dep of deps) {
// Resolve relative path
const resolved = path.resolve(path.dirname(file.path), dep)
const relResolved = path.relative(projectRoot, resolved).replace(/\\/g, '/')
// Find which subsystem this dep belongs to
// Try with and without .js extension
for (const candidate of [relResolved, relResolved + '.js', relResolved + '/index.js']) {
const toSub = fileToSubsystem[candidate]
if (toSub && toSub !== fromSub) {
subsystems[fromSub].deps.add(toSub)
subsystems[toSub].dependedBy.add(fromSub)
break
}
}
}
}
}
// ─── Loop History Cross-Reference ───
function crossReferenceLoops (subsystems, projectName) {
if (!fs.existsSync(LOOPS_FILE)) return
let loopsData
try { loopsData = JSON.parse(fs.readFileSync(LOOPS_FILE, 'utf8')) }
catch { return }
const projectLoops = (loopsData.loops || []).filter(l => l.project === projectName)
for (const loop of projectLoops) {
for (const f of (loop.filesTried || [])) {
const base = path.basename(f)
// Find which subsystem this file belongs to
for (const [sub, data] of Object.entries(subsystems)) {
if (data.files.some(sf => sf.endsWith(base) || sf.endsWith('/' + base))) {
if (!data.loopCount) data.loopCount = 0
if (!data.correctionCount) data.correctionCount = 0
if (!data.loopTypes) data.loopTypes = {}
data.loopCount++
data.correctionCount += loop.correctionCount
data.loopTypes[loop.type] = (data.loopTypes[loop.type] || 0) + 1
break
}
}
}
}
}
// ─── Staleness Detection ───
function detectChanges (subsystems, projectRoot, previousMap) {
if (!previousMap) return null
const prevFiles = new Set()
for (const data of Object.values(previousMap.subsystems || {})) {
for (const f of (data.files || [])) prevFiles.add(f)
}
const currentFiles = new Set()
for (const data of Object.values(subsystems)) {
for (const f of data.files) currentFiles.add(f)
}
const added = [...currentFiles].filter(f => !prevFiles.has(f))
const removed = [...prevFiles].filter(f => !currentFiles.has(f))
// Check modification times for existing files
const modified = []
if (previousMap.scannedAt) {
const lastScan = new Date(previousMap.scannedAt).getTime()
for (const f of currentFiles) {
if (prevFiles.has(f)) {
try {
const stat = fs.statSync(path.join(projectRoot, f))
if (stat.mtimeMs > lastScan) modified.push(f)
} catch {}
}
}
}
if (added.length === 0 && removed.length === 0 && modified.length === 0) return null
return { added, removed, modified }
}
// ─── Main ───
function mapProject (projectName, projectRoot, previousMaps) {
if (!fs.existsSync(projectRoot)) return null
const files = scanDir(projectRoot)
const subsystems = clusterSubsystems(files, projectRoot)
buildDependencyGraph(files, subsystems, projectRoot)
crossReferenceLoops(subsystems, projectName)
// Convert Sets to arrays for JSON
const serialized = {}
for (const [name, data] of Object.entries(subsystems)) {
serialized[name] = {
files: data.files,
fileCount: data.files.length,
deps: [...data.deps],
dependedBy: [...data.dependedBy],
loopCount: data.loopCount || 0,
correctionCount: data.correctionCount || 0,
loopTypes: data.loopTypes || {},
dangerLevel: data.loopCount > 10 ? 'high' : data.loopCount > 3 ? 'medium' : 'low',
}
}
const changes = detectChanges(subsystems, projectRoot, previousMaps?.[projectName])
return {
root: projectRoot,
scannedAt: new Date().toISOString(),
totalFiles: files.length,
subsystems: serialized,
changes,
}
}
function main () {
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true })
const verbose = process.argv.includes('--verbose')
const projectArg = process.argv.find(a => a.startsWith('--project='))
const targetProject = projectArg ? projectArg.split('=')[1] : null
// Load previous maps for change detection
let previousMaps = {}
if (fs.existsSync(OUTPUT_FILE)) {
try { previousMaps = JSON.parse(fs.readFileSync(OUTPUT_FILE, 'utf8')).projects || {} }
catch {}
}
const projects = targetProject
? { [targetProject]: PROJECT_ROOTS[targetProject] }
: PROJECT_ROOTS
console.log('═'.repeat(60))
console.log(' PROJECT MAPPER — Subsystem Autodiscovery')
console.log('═'.repeat(60))
const allMaps = {}
for (const [name, root] of Object.entries(projects)) {
if (!root) { console.log(` Unknown project: ${name}`); continue }
const map = mapProject(name, root, previousMaps)
if (!map) { console.log(` ${name}: directory not found`); continue }
allMaps[name] = map
const subs = Object.entries(map.subsystems).sort((a, b) => b[1].fileCount - a[1].fileCount)
console.log(`\n ${name} (${map.totalFiles} files, ${subs.length} subsystems)`)
console.log(' ' + '-'.repeat(56))
for (const [sub, data] of subs) {
const danger = data.dangerLevel === 'high' ? '⚠' : data.dangerLevel === 'medium' ? '~' : ' '
const loops = data.loopCount > 0 ? ` ${data.loopCount} loops` : ''
const deps = data.deps.length > 0 ? ` → ${data.deps.join(', ')}` : ''
console.log(` ${danger} ${sub.padEnd(22)} ${String(data.fileCount).padStart(3)} files${loops}${verbose ? deps : ''}`)
}
if (map.changes) {
console.log(`\n CHANGES since last scan:`)
if (map.changes.added.length > 0) console.log(` + ${map.changes.added.length} new files`)
if (map.changes.removed.length > 0) console.log(` - ${map.changes.removed.length} removed files`)
if (map.changes.modified.length > 0) console.log(` ~ ${map.changes.modified.length} modified files`)
if (verbose) {
for (const f of map.changes.added.slice(0, 5)) console.log(` + ${f}`)
for (const f of map.changes.removed.slice(0, 5)) console.log(` - ${f}`)
for (const f of map.changes.modified.slice(0, 5)) console.log(` ~ ${f}`)
}
}
}
// Save
const report = {
generated: new Date().toISOString(),
projectCount: Object.keys(allMaps).length,
projects: allMaps,
}
fs.writeFileSync(OUTPUT_FILE, JSON.stringify(report, null, 2))
console.log(`\nSaved: ${OUTPUT_FILE}`)
console.log('═'.repeat(60))
}
module.exports = { mapProject, PROJECT_ROOTS }
if (require.main === module) main()