import { existsSync } from 'fs'
import { resolve, join } from 'path'
import { scanStream } from './scan-stream.js'
import { TreeReducer } from './reducers/tree-reducer.js'
import { buildTree } from './tree.js'
import { parseSpecs, findAllDocs } from './doc-parser.js'
import { validate } from './validator.js'
import { detectDocCruft } from './doc-cruft.js'
import { buildIgnoreMatcher } from './ignore.js'
import { loadConfig, applyCliOverrides } from './config.js'
import { crawlDeps } from './deps-crawler.js'
// Re-export for consumers that want the push-based source directly
export { scanStream } from './scan-stream.js'
export { TreeReducer } from './reducers/tree-reducer.js'
/**
* Scan a directory and produce the complete code tree + validation report.
*
* Internally this composes scanStream() with a TreeReducer. Other consumers
* (UI updaters, metrics, doc-cruft) can attach their own listeners to the
* stream — that's the whole point of the refactor. This function is one
* convenience consumer that returns the fully-built tree.
*
* opts:
* maxDepth: number — project-discovery depth (default 2)
* ignore: string[] — extra ignore patterns (from --ignore CLI flag)
* shallow: boolean — if true, only parse the entry file per module.
* No cross-file analysis, no doc-cruft, no
* resolution context. Orders of magnitude faster
* for multi-project / recursive scans.
*/
export async function scan (rootDir, opts = {}) {
// Build ignore matcher up front so every scan stage uses the same predicate.
const ignore = buildIgnoreMatcher(rootDir, opts.ignore || [])
const shallow = !!opts.shallow
// Load per-project config (.codetree.json at rootDir). CLI overrides
// are threaded through opts.cli (set by the CLI entrypoint).
const baseConfig = loadConfig(rootDir)
const config = opts.cli ? applyCliOverrides(baseConfig, opts.cli) : baseConfig
// Drive the primary scan.
const graph = await new Promise((resolve, reject) => {
const source = scanStream(rootDir, { ...opts, ignore, shallow })
const tree = new TreeReducer().attach(source)
source.once('scan-end', () => resolve(tree.finalize()))
source.once('error', reject)
})
// Cross-filesystem dep crawling. Produces extra roots (explicit siblings
// + package.json file: deps + optional autoCrawl via sibling search).
// Each extra root is scanned independently, modules merged by name into
// the primary graph. The primary root, files list, and dirTree stay
// anchored to the primary — only the `modules` map grows.
const extraScans = []
const crawl = crawlDeps(rootDir, config, {})
if (crawl.roots.length > 0) {
for (const extraRoot of crawl.roots) {
try {
const extraGraph = await new Promise((res, rej) => {
const extraIgnore = buildIgnoreMatcher(extraRoot, opts.ignore || [])
const src = scanStream(extraRoot, { ...opts, ignore: extraIgnore, shallow })
const r = new TreeReducer().attach(src)
src.once('scan-end', () => res(r.finalize()))
src.once('error', rej)
})
extraScans.push({ root: extraRoot, graph: extraGraph, source: crawl.sources[extraRoot] })
} catch (err) {
// Extra roots shouldn't break the primary scan. Record but continue.
// eslint-disable-next-line no-console
console.warn('[codetree] extra-root scan failed for ' + extraRoot + ': ' + err.message)
}
}
}
// Merge each extra graph's modules into the primary graph. Modules already
// present (same name) in the primary are NOT overridden — primary wins.
// graph.modules is a Map (per TreeReducer.finalize). Flag extra-root modules
// so downstream consumers (UI, validator) can distinguish them.
let extraModulesAdded = 0
for (const extra of extraScans) {
if (!extra.graph.modules) continue
for (const [modName, modData] of extra.graph.modules) {
if (!graph.modules.has(modName)) {
graph.modules.set(modName, {
...modData,
_external: true,
_externalRoot: extra.root
})
extraModulesAdded++
}
}
}
const tree = buildTree(graph)
// Mark the tree so consumers know whether data is complete or shallow.
tree.shallow = shallow
// Attach the live graph as a non-enumerable property so consumers that need
// the incremental-update state (e.g. LiveStore in server.js) can reach it,
// but JSON serialization ignores it.
Object.defineProperty(tree, '__graph', { value: graph, enumerable: false })
// Record what crawling found so consumers can display or diagnose it.
tree.crawl = {
roots: crawl.roots,
unresolved: crawl.unresolved,
sources: crawl.sources,
addedModuleCount: extraModulesAdded
}
// Doc cruft walks every .md file and cross-references against the tree.
// Skip in shallow mode — it's one of the most expensive steps and the
// tree doesn't have the structural detail needed for meaningful checks.
if (!shallow) {
tree.docCruft = detectDocCruft(rootDir, tree, ignore, config)
}
// Expose the loaded config so downstream consumers (CLI, serve) can
// make informed decisions without re-loading.
tree.config = config
// Find and validate specs at the scan root (preserves existing behavior).
const docs = findAllDocs(rootDir)
let validation = null
let categories = null
if (docs.specDir) {
const specs = parseSpecs(docs.specDir)
validation = validate(tree, specs)
if (specs.system && specs.system.categories) {
categories = { ...specs.system.categories }
}
}
// Aggregate categories from spec/SYSTEM.md files nested INSIDE any discovered
// module. For a high-level scan like /storage/projects/code/, the curated
// SYSTEM.md sits 4+ levels deep (e.g. hypercore-kb/hyper-fleeky/pear-git-v3/
// spec/SYSTEM.md). Without this, the bubbles in the graph would lose all the
// semantic categories that were carefully defined in those spec files.
if (!categories) categories = {}
for (const mod of Object.values(tree.modules)) {
const modAbs = resolve(rootDir, mod.path)
const candidates = [join(modAbs, 'spec'), join(modAbs, 'docs', 'spec')]
for (const sd of candidates) {
if (!existsSync(join(sd, 'SYSTEM.md'))) continue
try {
const innerSpecs = parseSpecs(sd)
if (innerSpecs.system && innerSpecs.system.categories) {
for (const [cat, mods] of Object.entries(innerSpecs.system.categories)) {
if (!categories[cat]) categories[cat] = []
for (const m of mods) {
if (!categories[cat].includes(m)) categories[cat].push(m)
}
}
}
} catch {}
}
}
tree.validation = validation
tree.categories = Object.keys(categories).length > 0 ? categories : null
// Resolve every module to one category, once, here — curated SYSTEM.md
// groupings first, then the shared-dependency fallback. The CLI and both
// views need this mapping; computing it in each of them is how they drift.
tree.moduleCategories = assignCategories(tree, categories)
tree.docs = {
specDir: docs.specDir || null,
specCount: docs.specs.length,
readmeCount: docs.readmes.length
}
return tree
}
/**
* Map every module to exactly one category name.
*
* Two sources, in order:
* 1. curated — the groupings declared in spec/SYSTEM.md
* 2. derived — the dependency a module shares with the most other modules,
* which is the strongest relatedness signal available without authoring
* anything. Reported as "<dep> users" so it is obvious it was inferred.
*
* A module that matches neither lands in "Other".
*/
function assignCategories (tree, categories) {
const assigned = {}
for (const [cat, mods] of Object.entries(categories || {})) {
for (const m of mods) assigned[m] = cat
}
const shared = tree.system.sharedDeps || {}
const topDep = {}
for (const [dep, users] of Object.entries(shared)) {
for (const u of users) {
const prev = topDep[u]
if (!prev || users.length > shared[prev].length) topDep[u] = dep
}
}
for (const name of tree.system.modules) {
if (assigned[name]) continue
assigned[name] = topDep[name] ? topDep[name] + ' users' : 'Other'
}
return assigned
}