import { EventEmitter } from 'events'
import { readFileSync, readdirSync, statSync } from 'fs'
import { resolve, relative, join, basename, dirname } from 'path'
import {
detectProject,
findModuleBoundaries,
buildResolutionContext,
readProjectMeta,
resolveEntry,
getLang,
isSourceFile
} from './lang.js'
import { buildIgnoreMatcher } from './ignore.js'
/**
* Walk a project and emit events describing what was found.
*
* This is the push-based core of the scanner. Multiple consumers can
* subscribe to different events without forcing a second walk.
*
* const source = scanStream(rootDir, opts)
* source.on('file-parsed', e => ...)
* source.on('scan-end', e => ...)
*
* Events (all payload objects):
* scan-start { root, projectType, languages, resolutionContext }
* module-start { name, version, path, absPath, entry, meta, projectType,
* dependencies, devDependencies, importMaps, testCommand, type }
* file-discovered { modName, absPath, relPath, relInPkg, source, size }
* file-parsed { modName, relInPkg, relPath, parsed }
* import-edge { modName, fromRelInPkg, specifier, resolvedRelPath, implicit }
* module-end { name }
* scan-end { root, moduleCount, fileCount, dirTree }
* error Error
*
* The emitter returned from this function starts work on the next tick, giving
* callers a chance to attach listeners before the first event fires.
*
* There are NO timers anywhere in this file. Every async boundary is a
* promise/microtask or a listener callback.
*/
export function scanStream (rootDir, opts = {}) {
const emitter = new EventEmitter()
const root = resolve(rootDir)
// Defer the walk so callers can attach listeners first. queueMicrotask is
// push-based — no setTimeout, no Promise.resolve().then hack.
queueMicrotask(() => runScan(emitter, root, opts))
return emitter
}
function runScan (emitter, root, opts) {
try {
const ignore = opts.ignore || buildIgnoreMatcher(root)
const shallow = !!opts.shallow
const project = detectProject(root)
const resolutionContext = buildResolutionContext(root, project.type, ignore, { shallow })
resolutionContext.ignore = ignore
emitter.emit('scan-start', {
root,
projectType: project.type,
languages: project.languages,
shallow,
resolutionContext
})
const moduleDirs = findModuleBoundaries(root, project.type, {
maxDepth: opts.maxDepth || 2,
ignore
})
let moduleCount = 0
let fileCount = 0
const moduleAbsPaths = new Set(moduleDirs)
for (const pkgDir of moduleDirs) {
// Each module may be its own project type (monorepo case)
const modProject = detectProject(pkgDir)
const modType = modProject.type !== 'unknown' ? modProject.type : project.type
const meta = readProjectMeta(pkgDir, modType)
if (modType === 'js' && !meta) continue
const modName = (meta && meta.name) || basename(pkgDir)
const entry = resolveEntry(pkgDir, meta, modType)
emitter.emit('module-start', {
name: modName,
version: (meta && meta.version) || '0.0.0',
path: relative(root, pkgDir) || '.',
absPath: pkgDir,
entry,
meta,
projectType: modType,
dependencies: (meta && meta.dependencies) || {},
devDependencies: (meta && meta.devDependencies) || {},
importMaps: (meta && meta.imports) || {},
testCommand: meta && meta.scripts && meta.scripts.test ? meta.scripts.test : null,
type: (meta && meta.type) || 'commonjs'
})
if (shallow) {
// Shallow mode: parse ONLY the entry file (if any). No import resolution,
// no scanDir recursion, no cross-file analysis. This gives the UI enough
// to show every module + its public API without paying the full parse cost.
if (entry) {
fileCount += parseOneFile(emitter, entry, pkgDir, modName, root, ignore)
}
} else {
// Deep mode: walk entry + recursively scan module root, resolve imports.
const childModulePaths = new Set(
moduleDirs.filter(d => d !== pkgDir && d.startsWith(pkgDir + '/'))
)
const visited = new Set()
if (entry) {
fileCount += walkFile(emitter, entry, pkgDir, modName, visited, root, ignore, resolutionContext)
}
fileCount += scanDir(emitter, pkgDir, pkgDir, modName, visited, root, ignore, resolutionContext, childModulePaths)
}
emitter.emit('module-end', { name: modName })
moduleCount++
}
// Always build dirTree, even in shallow mode — the FS walk is cheap (just
// readdir+stat) compared to parsing, and the UI tree view needs structure.
// Without it the sidebar collapses to a flat list which is unreadable
// for monorepo-scale scans.
const dirTree = buildDirTree(root, moduleAbsPaths, ignore)
emitter.emit('scan-end', { root, moduleCount, fileCount, dirTree, shallow })
} catch (err) {
emitter.emit('error', err)
}
}
// Parse a single file (used by shallow mode). Emits file-discovered + file-parsed
// but does NOT follow imports or recurse.
function parseOneFile (emitter, filePath, pkgDir, modName, root, ignore) {
if (ignore(filePath)) return 0
let source
try { source = readFileSync(filePath, 'utf-8') } catch { return 0 }
const size = source.length
if (size > MAX_PARSE_SIZE) return 0
const relPath = relative(root, filePath)
const relInPkg = relative(pkgDir, filePath)
emitter.emit('file-discovered', { modName, absPath: filePath, relPath, relInPkg, size })
const lang = getLang(filePath)
if (!lang) return 1
try {
const parsed = lang.parse(source, relInPkg)
emitter.emit('file-parsed', { modName, relInPkg, relPath, parsed })
} catch (err) {
emitter.emit('file-parsed', { modName, relInPkg, relPath, parsed: { error: err.message, filePath: relInPkg } })
}
return 1
}
// --- Walking ---
// Skip parsing files larger than this. Generated bindings and minified bundles
// are almost always uninteresting for structural analysis, and their ASTs are
// the dominant memory cost on large recursive scans.
const MAX_PARSE_SIZE = 200 * 1024 // 200 KB
function walkFile (emitter, filePath, pkgDir, modName, visited, root, ignore, ctx) {
if (visited.has(filePath)) return 0
visited.add(filePath)
if (ignore(filePath)) return 0
let source, size
try {
source = readFileSync(filePath, 'utf-8')
size = source.length
} catch {
return 0
}
const relPath = relative(root, filePath)
const relInPkg = relative(pkgDir, filePath)
// For oversized files, emit the discovery event so the tree knows the file
// exists, but skip parsing + import resolution. Saves memory and CPU.
if (size > MAX_PARSE_SIZE) {
emitter.emit('file-discovered', { modName, absPath: filePath, relPath, relInPkg, size })
emitter.emit('file-parsed', { modName, relInPkg, relPath, parsed: { error: 'file too large to parse', filePath: relInPkg } })
return 1
}
// Deliberately do NOT include source in the event — subscribers that actually
// need the bytes can readFileSync from absPath. Sending source by default
// means every listener retains it, which blew up memory on recursive scans.
emitter.emit('file-discovered', {
modName, absPath: filePath, relPath, relInPkg, size
})
const lang = getLang(filePath)
if (!lang) return 1
// Parse eagerly and emit. (Workers and mtime cache are future optimizations
// that plug in here — they just change who produces the file-parsed event.)
let parsed = null
try {
parsed = lang.parse(source, relInPkg)
emitter.emit('file-parsed', { modName, relInPkg, relPath, parsed })
} catch (err) {
emitter.emit('file-parsed', { modName, relInPkg, relPath, parsed: { error: err.message, filePath: relInPkg } })
}
// Imported symbol names per specifier, taken from the AST parse. The edge
// carries these so downstream consumers can mate a specific demand (`-name`)
// against the specific export that supplies it, instead of re-deriving the
// association by string-matching specifiers later.
const namesBySpec = new Map()
if (parsed && Array.isArray(parsed.imports)) {
for (const imp of parsed.imports) {
if (!imp || !imp.specifier) continue
const prev = namesBySpec.get(imp.specifier) || []
namesBySpec.set(imp.specifier, prev.concat(imp.names || []))
}
}
// Resolve imports (explicit + implicit) and emit edges for each.
//
// The specifier list comes from the AST whenever the file parsed. The regex
// extractor in lang.js cannot see a multi-line import statement — its `.`
// does not cross newlines — so every wrapped `import {\n a,\n b\n} from 'x'`
// silently produced no edge at all. Fall back to the regex only when there
// is no usable parse (parse error, or a language with no AST import list).
let count = 1
const parsedOk = parsed && !parsed.error && Array.isArray(parsed.imports)
const explicit = parsedOk ? [...namesBySpec.keys()] : lang.extractImports(source)
for (const spec of explicit) {
const resolved = lang.resolveImport(filePath, spec, ctx)
if (!resolved) continue
emitter.emit('import-edge', {
modName,
fromRelInPkg: relInPkg,
specifier: spec,
resolvedRelPath: relative(root, resolved),
names: namesBySpec.get(spec) || [],
implicit: false
})
count += walkFile(emitter, resolved, pkgDir, modName, visited, root, ignore, ctx)
}
if (lang.extractImplicitImports) {
const implicit = lang.extractImplicitImports(source, ctx)
for (const spec of implicit) {
const resolved = lang.resolveImport(filePath, spec, ctx)
if (!resolved || resolved === filePath) continue
emitter.emit('import-edge', {
modName,
fromRelInPkg: relInPkg,
specifier: spec,
resolvedRelPath: relative(root, resolved),
implicit: true
})
count += walkFile(emitter, resolved, pkgDir, modName, visited, root, ignore, ctx)
}
}
return count
}
function scanDir (emitter, dir, pkgDir, modName, visited, root, ignore, ctx, childModulePaths) {
let entries
try { entries = readdirSync(dir) } catch { return 0 }
let count = 0
for (const entry of entries) {
if (entry.startsWith('.')) continue
const full = join(dir, entry)
if (ignore(full)) continue
if (childModulePaths && childModulePaths.has(full)) continue
let stat
try { stat = statSync(full) } catch { continue }
if (stat.isDirectory()) {
count += scanDir(emitter, full, pkgDir, modName, visited, root, ignore, ctx, childModulePaths)
} else if (isSourceFile(full)) {
count += walkFile(emitter, full, pkgDir, modName, visited, root, ignore, ctx)
}
}
return count
}
// --- Directory tree ---
function buildDirTree (root, moduleAbsPaths, ignore) {
return walkDir(root, root, moduleAbsPaths, 0, 8, ignore)
}
function walkDir (dir, root, moduleAbsPaths, depth, maxDepth, ignore) {
const name = basename(dir)
const relPath = relative(root, dir) || '.'
const node = {
name: depth === 0 ? basename(root) : name,
path: relPath,
type: moduleAbsPaths.has(dir) ? 'module-dir' : 'dir',
children: []
}
if (moduleAbsPaths.has(dir)) node.module = basename(dir)
if (depth >= maxDepth) return node
let entries
try { entries = readdirSync(dir).sort() } catch { return node }
// Only collect directories. File entries were a huge bloat in the dirTree
// payload (33 MB for /storage/projects/code/) and aren't used by the sidebar
// anymore — files for a focused module come from /api/file on demand.
for (const entry of entries) {
const full = join(dir, entry)
if (ignore(full)) continue
let stat
try { stat = statSync(full) } catch { continue }
if (!stat.isDirectory()) continue
const child = walkDir(full, root, moduleAbsPaths, depth + 1, maxDepth, ignore)
if (child.children.length > 0 || child.module) node.children.push(child)
}
return node
}