import { readFileSync, readdirSync, statSync, existsSync } from 'fs'
import { join, relative, resolve, extname } from 'path'
import { isExternalRef, isPlaceholder, loadConfig } from './config.js'
/**
* Detect documentation cruft by scanning all markdown files and
* cross-referencing their code references against the built tree.
*
* @param {string} rootDir - Project root directory
* @param {object} tree - Built code tree from buildTree()
* @param {Function} [ignore] - Ignore matcher from scanner.js
* @param {object} [config] - Loaded codetree config (optional; loaded from
* rootDir if not passed)
* @returns {object} Doc cruft report with per-doc details and summary
*/
export function detectDocCruft (rootDir, tree, ignore, config) {
const root = resolve(rootDir)
const ig = ignore || (() => false)
const cfg = config || loadConfig(root)
const mdFiles = findMarkdownFiles(root, ig)
// Build lookup indexes from the tree for fast cross-referencing
const index = buildCodeIndex(tree)
const docs = []
let totalMissing = 0
let totalValid = 0
let totalExternal = 0
for (const mdPath of mdFiles) {
const relPath = relative(root, mdPath)
let content
try { content = readFileSync(mdPath, 'utf-8') } catch { continue }
const lines = content.split('\n')
let mtime
try { mtime = statSync(mdPath).mtimeMs } catch { mtime = 0 }
// Extract code references (placeholder-filtered via config)
const refs = extractReferences(content, relPath, cfg)
// Cross-reference each against the code tree
const details = []
let valid = 0
let missing = 0
let external = 0
for (const ref of refs) {
const status = resolveReference(ref, index, root, cfg)
details.push({ ref: ref.value, type: ref.type, status, line: ref.line })
if (status === 'valid') valid++
else if (status === 'external') external++
else missing++
}
totalValid += valid
totalMissing += missing
totalExternal += external
// Staleness score: weighted combination of missing refs and age.
// External references (to known deps outside the scan) don't count
// toward missingRatio — they're expected to be out of scope.
const total = valid + missing + external
const resolvable = valid + missing
const missingRatio = resolvable > 0 ? missing / resolvable : 0
const ageDays = (Date.now() - mtime) / (1000 * 60 * 60 * 24)
const ageFactor = Math.min(ageDays / 90, 1) // caps at 90 days
const score = Math.round((missingRatio * 0.6 + ageFactor * 0.4) * 100) / 100
docs.push({
path: relPath,
lines: lines.length,
lastModified: new Date(mtime).toISOString().split('T')[0],
references: { total, valid, missing, external, details },
staleness: { score, missingRatio: Math.round(missingRatio * 100) / 100, ageDays: Math.round(ageDays) }
})
}
// Sort by staleness score descending (most stale first)
docs.sort((a, b) => b.staleness.score - a.staleness.score)
const staleDocs = docs.filter(d => d.staleness.score > 0.5).length
const freshDocs = docs.filter(d => d.staleness.score <= 0.3).length
return {
docs,
summary: {
totalDocs: docs.length,
totalLines: docs.reduce((sum, d) => sum + d.lines, 0),
freshDocs,
staleDocs,
validReferences: totalValid,
missingReferences: totalMissing,
externalReferences: totalExternal
}
}
}
// --- Markdown file discovery ---
function findMarkdownFiles (root, ignore) {
const files = []
walkForMd(root, files, 0, 6, ignore)
return files
}
function walkForMd (dir, files, depth, maxDepth, ignore) {
if (depth >= maxDepth) return
let entries
try { entries = readdirSync(dir) } catch { return }
for (const entry of entries) {
const full = join(dir, entry)
if (ignore(full)) continue
let stat
try { stat = statSync(full) } catch { continue }
if (stat.isDirectory()) {
walkForMd(full, files, depth + 1, maxDepth, ignore)
} else if (entry.endsWith('.md')) {
files.push(full)
}
}
}
// --- Code index construction ---
function buildCodeIndex (tree) {
const filePaths = new Set()
const classNames = new Set()
const functionNames = new Set()
const exportNames = new Set()
for (const mod of Object.values(tree.modules)) {
for (const [filePath, file] of Object.entries(mod.files)) {
// Index file paths (both package-relative and with module prefix)
filePaths.add(filePath)
if (mod.path !== '.') filePaths.add(mod.path + '/' + filePath)
if (file.classes) {
for (const cls of file.classes) classNames.add(cls.name)
}
if (file.functions) {
for (const fn of file.functions) functionNames.add(fn.name)
}
if (file.exports) {
for (const exp of file.exports) {
if (exp.name && exp.name !== '*' && exp.name !== 'default') exportNames.add(exp.name)
if (exp.localName) exportNames.add(exp.localName)
}
}
}
}
return { filePaths, classNames, functionNames, exportNames }
}
// --- Reference extraction from markdown ---
function extractReferences (content, docPath, config) {
const refs = []
const lines = content.split('\n')
const seen = new Set()
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
const lineNum = i + 1
// Backtick identifiers: `SomeClass`, `lib/foo.js`, `someFunction()`
const backtickPattern = /`([^`\n]+)`/g
let m
while ((m = backtickPattern.exec(line)) !== null) {
const val = m[1].trim()
if (val.length < 2 || val.length > 120) continue
if (seen.has(val)) continue
seen.add(val)
// Placeholder / example path — skip, not a real reference
if (config && isPlaceholder(val, config)) continue
const type = classifyReference(val)
if (type) refs.push({ value: val, type, line: lineNum })
}
// Markdown links to other files: [text](path.md) or [text](./path)
const linkPattern = /\[([^\]]*)\]\(([^)]+)\)/g
while ((m = linkPattern.exec(line)) !== null) {
const target = m[2].trim()
if (target.startsWith('http://') || target.startsWith('https://')) continue
if (target.startsWith('#')) continue // anchor links
if (seen.has('link:' + target)) continue
seen.add('link:' + target)
refs.push({ value: target, type: 'doc-link', line: lineNum })
}
}
// Code blocks: extract class names, imports, function names
// Recognize code blocks in any language we support. Leaving the language tag optional
// (and not restricting to js) lets us extract references from GDScript, C++, and GLSL
// snippets embedded in docs too.
const codeBlockPattern = /```(?:js|javascript|typescript|ts|gdscript|gd|cpp|c\+\+|c|glsl)?\n([\s\S]*?)```/g
let block
while ((block = codeBlockPattern.exec(content)) !== null) {
const code = block[1]
const blockStartLine = content.slice(0, block.index).split('\n').length
// import { X, Y } from 'module'
const importPattern = /import\s*\{([^}]+)\}\s*from\s*['"]([^'"]+)['"]/g
let imp
while ((imp = importPattern.exec(code)) !== null) {
const names = imp[1].split(',').map(n => n.trim()).filter(Boolean)
for (const name of names) {
if (seen.has(name)) continue
seen.add(name)
refs.push({ value: name, type: 'class-or-function', line: blockStartLine })
}
}
// new ClassName(
const ctorPattern = /new\s+([A-Z]\w+)\s*\(/g
let ctor
while ((ctor = ctorPattern.exec(code)) !== null) {
if (seen.has(ctor[1])) continue
seen.add(ctor[1])
refs.push({ value: ctor[1], type: 'class', line: blockStartLine })
}
}
return refs
}
const KNOWN_EXTENSIONS = new Set([
'js', 'mjs', 'cjs', 'json',
'gd', 'gdshader', 'tscn', 'tres',
'cpp', 'cc', 'cxx', 'c', 'h', 'hpp', 'hh', 'hxx',
'glsl', 'vert', 'frag', 'comp',
'md', 'txt', 'yaml', 'yml', 'toml'
])
function classifyReference (val) {
// File path: contains / or ends in a known source extension
const extMatch = val.match(/\.(\w{1,8})$/)
const hasKnownExt = extMatch && KNOWN_EXTENSIONS.has(extMatch[1].toLowerCase())
if (val.includes('/') || hasKnownExt) {
// Skip things that look like URLs, semver, or shell commands
if (val.includes('://') || /^\d+\.\d+/.test(val)) return null
if (val.startsWith('npm ') || val.startsWith('git ')) return null
return 'filepath'
}
// Class name: starts with uppercase letter
if (/^[A-Z][a-zA-Z0-9]+$/.test(val)) return 'class'
// Function/method call: ends with ()
if (val.endsWith('()') || val.endsWith('(...)')) {
return 'function'
}
// Property access: contains . like server.authManager
if (/^[a-z]\w+\.\w+/.test(val)) return 'property'
// Identifier: lowercase camelCase or snake_case, at least 4 chars to avoid noise
if (/^[a-z_]\w{3,}$/.test(val) && !isCommonWord(val)) return 'identifier'
return null
}
function isCommonWord (val) {
const common = new Set([
'true', 'false', 'null', 'undefined', 'async', 'await', 'const', 'function',
'return', 'export', 'import', 'class', 'this', 'super', 'default', 'string',
'number', 'boolean', 'object', 'array', 'error', 'open', 'closed', 'pending',
'active', 'admin', 'read', 'write', 'owner', 'mirror', 'user', 'name', 'value',
'type', 'status', 'title', 'body', 'path', 'data', 'test', 'node', 'bare',
'main', 'HEAD', 'master', 'refs'
])
return common.has(val)
}
// --- Cross-referencing ---
function resolveReference (ref, index, root, config) {
// External registry check — refs declared as external in .codetree.json
// resolve as 'external' instead of 'missing'. Only applies when the
// reference can't resolve locally; that way a local symbol shadowing
// an external still resolves as 'valid'.
const markExternalIfConfigured = (value) => {
if (config && isExternalRef(value, config)) return 'external'
return 'missing'
}
switch (ref.type) {
case 'filepath': {
// Clean up: remove trailing slashes, leading ./
let cleaned = ref.value.replace(/^\.\//, '').replace(/\/$/, '')
// Strip method call suffixes like lib/auth.js:authenticate()
cleaned = cleaned.replace(/:\w+\(.*\)$/, '')
// Check against indexed file paths
if (index.filePaths.has(cleaned)) return 'valid'
// Check if directory exists on disk
const absPath = resolve(root, cleaned)
if (existsSync(absPath)) return 'valid'
return markExternalIfConfigured(cleaned)
}
case 'class':
if (index.classNames.has(ref.value) || index.exportNames.has(ref.value)) return 'valid'
return markExternalIfConfigured(ref.value)
case 'function': {
const name = ref.value.replace(/\(.*\)$/, '')
if (index.functionNames.has(name) || index.exportNames.has(name)) return 'valid'
// External check against both bare name and the original foo() form.
if (config && (isExternalRef(name, config) || isExternalRef(ref.value, config))) {
return 'external'
}
return 'missing'
}
case 'class-or-function':
if (index.classNames.has(ref.value) || index.functionNames.has(ref.value) || index.exportNames.has(ref.value)) return 'valid'
return markExternalIfConfigured(ref.value)
case 'property':
case 'identifier':
// For identifiers, check if they appear as any known symbol
if (index.classNames.has(ref.value) || index.functionNames.has(ref.value) || index.exportNames.has(ref.value)) return 'valid'
if (config && isExternalRef(ref.value, config)) return 'external'
return 'unknown'
case 'doc-link': {
// Check if linked document exists on disk (relative to the doc's directory or root)
const target = ref.value.replace(/^\.\//, '')
const absPath = resolve(root, target)
if (existsSync(absPath)) return 'valid'
return markExternalIfConfigured(target)
}
default:
return 'unknown'
}
}