// Single-Pass Session Scanner — Knowledge Ratchet Phase 2
//
// Reads JSONL transcript files and extracts structured session data.
// Uses an mtime-based cache so unchanged files are loaded from disk
// cache instead of re-parsed. Typical speedup: 10s → <1s.
//
// This is a library — not a standalone script. Used by:
// - fuse-signals.js (sensor fusion scoring)
// - analyze-transcripts.js (deep sentiment analysis)
// - pipeline.js (merged pipeline)
const fs = require('fs')
const path = require('path')
const readline = require('readline')
const SESSIONS_DIRS = [
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb'),
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky'),
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky-pear-git')
]
const CACHE_DIR = path.join(__dirname, 'data', 'scan-cache')
// ─── Entry Extraction ───
function extractEntry (entry) {
const base = {
type: entry.type,
timestamp: entry.timestamp,
sessionId: entry.sessionId,
}
if (entry.type === 'user') {
const content = entry.message?.content
let text = ''
if (typeof content === 'string') text = content
else if (Array.isArray(content)) {
text = content.filter(c => c.type === 'text').map(c => c.text).join(' ')
}
return {
...base,
text,
permissionMode: entry.permissionMode,
cwd: entry.cwd,
gitBranch: entry.gitBranch,
isSidechain: entry.isSidechain || false,
isMeta: entry.isMeta || false,
toolUseResult: entry.toolUseResult,
sourceToolAssistantUUID: entry.sourceToolAssistantUUID,
}
}
if (entry.type === 'assistant') {
const content = entry.message?.content
let textChars = 0
let thinkingChars = 0
let toolUseCount = 0
let toolUseChars = 0
const toolNames = []
const editFiles = []
if (Array.isArray(content)) {
for (const c of content) {
if (c.type === 'text' && c.text) textChars += c.text.length
if (c.type === 'thinking' && c.thinking) thinkingChars += c.thinking.length
if (c.type === 'tool_use') {
toolUseCount++
toolNames.push(c.name)
if (c.input) toolUseChars += JSON.stringify(c.input).length
if ((c.name === 'Edit' || c.name === 'Write') && c.input?.file_path) {
editFiles.push(c.input.file_path)
}
}
}
}
const usage = entry.message?.usage
return {
...base,
textChars,
thinkingChars,
toolUseCount,
toolUseChars,
toolNames,
editFiles,
outputTokens: usage?.output_tokens || 0,
cacheRead: usage?.cache_read_input_tokens || 0,
cacheCreate: usage?.cache_creation_input_tokens || 0,
inputTokens: usage?.input_tokens || 0,
}
}
if (entry.type === 'system') {
return {
...base,
subtype: entry.subtype,
durationMs: entry.durationMs,
}
}
if (entry.type === 'queue-operation') {
return {
...base,
operation: entry.operation,
content: entry.content,
}
}
return base
}
// ─── Cache ───
function cacheKey (filePath) {
// Use filename as cache key (UUIDs are unique)
return path.basename(filePath, '.jsonl') + '.json'
}
function loadFromCache (filePath, fileMtime) {
const key = cacheKey(filePath)
const cachePath = path.join(CACHE_DIR, key)
if (!fs.existsSync(cachePath)) return null
try {
// Check if cache is newer than source file
const cacheStat = fs.statSync(cachePath)
if (cacheStat.mtimeMs < fileMtime) return null // source changed, cache stale
return JSON.parse(fs.readFileSync(cachePath, 'utf8'))
} catch {
return null
}
}
function saveToCache (filePath, session) {
if (!fs.existsSync(CACHE_DIR)) fs.mkdirSync(CACHE_DIR, { recursive: true })
const key = cacheKey(filePath)
const cachePath = path.join(CACHE_DIR, key)
try {
// Strip large fields that consumers don't need for most operations.
// Keep entries lean — consumers that need full text can re-scan.
const lean = {
sessionId: session.sessionId,
sessionDate: session.sessionDate,
filePath: session.filePath,
entries: session.entries.map(e => {
if (e.type === 'user') {
// Keep full text for correction detection, but cap at 500 chars
return { ...e, text: (e.text || '').slice(0, 500) }
}
if (e.type === 'assistant') {
// Drop toolNames array (redundant with toolUseCount), keep everything else
const { toolNames, ...rest } = e
return rest
}
return e
}),
}
fs.writeFileSync(cachePath, JSON.stringify(lean))
} catch {} // non-fatal
}
// ─── Session Scanner ───
async function scanSession (filePath) {
const stream = fs.createReadStream(filePath)
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity })
const entries = []
let sessionId = null
let sessionDate = null
for await (const line of rl) {
try {
const raw = JSON.parse(line)
const entry = extractEntry(raw)
entries.push(entry)
if (!sessionId && entry.sessionId) sessionId = entry.sessionId
if (!sessionDate && entry.timestamp) sessionDate = entry.timestamp
} catch {}
}
return { entries, sessionId, sessionDate, filePath }
}
async function scanAll (opts = {}) {
const { recentDays, targetSession, noCache } = opts
const cutoffDate = recentDays
? new Date(Date.now() - recentDays * 86400000).toISOString()
: null
// Collect all transcript files
const allFiles = []
for (const dir of SESSIONS_DIRS) {
if (!fs.existsSync(dir)) continue
const files = fs.readdirSync(dir)
.filter(f => f.endsWith('.jsonl') && !f.includes('subagent'))
.map(f => path.join(dir, f))
allFiles.push(...files)
}
if (targetSession) {
const match = allFiles.find(f => path.basename(f).includes(targetSession))
if (!match) return { sessions: [], skipped: 0, total: 0, cacheHits: 0 }
allFiles.length = 0
allFiles.push(match)
}
const sessions = []
let skipped = 0
let cacheHits = 0
let cacheMisses = 0
for (const file of allFiles) {
let fileMtime
try {
const stat = fs.statSync(file)
fileMtime = stat.mtimeMs
if (cutoffDate && stat.mtime.toISOString() < cutoffDate) {
skipped++
continue
}
} catch { continue }
// Try cache first
if (!noCache) {
const cached = loadFromCache(file, fileMtime)
if (cached) {
sessions.push(cached)
cacheHits++
continue
}
}
// Cache miss — full scan
const session = await scanSession(file)
if (session.entries.length > 0) {
sessions.push(session)
// Cache the result (don't cache the current session — it's still being written)
// Heuristic: if the file hasn't been modified in the last 60s, it's safe to cache
const ageMs = Date.now() - fileMtime
if (ageMs > 60000) {
saveToCache(file, session)
}
cacheMisses++
}
}
return { sessions, skipped, total: allFiles.length, cacheHits, cacheMisses }
}
module.exports = { scanAll, scanSession, SESSIONS_DIRS }