#!/usr/bin/env node
// Sensor Fusion Scorer — Knowledge Ratchet Phase 2
//
// Reads Claude Code JSONL transcripts and extracts 18 sensors per turn,
// normalizes each to [0,1] (1 = good, 0 = bad), applies weights,
// and computes a fused quality score per session.
//
// Usage:
// node fuse-signals.js # analyze all sessions
// node fuse-signals.js --recent=7 # only last 7 days
// node fuse-signals.js --session=UUID # single session
// node fuse-signals.js --verbose # print per-turn details
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const { scanAll, scanSession, SESSIONS_DIRS } = require('./scan-sessions')
const DATA_DIR = path.join(__dirname, 'data')
const OUTPUT_FILE = path.join(DATA_DIR, 'fused-scores.json')
const DETAIL_FILE = path.join(DATA_DIR, 'fused-turns-detail.json')
const WEIGHTS_FILE = path.join(DATA_DIR, 'sensor-weights.json')
const CORRECTIONS_LOG = path.join(__dirname, 'corrections.log')
// ─── Default Sensor Weights ───
// Higher weight = more influence on fused score.
// Negative-polarity sensors (correction, frustration, etc.) are inverted during normalization.
// These are the initial hand-tuned weights — the meta-ratchet will adjust them later
// based on which sensors best predict future corrections.
const DEFAULT_WEIGHTS = {
correction: 1.0, // Direct correction signal
approval: 0.8, // Direct approval signal
toolCallCount: 0.4, // Many tool calls = fumbling
turnDuration: 0.2, // Long turns = hard problem or fumbling
interruptions: 0.9, // User interrupted = strong frustration
userProvidesAnswer: 0.7, // User had to give the answer
messageLengthTrend: 0.3, // Shortening messages = disengagement
tokenEfficiency: 0.3, // Output tokens vs useful text
editSurvival: 0.5, // Edits that stuck vs reverted
buildErrors: 0.6, // Errors in tool results
permissionShift: 0.4, // Trust level changes
cacheHitRate: 0.1, // Context reuse (informational)
thinkingRatio: 0.2, // High thinking vs output = confusion
sidechainUsage: 0.3, // Sidechains = main path failed
queueDepth: 0.2, // Message queuing = user engagement
timeBetweenTurns: 0.1, // Time between user messages
}
function loadWeights () {
if (fs.existsSync(WEIGHTS_FILE)) {
try {
const saved = JSON.parse(fs.readFileSync(WEIGHTS_FILE, 'utf8'))
return { ...DEFAULT_WEIGHTS, ...saved }
} catch {}
}
return { ...DEFAULT_WEIGHTS }
}
// ─── LLM Classification Index ───
// Build a lookup from corrections.log: timestamp prefix → { type, source, confidence }
// LLM-confirmed signals get full weight; regex-only guesses get discounted.
function loadLLMIndex () {
const index = {} // key: timestamp prefix (first 20 chars) → { type, source, confidence }
if (!fs.existsSync(CORRECTIONS_LOG)) return index
const raw = fs.readFileSync(CORRECTIONS_LOG, 'utf8')
for (const entry of raw.split('\n---\n')) {
if (!entry.trim()) continue
const firstLine = entry.trim().split('\n')[0]
const tsMatch = firstLine.match(/^(\d{4}-\d{2}-\d{2}T[\d:.]+)/)
const typeMatch = firstLine.match(/\[(correction|approval)\]/)
if (!tsMatch || !typeMatch) continue
const key = tsMatch[1].slice(0, 20) // e.g. "2026-03-15T20:51:52."
const type = typeMatch[1]
let source = 'regex'
let confidence = 0.5
const llmMatch = firstLine.match(/\(llm, confidence: ([\d.]+)/)
const autoMatch = firstLine.match(/\(auto, confidence: ([\d.]+)/)
if (llmMatch) {
source = 'llm'
confidence = parseFloat(llmMatch[1])
} else if (autoMatch) {
source = 'auto'
confidence = parseFloat(autoMatch[1])
}
index[key] = { type, source, confidence }
}
return index
}
// ─── Correction/Approval Detection ───
// Loads tuned patterns from tune-regex.js output if available,
// falls back to hardcoded defaults otherwise.
const TUNED_PATTERNS_FILE = path.join(DATA_DIR, 'tuned-patterns.json')
function loadTunedPatterns () {
if (fs.existsSync(TUNED_PATTERNS_FILE)) {
try {
const tuned = JSON.parse(fs.readFileSync(TUNED_PATTERNS_FILE, 'utf8'))
// Merge high + medium tiers into correction patterns
const corrPatterns = []
for (const tier of ['high', 'medium']) {
for (const p of (tuned.correction?.[tier] || [])) {
corrPatterns.push(new RegExp(p.pattern, p.flags || 'i'))
}
}
const apprPatterns = []
for (const p of (tuned.approval?.high || [])) {
apprPatterns.push(new RegExp(p.pattern, p.flags || 'i'))
}
if (corrPatterns.length > 0) return { correction: corrPatterns, approval: apprPatterns }
} catch {}
}
return null
}
const _tuned = loadTunedPatterns()
const CORRECTION_PATTERNS = _tuned?.correction || [
/\b(no[,.]|wrong|not that|that's not|incorrect|stop doing)\b/i,
/\b(i said|i meant|i asked for|that's wrong)\b/i,
/\b(again|already told you|how many times|we discussed this)\b/i,
/\b(don't ever|never do that|stop|please don't|do not)\b/i,
/\b(why did you|why are you|that broke|you broke|you just)\b/i,
/\b(revert|undo|go back|roll back|put it back)\b/i,
/\b(let me (just |)do it|i'll (just |)do it|forget it|never\s*mind)\b/i,
]
const APPROVAL_PATTERNS = _tuned?.approval || [
/\b(perfect|exactly|great|lgtm|looks good|nice work|nailed it)\b/i,
/\b(that works|that's right|correct|good job|well done)\b/i,
/\b(ship it|merge it|push it|deploy it|commit it)\b/i,
/\b(thanks|thank you|yes)\b/i,
]
const USER_PROVIDES_ANSWER_PATTERNS = [
/\bthe (answer|solution|fix|issue|problem) is\b/i,
/\byou need to\b/i,
/\bhere'?s (what|how)\b/i,
/\bjust (do|use|add|remove|change)\b/i,
/```[\s\S]{20,}```/, // user pasting code blocks
]
// Load skip patterns from tuned config
const _skipFromTuned = (() => {
if (fs.existsSync(TUNED_PATTERNS_FILE)) {
try {
const tuned = JSON.parse(fs.readFileSync(TUNED_PATTERNS_FILE, 'utf8'))
if (tuned.skip?.length > 0) return tuned.skip.map(p => new RegExp(p, 'i'))
} catch {}
}
return null
})()
const SKIP_PATTERNS = _skipFromTuned || [
/^!/,
/^<command-name>/,
/^<system-reminder>/,
/^<task-notification>/,
/^<local-command/,
/^Base directory for this skill/,
/^Implement the following plan/,
/^This session is being continued/,
/^Service Unavailable/,
/^fleeky@\w+/,
/tool_use_id/,
/tool_result/,
/^Godot Engine v/,
/^Vulkan \d/,
/^\[Request interrupted/,
]
const INTERRUPTION_PATTERN = /^\[Request interrupted/
// ─── Text Helpers ───
function isRealUserMessage (text) {
if (!text || text.length < 3) return false
if (SKIP_PATTERNS.some(p => p.test(text))) return false
return true
}
// ─── Turn Builder ───
// A "turn" = one user message + all assistant responses until the next user message.
// Works with pre-extracted entries from scan-sessions.js.
function buildTurns (entries) {
const turns = []
let currentTurn = null
for (const entry of entries) {
if (entry.type === 'user') {
if (entry.toolUseResult || entry.sourceToolAssistantUUID) {
if (currentTurn) currentTurn.toolResults.push(entry)
continue
}
if (!isRealUserMessage(entry.text)) {
if (currentTurn) currentTurn.metaEntries.push(entry)
continue
}
if (currentTurn) turns.push(currentTurn)
currentTurn = {
userText: entry.text,
userTimestamp: entry.timestamp,
permissionMode: entry.permissionMode,
cwd: entry.cwd,
gitBranch: entry.gitBranch,
isSidechain: entry.isSidechain || false,
assistantMessages: [],
toolResults: [],
systemEntries: [],
metaEntries: [],
queueOps: [],
}
} else if (entry.type === 'assistant') {
if (currentTurn) currentTurn.assistantMessages.push(entry)
} else if (entry.type === 'system') {
if (currentTurn) currentTurn.systemEntries.push(entry)
} else if (entry.type === 'queue-operation') {
if (currentTurn) currentTurn.queueOps.push(entry)
}
}
if (currentTurn) turns.push(currentTurn)
return turns
}
// ─── Sensor Extractors ───
// Work with pre-extracted assistant entries from scan-sessions.js.
function extractToolCallCount (turn) {
return turn.assistantMessages.reduce((sum, msg) => sum + (msg.toolUseCount || 0), 0)
}
function extractEditFiles (turn) {
return turn.assistantMessages.flatMap(msg => msg.editFiles || [])
}
function extractTurnDuration (turn) {
for (const sys of turn.systemEntries) {
if (sys.subtype === 'turn_duration' && sys.durationMs) return sys.durationMs
}
return null
}
function extractTokenUsage (turn) {
let totalOutput = 0, totalCacheRead = 0, totalCacheCreate = 0, totalInput = 0
for (const msg of turn.assistantMessages) {
totalOutput += msg.outputTokens || 0
totalCacheRead += msg.cacheRead || 0
totalCacheCreate += msg.cacheCreate || 0
totalInput += msg.inputTokens || 0
}
return { totalOutput, totalCacheRead, totalCacheCreate, totalInput }
}
function extractThinkingVsOutput (turn) {
let thinkingChars = 0, outputChars = 0
for (const msg of turn.assistantMessages) {
thinkingChars += msg.thinkingChars || 0
outputChars += msg.textChars || 0
}
return { thinkingChars, outputChars }
}
function extractToolUseChars (turn) {
return turn.assistantMessages.reduce((sum, msg) => sum + (msg.toolUseChars || 0), 0)
}
function extractBuildErrors (turn) {
let errorCount = 0
for (const tr of turn.toolResults) {
const text = tr.text || ''
if (/error|Error|ERROR|FAIL|fatal|panic|exception|TypeError|ReferenceError|SyntaxError/i.test(text)) {
errorCount++
}
}
return errorCount
}
// ─── Per-Turn Scoring ───
function scoreTurn (turn, prevTurn, weights, llmIndex) {
const sensors = {}
// 1. Correction/Approval — with LLM confidence weighting
// Three signal quality tiers:
// LLM-confirmed (confidence 0.7+): full correction/approval weight
// Regex-detected, no LLM data: discounted (might be false positive)
// LLM says neutral but regex says correction: trust LLM, treat as neutral
const text = turn.userText
const regexCorrection = CORRECTION_PATTERNS.some(p => p.test(text))
const regexApproval = APPROVAL_PATTERNS.some(p => p.test(text))
// Check LLM index for this turn's timestamp
const tsKey = turn.userTimestamp?.slice(0, 20)
const llmSignal = tsKey ? llmIndex[tsKey] : null
let isCorrection = false
let isApproval = false
let signalConfidence = 0.5 // regex default
if (llmSignal) {
// LLM has classified this message — trust it over regex
isCorrection = llmSignal.type === 'correction' && llmSignal.confidence >= 0.5
isApproval = llmSignal.type === 'approval' && llmSignal.confidence >= 0.5
signalConfidence = llmSignal.confidence
} else {
// No LLM data — use regex but at reduced confidence
isCorrection = regexCorrection
isApproval = regexApproval
signalConfidence = 0.5
}
// Sensor values scaled by confidence:
// LLM correction at 0.9 confidence → sensor = 0.0 (strong signal)
// Regex correction at 0.5 confidence → sensor = 0.35 (weaker signal)
// No signal → sensor = 0.7 (neutral)
if (isCorrection) {
sensors.correction = 0.7 * (1 - signalConfidence) // 0.0 at conf=1.0, 0.35 at conf=0.5
} else {
sensors.correction = 0.7
}
if (isApproval) {
sensors.approval = 0.5 + 0.5 * signalConfidence // 1.0 at conf=1.0, 0.75 at conf=0.5
} else {
sensors.approval = 0.5
}
// 2. Tool call count — normalize: 0-3 calls = good, 4-8 = okay, 8+ = fumbling
const toolCalls = extractToolCallCount(turn)
sensors.toolCallCount = toolCalls <= 3 ? 1.0
: toolCalls <= 8 ? 1.0 - (toolCalls - 3) * 0.1
: Math.max(0.1, 1.0 - (toolCalls - 3) * 0.1)
// 3. Turn duration — normalize: <30s = good, 30s-5min = normal, >5min = concerning
const durationMs = extractTurnDuration(turn)
if (durationMs !== null) {
const durationSec = durationMs / 1000
sensors.turnDuration = durationSec <= 30 ? 1.0
: durationSec <= 300 ? 1.0 - (durationSec - 30) * 0.002
: Math.max(0.1, 0.46 - (durationSec - 300) * 0.001)
} else {
sensors.turnDuration = 0.7 // no data = assume okay
}
// 4. Interruptions
sensors.interruptions = INTERRUPTION_PATTERN.test(text) ? 0.0 : 1.0
// 5. User provides answer
const providesAnswer = USER_PROVIDES_ANSWER_PATTERNS.some(p => p.test(text))
sensors.userProvidesAnswer = providesAnswer ? 0.2 : 1.0
// 6. Message length trend (relative to previous turn)
if (prevTurn && prevTurn.userText.length > 30) {
const ratio = text.length / prevTurn.userText.length
// Shortening is potentially bad, but very short approvals are fine
if (ratio < 0.3 && !isApproval && text.length < 20) {
sensors.messageLengthTrend = 0.3 // terse response, not approval = disengagement
} else {
sensors.messageLengthTrend = Math.min(1.0, 0.5 + ratio * 0.5)
}
} else {
sensors.messageLengthTrend = 0.7 // no comparison = neutral
}
// 7. Token efficiency — output tokens vs useful content (text + tool calls)
const usage = extractTokenUsage(turn)
const { thinkingChars, outputChars } = extractThinkingVsOutput(turn)
// Tool use blocks are useful output too — estimate their char contribution
const toolUseChars = extractToolUseChars(turn)
const totalUsefulChars = outputChars + toolUseChars
if (usage.totalOutput > 0 && totalUsefulChars > 0) {
// Roughly 4 chars per token. Compare useful content tokens vs total output tokens
const expectedTokens = totalUsefulChars / 4
const efficiency = Math.min(1.0, expectedTokens / Math.max(1, usage.totalOutput))
sensors.tokenEfficiency = Math.max(0.2, efficiency)
} else {
sensors.tokenEfficiency = 0.7 // no data
}
// 8. Edit survival — checked by looking ahead (caller handles this)
// Default to 1.0, overridden by session-level analysis
sensors.editSurvival = 1.0
// 9. Build errors in tool results
const buildErrors = extractBuildErrors(turn)
sensors.buildErrors = buildErrors === 0 ? 1.0
: buildErrors === 1 ? 0.6
: buildErrors <= 3 ? 0.3
: 0.1
// 10. Permission mode shift
if (prevTurn && prevTurn.permissionMode && turn.permissionMode) {
const prev = prevTurn.permissionMode
const curr = turn.permissionMode
if (prev === 'acceptEdits' && curr === 'default') {
sensors.permissionShift = 0.1 // trust revoked = very bad signal
} else if (prev === 'default' && curr === 'acceptEdits') {
sensors.permissionShift = 1.0 // trust granted
} else {
sensors.permissionShift = 0.7 // no change
}
} else {
sensors.permissionShift = 0.7
}
// 11. Cache hit rate
const totalCache = usage.totalCacheRead + usage.totalCacheCreate
if (totalCache > 0) {
sensors.cacheHitRate = usage.totalCacheRead / totalCache
} else {
sensors.cacheHitRate = 0.5 // no cache data
}
// 12. Thinking ratio — high thinking relative to output = confusion
if (thinkingChars > 0 && outputChars > 0) {
const ratio = thinkingChars / (thinkingChars + outputChars)
// Some thinking is normal (0.3-0.5). Very high (>0.8) = confusion
sensors.thinkingRatio = ratio <= 0.5 ? 1.0
: ratio <= 0.8 ? 1.0 - (ratio - 0.5) * 2
: Math.max(0.1, 0.4 - (ratio - 0.8) * 2)
} else {
sensors.thinkingRatio = 0.7
}
// 13. Sidechain usage
sensors.sidechainUsage = turn.isSidechain ? 0.3 : 1.0
// 14. Queue depth (enqueue = user typing while Claude works = engagement OR frustration)
const enqueues = turn.queueOps.filter(q => q.operation === 'enqueue').length
sensors.queueDepth = enqueues === 0 ? 0.7
: enqueues === 1 ? 0.8 // single queue = engaged
: 0.5 // multiple queues = frustrated, typing faster than Claude works
// 15. Time between turns
if (prevTurn && prevTurn.userTimestamp && turn.userTimestamp) {
const deltaSec = (new Date(turn.userTimestamp) - new Date(prevTurn.userTimestamp)) / 1000
// Very fast response after long Claude turn could mean skipped output
// Very long gap could mean user left
sensors.timeBetweenTurns = deltaSec <= 5 ? 0.9 // quick = engaged
: deltaSec <= 60 ? 1.0 // normal
: deltaSec <= 300 ? 0.8 // reading
: 0.5 // long gap = might have left
} else {
sensors.timeBetweenTurns = 0.7
}
// ─── Fuse ───
let weightedSum = 0
let totalWeight = 0
for (const [sensor, value] of Object.entries(sensors)) {
const weight = weights[sensor] || 0
weightedSum += value * weight
totalWeight += weight
}
const fusedScore = totalWeight > 0 ? weightedSum / totalWeight : 0.5
return {
score: Math.round(fusedScore * 1000) / 1000,
sensors,
toolCalls,
editFiles: extractEditFiles(turn),
isCorrection,
isApproval,
textSnippet: text.slice(0, 150),
}
}
// ─── Edit Survival Analysis ───
// Look ahead: if an Edit to file F in turn N is followed by another Edit to F in turn N+1 or N+2,
// the first edit likely didn't survive (was a failed attempt).
function analyzeEditSurvival (turns, turnScores) {
for (let i = 0; i < turns.length; i++) {
const editFiles = extractEditFiles(turns[i])
if (editFiles.length === 0) continue
let reverted = 0
for (const fp of editFiles) {
// Check next 2 turns for re-edits to same file
for (let j = i + 1; j <= Math.min(i + 2, turns.length - 1); j++) {
const nextEdits = extractEditFiles(turns[j])
if (nextEdits.includes(fp)) {
// Same file edited again — possible revert
// Only count if the next turn had a correction
if (turnScores[j] && turnScores[j].isCorrection) {
reverted++
break
}
}
}
}
const survivalRate = editFiles.length > 0
? 1.0 - (reverted / editFiles.length)
: 1.0
if (turnScores[i]) {
turnScores[i].sensors.editSurvival = Math.max(0.1, survivalRate)
// Recompute fused score with updated edit survival
// (skip full recompute — just adjust proportionally)
}
}
}
// ─── Session Analysis ───
function analyzeSession (scannedSession, weights, llmIndex) {
const { entries, sessionId, sessionDate } = scannedSession
const turns = buildTurns(entries)
if (turns.length < 2) return null // need at least 2 turns for meaningful analysis
// Score each turn
const turnScores = []
for (let i = 0; i < turns.length; i++) {
const prevTurn = i > 0 ? turns[i - 1] : null
const score = scoreTurn(turns[i], prevTurn, weights, llmIndex)
turnScores.push(score)
}
// Post-process: edit survival
analyzeEditSurvival(turns, turnScores)
// Session-level aggregation
// Two components:
// 1. Per-turn average (70%) — weighted average of turn scores
// 2. Session-level signals (30%) — aggregate metrics that don't dilute with turn count
const perTurnScores = turnScores.map(t => t.score)
const turnAvg = perTurnScores.reduce((a, b) => a + b, 0) / perTurnScores.length
// Session-level signals (0-1 scale, 1 = good)
const sessionSignals = {}
// Correction ratio — what fraction of turns had corrections?
// 0% corrections = 1.0, 5% = 0.75, 10% = 0.5, 20%+ = 0.1
const correctionRatio = turnScores.filter(t => t.isCorrection).length / turnScores.length
sessionSignals.correctionRate = Math.max(0.1, 1.0 - correctionRatio * 5)
// Approval ratio — what fraction had approvals?
const approvalRatio = turnScores.filter(t => t.isApproval).length / turnScores.length
sessionSignals.approvalRate = Math.min(1.0, 0.3 + approvalRatio * 3)
// Correction-to-approval ratio — direct quality measure
const totalC = turnScores.filter(t => t.isCorrection).length
const totalA = turnScores.filter(t => t.isApproval).length
if (totalC + totalA > 0) {
sessionSignals.correctionApprovalRatio = totalA / (totalC + totalA)
} else {
sessionSignals.correctionApprovalRatio = 0.5
}
// Correction burst — consecutive corrections are much worse than spread out
let maxCorrectionStreak = 0
let streak = 0
for (const t of turnScores) {
if (t.isCorrection) { streak++; maxCorrectionStreak = Math.max(maxCorrectionStreak, streak) }
else streak = 0
}
sessionSignals.correctionBurst = maxCorrectionStreak <= 1 ? 1.0
: maxCorrectionStreak === 2 ? 0.6
: maxCorrectionStreak === 3 ? 0.3
: 0.1
// Message length trajectory across session
const userLengths = turns.map(t => t.userText.length).filter(l => l > 5)
if (userLengths.length >= 4) {
const half = Math.floor(userLengths.length / 2)
const firstAvg = userLengths.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondAvg = userLengths.slice(half).reduce((a, b) => a + b, 0) / (userLengths.length - half)
const ratio = firstAvg > 0 ? secondAvg / firstAvg : 1
sessionSignals.engagementTrajectory = ratio >= 0.5 ? Math.min(1.0, 0.5 + ratio * 0.3) : 0.3
} else {
sessionSignals.engagementTrajectory = 0.7
}
// Fuse: 70% per-turn average + 30% session-level
const sessionSignalAvg = Object.values(sessionSignals).reduce((a, b) => a + b, 0) / Object.values(sessionSignals).length
const avgScore = turnAvg * 0.7 + sessionSignalAvg * 0.3
// Trajectory: compare first half vs second half
let trajectory = 'stable'
if (perTurnScores.length >= 4) {
const half = Math.floor(perTurnScores.length / 2)
const firstHalf = perTurnScores.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondHalf = perTurnScores.slice(half).reduce((a, b) => a + b, 0) / (perTurnScores.length - half)
const delta = secondHalf - firstHalf
if (delta > 0.1) trajectory = 'improving'
else if (delta < -0.1) trajectory = 'declining'
}
// Sensor averages across all turns
const sensorNames = Object.keys(turnScores[0]?.sensors || {})
const sensorAverages = {}
for (const name of sensorNames) {
const values = turnScores.map(t => t.sensors[name]).filter(v => v !== undefined)
sensorAverages[name] = values.length > 0
? Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 1000) / 1000
: 0.5
}
// Count corrections and approvals
const corrections = turnScores.filter(t => t.isCorrection).length
const approvals = turnScores.filter(t => t.isApproval).length
const totalToolCalls = turnScores.reduce((sum, t) => sum + t.toolCalls, 0)
// Git branch and cwd (most common across turns)
const branches = turns.map(t => t.gitBranch).filter(Boolean)
const cwds = turns.map(t => t.cwd).filter(Boolean)
const gitBranch = mode(branches) || null
const cwd = mode(cwds) || null
// Derived metrics for clearer observability
const density = turns.length > 0 ? corrections / turns.length : 0
const approvalDensity = turns.length > 0 ? approvals / turns.length : 0
// Trusted-sensor-only per-turn score (top 5 sensors from audit)
// High signal-to-noise; not polluted by rare-event sensors
const TRUSTED_SENSORS = ['correction', 'approval', 'userProvidesAnswer', 'buildErrors', 'editSurvival']
let trustedSum = 0, trustedTurnCount = 0
for (const t of turnScores) {
const sensors = t.sensors
if (!sensors) continue
let sum = 0, count = 0
for (const k of TRUSTED_SENSORS) {
if (sensors[k] != null) { sum += sensors[k]; count++ }
}
if (count > 0) { trustedSum += (sum / count); trustedTurnCount++ }
}
const trustedScore = trustedTurnCount > 0
? Math.round((trustedSum / trustedTurnCount) * 1000) / 1000
: null
return {
sessionId,
date: sessionDate?.slice(0, 10),
timestamp: sessionDate,
gitBranch,
cwd,
turns: turns.length,
score: Math.round(avgScore * 1000) / 1000,
// Per-turn score: quality independent of session length
perTurnScore: Math.round(turnAvg * 1000) / 1000,
// Trusted score (top-5 sensors only — PRIMARY quality signal)
trustedScore,
// Session-level aggregate (what penalizes long sessions)
sessionSignalScore: Math.round(sessionSignalAvg * 1000) / 1000,
// Derived densities
density: Math.round(density * 1000) / 1000,
approvalDensity: Math.round(approvalDensity * 1000) / 1000,
// Correction burst — first-class metric. Length of longest run of consecutive
// corrections. 1 = healthy (isolated mistakes); 2+ = stuck pattern.
correctionBurst: maxCorrectionStreak,
corrections,
approvals,
totalToolCalls,
// Session length as first-class metric — longer = more ambitious work
sessionLength: {
turns: turns.length,
toolCalls: totalToolCalls,
toolsPerTurn: turns.length > 0 ? Math.round((totalToolCalls / turns.length) * 10) / 10 : 0
},
trajectory,
sessionSignals,
sensorAverages,
turnScores,
}
}
function mode (arr) {
if (arr.length === 0) return null
const counts = {}
for (const v of arr) counts[v] = (counts[v] || 0) + 1
return Object.entries(counts).sort((a, b) => b[1] - a[1])[0][0]
}
// ─── Cross-Session Analysis ───
function detectSessionRestarts (sessions) {
// Detect sessions that restart on the same topic (same branch + cwd within 2 hours)
// This signals abandonment of the previous session
const sorted = [...sessions].sort((a, b) => (a.timestamp || '').localeCompare(b.timestamp || ''))
const restarts = []
for (let i = 1; i < sorted.length; i++) {
const prev = sorted[i - 1]
const curr = sorted[i]
if (!prev.timestamp || !curr.timestamp) continue
const deltaSec = (new Date(curr.timestamp) - new Date(prev.timestamp)) / 1000
const sameContext = prev.gitBranch && curr.gitBranch &&
prev.gitBranch === curr.gitBranch &&
prev.cwd === curr.cwd
if (sameContext && deltaSec < 7200 && deltaSec > 60) {
restarts.push({
from: prev.sessionId,
to: curr.sessionId,
deltaSec: Math.round(deltaSec),
branch: curr.gitBranch,
})
// Penalize the abandoned session
if (prev.score) prev.score = Math.max(0.1, prev.score - 0.1)
}
}
return restarts
}
function computeWeeklyScores (sessions) {
const weeks = {}
for (const s of sessions) {
if (!s.date) continue
const d = new Date(s.date)
d.setDate(d.getDate() - d.getDay())
const weekKey = d.toISOString().slice(0, 10)
if (!weeks[weekKey]) {
weeks[weekKey] = {
sessions: 0,
totalScore: 0,
totalPerTurnScore: 0,
totalTrustedScore: 0,
trustedScoreSessions: 0,
totalTurns: 0,
totalDensity: 0,
totalCorrectionBurst: 0,
burstSessions: 0,
maxCorrectionBurst: 0,
corrections: 0,
approvals: 0,
toolCalls: 0
}
}
weeks[weekKey].sessions++
weeks[weekKey].totalScore += s.score
weeks[weekKey].totalPerTurnScore += (s.perTurnScore != null ? s.perTurnScore : s.score)
// Trusted score aggregate (NEW primary metric)
if (s.trustedScore != null) {
weeks[weekKey].totalTrustedScore += s.trustedScore
weeks[weekKey].trustedScoreSessions++
}
// Correction-burst aggregate — longest streak of consecutive corrections
if (s.correctionBurst != null) {
weeks[weekKey].totalCorrectionBurst += s.correctionBurst
weeks[weekKey].burstSessions++
weeks[weekKey].maxCorrectionBurst = Math.max(weeks[weekKey].maxCorrectionBurst, s.correctionBurst)
}
weeks[weekKey].totalTurns += s.turns
weeks[weekKey].totalDensity += (s.density != null ? s.density : (s.corrections / Math.max(1, s.turns)))
weeks[weekKey].corrections += s.corrections
weeks[weekKey].approvals += s.approvals
weeks[weekKey].toolCalls += s.totalToolCalls
}
for (const w of Object.values(weeks)) {
w.avgScore = Math.round((w.totalScore / w.sessions) * 1000) / 1000
w.avgPerTurnScore = Math.round((w.totalPerTurnScore / w.sessions) * 1000) / 1000
w.avgTrustedScore = w.trustedScoreSessions > 0
? Math.round((w.totalTrustedScore / w.trustedScoreSessions) * 1000) / 1000
: null
w.avgCorrectionBurst = w.burstSessions > 0
? Math.round((w.totalCorrectionBurst / w.burstSessions) * 100) / 100
: null
w.avgTurns = Math.round(w.totalTurns / w.sessions)
w.avgDensity = Math.round((w.totalDensity / w.sessions) * 1000) / 1000
}
return weeks
}
// ─── Main ───
async function main () {
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true })
const weights = loadWeights()
const llmIndex = loadLLMIndex()
console.log(`LLM index: ${Object.keys(llmIndex).length} classified messages`)
// CLI args
const recentArg = process.argv.find(a => a.startsWith('--recent='))
const sessionArg = process.argv.find(a => a.startsWith('--session='))
const verbose = process.argv.includes('--verbose')
const noCache = process.argv.includes('--no-cache')
const recentDays = recentArg ? parseInt(recentArg.split('=')[1]) : null
const targetSession = sessionArg ? sessionArg.split('=')[1] : null
// Results cache: keyed by filename + mtime + weights hash
// ~2-5KB per session (scored results) vs ~3MB per session (raw entries)
const RESULTS_CACHE_DIR = path.join(DATA_DIR, 'results-cache')
if (!fs.existsSync(RESULTS_CACHE_DIR)) fs.mkdirSync(RESULTS_CACHE_DIR, { recursive: true })
const weightsHash = crypto.createHash('md5')
.update(JSON.stringify(weights) + JSON.stringify(Object.keys(llmIndex).length))
.digest('hex').slice(0, 8)
// Collect all transcript files
const allFiles = []
for (const dir of SESSIONS_DIRS) {
if (!fs.existsSync(dir)) continue
const files = fs.readdirSync(dir)
.filter(f => f.endsWith('.jsonl') && !f.includes('subagent'))
.map(f => path.join(dir, f))
allFiles.push(...files)
}
if (targetSession) {
const match = allFiles.find(f => path.basename(f).includes(targetSession))
if (match) { allFiles.length = 0; allFiles.push(match) }
}
const sessions = []
let skipped = 0
let cacheHits = 0
let cacheMisses = 0
const cutoffDate = recentDays
? new Date(Date.now() - recentDays * 86400000).toISOString()
: null
for (const file of allFiles) {
let fileMtime
try {
const stat = fs.statSync(file)
fileMtime = stat.mtimeMs
if (cutoffDate && stat.mtime.toISOString() < cutoffDate) { skipped++; continue }
} catch { continue }
// Try results cache
const cacheFile = path.join(RESULTS_CACHE_DIR, path.basename(file, '.jsonl') + `-${weightsHash}.json`)
if (!noCache && fs.existsSync(cacheFile)) {
try {
const cacheStat = fs.statSync(cacheFile)
if (cacheStat.mtimeMs > fileMtime) {
// Cache is newer than source — use it
const cached = JSON.parse(fs.readFileSync(cacheFile, 'utf8'))
if (cached) { sessions.push(cached); cacheHits++; continue }
}
} catch {}
}
// Cache miss — full scan + analyze
const scanned = await scanSession(file)
const result = analyzeSession(scanned, weights, llmIndex)
if (result) {
sessions.push(result)
// Cache if file is older than 60s (not the active session)
if (Date.now() - fileMtime > 60000) {
try { fs.writeFileSync(cacheFile, JSON.stringify(result)) } catch {}
}
}
cacheMisses++
}
console.log(`Sessions: ${sessions.length} (${cacheHits} cached, ${cacheMisses} scanned, ${skipped} skipped)`)
// Sort by date
sessions.sort((a, b) => (a.timestamp || '').localeCompare(b.timestamp || ''))
// Cross-session analysis
const restarts = detectSessionRestarts(sessions)
// Weekly aggregation
const weekly = computeWeeklyScores(sessions)
// ─── Output ───
const report = {
generated: new Date().toISOString(),
totalSessions: sessions.length,
skippedFiles: skipped,
sensorWeights: weights,
weekly,
sessionRestarts: restarts,
sessions: sessions.map(s => ({
sessionId: s.sessionId,
date: s.date,
gitBranch: s.gitBranch,
cwd: s.cwd,
turns: s.turns,
score: s.score,
// Per-turn score (length-independent)
perTurnScore: s.perTurnScore,
// Trusted-score (top-5 sensors only) — PRIMARY quality metric
trustedScore: s.trustedScore,
sessionSignalScore: s.sessionSignalScore,
density: s.density,
approvalDensity: s.approvalDensity,
// Correction-burst: longest run of consecutive corrections (added 2026-04-13)
correctionBurst: s.correctionBurst,
sessionLength: s.sessionLength,
corrections: s.corrections,
approvals: s.approvals,
totalToolCalls: s.totalToolCalls,
trajectory: s.trajectory,
sensorAverages: s.sensorAverages,
// Omit per-turn details in the output to keep file size down
// Use --verbose to see them in console
})),
}
fs.writeFileSync(OUTPUT_FILE, JSON.stringify(report, null, 2))
// Write detailed per-turn sensor data for meta-ratchet weight learning
const detailData = {
generated: new Date().toISOString(),
sensorWeights: weights,
sessions: sessions.map(s => ({
sessionId: s.sessionId,
date: s.date,
cwd: s.cwd,
gitBranch: s.gitBranch,
turns: (s.turnScores || []).map(t => ({
score: t.score,
isCorrection: t.isCorrection,
isApproval: t.isApproval,
text: t.textSnippet?.slice(0, 200),
toolCalls: t.toolCalls,
editFiles: t.editFiles,
sensors: t.sensors,
})),
})),
}
fs.writeFileSync(DETAIL_FILE, JSON.stringify(detailData))
console.log(`\nWrote ${OUTPUT_FILE}`)
console.log(`Wrote ${DETAIL_FILE} (${detailData.sessions.reduce((s, sess) => s + sess.turns.length, 0)} turns)`)
// ─── Console Report ───
console.log(`\n${'═'.repeat(60)}`)
console.log(' SENSOR FUSION SCORECARD')
console.log('═'.repeat(60))
console.log(` Sessions analyzed: ${sessions.length}`)
if (skipped > 0) console.log(` Skipped (outside date range): ${skipped}`)
if (restarts.length > 0) console.log(` Session restarts detected: ${restarts.length}`)
// Weekly trend
console.log('\n WEEKLY TREND')
console.log(' ' + '-'.repeat(56))
for (const [week, data] of Object.entries(weekly).sort()) {
const bar = scoreBar(data.avgScore)
console.log(` ${week} ${bar} ${data.avgScore.toFixed(3)} (${data.sessions} sessions, ${data.corrections}✗ ${data.approvals}✓)`)
}
// Worst sessions
const worst = [...sessions].sort((a, b) => a.score - b.score).slice(0, 5)
if (worst.length > 0 && worst[0].score < 0.6) {
console.log('\n WORST SESSIONS')
console.log(' ' + '-'.repeat(56))
for (const s of worst) {
console.log(` ${s.date || '?'} [${s.score.toFixed(3)}] ${s.turns} turns, ${s.corrections}✗ ${s.approvals}✓ ${s.gitBranch || ''}`)
// Show which sensors dragged the score down
const lowSensors = Object.entries(s.sensorAverages)
.filter(([, v]) => v < 0.5)
.sort((a, b) => a[1] - b[1])
.slice(0, 3)
if (lowSensors.length > 0) {
console.log(` low: ${lowSensors.map(([k, v]) => `${k}=${v.toFixed(2)}`).join(', ')}`)
}
}
}
// Best sessions
const best = [...sessions].sort((a, b) => b.score - a.score).slice(0, 3)
if (best.length > 0) {
console.log('\n BEST SESSIONS')
console.log(' ' + '-'.repeat(56))
for (const s of best) {
console.log(` ${s.date || '?'} [${s.score.toFixed(3)}] ${s.turns} turns, ${s.corrections}✗ ${s.approvals}✓ ${s.gitBranch || ''}`)
}
}
// Sensor health — which sensors are consistently low across sessions?
if (sessions.length >= 3) {
const sensorNames = Object.keys(sessions[0]?.sensorAverages || {})
const globalSensorAvgs = {}
for (const name of sensorNames) {
const values = sessions.map(s => s.sensorAverages[name]).filter(v => v !== undefined)
globalSensorAvgs[name] = values.reduce((a, b) => a + b, 0) / values.length
}
const weakSensors = Object.entries(globalSensorAvgs)
.filter(([, v]) => v < 0.6)
.sort((a, b) => a[1] - b[1])
if (weakSensors.length > 0) {
console.log('\n WEAK SENSORS (avg < 0.6 across all sessions)')
console.log(' ' + '-'.repeat(56))
for (const [name, avg] of weakSensors) {
console.log(` ${scoreBar(avg)} ${avg.toFixed(3)} ${name}`)
}
}
}
// Verbose: per-turn details for each session
if (verbose) {
console.log('\n PER-TURN DETAILS')
console.log(' ' + '-'.repeat(56))
for (const s of sessions) {
console.log(`\n Session: ${s.sessionId?.slice(0, 8)} (${s.date})`)
for (let i = 0; i < (s.turnScores || []).length; i++) {
const t = s.turnScores[i]
const flags = []
if (t.isCorrection) flags.push('✗')
if (t.isApproval) flags.push('✓')
console.log(` [${i}] ${t.score.toFixed(3)} ${flags.join('')} tools=${t.toolCalls} "${t.textSnippet?.slice(0, 80)}"`)
}
}
}
console.log(`\n${'═'.repeat(60)}`)
}
function scoreBar (score) {
const filled = Math.round(score * 10)
const empty = 10 - filled
return '█'.repeat(filled) + '░'.repeat(empty)
}
main().catch(err => {
console.error('Fusion analysis failed:', err.message)
process.exit(1)
})