#!/usr/bin/env node
/**
* advanced-metrics.js — Compute derived metrics from turn-level detail
*
* Metrics:
* 1. time-to-correction — avg turns from bad turn to next correction
* 2. per-domain per-turn score — quality broken down by language/domain
* 3. context saturation — cumulative message/token growth per session
* 4. plan adherence — TODO completion vs creation (when tasks are tracked)
*
* Output: enriches sessions with derived metrics, writes to data/advanced-metrics.json
*/
const fs = require('fs')
const path = require('path')
const DATA_DIR = path.join(__dirname, '..', 'data')
function readJSON (name) {
const file = path.join(DATA_DIR, name)
if (!fs.existsSync(file)) return null
return JSON.parse(fs.readFileSync(file, 'utf8'))
}
/**
* Infer language/domain of a file path (subset of classify-files logic).
*/
function classify (file) {
if (!file) return { lang: 'unknown', domain: 'unknown' }
let lang = 'unknown'
if (/\.(cpp|cc|h|hpp)$/.test(file)) lang = 'cpp'
else if (/\.gd$/.test(file)) lang = 'gdscript'
else if (/\.(glsl|gdshader|shader)$/.test(file)) lang = 'glsl'
else if (/\.py$/.test(file)) lang = 'python'
else if (/\.js$/.test(file)) lang = 'javascript'
else if (/\.json$/.test(file)) lang = 'json'
else if (/\.md$/.test(file)) lang = 'markdown'
else if (/\.scm$/.test(file)) lang = 'scheme'
let domain = 'unknown'
if (file.includes('/code/godot/')) domain = 'godot'
else if (file.includes('/memory/') || file.includes('MEMORY.md')) domain = 'claude-memory'
else if (file.includes('.claude/')) domain = 'claude-config'
else if (file.includes('knowledge-ratchet')) domain = 'ratchet'
else if (file.includes('hyper-fleeky/') || /(pear-git|hypernote|hyper-|wirecore|codetree|trust-map)/.test(file)) domain = 'p2p-stack'
return { lang, domain }
}
/**
* 1. Time-to-correction per session.
* For each turn that scored low OR had a build error, find how many turns
* until the next correction (if any).
*/
function timeToCorrection (session) {
const turns = session.turns || []
if (turns.length === 0) return { samples: 0, avgTurns: null, quickRecoveryRate: null }
// Find "bad turn" indices — low score or build errors
const badIndices = []
for (let i = 0; i < turns.length; i++) {
const t = turns[i]
if (t.score < 0.5 || (t.sensors && t.sensors.buildErrors < 0.5)) badIndices.push(i)
}
// For each bad turn, time to next correction
const distances = []
for (const badIdx of badIndices) {
for (let j = badIdx + 1; j < turns.length; j++) {
if (turns[j].isCorrection) {
distances.push(j - badIdx)
break
}
}
}
if (distances.length === 0) return { samples: badIndices.length, avgTurns: null, quickRecoveryRate: null }
const avgTurns = distances.reduce((a, b) => a + b, 0) / distances.length
const quickRecoveryRate = distances.filter(d => d <= 2).length / distances.length
return {
samples: distances.length,
avgTurns: Math.round(avgTurns * 10) / 10,
quickRecoveryRate: Math.round(quickRecoveryRate * 100) / 100
}
}
/**
* 2. Per-domain per-turn score.
* Turns that touched a specific domain contribute to that domain's avg score.
*/
function perDomainScore (session) {
const turns = session.turns || []
const byDomain = {}
for (const t of turns) {
if (!t.editFiles || t.editFiles.length === 0) continue
// Classify each edited file's domain
const domains = new Set()
for (const f of t.editFiles) {
const { domain } = classify(f)
domains.add(domain)
}
for (const d of domains) {
if (!byDomain[d]) byDomain[d] = { totalScore: 0, count: 0 }
byDomain[d].totalScore += t.score
byDomain[d].count++
}
}
const result = {}
for (const [d, stats] of Object.entries(byDomain)) {
if (stats.count < 2) continue // need ≥2 turns for signal
result[d] = {
avgScore: Math.round((stats.totalScore / stats.count) * 1000) / 1000,
turnCount: stats.count
}
}
return result
}
/**
* 3. Context saturation — cumulative user-message length as proxy.
* Long sessions with lots of back-and-forth saturate context.
*/
function contextSaturation (session) {
const turns = session.turns || []
if (turns.length === 0) return { turns: 0, totalToolCalls: 0, saturationLevel: 'light' }
// Better proxy: combine turn count × avg tool calls per turn = total activity
// This correlates with context consumption more than user message length
const totalToolCalls = turns.reduce((s, t) => s + (t.toolCalls || 0), 0)
const toolsPerTurn = totalToolCalls / turns.length
// Cache hit rate (from sensors) = context being reused (healthy)
// Low cache hit = fresh context (consuming quota)
const cacheHits = turns.map(t => (t.sensors && t.sensors.cacheHitRate) || 0.5)
const avgCacheHit = cacheHits.reduce((a, b) => a + b, 0) / cacheHits.length
// Saturation = long session × many tools × low cache hits
// Based on actual session distributions we've seen:
// Light: <30 turns OR <100 tool calls
// Medium: 30-80 turns AND 100-500 tool calls
// Heavy: >80 turns OR >500 tool calls
let saturationLevel = 'light'
if (turns.length > 80 || totalToolCalls > 500) saturationLevel = 'heavy'
else if (turns.length > 30 && totalToolCalls > 100) saturationLevel = 'medium'
return {
turns: turns.length,
totalToolCalls,
toolsPerTurn: Math.round(toolsPerTurn * 10) / 10,
avgCacheHit: Math.round(avgCacheHit * 100) / 100,
saturationLevel
}
}
/**
* 4. Plan adherence — rough proxy via TaskCreate/TaskUpdate mentions.
* Counts "TaskCreate" mentions in tool calls vs "status.*completed" mentions.
*/
function planAdherence (session) {
const turns = session.turns || []
let created = 0
let completed = 0
let deleted = 0
// Plan adherence requires raw tool-call detail not present in turn summary.
// Use turn text as a weak proxy: count TaskCreate/TaskUpdate mentions in text.
// This is noisy but gives a directional signal.
for (const t of turns) {
const text = (t.text || '')
const tcMatches = (text.match(/TaskCreate/g) || []).length
const compMatches = (text.match(/status.*completed|TaskUpdate.*completed/gi) || []).length
const delMatches = (text.match(/status.*deleted/gi) || []).length
created += tcMatches
completed += compMatches
deleted += delMatches
}
if (created === 0) return { created: 0, completed: 0, deleted: 0, adherence: null }
// Adherence: completed / created (ignoring deleted — those were abandoned)
const adherence = created > 0 ? completed / created : null
return {
created,
completed,
deleted,
adherence: adherence != null ? Math.round(adherence * 100) / 100 : null
}
}
async function main () {
const detail = readJSON('fused-turns-detail.json')
const fused = readJSON('fused-scores.json')
if (!detail) { console.log('No fused-turns-detail.json'); process.exit(1) }
if (!fused) { console.log('No fused-scores.json'); process.exit(1) }
const detailSessions = detail.sessions || detail
const fusedSessionsById = {}
for (const s of (fused.sessions || [])) fusedSessionsById[s.sessionId] = s
const results = []
const aggregates = {
timeToCorrection: { totalSamples: 0, totalQuick: 0, avgDistances: [] },
perDomain: {},
saturation: { light: 0, medium: 0, heavy: 0 },
planAdherence: { totalCreated: 0, totalCompleted: 0, sessionsWithTasks: 0 }
}
for (const s of detailSessions) {
const ttc = timeToCorrection(s)
const perDom = perDomainScore(s)
const sat = contextSaturation(s)
const plan = planAdherence(s)
results.push({
sessionId: s.sessionId,
date: s.date,
ttc, perDom, sat, plan
})
// Aggregate
if (ttc.samples > 0) {
aggregates.timeToCorrection.totalSamples += ttc.samples
if (ttc.avgTurns !== null) aggregates.timeToCorrection.avgDistances.push(ttc.avgTurns)
if (ttc.quickRecoveryRate !== null) aggregates.timeToCorrection.totalQuick += ttc.quickRecoveryRate * ttc.samples
}
for (const [d, stats] of Object.entries(perDom)) {
if (!aggregates.perDomain[d]) aggregates.perDomain[d] = { totalScore: 0, totalTurns: 0, sessionCount: 0 }
aggregates.perDomain[d].totalScore += stats.avgScore * stats.turnCount
aggregates.perDomain[d].totalTurns += stats.turnCount
aggregates.perDomain[d].sessionCount++
}
aggregates.saturation[sat.saturationLevel]++
if (plan.created > 0) {
aggregates.planAdherence.totalCreated += plan.created
aggregates.planAdherence.totalCompleted += plan.completed
aggregates.planAdherence.sessionsWithTasks++
}
}
// Compute aggregates
const agg = {
timeToCorrection: {
samples: aggregates.timeToCorrection.totalSamples,
avgTurnsAcrossSessions: aggregates.timeToCorrection.avgDistances.length > 0
? Math.round((aggregates.timeToCorrection.avgDistances.reduce((a, b) => a + b, 0) / aggregates.timeToCorrection.avgDistances.length) * 10) / 10
: null,
quickRecoveryOverall: aggregates.timeToCorrection.totalSamples > 0
? Math.round((aggregates.timeToCorrection.totalQuick / aggregates.timeToCorrection.totalSamples) * 100) / 100
: null
},
perDomain: Object.fromEntries(
Object.entries(aggregates.perDomain).map(([d, s]) => [d, {
avgPerTurnScore: s.totalTurns > 0 ? Math.round((s.totalScore / s.totalTurns) * 1000) / 1000 : null,
turnCount: s.totalTurns,
sessionCount: s.sessionCount
}])
),
saturation: aggregates.saturation,
planAdherence: {
totalCreated: aggregates.planAdherence.totalCreated,
totalCompleted: aggregates.planAdherence.totalCompleted,
overallAdherence: aggregates.planAdherence.totalCreated > 0
? Math.round((aggregates.planAdherence.totalCompleted / aggregates.planAdherence.totalCreated) * 100) / 100
: null,
sessionsWithTasks: aggregates.planAdherence.sessionsWithTasks
}
}
const output = { generated: new Date().toISOString(), aggregate: agg, sessions: results }
fs.writeFileSync(path.join(DATA_DIR, 'advanced-metrics.json'), JSON.stringify(output, null, 2))
// Report
console.log('=== Advanced Metrics ===\n')
console.log('── Time to Correction ──')
console.log(` Samples: ${agg.timeToCorrection.samples} bad turns with follow-up corrections`)
console.log(` Avg turns until correction: ${agg.timeToCorrection.avgTurnsAcrossSessions}`)
console.log(` Quick recovery rate (≤2 turns): ${(agg.timeToCorrection.quickRecoveryOverall * 100).toFixed(0)}%`)
console.log('\n── Per-Domain Quality ──')
const sortedDom = Object.entries(agg.perDomain).sort((a, b) => b[1].turnCount - a[1].turnCount)
for (const [d, stats] of sortedDom) {
console.log(` ${d.padEnd(15)} score=${stats.avgPerTurnScore} (${stats.turnCount} turns, ${stats.sessionCount} sessions)`)
}
console.log('\n── Context Saturation ──')
console.log(` Light: ${agg.saturation.light} sessions`)
console.log(` Medium: ${agg.saturation.medium} sessions`)
console.log(` Heavy: ${agg.saturation.heavy} sessions`)
console.log('\n── Plan Adherence ──')
if (agg.planAdherence.overallAdherence !== null) {
console.log(` Tasks created: ${agg.planAdherence.totalCreated}`)
console.log(` Tasks completed: ${agg.planAdherence.totalCompleted}`)
console.log(` Adherence: ${(agg.planAdherence.overallAdherence * 100).toFixed(0)}%`)
console.log(` Sessions using tasks: ${agg.planAdherence.sessionsWithTasks}`)
} else {
console.log(' No TaskCreate/TaskUpdate data found')
}
}
if (require.main === module) {
main().catch(err => { console.error(err.message); process.exit(1) })
}
module.exports = { timeToCorrection, perDomainScore, contextSaturation, planAdherence }