#!/usr/bin/env node
// Ratchet Effectiveness Benchmarks — Knowledge Ratchet Phase 2
//
// Three validation benchmarks to determine if the system is working:
//
// T3: Coverage Map Accuracy
// Do high-uncertainty topics actually have higher correction rates?
// Correlate uncertainty with observed correction rate per topic.
//
// R1: Repeat Correction Rate
// For each principle, count violations per week. A working ratchet
// means the line goes down and stays down.
//
// R3: First-Attempt Success Rate
// What fraction of sessions complete with zero corrections?
// Track weekly. Ratchet = ratio improves over time.
//
// Usage:
// node benchmark-ratchet.js # all benchmarks
// node benchmark-ratchet.js t3 # just T3
// node benchmark-ratchet.js r1 # just R1
// node benchmark-ratchet.js r3 # just R3
const fs = require('fs')
const path = require('path')
const DATA_DIR = path.join(__dirname, 'data')
const CORRECTIONS_LOG = path.join(__dirname, 'corrections.log')
function loadJSON (file) {
if (!fs.existsSync(file)) return null
try { return JSON.parse(fs.readFileSync(file, 'utf8')) }
catch { return null }
}
// ─── Shared: Load corrections with timestamps ───
function loadCorrections () {
if (!fs.existsSync(CORRECTIONS_LOG)) return []
const raw = fs.readFileSync(CORRECTIONS_LOG, 'utf8')
return raw.split('\n---\n')
.filter(entry => entry.trim())
.map(entry => {
const lines = entry.trim().split('\n')
const tsMatch = lines[0]?.match(/^(\d{4}-\d{2}-\d{2}T[\d:.]+Z?)/)
const typeMatch = lines[0]?.match(/\[(correction|approval)\]/)
return {
timestamp: tsMatch?.[1] || null,
type: typeMatch?.[1] || null,
text: lines.slice(1).filter(l => !l.startsWith('cwd:') && !l.startsWith('session:')).join(' ').toLowerCase(),
cwd: lines.find(l => l.startsWith('cwd:'))?.replace('cwd:', '').trim() || '',
}
})
.filter(c => c.timestamp && c.type)
}
function weekKey (timestamp) {
const d = new Date(timestamp)
d.setDate(d.getDate() - d.getDay())
return d.toISOString().slice(0, 10)
}
// ═══════════════════════════════════════════════════════════
// T3: Coverage Map Accuracy
// ═══════════════════════════════════════════════════════════
function benchmarkT3 () {
const coverage = loadJSON(path.join(DATA_DIR, 'coverage-map.json'))
if (!coverage) {
console.log(' No coverage-map.json found. Run coverage-map.js first.')
return null
}
// Two validations:
// A) Does failureRisk correlate with correction rate? (should be ~1.0 by design)
// B) Does failureRisk predict FUTURE corrections? (the real test)
// Compare failureRisk computed from first-half data against correction rate in second-half.
const points = []
for (const [topic, data] of Object.entries(coverage.coverage)) {
const total = data.corrections + data.approvals
if (total < 2) continue
const correctionRate = data.corrections / total
points.push({
topic,
uncertainty: data.uncertainty,
failureRisk: data.failureRisk ?? 0.5,
correctionRate,
observations: data.observations,
corrections: data.corrections,
approvals: data.approvals,
status: data.status,
})
}
if (points.length < 5) {
console.log(' Not enough topics with data for correlation.')
return null
}
// A) Correlation: failureRisk vs correctionRate (sanity check — should be high)
const riskCorrelation = rankCorrelation(
points.map(p => p.failureRisk),
points.map(p => p.correctionRate)
)
// Also check old uncertainty formula for comparison
const uncertaintyCorrelation = rankCorrelation(
points.map(p => p.uncertainty),
points.map(p => p.correctionRate)
)
// B) Bin by failureRisk and compare correction rates
const bins = {
low: points.filter(p => p.failureRisk < 0.3),
medium: points.filter(p => p.failureRisk >= 0.3 && p.failureRisk < 0.6),
high: points.filter(p => p.failureRisk >= 0.6),
}
const binStats = {}
for (const [bin, pts] of Object.entries(bins)) {
if (pts.length === 0) continue
const avgCorrRate = pts.reduce((s, p) => s + p.correctionRate, 0) / pts.length
const avgRisk = pts.reduce((s, p) => s + p.failureRisk, 0) / pts.length
binStats[bin] = {
count: pts.length,
avgCorrectionRate: Math.round(avgCorrRate * 1000) / 1000,
avgFailureRisk: Math.round(avgRisk * 1000) / 1000,
}
}
// ─── Report ───
console.log('═'.repeat(60))
console.log(' T3: COVERAGE MAP ACCURACY')
console.log('═'.repeat(60))
console.log(` Topics with data: ${points.length}`)
console.log(` Spearman correlation (failureRisk vs correctionRate): ${riskCorrelation.toFixed(3)}`)
console.log(` Spearman correlation (old uncertainty vs correctionRate): ${uncertaintyCorrelation.toFixed(3)}`)
if (riskCorrelation > 0.5) {
console.log(' ✓ PASS — failureRisk correlates with actual correction rate')
} else if (riskCorrelation > 0.2) {
console.log(' ~ WEAK — Some correlation but not strong')
} else {
console.log(' ✗ FAIL — No meaningful correlation')
}
const improvement = riskCorrelation - uncertaintyCorrelation
if (improvement > 0.1) {
console.log(` ✓ New formula is better than old by ${improvement.toFixed(3)}`)
} else if (improvement < -0.1) {
console.log(` ✗ Old formula was actually better by ${(-improvement).toFixed(3)}`)
} else {
console.log(` ~ Similar performance (delta ${improvement.toFixed(3)})`)
}
console.log('\n Correction rate by failure risk level:')
for (const [bin, stats] of Object.entries(binStats)) {
const bar = '█'.repeat(Math.round(stats.avgCorrectionRate * 20))
console.log(` ${bin.padEnd(8)} (${String(stats.count).padStart(3)} topics): ${bar} ${(stats.avgCorrectionRate * 100).toFixed(1)}% corr rate (avg risk: ${stats.avgFailureRisk.toFixed(3)})`)
}
// Monotonicity check: do the bins increase?
const binOrder = ['low', 'medium', 'high']
const binRates = binOrder.map(b => binStats[b]?.avgCorrectionRate).filter(v => v !== undefined)
const isMonotonic = binRates.every((v, i) => i === 0 || v >= binRates[i - 1])
console.log(`\n Monotonicity: ${isMonotonic ? '✓ PASS — higher risk = higher correction rate' : '✗ FAIL — bins not monotonically increasing'}`)
// Show extreme topics
const sorted = [...points].sort((a, b) => b.failureRisk - a.failureRisk)
console.log('\n Highest failure risk topics:')
for (const p of sorted.slice(0, 5)) {
console.log(` ${p.topic.padEnd(35)} risk=${p.failureRisk.toFixed(3)} corr=${(p.correctionRate * 100).toFixed(0)}% (${p.corrections}✗ ${p.approvals}✓)`)
}
console.log('\n Lowest failure risk topics:')
for (const p of sorted.slice(-5)) {
console.log(` ${p.topic.padEnd(35)} risk=${p.failureRisk.toFixed(3)} corr=${(p.correctionRate * 100).toFixed(0)}% (${p.corrections}✗ ${p.approvals}✓)`)
}
return { riskCorrelation, uncertaintyCorrelation, improvement, bins: binStats, topicCount: points.length }
}
// Spearman rank correlation
function rankCorrelation (xs, ys) {
const n = xs.length
if (n < 3) return 0
const rankX = computeRanks(xs)
const rankY = computeRanks(ys)
let sumD2 = 0
for (let i = 0; i < n; i++) {
const d = rankX[i] - rankY[i]
sumD2 += d * d
}
return 1 - (6 * sumD2) / (n * (n * n - 1))
}
function computeRanks (values) {
const indexed = values.map((v, i) => ({ v, i }))
indexed.sort((a, b) => a.v - b.v)
const ranks = new Array(values.length)
for (let i = 0; i < indexed.length; i++) {
ranks[indexed[i].i] = i + 1
}
return ranks
}
// ═══════════════════════════════════════════════════════════
// R1: Repeat Correction Rate per Principle per Week
// ═══════════════════════════════════════════════════════════
function benchmarkR1 () {
// R1: Weekly correction DENSITY trend from fused scores
// Since we can't attribute corrections to specific principles without LLM
// semantic matching, we track the aggregate correction density per week.
// A working ratchet means density goes down over time.
const fused = loadJSON(path.join(DATA_DIR, 'fused-scores.json'))
if (!fused) {
console.log(' No fused-scores.json found.')
return null
}
// Group sessions by week
const weekly = {}
for (const session of fused.sessions) {
if (!session.date) continue
const week = weekKey(session.date)
if (!weekly[week]) weekly[week] = { sessions: 0, turns: 0, corrections: 0, approvals: 0, scores: [] }
weekly[week].sessions++
weekly[week].turns += session.turns
weekly[week].corrections += session.corrections
weekly[week].approvals += session.approvals
weekly[week].scores.push(session.score)
}
const allWeeks = Object.keys(weekly).sort()
if (allWeeks.length < 2) {
console.log(' Not enough weeks of data.')
return null
}
// ─── Report ───
console.log('═'.repeat(60))
console.log(' R1: WEEKLY QUALITY TREND (from fused scores)')
console.log('═'.repeat(60))
const densities = []
const fusedScores = []
for (const week of allWeeks) {
const w = weekly[week]
const density = w.turns > 0 ? w.corrections / w.turns : 0
const avgScore = w.scores.length > 0 ? w.scores.reduce((a, b) => a + b, 0) / w.scores.length : 0
densities.push(density)
fusedScores.push(avgScore)
const bar = '█'.repeat(Math.round((1 - density) * 10)) + '░'.repeat(Math.round(density * 10))
console.log(` ${week} ${bar} density=${(density * 100).toFixed(1)}% score=${avgScore.toFixed(3)} (${w.sessions} sess, ${w.turns} turns, ${w.corrections}✗ ${w.approvals}✓)`)
}
// Trend analysis
let trend = 'stable'
let anyRatcheted = false
let anySlipped = false
if (densities.length >= 4) {
const half = Math.floor(densities.length / 2)
const firstDensity = densities.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondDensity = densities.slice(half).reduce((a, b) => a + b, 0) / (densities.length - half)
const firstScore = fusedScores.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondScore = fusedScores.slice(half).reduce((a, b) => a + b, 0) / (fusedScores.length - half)
console.log('\n TREND')
console.log(' ' + '-'.repeat(56))
console.log(` First half: density=${(firstDensity * 100).toFixed(1)}% score=${firstScore.toFixed(3)}`)
console.log(` Second half: density=${(secondDensity * 100).toFixed(1)}% score=${secondScore.toFixed(3)}`)
const densityDelta = secondDensity - firstDensity
const scoreDelta = secondScore - firstScore
console.log(` Density delta: ${densityDelta >= 0 ? '+' : ''}${(densityDelta * 100).toFixed(1)}pp`)
console.log(` Score delta: ${scoreDelta >= 0 ? '+' : ''}${scoreDelta.toFixed(3)}`)
if (densityDelta < -2 && scoreDelta > 0.02) {
trend = 'ratcheted ↓'
anyRatcheted = true
console.log(' ✓ RATCHETING — correction density down, fused score up')
} else if (densityDelta > 2 && scoreDelta < -0.02) {
trend = 'SLIPPED ↑'
anySlipped = true
console.log(' ✗ SLIPPED — correction density up, fused score down')
} else {
console.log(' ~ STABLE')
}
}
console.log('\n SUMMARY')
console.log(' ' + '-'.repeat(56))
if (anyRatcheted) console.log(' ✓ Quality is ratcheting — fewer corrections, higher scores')
if (anySlipped) console.log(' ✗ Quality slipped — more corrections, lower scores')
if (!anyRatcheted && !anySlipped) console.log(' ~ No significant trend detected yet')
console.log(' Note: per-principle attribution requires LLM semantic matching (future work)')
return { trend, weekCount: allWeeks.length, weekly }
}
// ═══════════════════════════════════════════════════════════
// R3: First-Attempt Success Rate
// ═══════════════════════════════════════════════════════════
function benchmarkR3 () {
const fused = loadJSON(path.join(DATA_DIR, 'fused-scores.json'))
if (!fused) {
console.log(' No fused-scores.json found. Run fuse-signals.js first.')
return null
}
// Correction DENSITY: corrections per turn, not binary presence.
// A 200-turn session with 3 corrections (1.5%) is better than
// a 5-turn session with 2 corrections (40%).
// Per-session density
const sessionDensities = fused.sessions
.filter(s => s.turns >= 2)
.map(s => ({
...s,
density: s.turns > 0 ? s.corrections / s.turns : 0,
}))
// Group by week
const weekly = {}
for (const s of sessionDensities) {
if (!s.date) continue
const week = weekKey(s.date)
if (!weekly[week]) weekly[week] = {
sessions: 0, totalTurns: 0, totalCorrections: 0, totalApprovals: 0,
densities: [], scores: [], clean: 0,
}
weekly[week].sessions++
weekly[week].totalTurns += s.turns
weekly[week].totalCorrections += s.corrections
weekly[week].totalApprovals += s.approvals
weekly[week].densities.push(s.density)
weekly[week].scores.push(s.score)
if (s.corrections === 0) weekly[week].clean++
}
const weeks = Object.entries(weekly).sort()
// Overall stats
const totalTurns = sessionDensities.reduce((s, d) => s + d.turns, 0)
const totalCorr = sessionDensities.reduce((s, d) => s + d.corrections, 0)
const overallDensity = totalTurns > 0 ? totalCorr / totalTurns : 0
const cleanCount = sessionDensities.filter(s => s.corrections === 0).length
// ─── Report ───
console.log('═'.repeat(60))
console.log(' R3: SESSION QUALITY (correction density)')
console.log('═'.repeat(60))
console.log(` Sessions: ${sessionDensities.length} (${totalTurns} total turns)`)
console.log(` Total corrections: ${totalCorr}`)
console.log(` Overall density: ${(overallDensity * 100).toFixed(2)}% (corrections per turn)`)
console.log(` Clean sessions (0 corrections): ${cleanCount} (${(cleanCount / sessionDensities.length * 100).toFixed(0)}%)`)
console.log('\n WEEKLY BREAKDOWN')
console.log(' ' + '-'.repeat(60))
console.log(' week sess turns corr density clean% fused')
console.log(' ' + '-'.repeat(60))
const densities = []
for (const [week, data] of weeks) {
const weekDensity = data.totalTurns > 0 ? data.totalCorrections / data.totalTurns : 0
const cleanRate = data.clean / data.sessions
const avgScore = data.scores.reduce((a, b) => a + b, 0) / data.scores.length
densities.push(weekDensity)
// Bar: lower density = better (invert for display)
const quality = Math.max(0, 1 - weekDensity * 10) // 0% density = full bar, 10% = empty
const bar = '█'.repeat(Math.round(quality * 10)) + '░'.repeat(10 - Math.round(quality * 10))
console.log(` ${week} ${String(data.sessions).padStart(4)} ${String(data.totalTurns).padStart(5)} ${String(data.totalCorrections).padStart(4)} ${bar} ${(weekDensity * 100).toFixed(1)}% ${(cleanRate * 100).toFixed(0).padStart(4)}% ${avgScore.toFixed(3)}`)
}
// Trend
if (densities.length >= 4) {
const half = Math.floor(densities.length / 2)
const firstAvg = densities.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondAvg = densities.slice(half).reduce((a, b) => a + b, 0) / (densities.length - half)
console.log('\n TREND (lower density = better)')
console.log(' ' + '-'.repeat(60))
console.log(` First half avg density: ${(firstAvg * 100).toFixed(2)}%`)
console.log(` Second half avg density: ${(secondAvg * 100).toFixed(2)}%`)
const delta = secondAvg - firstAvg
console.log(` Delta: ${delta >= 0 ? '+' : ''}${(delta * 100).toFixed(2)}pp`)
if (delta < -1) {
console.log(' ✓ IMPROVING — Fewer corrections per turn over time')
} else if (delta > 1) {
console.log(' ✗ DECLINING — More corrections per turn over time')
} else {
console.log(' ~ STABLE')
}
}
// By session length — density normalizes this, so both should be comparable
const shortSessions = sessionDensities.filter(s => s.turns <= 10)
const longSessions = sessionDensities.filter(s => s.turns > 10)
if (shortSessions.length > 0 && longSessions.length > 0) {
const shortDensity = shortSessions.reduce((s, d) => s + d.density, 0) / shortSessions.length
const longDensity = longSessions.reduce((s, d) => s + d.density, 0) / longSessions.length
console.log('\n BY SESSION LENGTH (density should be comparable)')
console.log(' ' + '-'.repeat(60))
console.log(` Short (≤10 turns): ${(shortDensity * 100).toFixed(2)}% density (${shortSessions.length} sessions)`)
console.log(` Long (>10 turns): ${(longDensity * 100).toFixed(2)}% density (${longSessions.length} sessions)`)
if (Math.abs(shortDensity - longDensity) < 2) {
console.log(' ✓ Density comparable — metric normalizes for session length')
} else {
console.log(` ~ ${shortDensity < longDensity ? 'Short' : 'Long'} sessions have lower density`)
}
}
return { overallDensity, cleanRate: cleanCount / sessionDensities.length, weekCount: weeks.length }
}
// ═══════════════════════════════════════════════════════════
// R2: Novel vs Repeat Corrections
// ═══════════════════════════════════════════════════════════
// Infer domain from a cwd path.
function cwdToDomain (cwd) {
if (!cwd) return 'unknown'
if (cwd.includes('/code/godot/')) return 'godot'
if (cwd.includes('/hyper-fleeky/')) return 'p2p-stack'
if (cwd.includes('knowledge-ratchet')) return 'ratchet'
if (cwd.includes('hypercore-kb')) return 'p2p-stack'
return 'unknown'
}
function benchmarkR2 () {
const detail = loadJSON(path.join(DATA_DIR, 'fused-turns-detail.json'))
if (!detail) { console.log(' No detail data.'); return null }
// Collect all correction texts chronologically, tagged with domain
const allCorrections = []
for (const sess of detail.sessions) {
const domain = cwdToDomain(sess.cwd)
for (const turn of (sess.turns || [])) {
if (turn.isCorrection && turn.text) {
allCorrections.push({
text: turn.text,
date: sess.date,
session: sess.sessionId,
domain
})
}
}
}
if (allCorrections.length < 5) { console.log(' Not enough corrections.'); return null }
// For each correction, check similarity to all PREVIOUS corrections
// Jaccard similarity on word sets, threshold 0.4 = repeat
function wordSet (text) {
return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter(w => w.length > 2))
}
function jaccard (a, b) {
const intersection = [...a].filter(w => b.has(w)).length
const union = new Set([...a, ...b]).size
return union > 0 ? intersection / union : 0
}
let novel = 0
let repeat = 0
const novelByWeek = {}
const repeatByWeek = {}
const pastSets = []
// Per-domain tracking
const byDomain = {}
for (const corr of allCorrections) {
const words = wordSet(corr.text)
const week = weekKey(corr.date || '2026-01-01')
// Check against all previous corrections
let isRepeat = false
for (const past of pastSets) {
if (jaccard(words, past) > 0.4) {
isRepeat = true
break
}
}
if (isRepeat) {
repeat++
repeatByWeek[week] = (repeatByWeek[week] || 0) + 1
} else {
novel++
novelByWeek[week] = (novelByWeek[week] || 0) + 1
}
pastSets.push(words)
// Per-domain aggregation
if (!byDomain[corr.domain]) byDomain[corr.domain] = { novel: 0, repeat: 0 }
if (isRepeat) byDomain[corr.domain].repeat++
else byDomain[corr.domain].novel++
}
// Compute per-domain novel rate
const perDomain = {}
for (const [d, stats] of Object.entries(byDomain)) {
const total = stats.novel + stats.repeat
perDomain[d] = {
novel: stats.novel,
repeat: stats.repeat,
total,
novelRate: total > 0 ? stats.novel / total : null
}
}
const allWeeks = [...new Set([...Object.keys(novelByWeek), ...Object.keys(repeatByWeek)])].sort()
// ─── Report ───
console.log('═'.repeat(60))
console.log(' R2: NOVEL vs REPEAT CORRECTIONS')
console.log('═'.repeat(60))
console.log(` Total corrections: ${allCorrections.length}`)
console.log(` Novel: ${novel} (${(novel / allCorrections.length * 100).toFixed(0)}%)`)
console.log(` Repeat: ${repeat} (${(repeat / allCorrections.length * 100).toFixed(0)}%)`)
const novelRate = novel / allCorrections.length
if (novelRate > 0.8) {
console.log(' ✓ RATCHETING — >80% of corrections are novel (old mistakes eliminated)')
} else if (novelRate > 0.5) {
console.log(' ~ PARTIAL — Some repeat corrections still occurring')
} else {
console.log(' ✗ NOT RATCHETING — >50% of corrections are repeats')
}
console.log('\n WEEKLY (novel / repeat)')
console.log(' ' + '-'.repeat(60))
for (const week of allWeeks) {
const n = novelByWeek[week] || 0
const r = repeatByWeek[week] || 0
const total = n + r
const pct = total > 0 ? (n / total * 100).toFixed(0) : 'n/a'
const bar = total > 0
? '█'.repeat(Math.round(n / total * 10)) + '░'.repeat(Math.round(r / total * 10))
: ''
console.log(` ${week} ${bar} novel=${n} repeat=${r} (${pct}% novel)`)
}
// Trend
if (allWeeks.length >= 4) {
const half = Math.floor(allWeeks.length / 2)
const firstNovel = allWeeks.slice(0, half).reduce((s, w) => s + (novelByWeek[w] || 0), 0)
const firstRepeat = allWeeks.slice(0, half).reduce((s, w) => s + (repeatByWeek[w] || 0), 0)
const secondNovel = allWeeks.slice(half).reduce((s, w) => s + (novelByWeek[w] || 0), 0)
const secondRepeat = allWeeks.slice(half).reduce((s, w) => s + (repeatByWeek[w] || 0), 0)
const firstRate = (firstNovel + firstRepeat) > 0 ? firstNovel / (firstNovel + firstRepeat) : 0
const secondRate = (secondNovel + secondRepeat) > 0 ? secondNovel / (secondNovel + secondRepeat) : 0
console.log(`\n First half novel rate: ${(firstRate * 100).toFixed(0)}%`)
console.log(` Second half novel rate: ${(secondRate * 100).toFixed(0)}%`)
const delta = secondRate - firstRate
console.log(` Delta: ${delta >= 0 ? '+' : ''}${(delta * 100).toFixed(0)}pp`)
}
// Print per-domain breakdown
console.log('\n BY DOMAIN (novel rate — higher means ratchet preventing repeats)')
console.log(' ' + '-'.repeat(60))
for (const [d, stats] of Object.entries(perDomain).sort((a, b) => b[1].total - a[1].total)) {
if (stats.total < 5) continue
const pct = (stats.novelRate * 100).toFixed(1)
console.log(` ${d.padEnd(15)} ${pct}% novel (${stats.novel}/${stats.total})`)
}
return { novel, repeat, novelRate, weekCount: allWeeks.length, perDomain }
}
// ═══════════════════════════════════════════════════════════
// R4: Loop Resolution Speed
// ═══════════════════════════════════════════════════════════
function benchmarkR4 () {
const loops = loadJSON(path.join(DATA_DIR, 'loops.json'))
if (!loops) { console.log(' No loops data.'); return null }
// Compute resolution RATE — resolved vs total (catches stuck loops)
const allLoops = loops.loops.filter(l => l.date)
const resolved = allLoops.filter(l => l.resolved)
const unresolvedCount = allLoops.length - resolved.length
const resolutionRate = allLoops.length > 0 ? resolved.length / allLoops.length : 0
if (resolved.length < 5) { console.log(' Not enough resolved loops.'); return null }
// Duration percentiles (quick fixes vs sagas)
const sortedDurations = resolved.map(l => l.duration).sort((a, b) => a - b)
const p50 = sortedDurations[Math.floor(sortedDurations.length * 0.5)]
const p90 = sortedDurations[Math.floor(sortedDurations.length * 0.9)]
const longHaulRate = sortedDurations.filter(d => d > 10).length / sortedDurations.length
// Group by type and week, track avg duration
const byTypeWeek = {}
for (const l of resolved) {
const d = new Date(l.date)
d.setDate(d.getDate() - d.getDay())
const week = d.toISOString().slice(0, 10)
if (!byTypeWeek[l.type]) byTypeWeek[l.type] = {}
if (!byTypeWeek[l.type][week]) byTypeWeek[l.type][week] = []
byTypeWeek[l.type][week].push(l.duration)
}
// Overall weekly avg
const overallWeekly = {}
for (const l of resolved) {
const d = new Date(l.date)
d.setDate(d.getDate() - d.getDay())
const week = d.toISOString().slice(0, 10)
if (!overallWeekly[week]) overallWeekly[week] = []
overallWeekly[week].push(l.duration)
}
const allWeeks = Object.keys(overallWeekly).sort()
console.log('═'.repeat(60))
console.log(' R4: LOOP RESOLUTION')
console.log('═'.repeat(60))
console.log(` Total loops: ${allLoops.length} (resolved: ${resolved.length}, stuck: ${unresolvedCount})`)
console.log(` Resolution rate: ${(resolutionRate * 100).toFixed(1)}% ${resolutionRate >= 0.8 ? '✓' : resolutionRate >= 0.6 ? '~' : '✗'}`)
console.log(` Avg duration: ${(resolved.reduce((s, l) => s + l.duration, 0) / resolved.length).toFixed(1)} turns`)
console.log(` P50 duration: ${p50} turns (half resolved this fast or faster)`)
console.log(` P90 duration: ${p90} turns (tail is here — long sagas)`)
console.log(` Long-haul rate: ${(longHaulRate * 100).toFixed(1)}% (fraction of loops >10 turns)`)
console.log('\n WEEKLY AVG RESOLUTION (turns to fix)')
console.log(' ' + '-'.repeat(60))
const weeklyAvgs = []
for (const week of allWeeks) {
const durations = overallWeekly[week]
const avg = durations.reduce((a, b) => a + b, 0) / durations.length
weeklyAvgs.push(avg)
const bar = '█'.repeat(Math.min(20, Math.round(avg)))
console.log(` ${week} ${bar} ${avg.toFixed(1)} turns (${durations.length} loops)`)
}
// Trend
if (weeklyAvgs.length >= 4) {
const half = Math.floor(weeklyAvgs.length / 2)
const firstAvg = weeklyAvgs.slice(0, half).reduce((a, b) => a + b, 0) / half
const secondAvg = weeklyAvgs.slice(half).reduce((a, b) => a + b, 0) / (weeklyAvgs.length - half)
console.log(`\n First half avg: ${firstAvg.toFixed(1)} turns`)
console.log(` Second half avg: ${secondAvg.toFixed(1)} turns`)
const delta = secondAvg - firstAvg
console.log(` Delta: ${delta >= 0 ? '+' : ''}${delta.toFixed(1)} turns`)
if (delta < -1) {
console.log(' ✓ IMPROVING — Loops resolving faster')
} else if (delta > 1) {
console.log(' ✗ SLOWER — Loops taking longer to resolve')
} else {
console.log(' ~ STABLE')
}
}
// By type
console.log('\n BY TYPE (first half → second half)')
console.log(' ' + '-'.repeat(60))
for (const [type, weeks] of Object.entries(byTypeWeek).sort()) {
const sorted = Object.entries(weeks).sort()
const allDurations = sorted.flatMap(([, d]) => d)
if (allDurations.length < 3) continue
const half = Math.floor(sorted.length / 2)
const firstDurations = sorted.slice(0, half).flatMap(([, d]) => d)
const secondDurations = sorted.slice(half).flatMap(([, d]) => d)
const firstAvg = firstDurations.length > 0 ? firstDurations.reduce((a, b) => a + b, 0) / firstDurations.length : 0
const secondAvg = secondDurations.length > 0 ? secondDurations.reduce((a, b) => a + b, 0) / secondDurations.length : 0
const delta = secondAvg - firstAvg
const arrow = delta < -1 ? '↓' : delta > 1 ? '↑' : '='
console.log(` ${arrow} ${type.padEnd(20)} ${firstAvg.toFixed(1)} → ${secondAvg.toFixed(1)} turns (${delta >= 0 ? '+' : ''}${delta.toFixed(1)})`)
}
return {
avgResolution: resolved.reduce((s, l) => s + l.duration, 0) / resolved.length,
resolutionRate,
p50,
p90,
longHaulRate,
totalLoops: allLoops.length,
resolvedCount: resolved.length,
unresolvedCount,
weekCount: allWeeks.length
}
}
// ─── Main ───
const RATCHET_ACTIVE_SINCE = '2026-04-07' // date the full system went live
function main () {
const target = process.argv[2]
const results = {}
console.log(`Ratchet active since: ${RATCHET_ACTIVE_SINCE}\n`)
if (!target || target === 't3') {
results.t3 = benchmarkT3()
console.log('')
}
if (!target || target === 'r1') {
results.r1 = benchmarkR1()
console.log('')
}
if (!target || target === 'r2') {
results.r2 = benchmarkR2()
console.log('')
}
if (!target || target === 'r3') {
results.r3 = benchmarkR3()
console.log('')
}
if (!target || target === 'r4') {
results.r4 = benchmarkR4()
console.log('')
}
// Save current results
const reportFile = path.join(DATA_DIR, 'benchmark-results.json')
const report = {
timestamp: new Date().toISOString(),
ratchetActiveSince: RATCHET_ACTIVE_SINCE,
...results,
}
fs.writeFileSync(reportFile, JSON.stringify(report, null, 2))
// Append to history (trend tracking)
const historyFile = path.join(DATA_DIR, 'benchmark-history.json')
let history = []
if (fs.existsSync(historyFile)) {
try { history = JSON.parse(fs.readFileSync(historyFile, 'utf8')) } catch {}
}
const snapshot = {
date: new Date().toISOString().slice(0, 10),
timestamp: new Date().toISOString(),
novelRate: results.r2?.novelRate ?? null,
repeatRate: results.r2 ? 1 - results.r2.novelRate : null,
// Per-domain novel rates (R2 broken down by domain)
novelRateByDomain: results.r2?.perDomain ?? null,
density: results.r3?.overallDensity ?? null,
// R4 enhanced fields
avgResolution: results.r4?.avgResolution ?? null,
resolutionRate: results.r4?.resolutionRate ?? null,
p50Duration: results.r4?.p50 ?? null,
p90Duration: results.r4?.p90 ?? null,
longHaulRate: results.r4?.longHaulRate ?? null,
totalLoopsAll: results.r4?.totalLoops ?? null,
stuckLoops: results.r4?.unresolvedCount ?? null,
// Trusted quality (from fused-scores latest)
trustedScore: null,
perTurnScore: null,
avgTurns: null,
saturationLevels: null,
// Legacy fields
totalLoops: results.r4?.weekCount ?? null,
sessions: results.r3?.weekCount ?? null,
rulesCount: null,
}
// Load rules count
const rulesFile = path.join(DATA_DIR, 'learned-rules.json')
if (fs.existsSync(rulesFile)) {
try { snapshot.rulesCount = JSON.parse(fs.readFileSync(rulesFile, 'utf8')).totalRules } catch {}
}
// Load latest trustedScore/perTurnScore from fused-scores weekly
const fusedFile = path.join(DATA_DIR, 'fused-scores.json')
if (fs.existsSync(fusedFile)) {
try {
const fused = JSON.parse(fs.readFileSync(fusedFile, 'utf8'))
const weeks = Object.entries(fused.weekly || {}).sort()
if (weeks.length > 0) {
const latest = weeks[weeks.length - 1][1]
snapshot.trustedScore = latest.avgTrustedScore ?? null
snapshot.perTurnScore = latest.avgPerTurnScore ?? null
snapshot.avgTurns = latest.avgTurns ?? null
}
} catch {}
}
// Load saturation distribution from advanced metrics
const advFile = path.join(DATA_DIR, 'advanced-metrics.json')
if (fs.existsSync(advFile)) {
try {
const adv = JSON.parse(fs.readFileSync(advFile, 'utf8'))
snapshot.saturationLevels = adv.aggregate?.saturation ?? null
} catch {}
}
// Dedup by date — only one entry per day
history = history.filter(h => h.date !== snapshot.date)
history.push(snapshot)
// Keep last 365 days
if (history.length > 365) history = history.slice(-365)
fs.writeFileSync(historyFile, JSON.stringify(history, null, 2))
console.log(`Results saved: ${reportFile}`)
console.log(`History: ${history.length} entries in ${historyFile}`)
}
main()