#!/usr/bin/env node
/**
* hook-utility.js — Falsify hook utility by joining fires with corrections
*
* Analysis:
* For each rule-fire recorded in rule-fires.jsonl:
* Find corrections in the same session that came AFTER the fire.
* If a correction occurred within N turns → rule fired but did NOT prevent it.
* If no correction within N turns → rule may have prevented one.
*
* Per hook (attention-router, preflight):
* total_fires — how often the hook fired
* follow_up_correction — fires followed by correction within window
* no_correction — fires NOT followed by correction (plausibly helpful)
*
* Per rule/atom:
* same analysis at atom level — which specific rules are useful
*
* NOTE: This is OBSERVATIONAL, not causal. We can't prove a rule PREVENTED
* a correction — only that fires correlate with outcomes.
*
* Usage:
* node atomspace/hook-utility.js # report
* node atomspace/hook-utility.js --window=5 # corrections within N turns = "failed"
* node atomspace/hook-utility.js --json # machine-readable output
*/
const fs = require('fs')
const path = require('path')
const FIRES_FILE = path.join(__dirname, '..', 'data', 'rule-fires.jsonl')
const CORRECTIONS_LOG = path.join(__dirname, '..', 'corrections.log')
const SESSION_STATE_FILE = '/tmp/claude-session-score.json'
const DEFAULT_WINDOW_SECONDS = 600 // 10 minutes default
function readFires () {
if (!fs.existsSync(FIRES_FILE)) return []
const content = fs.readFileSync(FIRES_FILE, 'utf8')
return content.trim().split('\n').filter(Boolean).map(l => {
try { return JSON.parse(l) } catch { return null }
}).filter(Boolean)
}
/**
* Parse corrections.log into array of { timestamp, type, message, cwd }.
* Format:
* ---
* 2026-04-13T20:29:28.857Z [correction]
* message text...
* cwd: /path
*/
function readCorrections () {
if (!fs.existsSync(CORRECTIONS_LOG)) return []
const content = fs.readFileSync(CORRECTIONS_LOG, 'utf8')
const blocks = content.split('\n---\n').filter(Boolean)
const entries = []
for (const block of blocks) {
const lines = block.trim().split('\n')
if (lines.length < 1) continue
const m = lines[0].match(/^(\d{4}-\d{2}-\d{2}T[\d:.]+Z)\s*\[(correction|approval)\]/)
if (!m) continue
const timestamp = m[1]
const type = m[2]
const cwdLine = lines.find(l => l.startsWith('cwd:'))
const cwd = cwdLine ? cwdLine.slice(4).trim() : null
const message = lines.slice(1).filter(l => !l.startsWith('cwd:')).join('\n').trim()
entries.push({ timestamp, type, message, cwd, ts: new Date(timestamp).getTime() })
}
return entries
}
/**
* For each fire, check if a correction followed in the same cwd within window.
*/
function analyzeFires (fires, corrections, windowSec) {
const windowMs = windowSec * 1000
// Only analyze corrections (not approvals)
const correctionEvents = corrections.filter(c => c.type === 'correction')
const results = []
for (const fire of fires) {
const fireTs = new Date(fire.timestamp).getTime()
// Find corrections in the window that came AFTER this fire
const followUp = correctionEvents.filter(c =>
c.ts > fireTs && c.ts <= fireTs + windowMs
)
results.push({ ...fire, fireTs, followUpCount: followUp.length, hadCorrection: followUp.length > 0 })
}
return results
}
/**
* Aggregate utility by hook and by atom.
*/
function aggregate (analyzed) {
const byHook = {}
const byAtom = {}
for (const r of analyzed) {
// Hook-level
if (!byHook[r.hook]) byHook[r.hook] = { fires: 0, followedByCorrection: 0, cleanFires: 0 }
byHook[r.hook].fires++
if (r.hadCorrection) byHook[r.hook].followedByCorrection++
else byHook[r.hook].cleanFires++
// Atom-level
const key = r.atom
if (!byAtom[key]) byAtom[key] = { fires: 0, followedByCorrection: 0, cleanFires: 0 }
byAtom[key].fires++
if (r.hadCorrection) byAtom[key].followedByCorrection++
else byAtom[key].cleanFires++
}
// Compute "clean rate" — fires not followed by correction
for (const stats of Object.values(byHook)) {
stats.cleanRate = stats.fires > 0 ? stats.cleanFires / stats.fires : 0
}
for (const stats of Object.values(byAtom)) {
stats.cleanRate = stats.fires > 0 ? stats.cleanFires / stats.fires : 0
}
return { byHook, byAtom }
}
function main () {
const args = process.argv.slice(2)
const windowArg = args.find(a => a.startsWith('--window='))
const windowSec = windowArg ? parseInt(windowArg.split('=')[1]) : DEFAULT_WINDOW_SECONDS
const jsonOutput = args.includes('--json')
const fires = readFires()
const corrections = readCorrections()
if (fires.length === 0) {
console.log('No fires logged yet. Hooks need to fire first.')
process.exit(0)
}
const analyzed = analyzeFires(fires, corrections, windowSec)
const { byHook, byAtom } = aggregate(analyzed)
if (jsonOutput) {
console.log(JSON.stringify({ byHook, byAtom, windowSec, totalFires: fires.length, totalCorrections: corrections.length }, null, 2))
return
}
console.log('=== Hook Utility Analysis ===')
console.log(`Window: ${windowSec}s (correction within this time after fire = "followed by correction")`)
console.log(`Total fires logged: ${fires.length}`)
console.log(`Total corrections logged: ${corrections.length}`)
console.log('')
console.log('── Per Hook ──')
for (const [hook, stats] of Object.entries(byHook).sort((a, b) => b[1].fires - a[1].fires)) {
const pct = (stats.cleanRate * 100).toFixed(1)
console.log(` ${hook.padEnd(22)} fires=${String(stats.fires).padStart(4)} followed-by-correction=${String(stats.followedByCorrection).padStart(3)} clean=${String(stats.cleanFires).padStart(3)} clean-rate=${pct}%`)
}
console.log('')
console.log('── Top 10 "good" atoms (high clean rate, ≥3 fires) ──')
const good = Object.entries(byAtom)
.filter(([a, s]) => s.fires >= 3)
.sort((a, b) => b[1].cleanRate - a[1].cleanRate)
.slice(0, 10)
for (const [atom, stats] of good) {
const pct = (stats.cleanRate * 100).toFixed(0)
const label = atom.length > 60 ? atom.slice(0, 60) + '…' : atom
console.log(` clean=${pct}% (${stats.cleanFires}/${stats.fires}) ${label}`)
}
console.log('')
console.log('── Bottom 10 "wasteful" atoms (low clean rate, ≥3 fires) ──')
const bad = Object.entries(byAtom)
.filter(([a, s]) => s.fires >= 3)
.sort((a, b) => a[1].cleanRate - b[1].cleanRate)
.slice(0, 10)
for (const [atom, stats] of bad) {
const pct = (stats.cleanRate * 100).toFixed(0)
const label = atom.length > 60 ? atom.slice(0, 60) + '…' : atom
console.log(` clean=${pct}% (${stats.cleanFires}/${stats.fires}) ${label}`)
}
console.log('')
console.log('── Caveat ──')
console.log(' This is OBSERVATIONAL — we only see correlation between fires and subsequent')
console.log(' corrections. We cannot prove the fires PREVENTED corrections, only that atoms')
console.log(' with high clean-rate are associated with better outcomes.')
console.log(' Low clean-rate atoms may be wasteful fires OR may be firing on genuinely')
console.log(' risky contexts where corrections happen regardless.')
}
if (require.main === module) {
main()
}
module.exports = { readFires, readCorrections, analyzeFires, aggregate }