#!/usr/bin/env node
// Regression Checker
// Reads corrections.log and scores principles by correction rate
// Flags principles where corrections are rising despite enforcement
// Outputs scorecard.tsv and regression-report.json
const fs = require('fs')
const path = require('path')
const CORRECTIONS_LOG = path.join(__dirname, 'corrections.log')
const PRINCIPLES_DATA = path.join(__dirname, 'data', 'principles.json')
const OVERRIDES_FILE = path.join(__dirname, 'data', 'overrides.json')
const FUSED_SCORES_FILE = path.join(__dirname, 'data', 'fused-scores.json')
const SCORECARD_FILE = path.join(__dirname, 'data', 'scorecard.tsv')
const REPORT_FILE = path.join(__dirname, 'data', 'regression-report.json')
const QUESTIONS_DIR = path.join(__dirname, 'questions')
function loadCorrections () {
if (!fs.existsSync(CORRECTIONS_LOG)) return []
const raw = fs.readFileSync(CORRECTIONS_LOG, 'utf8')
return raw.split('\n---\n')
.filter(entry => entry.trim())
.map(entry => {
const lines = entry.trim().split('\n')
const timestampLine = lines.find(l => /^\d{4}-\d{2}-\d{2}/.test(l))
const typeLine = lines.find(l => l.includes('[correction]') || l.includes('[approval]'))
return {
timestamp: timestampLine ? timestampLine.split(' ')[0] : null,
type: typeLine?.includes('[correction]') ? 'correction' : 'approval',
text: lines.filter(l => !l.startsWith('cwd:') && l !== timestampLine).join(' '),
raw: entry
}
})
.filter(c => c.timestamp)
}
function loadPrinciples () {
if (!fs.existsSync(PRINCIPLES_DATA)) return []
return JSON.parse(fs.readFileSync(PRINCIPLES_DATA, 'utf8'))
}
function loadOverrides () {
if (!fs.existsSync(OVERRIDES_FILE)) return {}
try {
return JSON.parse(fs.readFileSync(OVERRIDES_FILE, 'utf8'))
} catch { return {} }
}
// Group corrections by week
function groupByWeek (corrections) {
const weeks = {}
for (const c of corrections) {
const date = new Date(c.timestamp)
const weekStart = new Date(date)
weekStart.setDate(date.getDate() - date.getDay())
const weekKey = weekStart.toISOString().slice(0, 10)
if (!weeks[weekKey]) weeks[weekKey] = { corrections: 0, approvals: 0 }
if (c.type === 'correction') weeks[weekKey].corrections++
else weeks[weekKey].approvals++
}
return weeks
}
// Score each principle by how many recent corrections overlap with its domain
function scorePrinciples (principles, corrections, overrides) {
const scores = []
for (const principle of principles) {
if (principle.status !== 'active') continue
const strip = w => w.replace(/[^a-z0-9]/g, '')
const principleWords = principle.principle.toLowerCase().split(/\s+/).map(strip).filter(w => w.length > 2)
const factWords = (principle.supportingFacts || []).join(' ').toLowerCase().split(/[\s_-]+/).map(strip).filter(w => w.length > 2)
const allWords = [...new Set([...principleWords, ...factWords])]
let relevantCorrections = 0
let relevantApprovals = 0
for (const c of corrections) {
const cWords = c.text.toLowerCase().split(/\s+/).map(strip).filter(w => w.length > 2)
const overlap = cWords.filter(w => allWords.includes(w))
if (overlap.length >= 2) {
if (c.type === 'correction') relevantCorrections++
else relevantApprovals++
}
}
const overrideCount = overrides[principle.principle]?.count || 0
const total = relevantCorrections + relevantApprovals
const score = total > 0 ? relevantApprovals / total : 1.0 // no data = assume good
scores.push({
principle: principle.principle,
corrections: relevantCorrections,
approvals: relevantApprovals,
overrides: overrideCount,
score,
status: score < 0.3 ? 'DEGRADED' : score < 0.6 ? 'WEAK' : 'HEALTHY'
})
}
return scores.sort((a, b) => a.score - b.score)
}
function loadFusedScores () {
if (!fs.existsSync(FUSED_SCORES_FILE)) return null
try { return JSON.parse(fs.readFileSync(FUSED_SCORES_FILE, 'utf8')) }
catch { return null }
}
function writeScorecard (scores, weeks, fusedWeekly) {
const lines = ['principle\tcorrections\tapprovals\toverrides\tscore\tstatus']
for (const s of scores) {
lines.push(`${s.principle.slice(0, 60)}\t${s.corrections}\t${s.approvals}\t${s.overrides}\t${s.score.toFixed(2)}\t${s.status}`)
}
lines.push('')
lines.push('# Weekly trend (corrections + fused score)')
lines.push('week\tcorrections\tapprovals\tfused_score')
for (const [week, data] of Object.entries(weeks).sort()) {
const fused = fusedWeekly?.[week]?.avgScore
lines.push(`${week}\t${data.corrections}\t${data.approvals}\t${fused != null ? fused.toFixed(3) : 'n/a'}`)
}
fs.writeFileSync(SCORECARD_FILE, lines.join('\n') + '\n')
}
function writeQuestion (id, text) {
if (!fs.existsSync(QUESTIONS_DIR)) fs.mkdirSync(QUESTIONS_DIR, { recursive: true })
const file = path.join(QUESTIONS_DIR, `${id}.md`)
fs.writeFileSync(file, `---\nid: ${id}\ncreated: ${new Date().toISOString()}\nstatus: pending\n---\n\n${text}`)
}
function main () {
const principles = loadPrinciples()
const overrides = loadOverrides()
// PRIMARY DATA SOURCE: fused-scores.json (LLM-weighted, per-session)
const fusedData = loadFusedScores()
if (!fusedData) {
console.log('No fused-scores.json found. Run fuse-signals.js first.')
return
}
const fusedWeekly = fusedData.weekly || {}
const totalCorr = fusedData.sessions.reduce((s, sess) => s + sess.corrections, 0)
const totalAppr = fusedData.sessions.reduce((s, sess) => s + sess.approvals, 0)
console.log(`Fused scores: ${fusedData.sessions.length} sessions, ${totalCorr} corrections, ${totalAppr} approvals`)
console.log(`Active principles: ${principles.filter(p => p.status === 'active').length}`)
// Score principles — use fused session data instead of corrections.log
// Group sessions by week for weekly corrections/approvals
const weeks = {}
for (const session of fusedData.sessions) {
if (!session.date) continue
const d = new Date(session.date)
d.setDate(d.getDate() - d.getDay())
const weekKey = d.toISOString().slice(0, 10)
if (!weeks[weekKey]) weeks[weekKey] = { corrections: 0, approvals: 0 }
weeks[weekKey].corrections += session.corrections
weeks[weekKey].approvals += session.approvals
}
// Principle scoring using fused session data
// Note: without per-correction text, we can't match principles to specific corrections.
// Instead, report overall session quality trends. Principle-level attribution requires
// LLM semantic matching (future work).
const scores = scorePrinciples(principles, [], overrides) // empty corrections — no keyword matching
// Write scorecard
writeScorecard(scores, weeks, fusedWeekly)
// Print results
console.log('\n═══ PRINCIPLE SCORECARD ═══')
if (scores.length === 0) {
console.log(' (no active principles)')
}
for (const s of scores) {
const icon = s.status === 'HEALTHY' ? '✓' : s.status === 'WEAK' ? '~' : '✗'
console.log(` ${icon} [${s.score.toFixed(2)}] ${s.principle.slice(0, 60)}`)
console.log(` corrections: ${s.corrections} approvals: ${s.approvals} overrides: ${s.overrides}`)
}
console.log('\n═══ WEEKLY TREND (from fused scores) ═══')
for (const [week, data] of Object.entries(weeks).sort()) {
const fused = fusedWeekly?.[week]?.avgScore
const density = (data.corrections + data.approvals) > 0
? (data.corrections / (data.corrections + data.approvals) * 100).toFixed(0)
: 'n/a'
const fusedStr = fused != null ? ` fused: ${fused.toFixed(3)}` : ''
console.log(` ${week} corr: ${String(data.corrections).padStart(4)} appr: ${String(data.approvals).padStart(4)} corr%: ${density}%${fusedStr}`)
}
// Flag degraded principles
const degraded = scores.filter(s => s.status === 'DEGRADED')
if (degraded.length > 0) {
console.log('\n⚠ DEGRADED PRINCIPLES — correction rate is high despite enforcement:')
for (const d of degraded) {
console.log(` ✗ ${d.principle}`)
writeQuestion(
`regression-${Date.now()}`,
`Principle is degraded (score: ${d.score.toFixed(2)}):\n\n` +
`"${d.principle}"\n\n` +
`${d.corrections} corrections, ${d.approvals} approvals, ${d.overrides} overrides.\n\n` +
`The principle isn't preventing mistakes. Should I:\n` +
`1. SHARPEN it (make it more specific)\n` +
`2. ESCALATE it (move from context to enforcement hook)\n` +
`3. RETIRE it (it's no longer relevant)\n`
)
}
}
// Fused score summary
if (fusedWeekly) {
const fusedWeeks = Object.entries(fusedWeekly).sort()
if (fusedWeeks.length >= 2) {
const recent = fusedWeeks[fusedWeeks.length - 1][1]
const prev = fusedWeeks[fusedWeeks.length - 2][1]
const delta = recent.avgScore - prev.avgScore
console.log('\n═══ FUSED QUALITY TREND ═══')
console.log(` Previous week: ${prev.avgScore.toFixed(3)} (${prev.sessions} sessions)`)
console.log(` Current week: ${recent.avgScore.toFixed(3)} (${recent.sessions} sessions)`)
console.log(` Delta: ${delta >= 0 ? '+' : ''}${delta.toFixed(3)}`)
if (delta < -0.05) console.log(' ⚠ FUSED QUALITY DECLINING')
else if (delta > 0.05) console.log(' ✓ FUSED QUALITY IMPROVING')
else console.log(' ~ STABLE')
}
}
// Save report
const report = {
timestamp: new Date().toISOString(),
totalCorrections: corrections.length,
fusedWeekly,
scores,
weeks,
degradedCount: degraded.length
}
fs.writeFileSync(REPORT_FILE, JSON.stringify(report, null, 2))
console.log(`\nScorecard: ${SCORECARD_FILE}`)
console.log(`Report: ${REPORT_FILE}`)
}
main()