#!/usr/bin/env node
// Meta-Ratchet — Self-Adjusting Sensor Weights
// Knowledge Ratchet Phase 2, Item #4
//
// Analyzes per-turn sensor data to determine which sensors best predict
// future corrections. Adjusts weights so predictive sensors have more
// influence on the fused score.
//
// Algorithm:
// For each correction at turn T:
// Look back at turns T-1, T-2, T-3
// For each sensor, if the value was LOW (< threshold), it predicted the correction
// That sensor's predictive_hits++
//
// For each approval at turn T (after 3+ non-correction turns):
// Look back at turns T-1, T-2, T-3
// For each sensor, if the value was HIGH (> threshold), it correctly indicated quality
// That sensor's approval_hits++
//
// Weight adjustment:
// predictive_rate = predictive_hits / total_corrections
// If predictive_rate > 0.5 → sensor is useful, increase weight
// If predictive_rate < 0.2 → sensor is noise, decrease weight
//
// Usage:
// node meta-ratchet.js # analyze and adjust weights
// node meta-ratchet.js --dry-run # analyze without saving
// node meta-ratchet.js --verbose # show per-sensor details
const fs = require('fs')
const path = require('path')
const DATA_DIR = path.join(__dirname, 'data')
const DETAIL_FILE = path.join(DATA_DIR, 'fused-turns-detail.json')
const WEIGHTS_FILE = path.join(DATA_DIR, 'sensor-weights.json')
const HISTORY_FILE = path.join(DATA_DIR, 'weight-history.json')
// How far back to look for predictive signals
const LOOKBACK = 3
// Sensor thresholds — below this value, the sensor is "signaling"
// (i.e., it detected something potentially wrong)
const SIGNAL_THRESHOLD = 0.6
// Sensors that are inverted (high = bad) or not meaningful for prediction
const EXCLUDED_SENSORS = new Set([
'cacheHitRate', // informational only, not predictive
'timeBetweenTurns', // user behavior, not Claude quality
])
// Weight adjustment bounds
const MIN_WEIGHT = 0.05
const MAX_WEIGHT = 2.0
const ADJUST_STEP = 0.05 // how much to adjust per run
function loadDetail () {
if (!fs.existsSync(DETAIL_FILE)) {
console.error(`Detail file not found: ${DETAIL_FILE}`)
console.error('Run fuse-signals.js first.')
process.exit(1)
}
return JSON.parse(fs.readFileSync(DETAIL_FILE, 'utf8'))
}
function loadWeightHistory () {
if (!fs.existsSync(HISTORY_FILE)) return []
try { return JSON.parse(fs.readFileSync(HISTORY_FILE, 'utf8')) }
catch { return [] }
}
function analyze (data) {
const currentWeights = data.sensorWeights || {}
// Get all sensor names from first turn with data
let sensorNames = []
for (const sess of data.sessions) {
for (const turn of sess.turns) {
if (turn.sensors) {
sensorNames = Object.keys(turn.sensors).filter(s => !EXCLUDED_SENSORS.has(s))
break
}
}
if (sensorNames.length > 0) break
}
// Initialize tracking per sensor
const stats = {}
for (const name of sensorNames) {
stats[name] = {
// How many corrections had this sensor signaling in the previous turns?
predictiveHits: 0,
// How many approvals had this sensor healthy in the previous turns?
approvalHits: 0,
// Total corrections/approvals we checked
totalCorrections: 0,
totalApprovals: 0,
// Raw sensor values at correction/approval points (for distribution analysis)
valuesAtCorrection: [],
valuesBeforeCorrection: [],
valuesBeforeApproval: [],
}
}
let totalCorrections = 0
let totalApprovals = 0
// Analyze each session's turns
for (const sess of data.sessions) {
const turns = sess.turns
if (turns.length < LOOKBACK + 1) continue
for (let i = LOOKBACK; i < turns.length; i++) {
const turn = turns[i]
if (!turn.sensors) continue
if (turn.isCorrection) {
totalCorrections++
// Look back at previous turns
for (const name of sensorNames) {
stats[name].totalCorrections++
// Collect sensor values from lookback window
let signaled = false
for (let j = i - LOOKBACK; j < i; j++) {
if (j < 0 || !turns[j]?.sensors) continue
const value = turns[j].sensors[name]
if (value !== undefined) {
stats[name].valuesBeforeCorrection.push(value)
if (value < SIGNAL_THRESHOLD) {
signaled = true
}
}
}
// Also check the correction turn itself
const currentValue = turn.sensors[name]
if (currentValue !== undefined) {
stats[name].valuesAtCorrection.push(currentValue)
}
if (signaled) {
stats[name].predictiveHits++
}
}
}
if (turn.isApproval) {
// Only count "strong" approvals — ones after at least 3 non-correction turns
const recentTurns = turns.slice(Math.max(0, i - 3), i)
const hasRecentCorrection = recentTurns.some(t => t.isCorrection)
if (hasRecentCorrection) continue // approval after correction = recovery, not quality signal
totalApprovals++
for (const name of sensorNames) {
stats[name].totalApprovals++
// Check if sensors were healthy in lookback window
let healthy = true
for (let j = i - LOOKBACK; j < i; j++) {
if (j < 0 || !turns[j]?.sensors) continue
const value = turns[j].sensors[name]
if (value !== undefined) {
stats[name].valuesBeforeApproval.push(value)
if (value < SIGNAL_THRESHOLD) {
healthy = false
}
}
}
if (healthy) {
stats[name].approvalHits++
}
}
}
}
}
// Compute predictive rates and new weights
const results = {}
for (const name of sensorNames) {
const s = stats[name]
const predictiveRate = s.totalCorrections > 0
? s.predictiveHits / s.totalCorrections
: 0
const approvalRate = s.totalApprovals > 0
? s.approvalHits / s.totalApprovals
: 0
// Distribution stats
const avgBeforeCorr = s.valuesBeforeCorrection.length > 0
? s.valuesBeforeCorrection.reduce((a, b) => a + b, 0) / s.valuesBeforeCorrection.length
: null
const avgBeforeAppr = s.valuesBeforeApproval.length > 0
? s.valuesBeforeApproval.reduce((a, b) => a + b, 0) / s.valuesBeforeApproval.length
: null
// Separation: how different is the sensor before corrections vs approvals?
// High separation = sensor discriminates well between good and bad turns
const separation = (avgBeforeCorr !== null && avgBeforeAppr !== null)
? avgBeforeAppr - avgBeforeCorr
: 0
// Weight adjustment logic
const currentWeight = currentWeights[name] || 0.3
let newWeight = currentWeight
let reason = 'stable'
if (s.totalCorrections >= 5) { // need enough data to judge
if (predictiveRate > 0.5 && separation > 0.05) {
// Sensor predicts corrections well AND differentiates — increase weight
newWeight = Math.min(MAX_WEIGHT, currentWeight + ADJUST_STEP)
reason = 'predictive (↑)'
} else if (predictiveRate < 0.2 && separation < 0.02) {
// Sensor rarely signals before corrections AND doesn't differentiate — decrease weight
newWeight = Math.max(MIN_WEIGHT, currentWeight - ADJUST_STEP)
reason = 'noise (↓)'
} else if (predictiveRate > 0.3 && separation > 0.03) {
// Moderately predictive — keep or slight increase
newWeight = Math.min(MAX_WEIGHT, currentWeight + ADJUST_STEP * 0.5)
reason = 'moderate (↗)'
}
} else {
reason = 'insufficient data'
}
results[name] = {
currentWeight: Math.round(currentWeight * 1000) / 1000,
newWeight: Math.round(newWeight * 1000) / 1000,
predictiveRate: Math.round(predictiveRate * 1000) / 1000,
approvalRate: Math.round(approvalRate * 1000) / 1000,
separation: Math.round(separation * 1000) / 1000,
avgBeforeCorrection: avgBeforeCorr !== null ? Math.round(avgBeforeCorr * 1000) / 1000 : null,
avgBeforeApproval: avgBeforeAppr !== null ? Math.round(avgBeforeAppr * 1000) / 1000 : null,
predictiveHits: s.predictiveHits,
totalCorrections: s.totalCorrections,
approvalHits: s.approvalHits,
totalApprovals: s.totalApprovals,
reason,
}
}
return { results, totalCorrections, totalApprovals, sensorNames }
}
function main () {
const dryRun = process.argv.includes('--dry-run')
const verbose = process.argv.includes('--verbose')
const data = loadDetail()
const { results, totalCorrections, totalApprovals, sensorNames } = analyze(data)
// ─── Console Report ───
console.log('═'.repeat(60))
console.log(' META-RATCHET — Sensor Weight Analysis')
console.log('═'.repeat(60))
console.log(` Turns analyzed: ${data.sessions.reduce((s, sess) => s + sess.turns.length, 0)}`)
console.log(` Corrections: ${totalCorrections}`)
console.log(` Strong approvals: ${totalApprovals}`)
console.log(` Lookback window: ${LOOKBACK} turns`)
console.log(` Signal threshold: < ${SIGNAL_THRESHOLD}`)
// Sort by predictive rate (most predictive first)
const sorted = Object.entries(results)
.sort((a, b) => b[1].predictiveRate - a[1].predictiveRate)
console.log('\n SENSOR PREDICTIVENESS')
console.log(' ' + '-'.repeat(56))
console.log(' sensor pred% sep weight → new reason')
console.log(' ' + '-'.repeat(56))
for (const [name, r] of sorted) {
const predBar = '█'.repeat(Math.round(r.predictiveRate * 10)) + '░'.repeat(10 - Math.round(r.predictiveRate * 10))
const arrow = r.newWeight > r.currentWeight ? '↑'
: r.newWeight < r.currentWeight ? '↓'
: '='
console.log(` ${predBar} ${name.padEnd(22)} ${(r.predictiveRate * 100).toFixed(0).padStart(3)}% ${r.separation.toFixed(3).padStart(6)} ${r.currentWeight.toFixed(2).padStart(5)} ${arrow} ${r.newWeight.toFixed(2).padStart(5)} ${r.reason}`)
}
if (verbose) {
console.log('\n DETAILED SENSOR DISTRIBUTIONS')
console.log(' ' + '-'.repeat(56))
for (const [name, r] of sorted) {
console.log(` ${name}:`)
console.log(` avg before correction: ${r.avgBeforeCorrection?.toFixed(3) || 'n/a'}`)
console.log(` avg before approval: ${r.avgBeforeApproval?.toFixed(3) || 'n/a'}`)
console.log(` predictive hits: ${r.predictiveHits}/${r.totalCorrections}`)
console.log(` approval hits: ${r.approvalHits}/${r.totalApprovals}`)
}
}
// Weight changes summary
const changed = sorted.filter(([, r]) => r.newWeight !== r.currentWeight)
if (changed.length > 0) {
console.log('\n WEIGHT ADJUSTMENTS')
console.log(' ' + '-'.repeat(56))
for (const [name, r] of changed) {
const delta = r.newWeight - r.currentWeight
console.log(` ${name}: ${r.currentWeight.toFixed(3)} → ${r.newWeight.toFixed(3)} (${delta >= 0 ? '+' : ''}${delta.toFixed(3)}) — ${r.reason}`)
}
} else {
console.log('\n No weight adjustments needed.')
}
// Save new weights
if (!dryRun && changed.length > 0) {
const newWeights = {}
for (const [name, r] of Object.entries(results)) {
newWeights[name] = r.newWeight
}
fs.writeFileSync(WEIGHTS_FILE, JSON.stringify(newWeights, null, 2))
console.log(`\n Saved: ${WEIGHTS_FILE}`)
// Append to weight history for trend tracking
const history = loadWeightHistory()
history.push({
timestamp: new Date().toISOString(),
corrections: totalCorrections,
approvals: totalApprovals,
weights: newWeights,
changes: changed.map(([name, r]) => ({
sensor: name,
from: r.currentWeight,
to: r.newWeight,
reason: r.reason,
})),
})
// Keep last 100 history entries
while (history.length > 100) history.shift()
fs.writeFileSync(HISTORY_FILE, JSON.stringify(history, null, 2))
console.log(` Saved: ${HISTORY_FILE} (${history.length} entries)`)
} else if (dryRun) {
console.log('\n (dry run — weights not saved)')
}
console.log(`\n${'═'.repeat(60)}`)
}
main()