knowledge-ratchet / rule-extractor.js

#!/usr/bin/env node

// Rule Extractor — Auto-Generate Rules from Resolved Loops
// Knowledge Ratchet Phase 2
//
// Reads loops.json and extracts specific, actionable rules from resolved loops.
// Each rule has:
//   trigger: { project, subsystem, symptom (from corrections) }
//   action: what fixed it (from resolution files + approval text)
//   antipattern: what was tried and failed (from loop corrections)
//   source: which loop taught this
//
// Rules accumulate in data/learned-rules.json.
// Loaded by loop-briefing.js and preflight-review.js as directives.
//
// Usage:
//   node rule-extractor.js           # extract rules from all resolved loops
//   node rule-extractor.js --verbose # show rule details

const fs = require('fs')
const path = require('path')
const crypto = require('crypto')

const DATA_DIR = path.join(__dirname, 'data')
const LOOPS_FILE = path.join(DATA_DIR, 'loops.json')
const RULES_FILE = path.join(DATA_DIR, 'learned-rules.json')

// ─── Content-addressed rule IDs ───
// Previously: `rule-${project}-${type}-${rules.length}` — index-based,
// unstable across extraction runs (rules.length shifts when loops are
// reordered or new ones appear). This caused merge-based dedup to
// fail: the same underlying loop produced a new rule ID on every
// pipeline run, and mergeRules treated each as a new rule. Result:
// corpus inflation to ~4x the actual number of distinct loops.
//
// Fix: derive the ID from the loop's fingerprint (which is stable for
// a given loop). Same loop → same ID → merge is idempotent.
//
// The fingerprint is already generated by loop-detector.js:buildFingerprint
// and stored on every rule as source.loopFingerprint.

function ruleId (ruleOrLoop) {
  const fp = ruleOrLoop.fingerprint || ruleOrLoop.source?.loopFingerprint || ''
  const project = ruleOrLoop.project || ruleOrLoop.trigger?.project || 'unknown'
  const type = ruleOrLoop.type || ruleOrLoop.trigger?.type || 'other'
  const hash = crypto.createHash('md5').update(fp).digest('hex').slice(0, 8)
  return `rule-${project}-${type}-${hash}`
}

function loadJSON (file) {
  if (!fs.existsSync(file)) return null
  try { return JSON.parse(fs.readFileSync(file, 'utf8')) }
  catch { return null }
}

// ─── Quality Classification ───
// A rule's quality is a function of whether it has actionable content:
//   specific            — concrete symptom + concrete fix files
//   partial             — one of the two is missing (still injectable)
//   needs-investigation — both antipattern and fix-files are the fallback
//                         values, meaning the regex extractor couldn't
//                         distill anything actionable. These rules are
//                         held back from injection by loop-briefing.js
//                         so they don't pollute briefings with noise.
//
// The two "fallback" strings are defined where they're produced:
//   buildAntipattern returns "Slow down and verify each step" when no
//   correction regex matched any pattern.
//   buildActionDescription falls back to "Read the target files completely
//   before making changes" when there's no fix, no approval text, and the
//   loop wasn't long enough to trigger the "planning mode early" branch.
// We detect via antipattern.description (stable sentinel) and fixFiles
// length (the load-bearing signal).

const GENERIC_ANTIPATTERN = 'Slow down and verify each step'
const QUALITY_RANK = { specific: 0, partial: 1, 'needs-investigation': 2 }

function computeQuality (rule) {
  const hasGenericAntipattern = rule.antipattern?.description === GENERIC_ANTIPATTERN
  const hasNoFixFiles = !rule.action?.fixFiles || rule.action.fixFiles.length === 0
  if (hasGenericAntipattern && hasNoFixFiles) return 'needs-investigation'
  if (hasGenericAntipattern || hasNoFixFiles) return 'partial'
  return 'specific'
}

// ─── Symptom Extraction ───
// Extract the core symptom from correction text

function extractSymptom (corrections) {
  if (corrections.length === 0) return 'unknown'

  // Use the first correction as the primary symptom
  const text = corrections[0].toLowerCase()

  // Try to extract a concise symptom
  if (/invisible|not (visible|showing|rendering)|can.t see|don.t see/.test(text)) return 'not rendering/visible'
  if (/flat|just flat|totally flat/.test(text)) return 'output is flat/empty'
  if (/crash|exception|error|fail/.test(text)) return 'runtime crash/error'
  if (/same|still|no change|didn.t (work|help|fix)/.test(text)) return 'fix did not work'
  if (/worse|regression|broke/.test(text)) return 'change made it worse'
  if (/wrong|reversed|flipped|incorrect/.test(text)) return 'output is wrong/reversed'
  if (/why (are|is|did)|approach/.test(text)) return 'wrong approach/design'
  if (/audit|planning|think deeper/.test(text)) return 'needs deeper analysis'
  if (/stop/.test(text)) return 'going in wrong direction'
  if (/corrupt/.test(text)) return 'data corruption'

  // Fallback: first 60 chars of first correction
  return corrections[0].slice(0, 60).toLowerCase()
}

// ─── Rule Generation ───

function extractRules (loops) {
  const rules = []
  const seen = new Set() // dedup by fingerprint

  for (const loop of loops) {
    if (!loop.resolved) continue
    if (!loop.corrections || loop.corrections.length === 0) continue
    if (!loop.resolution) continue

    // Dedup: same fingerprint = same rule
    if (seen.has(loop.fingerprint)) continue
    seen.add(loop.fingerprint)

    const symptom = extractSymptom(loop.corrections)
    const fixFiles = (loop.resolution.files || []).map(f => path.basename(f))
    const approval = loop.resolution.approval || null

    // Build the rule — id is content-addressed via ruleId(loop) so the
    // same loop always produces the same id across extraction runs
    const rule = {
      id: ruleId(loop),
      trigger: {
        project: loop.project,
        type: loop.type,
        symptom,
        files: (loop.filesTried || []).slice(0, 5).map(f => path.basename(f)),
      },
      action: {
        fixFiles,
        description: buildActionDescription(symptom, fixFiles, approval, loop),
      },
      antipattern: {
        description: buildAntipattern(loop.corrections),
        corrections: loop.corrections.slice(0, 3),
      },
      source: {
        loopFingerprint: loop.fingerprint,
        date: loop.date,
        duration: loop.duration,
        correctionCount: loop.correctionCount,
      },
    }
    rule.quality = computeQuality(rule)

    rules.push(rule)
  }

  return rules
}

function buildActionDescription (symptom, fixFiles, approval, loop) {
  const parts = []

  if (fixFiles.length > 0) {
    parts.push(`Fix was in: ${fixFiles.join(', ')}`)
  }

  if (approval) {
    parts.push(`Resolution: "${approval.slice(0, 120)}"`)
  }

  if (loop.duration > 10) {
    parts.push(`This took ${loop.duration} turns to resolve — enter planning mode early`)
  }

  if (parts.length === 0) {
    parts.push('Read the target files completely before making changes')
  }

  return parts.join('. ')
}

function buildAntipattern (corrections) {
  // Extract what NOT to do from the correction sequence
  const patterns = []

  if (corrections.some(c => /stop/i.test(c))) {
    patterns.push('Do not continue without reading the code first')
  }
  if (corrections.some(c => /audit|planning/i.test(c))) {
    patterns.push('Do not implement without planning — enter planning mode')
  }
  if (corrections.some(c => /same|still/i.test(c))) {
    patterns.push('Do not apply the same approach again if it already failed')
  }
  if (corrections.some(c => /worse|regression|broke/i.test(c))) {
    patterns.push('Verify the fix does not break existing functionality')
  }
  if (corrections.some(c => /corrupt/i.test(c))) {
    patterns.push('Check for data corruption before and after changes')
  }

  return patterns.length > 0 ? patterns.join('. ') : 'Slow down and verify each step'
}

// ─── Merge with existing rules ───

function mergeRules (newRules, existingRules) {
  const existingIds = new Set(existingRules.map(r => r.id))
  const merged = [...existingRules]

  let added = 0
  for (const rule of newRules) {
    if (!existingIds.has(rule.id)) {
      merged.push(rule)
      added++
    }
  }

  return { merged, added }
}

// ─── Main ───

function main () {
  const verbose = process.argv.includes('--verbose')

  const loopsData = loadJSON(LOOPS_FILE)
  if (!loopsData) {
    console.log('No loops.json found. Run loop-detector.js first.')
    return
  }

  const existingRules = loadJSON(RULES_FILE)?.rules || []
  const newRules = extractRules(loopsData.loops || [])
  const { merged, added } = mergeRules(newRules, existingRules)

  // ─── Content-addressed dedup pass ───
  // Group by loop fingerprint, collapse duplicates, and normalize all
  // IDs to the content-addressed form. This handles:
  //   - One-shot migration: pre-Change-4 corpora have index-based IDs
  //     like `rule-dc-terrain-other-3` which don't survive re-extraction,
  //     so the corpus accumulates ~4x duplicates. This pass collapses
  //     them to one representative per fingerprint.
  //   - Defensive: future pipeline runs stay idempotent. If anything
  //     ever produces a duplicate again, it gets collapsed here.
  // Rules without a fingerprint are kept as-is (can't dedup them safely).
  // When duplicates exist, the rule with the most recent source.date wins.
  const byFingerprint = new Map()
  const noFingerprint = []
  let duplicatesDropped = 0

  for (const rule of merged) {
    const fp = rule.source?.loopFingerprint
    if (!fp) {
      noFingerprint.push(rule)
      continue
    }
    // Normalize id to content-addressed form
    rule.id = ruleId({
      project: rule.trigger.project,
      type: rule.trigger.type,
      fingerprint: fp
    })
    const existing = byFingerprint.get(fp)
    if (!existing) {
      byFingerprint.set(fp, rule)
    } else {
      duplicatesDropped++
      // Tiebreak: keep the one with the most recent source.date
      const existingDate = existing.source?.date || ''
      const newDate = rule.source?.date || ''
      if (newDate > existingDate) {
        byFingerprint.set(fp, rule)
      }
    }
  }

  const deduped = [...Array.from(byFingerprint.values()), ...noFingerprint]

  // ─── Drop unknown-project rules (Change 5) ───
  // Rules with trigger.project === 'unknown' can never fire at runtime.
  // loop-briefing.js matches rules by specific project name, and
  // classify-project.js's walker either finds a .git (returning a real
  // project name) or fails completely. No live session ever classifies
  // as 'unknown', so these rules are unreachable by design. Dropping them
  // is a one-shot migration (catches pre-Change-4 stale unknowns) AND a
  // permanent filter (prevents future unknown rules from accumulating).
  const filtered = deduped.filter(r => r.trigger.project !== 'unknown')
  const unknownDropped = deduped.length - filtered.length

  // Backfill quality on every rule in the filtered output. This catches
  // pre-Change 2 rules that lack a .quality field, and also refreshes
  // the tag for any rule whose antipattern or fixFiles changed since
  // the last extraction.
  for (const rule of filtered) {
    rule.quality = computeQuality(rule)
  }

  // Group by project
  const byProject = {}
  for (const rule of filtered) {
    const proj = rule.trigger.project
    if (!byProject[proj]) byProject[proj] = []
    byProject[proj].push(rule)
  }

  console.log('═'.repeat(60))
  console.log('  RULE EXTRACTOR — Learned Rules from Loops')
  console.log('═'.repeat(60))
  console.log(`  Resolved loops analyzed: ${loopsData.loops.filter(l => l.resolved).length}`)
  console.log(`  Rules extracted: ${newRules.length}`)
  console.log(`  New rules added: ${added}`)
  console.log(`  Pre-dedup total: ${merged.length}`)
  console.log(`  Duplicates dropped: ${duplicatesDropped}`)
  console.log(`  Unknown-project rules dropped: ${unknownDropped}`)
  console.log(`  Total rules: ${filtered.length}`)

  console.log('\n  BY PROJECT')
  console.log('  ' + '-'.repeat(56))
  for (const [proj, rules] of Object.entries(byProject).sort((a, b) => b[1].length - a[1].length)) {
    console.log(`  ${proj}: ${rules.length} rules`)
    if (verbose) {
      for (const r of rules.slice(0, 5)) {
        console.log(`    [${r.trigger.type}] "${r.trigger.symptom}"`)
        console.log(`      → ${r.action.description.slice(0, 100)}`)
        if (r.antipattern.description) {
          console.log(`      ✗ ${r.antipattern.description.slice(0, 100)}`)
        }
      }
    }
  }

  // Show top rules by severity (longest loops)
  const bySeverity = [...filtered].sort((a, b) => b.source.duration - a.source.duration)

  console.log('\n  HIGHEST SEVERITY RULES (longest loops)')
  console.log('  ' + '-'.repeat(56))
  for (const r of bySeverity.slice(0, 10)) {
    console.log(`  [${r.source.duration} turns] ${r.trigger.project}/${r.trigger.type}: "${r.trigger.symptom}"`)
    console.log(`    → ${r.action.description.slice(0, 120)}`)
  }

  // Save
  const report = {
    generated: new Date().toISOString(),
    totalRules: filtered.length,
    rules: filtered,
  }
  fs.writeFileSync(RULES_FILE, JSON.stringify(report, null, 2))
  console.log(`\nSaved: ${RULES_FILE}`)
  console.log('═'.repeat(60))
}

module.exports = { extractRules, mergeRules, computeQuality, QUALITY_RANK, GENERIC_ANTIPATTERN, ruleId }

if (require.main === module) main()