knowledge-ratchet / extract-patterns.js

#!/usr/bin/env node

// Pattern & Principle Extractor
// Reads feedback_*.md files, groups by topic, outputs structured patterns and principles
// Used by the ratchet pipeline to generate candidate principles from raw corrections

const fs = require('fs')
const path = require('path')

const MEMORY_DIR = '/home/fleeky/.claude/projects/-storage-projects-code-hypercore-kb/memory'
const OUTPUT_DIR = path.join(__dirname, 'data')
const PATTERNS_FILE = path.join(OUTPUT_DIR, 'patterns.json')
const PRINCIPLES_FILE = path.join(OUTPUT_DIR, 'principles.json')

// Topic keywords for grouping
const TOPIC_KEYWORDS = {
  'verification': ['verify', 'test', 'check', 'confirm', 'assume', 'investigate', 'repro', 'binary', 'background process'],
  'deployment': ['deploy', 'push', 'pear', 'pgh', 'scp', 'ssh', 'rsync', 'stortron', 'ci', 'pipeline', 'sidecar', 'launcher', 'service script'],
  'persistence': ['storage', 'delete', 'destroy', 'identity', 'data', 'rocksdb', 'hyperdrive', 'cleanup', 'gc', 'append-only'],
  'discipline': ['fix', 'skip', 'kludge', 'bypass', 'workaround', 'shortcut', 'band-aid', 'batch', 'commit', 'incremental', 'step back', 'messy'],
  'system-knowledge': ['understand', 'api', 'winding', 'version', 'bump', 'config', 'server', 'restart', 'nested', 'repo', 'git root'],
  'communication': ['apolog', 'sorry']
}

function readFeedbackFiles () {
  const files = fs.readdirSync(MEMORY_DIR)
    .filter(f => f.startsWith('feedback_') && f.endsWith('.md'))
    .sort() // alphabetical = stable order
    .map(f => {
      const content = fs.readFileSync(path.join(MEMORY_DIR, f), 'utf8')
      const name = f.replace('feedback_', '').replace('.md', '')
      return { file: f, name, content }
    })
  return files
}

function extractTopics (content) {
  const lower = content.toLowerCase()
  const topics = []
  for (const [topic, keywords] of Object.entries(TOPIC_KEYWORDS)) {
    if (keywords.some(kw => lower.includes(kw))) {
      topics.push(topic)
    }
  }
  return topics.length > 0 ? topics : ['uncategorized']
}

function extractCoreRule (content) {
  // Pull the main rule from the feedback file
  // Look for the first bold statement or the content after the frontmatter
  const lines = content.split('\n')
  let inFrontmatter = false
  let rule = ''

  for (const line of lines) {
    if (line.trim() === '---') {
      inFrontmatter = !inFrontmatter
      continue
    }
    if (inFrontmatter) continue

    const trimmed = line.trim()
    if (!trimmed) continue

    // First non-empty line after frontmatter, or first bold line
    if (trimmed.startsWith('**') || trimmed.startsWith('#')) {
      rule = trimmed.replace(/\*\*/g, '').replace(/^#+\s*/, '')
      break
    }
    if (!rule) {
      rule = trimmed
      break
    }
  }

  return rule
}

function extractWhy (content) {
  const whyMatch = content.match(/\*\*Why:\*\*\s*(.+?)(?:\n\n|\*\*How)/s)
  if (whyMatch) return whyMatch[1].trim().split('\n')[0]

  const whySection = content.match(/Why:.*?\n(.+?)(?:\n\n|How to apply)/s)
  if (whySection) return whySection[1].trim().split('\n')[0]

  return ''
}

function groupByTopic (facts) {
  const groups = {}
  for (const fact of facts) {
    for (const topic of fact.topics) {
      if (!groups[topic]) groups[topic] = []
      groups[topic].push(fact)
    }
  }
  return groups
}

function extractPatterns (groups) {
  const patterns = []

  for (const [topic, facts] of Object.entries(groups)) {
    if (facts.length < 2) continue // need at least 2 facts to form a pattern

    // Find the common thread across facts in this group
    const rules = facts.map(f => f.rule)
    const whys = facts.map(f => f.why).filter(Boolean)

    patterns.push({
      topic,
      factCount: facts.length,
      facts: facts.map(f => f.name),
      rules,
      whys,
      extractedAt: new Date().toISOString()
    })
  }

  return patterns
}

function crystallizePrinciples (patterns) {
  // Map well-populated patterns to principles
  // These are the cross-cutting rules validated by the retrospective test
  const principleMap = {
    'discipline': {
      principle: 'When something breaks, fix the system — never work around it. One change, one verify, one commit.',
      coveredPatterns: ['discipline'],
      strength: 'high'
    },
    'verification': {
      principle: 'Verify before acting — understand the full system, confirm state, never assume.',
      coveredPatterns: ['verification'],
      strength: 'high'
    },
    'deployment': {
      principle: 'Use established tools and paths. Never invent ad-hoc alternatives for deployment.',
      coveredPatterns: ['deployment'],
      strength: 'high'
    },
    'persistence': {
      principle: 'Never destroy persistent state. Every storage decision must answer: what is the deletion story?',
      coveredPatterns: ['persistence'],
      strength: 'high'
    },
    'system-knowledge': {
      principle: 'Understand the full system before modifying any part. Research APIs, check downstream impact.',
      coveredPatterns: ['system-knowledge'],
      strength: 'medium'
    }
  }

  const principles = []
  for (const pattern of patterns) {
    const mapped = principleMap[pattern.topic]
    if (mapped) {
      principles.push({
        ...mapped,
        supportingFacts: pattern.facts,
        factCount: pattern.factCount,
        validatedAt: new Date().toISOString(),
        contradictions: [],
        overrideCount: 0,
        status: 'active'
      })
    }
  }

  return principles
}

function main () {
  if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR, { recursive: true })

  // Step 1: Read all feedback files
  const feedbackFiles = readFeedbackFiles()
  console.log(`Read ${feedbackFiles.length} feedback files`)

  // Step 2: Extract facts with topics
  const facts = feedbackFiles.map(f => ({
    name: f.name,
    file: f.file,
    rule: extractCoreRule(f.content),
    why: extractWhy(f.content),
    topics: extractTopics(f.content),
    content: f.content
  }))

  // Step 3: Group by topic
  const groups = groupByTopic(facts)
  console.log(`\nTopic groups:`)
  for (const [topic, members] of Object.entries(groups)) {
    console.log(`  ${topic}: ${members.length} facts`)
    for (const m of members) {
      console.log(`    - ${m.name}`)
    }
  }

  // Step 4: Extract patterns
  const patterns = extractPatterns(groups)
  fs.writeFileSync(PATTERNS_FILE, JSON.stringify(patterns, null, 2))
  console.log(`\nExtracted ${patterns.length} patterns → ${PATTERNS_FILE}`)

  // Step 5: Crystallize principles
  const principles = crystallizePrinciples(patterns)
  fs.writeFileSync(PRINCIPLES_FILE, JSON.stringify(principles, null, 2))
  console.log(`Crystallized ${principles.length} principles → ${PRINCIPLES_FILE}`)

  // Step 6: Print summary
  console.log('\n═══ EXTRACTED PRINCIPLES ═══')
  for (const p of principles) {
    console.log(`\n[${p.strength}] ${p.principle}`)
    console.log(`  Supported by ${p.factCount} facts: ${p.supportingFacts.join(', ')}`)
  }

  return { facts, patterns, principles }
}

main()