#!/usr/bin/env node
// Regex Tuner — LLM-Driven Pattern Discovery
// Knowledge Ratchet Phase 2
//
// Reads LLM-classified corrections and approvals from corrections.log,
// discovers new high-precision patterns that the regex missed,
// measures per-pattern precision, and outputs a tiered pattern config.
//
// The feedback loop:
// regex detects (fast) → LLM validates (accurate) → tuner learns → regex improves
//
// Usage:
// node tune-regex.js # analyze and output tuned patterns
// node tune-regex.js --dry-run # analyze without saving
// node tune-regex.js --verbose # show all candidate patterns
const fs = require('fs')
const path = require('path')
const CORRECTIONS_LOG = path.join(__dirname, 'corrections.log')
const DATA_DIR = path.join(__dirname, 'data')
const OUTPUT_FILE = path.join(DATA_DIR, 'tuned-patterns.json')
// Current regex patterns (for precision measurement)
const CORRECTION_PATTERNS = [
{ re: /\b(no[,.]|wrong|not that|that's not|incorrect|stop doing)\b/i, id: 'explicit-no' },
{ re: /\b(i said|i meant|i asked for|that's wrong)\b/i, id: 'repeating-self' },
{ re: /\b(again|already told you|how many times|we discussed this)\b/i, id: 'frustration-repeat' },
{ re: /\b(don't ever|never do that|stop|please don't|do not)\b/i, id: 'prohibition' },
{ re: /\b(why did you|why are you|that broke|you broke|you just)\b/i, id: 'blame-question' },
{ re: /\b(revert|undo|go back|roll back|put it back)\b/i, id: 'revert-request' },
{ re: /\b(let me (just |)do it|i'll (just |)do it|forget it|never\s*mind)\b/i, id: 'giving-up' },
]
const APPROVAL_PATTERNS = [
{ re: /\b(perfect|exactly|great|lgtm|looks good|nice work|nailed it)\b/i, id: 'strong-approval' },
{ re: /\b(that works|that's right|correct|good job|well done)\b/i, id: 'confirmation' },
{ re: /\b(ship it|merge it|push it|deploy it|commit it)\b/i, id: 'action-approval' },
{ re: /\b(thanks|thank you)\b/i, id: 'thanks' },
]
// ─── Data Loading ───
function loadEntries () {
if (!fs.existsSync(CORRECTIONS_LOG)) return { corrections: [], approvals: [] }
const raw = fs.readFileSync(CORRECTIONS_LOG, 'utf8')
const corrections = []
const approvals = []
for (const entry of raw.split('\n---\n')) {
if (!entry.trim()) continue
const lines = entry.trim().split('\n')
const firstLine = lines[0]
const tsMatch = firstLine.match(/^(\d{4}-\d{2}-\d{2}T[\d:.]+)/)
const typeMatch = firstLine.match(/\[(correction|approval)\]/)
if (!tsMatch || !typeMatch) continue
const text = lines.slice(1)
.filter(l => !l.startsWith('cwd:') && !l.startsWith('session:') && !l.startsWith('reason:'))
.join(' ').trim()
if (!text || text.length < 3) continue
const isLLM = firstLine.includes('(llm,')
const confMatch = firstLine.match(/confidence: ([\d.]+)/)
const confidence = confMatch ? parseFloat(confMatch[1]) : (isLLM ? 0.7 : 0.5)
const record = { text, source: isLLM ? 'llm' : 'regex', confidence, timestamp: tsMatch[1] }
if (typeMatch[1] === 'correction') corrections.push(record)
else approvals.push(record)
}
return { corrections, approvals }
}
// ─── Pattern Precision Measurement ───
function measurePatternPrecision (corrections, approvals) {
const stats = {}
for (const pat of CORRECTION_PATTERNS) {
stats[pat.id] = { fires: 0, llmAgrees: 0, llmDisagrees: 0, texts: [] }
}
// Check LLM-classified entries against each pattern
const llmEntries = [...corrections.filter(c => c.source === 'llm'), ...approvals.filter(a => a.source === 'llm')]
for (const entry of llmEntries) {
for (const pat of CORRECTION_PATTERNS) {
if (pat.re.test(entry.text)) {
stats[pat.id].fires++
const isActualCorrection = corrections.some(c => c === entry)
if (isActualCorrection) stats[pat.id].llmAgrees++
else {
stats[pat.id].llmDisagrees++
stats[pat.id].texts.push(entry.text.slice(0, 100))
}
}
}
}
return stats
}
// ─── New Pattern Discovery ───
function discoverPatterns (corrections, approvals) {
// LLM-confirmed corrections that no existing regex catches
const llmCorrections = corrections.filter(c => c.source === 'llm' && c.confidence >= 0.7)
const llmApprovals = approvals.filter(a => a.source === 'llm' && a.confidence >= 0.7)
const missed = llmCorrections.filter(c => {
return !CORRECTION_PATTERNS.some(p => p.re.test(c.text))
})
// Extract common n-grams from missed corrections
const bigramCounts = {}
const trigramCounts = {}
const phraseCounts = {}
for (const entry of missed) {
const words = entry.text.toLowerCase()
.replace(/[^a-z0-9\s']/g, ' ')
.split(/\s+/)
.filter(w => w.length > 1)
// Bigrams
for (let i = 0; i < words.length - 1; i++) {
const bg = words[i] + ' ' + words[i + 1]
bigramCounts[bg] = (bigramCounts[bg] || 0) + 1
}
// Trigrams
for (let i = 0; i < words.length - 2; i++) {
const tg = words[i] + ' ' + words[i + 1] + ' ' + words[i + 2]
trigramCounts[tg] = (trigramCounts[tg] || 0) + 1
}
// Known correction phrases (manually seeded from analysis)
const knownPhrases = [
/\b(it is the same|it's the same|still the same)\b/i,
/\b(still (flat|broken|wrong|missing|empty|invisible))\b/i,
/\b(i think the .+ is|i think .+ are)\b/i,
/\b(don't see any|can't see|cannot see|not visible|invisible)\b/i,
/\b(you flipped|you swapped|you mixed up|you confused)\b/i,
/\b(just (flat|empty|blank|black|white|nothing))\b/i,
/\b(crash|crashes|crashing|crashed)\b/i,
/\b(artifact|artifacts|glitch|glitches)\b/i,
/\b(not (working|rendering|updating|showing|displaying))\b/i,
/\b(check (the )?(screenshot|image|output|result|log))\b/i,
/\b(hmm|hmmm)\b/i,
/\b(that didn't|that does not|that doesn't) (work|help|fix|change)\b/i,
/\b(same (problem|issue|bug|error|result))\b/i,
/\b(why (is|are|does|did) (it|the|this))\b/i,
/\b(audit|think deeper|planning mode)\b/i,
]
for (const phrase of knownPhrases) {
const key = phrase.source
if (phrase.test(entry.text)) {
phraseCounts[key] = (phraseCounts[key] || 0) + 1
}
}
}
// Score each candidate pattern by:
// hits: how many LLM corrections it catches
// falsePositives: how many LLM approvals it also catches
// precision: hits / (hits + falsePositives)
const candidates = [
{ id: 'same-result', re: /\b(it is the same|it's the same|still the same|same (problem|issue|bug|error|result))\b/i },
{ id: 'still-broken', re: /\b(still (flat|broken|wrong|missing|empty|invisible|not))\b/i },
{ id: 'not-visible', re: /\b(don't see|can't see|cannot see|not visible|invisible|not showing|not rendering)\b/i },
{ id: 'wrong-action', re: /\b(you flipped|you swapped|you mixed up|you confused)\b/i },
{ id: 'runtime-issue', re: /\b(crash|crashes|crashing|crashed)\b/i },
{ id: 'visual-artifact', re: /\b(artifact|artifacts|glitch|glitches)\b/i },
{ id: 'not-working', re: /\b(not (working|rendering|updating|showing|displaying))\b/i },
{ id: 'check-output', re: /\b(check (the )?(screenshot|image|output|result))\b/i },
{ id: 'didnt-work', re: /\b(that didn't|that does not|that doesn't) (work|help|fix|change)\b/i },
{ id: 'questioning-why', re: /\b(why (is|are|does|did) (it|the|this))\b/i },
{ id: 'hesitation', re: /\b(hmm+)\b/i },
{ id: 'quality-pushback', re: /\b(audit|think deeper|planning mode)\b/i },
{ id: 'just-empty', re: /\b(just (flat|empty|blank|black|white|nothing))\b/i },
]
const candidateStats = []
for (const cand of candidates) {
const hits = llmCorrections.filter(c => cand.re.test(c.text)).length
const falsePos = llmApprovals.filter(a => cand.re.test(a.text)).length
const total = hits + falsePos
const precision = total > 0 ? hits / total : 0
candidateStats.push({
id: cand.id,
pattern: cand.re.source,
flags: cand.re.flags,
hits,
falsePositives: falsePos,
precision: Math.round(precision * 1000) / 1000,
recall: llmCorrections.length > 0 ? Math.round((hits / llmCorrections.length) * 1000) / 1000 : 0,
})
}
return {
missed: missed.length,
totalLLMCorrections: llmCorrections.length,
totalLLMApprovals: llmApprovals.length,
topBigrams: Object.entries(bigramCounts).sort((a, b) => b[1] - a[1]).slice(0, 15),
candidates: candidateStats.sort((a, b) => b.hits - a.hits),
}
}
// ─── Output Tuned Patterns ───
function buildTunedPatterns (patternPrecision, discovery) {
const tuned = {
generated: new Date().toISOString(),
correction: { high: [], medium: [], low: [] },
approval: { high: [], medium: [], low: [] },
}
// Existing correction patterns — tier by precision
for (const pat of CORRECTION_PATTERNS) {
const stats = patternPrecision[pat.id]
const precision = stats.fires > 0 ? stats.llmAgrees / stats.fires : 1.0
const entry = { id: pat.id, pattern: pat.re.source, flags: pat.re.flags, precision, fires: stats.fires }
if (precision >= 0.8 || stats.fires < 3) {
tuned.correction.high.push(entry) // high precision or insufficient data to judge
} else if (precision >= 0.5) {
tuned.correction.medium.push(entry)
} else {
tuned.correction.low.push(entry)
}
}
// New discovered patterns — only promote if precision > 70% and hits > 2
for (const cand of discovery.candidates) {
if (cand.hits < 2) continue
const entry = { id: cand.id, pattern: cand.pattern, flags: cand.flags, precision: cand.precision, hits: cand.hits, source: 'discovered' }
if (cand.precision >= 0.8) {
tuned.correction.high.push(entry)
} else if (cand.precision >= 0.5) {
tuned.correction.medium.push(entry)
}
// Don't add low-precision discovered patterns — not worth the noise
}
// Existing approval patterns (all high tier for now — no LLM data to judge)
for (const pat of APPROVAL_PATTERNS) {
tuned.approval.high.push({ id: pat.id, pattern: pat.re.source, flags: pat.re.flags })
}
return tuned
}
// ─── Main ───
function main () {
const dryRun = process.argv.includes('--dry-run')
const verbose = process.argv.includes('--verbose')
const { corrections, approvals } = loadEntries()
const llmCorrections = corrections.filter(c => c.source === 'llm')
const llmApprovals = approvals.filter(a => a.source === 'llm')
console.log('═'.repeat(60))
console.log(' REGEX TUNER — LLM-Driven Pattern Discovery')
console.log('═'.repeat(60))
console.log(` Corrections in log: ${corrections.length} (${llmCorrections.length} LLM, ${corrections.length - llmCorrections.length} regex)`)
console.log(` Approvals in log: ${approvals.length} (${llmApprovals.length} LLM, ${approvals.length - llmApprovals.length} regex)`)
// 1. Measure existing pattern precision
const patternPrecision = measurePatternPrecision(corrections, approvals)
console.log('\n EXISTING PATTERN PRECISION (vs LLM ground truth)')
console.log(' ' + '-'.repeat(56))
for (const [id, stats] of Object.entries(patternPrecision)) {
const precision = stats.fires > 0 ? (stats.llmAgrees / stats.fires * 100).toFixed(0) : 'n/a'
const status = stats.fires === 0 ? '?' : stats.fires > 0 && stats.llmDisagrees === 0 ? '✓' : '~'
console.log(` ${status} ${id.padEnd(22)} fires=${String(stats.fires).padStart(3)} agree=${String(stats.llmAgrees).padStart(3)} disagree=${String(stats.llmDisagrees).padStart(2)} precision=${precision}%`)
if (verbose && stats.texts.length > 0) {
for (const t of stats.texts.slice(0, 2)) {
console.log(` false+: "${t}"`)
}
}
}
// 2. Discover new patterns
const discovery = discoverPatterns(corrections, approvals)
console.log(`\n PATTERN DISCOVERY`)
console.log(' ' + '-'.repeat(56))
console.log(` LLM corrections missed by regex: ${discovery.missed}/${discovery.totalLLMCorrections} (${(discovery.missed / Math.max(1, discovery.totalLLMCorrections) * 100).toFixed(0)}%)`)
if (verbose) {
console.log('\n Top bigrams in missed corrections:')
for (const [bg, count] of discovery.topBigrams) {
console.log(` ${String(count).padStart(3)}x "${bg}"`)
}
}
console.log('\n CANDIDATE PATTERNS')
console.log(' ' + '-'.repeat(56))
console.log(' id hits false+ prec% recall% tier')
console.log(' ' + '-'.repeat(56))
for (const c of discovery.candidates) {
const tier = c.precision >= 0.8 && c.hits >= 2 ? 'HIGH'
: c.precision >= 0.5 && c.hits >= 2 ? 'medium'
: c.hits < 2 ? 'skip (low n)'
: 'skip (low prec)'
console.log(` ${c.id.padEnd(25)} ${String(c.hits).padStart(4)} ${String(c.falsePositives).padStart(5)} ${(c.precision * 100).toFixed(0).padStart(4)}% ${(c.recall * 100).toFixed(1).padStart(6)}% ${tier}`)
}
// 3. Build tuned patterns
const tuned = buildTunedPatterns(patternPrecision, discovery)
console.log(`\n TUNED PATTERN SUMMARY`)
console.log(' ' + '-'.repeat(56))
console.log(` Correction high: ${tuned.correction.high.length} patterns`)
console.log(` Correction medium: ${tuned.correction.medium.length} patterns`)
console.log(` Correction low: ${tuned.correction.low.length} patterns`)
console.log(` Approval high: ${tuned.approval.high.length} patterns`)
if (tuned.correction.high.filter(p => p.source === 'discovered').length > 0) {
console.log('\n NEW patterns promoted to high tier:')
for (const p of tuned.correction.high.filter(p => p.source === 'discovered')) {
console.log(` + ${p.id} (precision=${(p.precision * 100).toFixed(0)}%, hits=${p.hits})`)
}
}
// Save
if (!dryRun) {
fs.writeFileSync(OUTPUT_FILE, JSON.stringify(tuned, null, 2))
console.log(`\n Saved: ${OUTPUT_FILE}`)
} else {
console.log('\n (dry run — not saved)')
}
console.log(`\n${'═'.repeat(60)}`)
}
main()