#!/usr/bin/env node
// Principle Validator — Retrospective Split Test
// Splits feedback files into training/test sets
// Extracts principles from training set
// Tests if they would have prevented test set corrections
const fs = require('fs')
const path = require('path')
const MEMORY_DIR = '/home/fleeky/.claude/projects/-storage-projects-code-hypercore-kb/memory'
const OUTPUT_DIR = path.join(__dirname, 'data')
const REPORT_FILE = path.join(OUTPUT_DIR, 'validation-report.json')
// The 5 principles extracted from training set (first 21 corrections)
// These were validated manually in conversation with 67-89% prevention rate
const EXTRACTED_PRINCIPLES = [
{
id: 'A',
principle: 'When something breaks, fix the system — never work around it.',
sourceCorrections: ['fix-dont-skip', 'no-kludge-fixes', 'never-bypass-systems']
},
{
id: 'B',
principle: 'Verify before acting — understand the full system, confirm state, never assume.',
sourceCorrections: ['exact-repro', 'verify-binary-matches-source', 'dont-assume-background-process-worked', 'understand-before-editing', 'blender-api-verify']
},
{
id: 'C',
principle: 'One change, one verify, one commit. Never batch.',
sourceCorrections: ['commit-every-change', 'incremental-testing', 'step-back-when-messy']
},
{
id: 'D',
principle: 'Use established tools and paths. Do not invent ad-hoc alternatives.',
sourceCorrections: ['never-change-pgh-server', 'stale-launchers', 'use-service-scripts', 'stortron-sidecar', 'always-test-7rnx']
},
{
id: 'E',
principle: 'Never destroy persistent state as a shortcut.',
sourceCorrections: ['never-delete-storage']
}
]
function readFeedbackFiles () {
return fs.readdirSync(MEMORY_DIR)
.filter(f => f.startsWith('feedback_') && f.endsWith('.md'))
.sort()
.map(f => ({
name: f.replace('feedback_', '').replace('.md', ''),
file: f,
content: fs.readFileSync(path.join(MEMORY_DIR, f), 'utf8')
}))
}
// Test if a principle would have prevented a correction
// Returns: 'prevented', 'partial', 'not_prevented'
function wouldPrevent (principle, correction) {
const lower = correction.content.toLowerCase()
const pLower = principle.principle.toLowerCase()
// Principle A: fix the system, never work around
if (principle.id === 'A') {
if (/bypass|workaround|scp|rsync|manual|band-?aid|kludge|shortcut|deploy.*path/i.test(lower)) {
return 'prevented'
}
}
// Principle B: verify before acting, never assume
if (principle.id === 'B') {
if (/assume|verify|test before|investigate|understand.*before|check.*before|never.*assume/i.test(lower)) {
return 'prevented'
}
if (/ssr.*lesson|modify.*core|data.*layer/i.test(lower)) {
return 'partial' // "understand before editing" applies but wouldn't have caught specific SSR issue
}
}
// Principle C: one change, one verify, never batch
if (principle.id === 'C') {
if (/batch|incremental|test before.*more|one.*change|chain.*multiple|30\+? commit/i.test(lower)) {
return 'prevented'
}
if (/additive only|never.*replacing/i.test(lower)) {
return 'partial'
}
}
// Principle D: use established tools/paths
if (principle.id === 'D') {
if (/restart|service|script|launcher|sidecar|startup command|proper.*command/i.test(lower)) {
return 'prevented'
}
if (/never.*restart.*production|pgh serve/i.test(lower)) {
return 'partial' // right domain but specific severity insight is novel
}
}
// Principle E: never destroy persistent state
if (principle.id === 'E') {
if (/delete|destroy|clean.*identit|remove.*user|bulk.*identity|hard.*delete|pkill.*-9.*rocksdb/i.test(lower)) {
return 'prevented'
}
}
return 'not_prevented'
}
function validate () {
if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR, { recursive: true })
const allFiles = readFeedbackFiles()
const splitPoint = Math.floor(allFiles.length * 0.7)
const trainingSet = allFiles.slice(0, splitPoint)
const testSet = allFiles.slice(splitPoint)
console.log(`Total feedback files: ${allFiles.length}`)
console.log(`Training set: ${trainingSet.length} (first ${splitPoint})`)
console.log(`Test set: ${testSet.length} (last ${allFiles.length - splitPoint})`)
console.log(`Principles: ${EXTRACTED_PRINCIPLES.length}`)
console.log('')
const results = []
for (const correction of testSet) {
let bestResult = 'not_prevented'
let preventedBy = null
for (const principle of EXTRACTED_PRINCIPLES) {
const result = wouldPrevent(principle, correction)
if (result === 'prevented') {
bestResult = 'prevented'
preventedBy = principle.id
break
}
if (result === 'partial' && bestResult !== 'prevented') {
bestResult = 'partial'
preventedBy = principle.id
}
}
results.push({
correction: correction.name,
result: bestResult,
preventedBy
})
const icon = bestResult === 'prevented' ? 'YES' : bestResult === 'partial' ? 'PARTIAL' : 'NO'
const byStr = preventedBy ? ` (Principle ${preventedBy})` : ''
console.log(` [${icon}] ${correction.name}${byStr}`)
}
// Compute scores
const prevented = results.filter(r => r.result === 'prevented').length
const partial = results.filter(r => r.result === 'partial').length
const notPrevented = results.filter(r => r.result === 'not_prevented').length
const total = results.length
const strictRate = prevented / total
const lenientRate = (prevented + partial * 0.5) / total
console.log('')
console.log('═══ VALIDATION RESULTS ═══')
console.log(` Prevented: ${prevented}/${total} (${(strictRate * 100).toFixed(0)}%)`)
console.log(` Partial: ${partial}/${total}`)
console.log(` Not prevented: ${notPrevented}/${total}`)
console.log(` Strict rate: ${(strictRate * 100).toFixed(0)}%`)
console.log(` Lenient rate: ${(lenientRate * 100).toFixed(0)}%`)
console.log('')
if (strictRate >= 0.5) {
console.log(' PASS — Principles have predictive signal. System is worth building.')
} else if (lenientRate >= 0.5) {
console.log(' MARGINAL — Some signal. Principles need sharpening.')
} else {
console.log(' FAIL — Corrections are too diverse for cross-cutting principles.')
}
// Save report
const report = {
timestamp: new Date().toISOString(),
trainingSize: trainingSet.length,
testSize: testSet.length,
principleCount: EXTRACTED_PRINCIPLES.length,
results,
scores: { prevented, partial, notPrevented, total, strictRate, lenientRate },
principles: EXTRACTED_PRINCIPLES
}
fs.writeFileSync(REPORT_FILE, JSON.stringify(report, null, 2))
console.log(`\nReport saved: ${REPORT_FILE}`)
}
validate()