| … | … | @@ -50,8 +50,8 @@ node benchmark-ratchet.js |
| 50 | 50 | | Novel correction rate | 86.9% | >90% | Old mistakes eliminated (higher = better) | |
| 51 | 51 | | Correction density | 21.2% | <15% | Corrections per turn (lower = better) | |
| 52 | 52 | | Loop resolution | 8.2 turns | <6 turns | How fast loops resolve (lower = better) | |
| 53 | | Learned rules | 271 | growing | Auto-extracted from resolved loops (content-addressed, idempotent across runs) | | |
| 54 | | Rule quality | 47.2% specific / 39.9% partial / 12.9% needs-investigation | more specific | Quality-aware injection prioritizes actionable rules | | |
| 53 | | Learned rules | 251 | growing | Auto-extracted from resolved loops (content-addressed, idempotent across runs, unknown-project filtered) | | |
| 54 | | Rule quality | 49.8% specific / 39.0% partial / 11.2% needs-investigation | more specific | Quality-aware injection prioritizes actionable rules | | |
| 55 | 55 | |
| 56 | 56 | ## Architecture |
| 57 | 57 |
| … | … | @@ -182,13 +182,104 @@ directory after backfill. The briefing header reads |
| 182 | 182 | rules all have at least one concrete signal (concrete fix files or a |
| 183 | 183 | concrete antipattern — not both missing). |
| 184 | 184 | |
| 185 | ## Change 3 — Wire `findSimilarLoops` into `loop-briefing.js` | |
| 185 | ## Change 3 — Cross-project similar-loop surfacing | |
| 186 | ||
| 187 | Shipped. Original intent: wire `findSimilarLoops`. When implemented, it | |
| 188 | turned out `findSimilarLoops`'s scoring (+2 per file overlap, threshold | |
| 189 | ≥3) expects a rich current fingerprint to work for cross-project matches. | |
| 190 | At SessionStart we only have project danger-file history, which typically | |
| 191 | yields 1-file overlaps with other-project loops (insufficient to clear | |
| 192 | threshold unless same type). | |
| 193 | ||
| 194 | Rather than force-fit `findSimilarLoops`, an inline custom matcher was | |
| 195 | added to `loop-briefing.js`: for each resolved loop in other projects, | |
| 196 | count file-basename overlap with the current project's danger files, | |
| 197 | keep any with ≥1 overlap, sort by overlap count then by duration, show | |
| 198 | top 3. `loop-detector.js` exports `buildFingerprint` for future use. | |
| 199 | ||
| 200 | **Verified:** hypercore-kb session surfaces 27 cross-project matches | |
| 201 | (dc-terrain, pear-git, hyper-social, hyper-tps) sharing files like | |
| 202 | `MEMORY.md`, `dashboard.js`, `rt-varying-crash-fix.md`. The top 3 are | |
| 203 | shown in a new `CROSS-PROJECT SIMILAR LOOPS` section in the briefing. | |
| 204 | ||
| 205 | ## Change 5 — Drop unknown-project rules | |
| 206 | ||
| 207 | Shipped. Discovered while inspecting the 21 remaining unknowns after | |
| 208 | Change 4's dedup. The rules fall into categories: | |
| 209 | - Personal-context bleed (one rule contained "Uphold crypto sale" | |
| 210 | text — tax filing content captured as a correction) | |
| 211 | - Planning/audit chatter misclassified as corrections | |
| 212 | - Real loops stuck unknown because session cwd was missing or /tmp | |
| 213 | - Rules referencing files that belong to real projects but whose | |
| 214 | session context wasn't captured (e.g., fingerprints with | |
| 215 | `benchmark-ratchet`, `promote`, `regress` which are knowledge-ratchet | |
| 216 | files, but the session cwd didn't resolve via the git walker) | |
| 217 | ||
| 218 | None of these can ever fire: `loop-briefing.js` matches by exact project | |
| 219 | name, and live sessions now always classify to a real project via the | |
| 220 | git-root walker. Unknown-project rules are unreachable by construction. | |
| 221 | ||
| 222 | Fix: permanent filter in `rule-extractor.js`'s dedup pass. | |
| 223 | `const filtered = deduped.filter(r => r.trigger.project !== 'unknown')`. | |
| 224 | Catches both the existing 21 stale unknowns AND any future unknown rules | |
| 225 | that might be extracted from loops without usable cwd context. | |
| 226 | ||
| 227 | **Verified:** 271 → 251 rules (−21 dropped, +1 run-2 new loop dropped). | |
| 228 | 0 unknown rules remain. Pipeline still idempotent at 2.6s on re-run. | |
| 229 | ||
| 230 | ## Change 6 — Shadow harness for briefing comparison | |
| 231 | ||
| 232 | Shipped. Goal: longitudinal measurement of Change 2's quality filter | |
| 233 | and quality reordering by logging what the NEW briefing injected vs what | |
| 234 | a PRE-CHANGE-2 BASELINE briefing would have injected. | |
| 235 | ||
| 236 | Implementation: in `loop-briefing.js`, after computing the injectable | |
| 237 | rule set, also compute `shadowBaselineRules` = project rules sorted by | |
| 238 | `source.duration` (severity proxy), top 5. Log diff to | |
| 239 | `data/briefing-shadow.jsonl` on every briefing invocation. | |
| 240 | ||
| 241 | Record shape: | |
| 242 | ||
| 243 | ```json | |
| 244 | { | |
| 245 | "timestamp": "2026-04-11T19:02:06.079Z", | |
| 246 | "project": "hypercore-kb", | |
| 247 | "cwd": "/storage/projects/code/hypercore-kb", | |
| 248 | "counts": { | |
| 249 | "projectRules": 15, | |
| 250 | "injected": 5, | |
| 251 | "baseline": 5, | |
| 252 | "needsInvestigation": 6, | |
| 253 | "crossProjectSurfaced": 3, | |
| 254 | "crossProjectTotal": 27 | |
| 255 | }, | |
| 256 | "overlap": 1, | |
| 257 | "removedByNew": ["rule-...", "rule-...", ...], | |
| 258 | "addedByNew": ["rule-...", "rule-...", ...] | |
| 259 | } | |
| 260 | ``` | |
| 261 | ||
| 262 | **Initial result for hypercore-kb:** overlap = 1 out of 5. Change 2's | |
| 263 | quality-tier reordering is significantly different from the baseline — | |
| 264 | it suppressed 4 rules (quality-doubt / wrong-output / runtime-error / | |
| 265 | other types whose duration would have put them in baseline's top 5) | |
| 266 | and promoted 4 rules (needs-rethink / other types whose quality was | |
| 267 | higher). | |
| 186 | 268 | |
| 187 | Still deferred. The function exists (`loop-detector.js:220`) and works. | |
| 188 | The wire-up is ~5 lines (import and call after the project filter). | |
| 189 | After Change 1, cross-project retrieval is less urgent — most loops | |
| 190 | now have specific project tags. Revisit once we see whether cross-project | |
| 191 | rule transfer is still a measurable gap. | |
| 269 | How to read the log longitudinally: | |
| 270 | ||
| 271 | - `overlap` average over N sessions — measures how different | |
| 272 | Change 2 is from baseline. 0 = completely different, 5 = identical. | |
| 273 | - `removedByNew` accumulated — rules the quality filter rejected. | |
| 274 | If any of these show up in later correction sequences, Change 2 | |
| 275 | was suppressing useful rules (false negatives). | |
| 276 | - `addedByNew` accumulated — rules promoted by quality reordering. | |
| 277 | If these correlate with fewer corrections in the session, Change 2 | |
| 278 | is surfacing useful rules. | |
| 279 | - `crossProjectSurfaced` — how often Change 3 actually contributes. | |
| 280 | ||
| 281 | The harness doesn't produce a verdict on its own. It accumulates data | |
| 282 | for later offline analysis. Ship it, wait 2-3 weeks, read the log. | |
| 192 | 283 | |
| 193 | 284 | ## Change 4 — Content-addressed rule IDs + dedup pass |
| 194 | 285 | |
| … | … | @@ -243,22 +334,24 @@ linearly with pipeline invocations. |
| 243 | 334 | - The benchmark history still shows old inflated numbers (462 → 498 → |
| 244 | 335 | 648 → 953 → 1048 → 271). Future entries will be stable. |
| 245 | 336 | |
| 246 | ## Deferred | |
| 337 | ## Deferred (still) | |
| 247 | 338 | |
| 248 | ### Cleanup of 21 remaining `unknown` rules | |
| 339 | ### Rule-extractor's "dead rule" eviction based on hit/miss | |
| 249 | 340 | |
| 250 | The 108-rule unknown bucket has collapsed to 21 after Change 4's dedup | |
| 251 | (most of the 108 were duplicates of the same underlying unknown loops). | |
| 252 | They remain in the corpus as `needs-investigation` and are held back from | |
| 253 | injection by Change 2. A cleanup migration would drop them entirely but | |
| 254 | loses the audit trail. Scoped separately. | |
| 341 | Rules never fire-tracked. The original "live rule scoring" (Change C in | |
| 342 | the pre-audit plan) stayed deferred because the counterfactual problem | |
| 343 | (attributing a loop to any one injected rule) is genuinely hard. The | |
| 344 | shadow harness (Change 6) now at least captures which rules were | |
| 345 | injected, so a future change could tally per-rule firing frequency and | |
| 346 | hit-rate by correlating rule IDs in `removedByNew`/`addedByNew` with | |
| 347 | subsequent correction sequences in the same session. Not in scope. | |
| 255 | 348 | |
| 256 | ### Shadow harness for real measurement | |
| 349 | ### Correction-burst regex fix (not mine) | |
| 257 | 350 | |
| 258 | Still not built. The dashboard's benchmark trend is only 5 days old and | |
| 259 | now has a synthetic discontinuity from Change 4's collapse. Real before/ | |
| 260 | after measurement of the changes requires either waiting 1-2 weeks for a | |
| 261 | clean history or building the shadow harness. | |
| 351 | `realtime-score.js` has an uncommitted fix for the learning-rate | |
| 352 | computation that collapses consecutive corrections into a single | |
| 353 | "burst." Not mine, left in the working tree as pre-existing work to | |
| 354 | avoid committing code I didn't author. | |
| 262 | 355 | |
| 263 | 356 | ## Measurement caveats |
| 264 | 357 | |
| … | … | @@ -329,12 +422,16 @@ conversation transcript if anyone wants the reasoning in detail. |
| 329 | 422 | |
| 330 | 423 | **Changed:** |
| 331 | 424 | - `knowledge-ratchet/classify-project.js` (new, 147 LOC) — Change 1 |
| 332 | - `knowledge-ratchet/loop-detector.js` — imports shared classifier | |
| 333 | - `knowledge-ratchet/loop-briefing.js` — imports shared classifier + quality-aware injection | |
| 334 | - `knowledge-ratchet/rule-extractor.js` — `computeQuality` + backfill + content-addressed IDs + dedup pass | |
| 425 | - `knowledge-ratchet/loop-detector.js` — imports shared classifier, exports `buildFingerprint` | |
| 426 | - `knowledge-ratchet/loop-briefing.js` — imports shared classifier + quality-aware injection + inline cross-project matcher + shadow harness logging | |
| 427 | - `knowledge-ratchet/rule-extractor.js` — `computeQuality` + backfill + content-addressed IDs + dedup pass + unknown-project filter | |
| 428 | ||
| 429 | **Data added:** | |
| 430 | - `data/briefing-shadow.jsonl` — append-only shadow harness log (new) | |
| 335 | 431 | |
| 336 | 432 | **Not changed (but modified in working tree by other work — not staged):** |
| 337 | 433 | - `realtime-score.js` — pre-existing uncommitted fix for correction-burst counting |
| 338 | 434 | - Various data files under `data/` — pipeline artifacts |
| 339 | 435 | |
| 340 | **Total net: ~280 LOC across 4 files across 3 code commits.** | |
| 436 | **Total: ~380 LOC across 4 source files, 4 code commits | |
| 437 | (Change 1+2, audit doc, Change 4, Changes 3+5+6).** |
| … | … | @@ -15,6 +15,7 @@ const { computeQuality, QUALITY_RANK } = require('./rule-extractor') |
| 15 | 15 | |
| 16 | 16 | const LOOPS_FILE = path.join(__dirname, 'data', 'loops.json') |
| 17 | 17 | const RULES_FILE = path.join(__dirname, 'data', 'learned-rules.json') |
| 18 | const SHADOW_LOG = path.join(__dirname, 'data', 'briefing-shadow.jsonl') | |
| 18 | 19 | |
| 19 | 20 | // classifyProject is imported from classify-project.js (shared with |
| 20 | 21 | // loop-detector.js). It always returns a non-empty string — either a |
| … | … | @@ -104,6 +105,61 @@ function main () { |
| 104 | 105 | } |
| 105 | 106 | } |
| 106 | 107 | |
| 108 | // ─── Cross-project similar loops (Change 3) ─── | |
| 109 | // Surface resolved loops from OTHER projects whose filesTried overlap with | |
| 110 | // this project's danger files. The goal: a rule learned in dc-terrain about | |
| 111 | // MEMORY.md fumbling surfaces when working in hypercore-kb, because they're | |
| 112 | // the same class of problem. | |
| 113 | // | |
| 114 | // Note: loop-detector's findSimilarLoops exists but its scoring expects a | |
| 115 | // rich current fingerprint (≥2 file overlap to clear threshold for cross- | |
| 116 | // project). At SessionStart we only have project danger-file history, which | |
| 117 | // typically yields 1-file overlaps with other-project loops. So we do an | |
| 118 | // inline match with a lower threshold (≥1 overlap) specific to this use | |
| 119 | // case, rather than hammer findSimilarLoops's threshold to fit. | |
| 120 | // | |
| 121 | // Matching uses raw basenames (no extension stripping) since danger files | |
| 122 | // often have meaningful extensions like .md that loop-detector's | |
| 123 | // buildFingerprint preserves. | |
| 124 | const dangerBasenamesSet = new Set(Object.keys(dangerFiles)) | |
| 125 | const crossProjectMatches = [] | |
| 126 | for (const l of (loopsData.loops || [])) { | |
| 127 | if (l.project === project) continue | |
| 128 | if (!l.resolved) continue | |
| 129 | const lBasenames = (l.filesTried || []).map(f => path.basename(f)) | |
| 130 | const overlapFiles = lBasenames.filter(f => dangerBasenamesSet.has(f)) | |
| 131 | if (overlapFiles.length >= 1) { | |
| 132 | crossProjectMatches.push({ loop: l, overlap: overlapFiles.length, overlapFiles }) | |
| 133 | } | |
| 134 | } | |
| 135 | crossProjectMatches.sort((a, b) => { | |
| 136 | // Primary: more file overlap is better | |
| 137 | if (b.overlap !== a.overlap) return b.overlap - a.overlap | |
| 138 | // Tiebreak: longer loops = higher severity | |
| 139 | return (b.loop.duration || 0) - (a.loop.duration || 0) | |
| 140 | }) | |
| 141 | const topCross = crossProjectMatches.slice(0, 3) | |
| 142 | ||
| 143 | if (topCross.length > 0) { | |
| 144 | lines.push('') | |
| 145 | lines.push(`CROSS-PROJECT SIMILAR LOOPS (${crossProjectMatches.length} by danger-file overlap, top 3):`) | |
| 146 | for (const m of topCross) { | |
| 147 | const l = m.loop | |
| 148 | lines.push(` ${l.date} [${l.project}/${l.type}] ${l.duration} turns, shared: ${m.overlapFiles.slice(0, 3).join(', ')}`) | |
| 149 | lines.push(` "${(l.corrections[0] || '').slice(0, 90)}"`) | |
| 150 | if (l.resolution?.files?.length > 0) { | |
| 151 | lines.push(` fix: ${l.resolution.files.map(f => path.basename(f)).slice(0, 3).join(', ')}`) | |
| 152 | } | |
| 153 | } | |
| 154 | } | |
| 155 | ||
| 156 | // Pre-declare shadow harness collectors. These are populated inside | |
| 157 | // the rules-loading block below and logged at the end of main(). | |
| 158 | let shadowBaselineIds = [] | |
| 159 | let shadowNewIds = [] | |
| 160 | let shadowProjectRuleCount = 0 | |
| 161 | let shadowNeedsInvestigationCount = 0 | |
| 162 | ||
| 107 | 163 | // Load learned rules for this project. |
| 108 | 164 | // |
| 109 | 165 | // Quality-aware injection (Change 2): |
| … | … | @@ -137,6 +193,21 @@ function main () { |
| 137 | 193 | const injectable = sortedRules.filter(r => r.quality !== 'needs-investigation') |
| 138 | 194 | const needsInvestigationCount = sortedRules.length - injectable.length |
| 139 | 195 | |
| 196 | // ─── Shadow harness collection (Change 6) ─── | |
| 197 | // Compute the BASELINE rule set — what the pre-Change-2 briefing | |
| 198 | // would have injected — so we can log the diff for longitudinal | |
| 199 | // comparison. Baseline = filter by project exact match, sort by | |
| 200 | // source.duration (severity proxy), take top 5. No quality filter, | |
| 201 | // no cross-project, matching the pre-Change-2 behavior in order to | |
| 202 | // measure what the quality filter and quality reorder changed. | |
| 203 | const shadowBaselineRules = [...projectRules] | |
| 204 | .sort((a, b) => (b.source?.duration || 0) - (a.source?.duration || 0)) | |
| 205 | .slice(0, 5) | |
| 206 | shadowBaselineIds = shadowBaselineRules.map(r => r.id) | |
| 207 | shadowNewIds = injectable.slice(0, 5).map(r => r.id) | |
| 208 | shadowProjectRuleCount = projectRules.length | |
| 209 | shadowNeedsInvestigationCount = needsInvestigationCount | |
| 210 | ||
| 140 | 211 | if (injectable.length > 0) { |
| 141 | 212 | const header = needsInvestigationCount > 0 |
| 142 | 213 | ? `LEARNED RULES (${injectable.length} actionable, ${needsInvestigationCount} need investigation):` |
| … | … | @@ -165,6 +236,37 @@ function main () { |
| 165 | 236 | lines.push('═'.repeat(50)) |
| 166 | 237 | lines.push('') |
| 167 | 238 | |
| 239 | // ─── Shadow harness log (Change 6) ─── | |
| 240 | // Append one JSONL record capturing what this briefing injected vs | |
| 241 | // what a pre-Change-2 baseline briefing would have injected. Read | |
| 242 | // longitudinally to answer: "is the quality filter removing rules | |
| 243 | // that would have helped?" (false negatives) and "is quality | |
| 244 | // reordering surfacing rules that wouldn't have been seen?" | |
| 245 | // (effective promotions). This does not block injection — it's a | |
| 246 | // side-effect log for measurement. | |
| 247 | try { | |
| 248 | const overlap = shadowNewIds.filter(id => shadowBaselineIds.includes(id)) | |
| 249 | const removedByNew = shadowBaselineIds.filter(id => !shadowNewIds.includes(id)) | |
| 250 | const addedByNew = shadowNewIds.filter(id => !shadowBaselineIds.includes(id)) | |
| 251 | const shadowEntry = { | |
| 252 | timestamp: new Date().toISOString(), | |
| 253 | project, | |
| 254 | cwd, | |
| 255 | counts: { | |
| 256 | projectRules: shadowProjectRuleCount, | |
| 257 | injected: shadowNewIds.length, | |
| 258 | baseline: shadowBaselineIds.length, | |
| 259 | needsInvestigation: shadowNeedsInvestigationCount, | |
| 260 | crossProjectSurfaced: topCross.length, | |
| 261 | crossProjectTotal: crossProjectMatches.length | |
| 262 | }, | |
| 263 | overlap: overlap.length, | |
| 264 | removedByNew, // rules baseline would have shown but we suppressed | |
| 265 | addedByNew // rules we surfaced by quality reordering | |
| 266 | } | |
| 267 | fs.appendFileSync(SHADOW_LOG, JSON.stringify(shadowEntry) + '\n') | |
| 268 | } catch {} | |
| 269 | ||
| 168 | 270 | // Mark as delivered so we don't repeat |
| 169 | 271 | fs.writeFileSync(BRIEFING_FLAG, new Date().toISOString()) |
| 170 | 272 |
| … | … | @@ -361,6 +361,6 @@ function main () { |
| 361 | 361 | console.log('═'.repeat(60)) |
| 362 | 362 | } |
| 363 | 363 | |
| 364 | module.exports = { extractLoops, findSimilarLoops } | |
| 364 | module.exports = { extractLoops, findSimilarLoops, buildFingerprint } | |
| 365 | 365 | |
| 366 | 366 | if (require.main === module) main() |
| … | … | @@ -280,17 +280,28 @@ function main () { |
| 280 | 280 | |
| 281 | 281 | const deduped = [...Array.from(byFingerprint.values()), ...noFingerprint] |
| 282 | 282 | |
| 283 | // Backfill quality on every rule in the deduped output. This catches | |
| 283 | // ─── Drop unknown-project rules (Change 5) ─── | |
| 284 | // Rules with trigger.project === 'unknown' can never fire at runtime. | |
| 285 | // loop-briefing.js matches rules by specific project name, and | |
| 286 | // classify-project.js's walker either finds a .git (returning a real | |
| 287 | // project name) or fails completely. No live session ever classifies | |
| 288 | // as 'unknown', so these rules are unreachable by design. Dropping them | |
| 289 | // is a one-shot migration (catches pre-Change-4 stale unknowns) AND a | |
| 290 | // permanent filter (prevents future unknown rules from accumulating). | |
| 291 | const filtered = deduped.filter(r => r.trigger.project !== 'unknown') | |
| 292 | const unknownDropped = deduped.length - filtered.length | |
| 293 | ||
| 294 | // Backfill quality on every rule in the filtered output. This catches | |
| 284 | 295 | // pre-Change 2 rules that lack a .quality field, and also refreshes |
| 285 | 296 | // the tag for any rule whose antipattern or fixFiles changed since |
| 286 | 297 | // the last extraction. |
| 287 | for (const rule of deduped) { | |
| 298 | for (const rule of filtered) { | |
| 288 | 299 | rule.quality = computeQuality(rule) |
| 289 | 300 | } |
| 290 | 301 | |
| 291 | 302 | // Group by project |
| 292 | 303 | const byProject = {} |
| 293 | for (const rule of deduped) { | |
| 304 | for (const rule of filtered) { | |
| 294 | 305 | const proj = rule.trigger.project |
| 295 | 306 | if (!byProject[proj]) byProject[proj] = [] |
| 296 | 307 | byProject[proj].push(rule) |
| … | … | @@ -304,7 +315,8 @@ function main () { |
| 304 | 315 | console.log(` New rules added: ${added}`) |
| 305 | 316 | console.log(` Pre-dedup total: ${merged.length}`) |
| 306 | 317 | console.log(` Duplicates dropped: ${duplicatesDropped}`) |
| 307 | console.log(` Total rules: ${deduped.length}`) | |
| 318 | console.log(` Unknown-project rules dropped: ${unknownDropped}`) | |
| 319 | console.log(` Total rules: ${filtered.length}`) | |
| 308 | 320 | |
| 309 | 321 | console.log('\n BY PROJECT') |
| 310 | 322 | console.log(' ' + '-'.repeat(56)) |
| … | … | @@ -322,7 +334,7 @@ function main () { |
| 322 | 334 | } |
| 323 | 335 | |
| 324 | 336 | // Show top rules by severity (longest loops) |
| 325 | const bySeverity = [...deduped].sort((a, b) => b.source.duration - a.source.duration) | |
| 337 | const bySeverity = [...filtered].sort((a, b) => b.source.duration - a.source.duration) | |
| 326 | 338 | |
| 327 | 339 | console.log('\n HIGHEST SEVERITY RULES (longest loops)') |
| 328 | 340 | console.log(' ' + '-'.repeat(56)) |
| … | … | @@ -334,8 +346,8 @@ function main () { |
| 334 | 346 | // Save |
| 335 | 347 | const report = { |
| 336 | 348 | generated: new Date().toISOString(), |
| 337 | totalRules: deduped.length, | |
| 338 | rules: deduped, | |
| 349 | totalRules: filtered.length, | |
| 350 | rules: filtered, | |
| 339 | 351 | } |
| 340 | 352 | fs.writeFileSync(RULES_FILE, JSON.stringify(report, null, 2)) |
| 341 | 353 | console.log(`\nSaved: ${RULES_FILE}`) |