#!/usr/bin/env node
// LLM-Augmented Transcript Analyzer
// Reads conversation turns the regex couldn't classify
// Batch-sends them to an LLM for semantic/sentiment analysis
// Maintains a cursor so it never re-reads processed data
//
// Dedup strategy:
// - cursor.json tracks last-processed timestamp per session file
// - Only new/unprocessed messages get sent to LLM
// - Results merge into corrections.log with source=llm tag
const fs = require('fs')
const path = require('path')
const readline = require('readline')
const { execSync } = require('child_process')
const SESSIONS_DIRS = [
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb'),
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky'),
path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky-pear-git')
]
const DATA_DIR = path.join(__dirname, 'data')
const CURSOR_FILE = path.join(DATA_DIR, 'llm-cursor.json')
const LOG_FILE = path.join(__dirname, 'corrections.log')
const BATCH_DIR = path.join(DATA_DIR, 'llm-batches')
// Messages the regex already classified — skip these
const REGEX_CLASSIFIED = [
/\b(no|wrong|not that|that's not|incorrect|stop doing)\b/i,
/\b(i said|i meant|i asked for|that's wrong)\b/i,
/\b(it's actually|you should use|the function is|look at|use .+ instead)\b/i,
/\b(again|already told you|how many times|we discussed this)\b/i,
/\b(don't ever|never do that|stop|please don't|do not)\b/i,
/\b(perfect|exactly|great|yes|lgtm|looks good|nice|thanks|thank you)\b/i,
/\b(that works|that's right|correct|good job)\b/i,
]
// Skip patterns — not user conversation
const SKIP_PATTERNS = [
/^!/,
/^<command-name>/,
/^<system-reminder>/,
/^<task-notification>/,
/^Base directory for this skill/,
/^Implement the following plan/,
/^This session is being continued/,
/^Service Unavailable/,
/^fleeky@\w+ /, // terminal output pastes
/tool_use_id/,
/tool_result/,
/^Godot Engine v/,
/^Vulkan \d/,
]
// Interruptions are their own signal — track separately
const INTERRUPTION_PATTERN = /^\[Request interrupted by user/
function loadCursor () {
if (!fs.existsSync(CURSOR_FILE)) return {}
try { return JSON.parse(fs.readFileSync(CURSOR_FILE, 'utf8')) }
catch { return {} }
}
function saveCursor (cursor) {
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true })
fs.writeFileSync(CURSOR_FILE, JSON.stringify(cursor, null, 2))
}
function isRegexClassified (text) {
return REGEX_CLASSIFIED.some(p => p.test(text))
}
function shouldSkip (text) {
return SKIP_PATTERNS.some(p => p.test(text))
}
async function extractUnclassified (filePath, lastTimestamp) {
const stream = fs.createReadStream(filePath)
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity })
const unclassified = []
let prevAssistantText = ''
for await (const line of rl) {
try {
const entry = JSON.parse(line)
// Track assistant responses for context
if (entry.type === 'assistant') {
const content = entry.message?.content
if (Array.isArray(content)) {
prevAssistantText = content
.filter(c => c.type === 'text')
.map(c => c.text)
.join(' ')
.slice(0, 500)
}
continue
}
if (entry.type !== 'user') continue
// Skip if before cursor
if (lastTimestamp && entry.timestamp && entry.timestamp <= lastTimestamp) continue
const content = entry.message?.content
let text = ''
if (typeof content === 'string') text = content
else if (Array.isArray(content)) {
text = content.filter(c => c.type === 'text').map(c => c.text).join(' ')
}
if (!text || text.length < 5) continue
if (shouldSkip(text)) continue
// Track interruptions as frustration signal (don't send to LLM)
if (INTERRUPTION_PATTERN.test(text)) {
unclassified.push({
timestamp: entry.timestamp,
text: '[user interrupted]',
prevAssistant: prevAssistantText.slice(0, 300),
session: entry.sessionId,
cwd: entry.cwd || '',
preClassified: { type: 'correction', confidence: 0.4, reason: 'user interrupted response' }
})
continue
}
// Don't filter by regex — send everything to LLM
// When LLM disagrees with regex, that's the sharpening signal
const regexType = isRegexClassified(text)
? (REGEX_CLASSIFIED.slice(0, 5).some(p => p.test(text)) ? 'regex-correction' : 'regex-approval')
: null
unclassified.push({
timestamp: entry.timestamp,
text: text.slice(0, 500),
prevAssistant: prevAssistantText.slice(0, 300),
session: entry.sessionId,
cwd: entry.cwd,
regexClassification: regexType
})
} catch {}
}
return unclassified
}
function buildPrompt (batch) {
const messages = batch.map((m, i) => {
const regexNote = m.regexClassification ? `\n REGEX CLASSIFIED AS: ${m.regexClassification}` : ''
return `[${i}] USER: "${m.text.slice(0, 300)}"\n CONTEXT (Claude said before): "${m.prevAssistant.slice(0, 200)}"${regexNote}`
}).join('\n\n')
return `You are analyzing a user's messages to an AI coding assistant. For each message, classify:
1. Is this a CORRECTION? (user is telling the AI it did something wrong, is on the wrong track, misunderstood, needs to change approach)
2. Is this an APPROVAL? (user is confirming the AI did the right thing, accepting work, expressing satisfaction)
3. Is this NEUTRAL? (just a request, question, or instruction with no quality signal)
Subtle corrections include:
- Redirections: "let's try a different approach" (implies current approach is wrong)
- Explanations of why something is wrong without using "wrong"
- User providing the answer themselves (implies AI couldn't)
- Frustration markers: "...", short terse responses, sighs
- Implicit dissatisfaction: "hmm", "I'm not sure about that"
- Architecture pushback: "that's not the right way to structure this"
- Quality complaints: "this feels fragile", "that's too complex"
Some messages include a REGEX CLASSIFIED AS note — this is what a simple keyword matcher thought. You may AGREE or DISAGREE. A message with "yes" that's actually "yes but that's wrong" should be correction, not approval. Trust the full context over keywords.
For each message, respond with EXACTLY one line in this format:
[index] TYPE(correction|approval|neutral) CONFIDENCE(0.0-1.0) REASON(short phrase)
Messages:
${messages}
Respond with ONLY the classification lines, nothing else.`
}
function parseLLMResponse (response, batch) {
const results = []
const lines = response.split('\n').filter(l => l.trim())
for (const line of lines) {
// Handle both formats:
// [0] correction 0.8 reason text
// [0] CORRECTION(0.8) reason text
const match = line.match(/\[(\d+)\]\s*(\w+?)[\s(]+([\d.]+)\)?\s*(.*)/i)
if (!match) continue
const idx = parseInt(match[1])
const type = match[2].toLowerCase()
const confidence = parseFloat(match[3])
const reason = match[4].trim()
if (idx >= batch.length) continue
if (type === 'neutral') continue // don't log neutrals
if (confidence < 0.4) continue // low confidence = skip
// Track disagreements with regex
const regexSaid = batch[idx].regexClassification
const disagreement = regexSaid && (
(regexSaid === 'regex-correction' && type === 'approval') ||
(regexSaid === 'regex-approval' && type === 'correction')
)
results.push({
...batch[idx],
type,
confidence,
reason,
source: 'llm',
regexDisagreement: disagreement ? regexSaid : null
})
}
return results
}
async function main () {
if (!fs.existsSync(BATCH_DIR)) fs.mkdirSync(BATCH_DIR, { recursive: true })
const cursor = loadCursor()
let totalUnclassified = 0
let allUnclassified = []
// Collect unclassified messages from all sessions
for (const dir of SESSIONS_DIRS) {
if (!fs.existsSync(dir)) continue
const files = fs.readdirSync(dir)
.filter(f => f.endsWith('.jsonl') && !f.includes('subagent'))
.map(f => path.join(dir, f))
for (const file of files) {
const fileKey = path.basename(file)
const lastTs = cursor[fileKey] || null
const messages = await extractUnclassified(file, lastTs)
if (messages.length > 0) {
allUnclassified.push(...messages)
// DON'T advance cursor here — advance only after LLM processes
// This way, messages that were collected but not LLM-analyzed
// will be seen again on the next run
}
}
}
totalUnclassified = allUnclassified.length
let totalClassified = 0
let totalCorrections = 0
let totalApprovals = 0
console.log(`Found ${totalUnclassified} unclassified messages (regex couldn't categorize)`)
if (totalUnclassified === 0) {
console.log('Nothing new to analyze.')
saveCursor(cursor)
return
}
// Separate pre-classified (interruptions) from ones needing LLM
const preClassified = allUnclassified.filter(m => m.preClassified)
const needsLLM = allUnclassified.filter(m => !m.preClassified)
// Log pre-classified directly
for (const r of preClassified) {
const pc = r.preClassified
const logLine = `\n---\n${r.timestamp} [${pc.type}] (auto, confidence: ${pc.confidence})\n${r.text}\nreason: ${pc.reason}\ncwd: ${r.cwd}\nsession: ${r.session}\n`
fs.appendFileSync(LOG_FILE, logLine)
if (pc.type === 'correction') totalCorrections++
else totalApprovals++
totalClassified++
}
if (preClassified.length > 0) {
console.log(`Pre-classified: ${preClassified.length} (interruptions etc.)`)
}
// Batch the rest for LLM analysis
const BATCH_SIZE = 20
const batches = []
for (let i = 0; i < needsLLM.length; i += BATCH_SIZE) {
batches.push(needsLLM.slice(i, i + BATCH_SIZE))
}
const processCount = Math.min(batches.length, BATCH_LIMIT)
console.log(`Processing ${processCount}/${batches.length} batches of ~${BATCH_SIZE} messages each (--limit=${BATCH_LIMIT})`)
for (let i = 0; i < processCount; i++) {
const batch = batches[i]
const prompt = buildPrompt(batch)
// Save batch for debugging
const batchFile = path.join(BATCH_DIR, `batch-${Date.now()}-${i}.txt`)
fs.writeFileSync(batchFile, prompt)
// Call LLM via claude CLI
try {
// Write prompt to temp file to avoid shell escaping issues
const tmpFile = path.join(BATCH_DIR, `_current-prompt.txt`)
fs.writeFileSync(tmpFile, prompt)
const response = execSync(
`cat "${tmpFile}" | claude --print --model haiku`,
{ encoding: 'utf8', timeout: 60000, stdio: ['pipe', 'pipe', 'pipe'] }
)
const results = parseLLMResponse(response, batch)
// Append to corrections log
let batchDisagreements = 0
for (const r of results) {
const disagreeTag = r.regexDisagreement ? ` DISAGREE:regex-said-${r.regexDisagreement}` : ''
const logLine = `\n---\n${r.timestamp} [${r.type}] (llm, confidence: ${r.confidence}${disagreeTag})\n${r.text}\nreason: ${r.reason}\ncwd: ${r.cwd}\nsession: ${r.session}\n`
fs.appendFileSync(LOG_FILE, logLine)
if (r.regexDisagreement) batchDisagreements++
if (r.type === 'correction') totalCorrections++
else totalApprovals++
totalClassified++
}
if (batchDisagreements > 0) {
console.log(` ⚡ ${batchDisagreements} regex disagreement(s) — regex was wrong`)
}
console.log(` Batch ${i + 1}/${batches.length}: ${results.length} classified (${results.filter(r => r.type === 'correction').length} corrections, ${results.filter(r => r.type === 'approval').length} approvals)`)
} catch (err) {
console.error(` Batch ${i + 1} failed: ${err.message.slice(0, 100)}`)
// Save cursor up to this point so we don't re-process on retry
saveCursor(cursor)
}
}
// Advance cursor only for messages we actually processed
// Pre-classified messages + LLM-analyzed batches
const processedMessages = [
...preClassified,
...needsLLM.slice(0, processCount * BATCH_SIZE)
]
for (const m of processedMessages) {
if (!m.session) continue
// Find which file this session belongs to
for (const dir of SESSIONS_DIRS) {
const fileKey = `${m.session}.jsonl`
const fullPath = path.join(dir, fileKey)
if (fs.existsSync(fullPath)) {
if (!cursor[fileKey] || m.timestamp > cursor[fileKey]) {
cursor[fileKey] = m.timestamp
}
break
}
}
}
saveCursor(cursor)
console.log(`\n═══ LLM ANALYSIS COMPLETE ═══`)
console.log(` Messages analyzed: ${totalUnclassified}`)
console.log(` New classifications: ${totalClassified}`)
console.log(` Corrections found: ${totalCorrections}`)
console.log(` Approvals found: ${totalApprovals}`)
console.log(` Cursor saved — next run will only process new messages`)
}
// CLI
const command = process.argv[2]
const limitArg = process.argv.find(a => a.startsWith('--limit='))
const BATCH_LIMIT = limitArg ? parseInt(limitArg.split('=')[1]) : 10 // default 10 batches = 200 messages
if (command === 'reset') {
if (fs.existsSync(CURSOR_FILE)) {
fs.unlinkSync(CURSOR_FILE)
console.log('Cursor reset. Next run will process all messages.')
} else {
console.log('No cursor to reset.')
}
} else if (command === 'status') {
const cursor = loadCursor()
const entries = Object.entries(cursor)
console.log(`Cursor tracks ${entries.length} session files`)
if (entries.length > 0) {
const latest = entries.sort((a, b) => b[1].localeCompare(a[1]))[0]
console.log(`Latest processed: ${latest[0]} at ${latest[1]}`)
}
} else {
main().catch(err => {
console.error('LLM analysis failed:', err.message)
process.exit(1)
})
}