knowledge-ratchet / scan-transcripts.js

#!/usr/bin/env node

// Transcript Scanner
// Reads Claude Code session JSONL files, extracts user messages,
// detects corrections and approvals, appends to corrections.log

const fs = require('fs')
const path = require('path')
const readline = require('readline')

const SESSIONS_DIRS = [
  path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb'),
  path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky'),
  path.join(process.env.HOME, '.claude/projects/-storage-projects-code-hypercore-kb-hyper-fleeky-pear-git')
]

const LOG_FILE = path.join(__dirname, 'corrections.log')

const CORRECTION_PATTERNS = [
  /\b(no[,.]|wrong|not that|that's not|that is not|incorrect|stop doing)\b/i,
  /\b(i said|i meant|i asked for|that's wrong|that is wrong)\b/i,
  /\b(it's actually|you should use|the function is|look at|use .+ instead)\b/i,
  /\b(again|already told you|how many times|we discussed this)\b/i,
  /\b(don't ever|never do that|stop|please don't|do not)\b/i,
  /\b(why did you|why are you|that broke|you broke|you just)\b/i,
  /\b(revert|undo|go back|roll back|put it back)\b/i
]

const APPROVAL_PATTERNS = [
  /\b(perfect|exactly|great|yes|lgtm|looks good|nice work|thanks|thank you)\b/i,
  /\b(that works|that's right|correct|good job|well done|nailed it)\b/i,
  /\b(ship it|merge it|push it|deploy it|commit it)\b/i
]

// Skip patterns — messages that look like corrections but aren't
const SKIP_PATTERNS = [
  /^!/, // shell commands
  /^\s*$/, // empty
  /^\//, // slash commands
  /^</, // XML/system tags
]

function isUserText (entry) {
  if (entry.type !== 'user') return false
  const content = entry.message?.content
  if (!content) return false
  if (typeof content === 'string') return content.length > 3
  if (Array.isArray(content)) {
    return content.some(c => c.type === 'text' && c.text?.length > 3)
  }
  return false
}

function extractText (entry) {
  const content = entry.message?.content
  if (typeof content === 'string') return content
  if (Array.isArray(content)) {
    return content
      .filter(c => c.type === 'text')
      .map(c => c.text)
      .join(' ')
  }
  return ''
}

function classifyMessage (text) {
  if (SKIP_PATTERNS.some(p => p.test(text))) return null
  if (text.length < 4) return null

  const isCorrection = CORRECTION_PATTERNS.some(p => p.test(text))
  const isApproval = APPROVAL_PATTERNS.some(p => p.test(text))

  // If both match (e.g. "no that's not right, use X instead — thanks"), prefer correction
  if (isCorrection) return 'correction'
  if (isApproval) return 'approval'
  return null
}

async function scanFile (filePath) {
  const results = []

  const stream = fs.createReadStream(filePath)
  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity })

  for await (const line of rl) {
    try {
      const entry = JSON.parse(line)
      if (!isUserText(entry)) continue

      const text = extractText(entry)
      const classification = classifyMessage(text)
      if (!classification) continue

      results.push({
        timestamp: entry.timestamp || new Date().toISOString(),
        type: classification,
        text: text.slice(0, 500),
        session: entry.sessionId || path.basename(filePath, '.jsonl'),
        cwd: entry.cwd || ''
      })
    } catch {}
  }

  return results
}

async function scanAll () {
  let allResults = []
  let filesScanned = 0
  let sessionsFound = 0

  for (const dir of SESSIONS_DIRS) {
    if (!fs.existsSync(dir)) continue

    // Find all JSONL files (skip subagents — they're not user conversations)
    const files = fs.readdirSync(dir)
      .filter(f => f.endsWith('.jsonl'))
      .map(f => path.join(dir, f))

    for (const file of files) {
      filesScanned++
      const results = await scanFile(file)
      if (results.length > 0) {
        sessionsFound++
        allResults.push(...results)
      }
    }
  }

  // Sort by timestamp
  allResults.sort((a, b) => a.timestamp.localeCompare(b.timestamp))

  // Deduplicate against existing log
  let existing = new Set()
  if (fs.existsSync(LOG_FILE)) {
    const log = fs.readFileSync(LOG_FILE, 'utf8')
    // Extract timestamps + first few words as dedup key
    for (const entry of log.split('\n---\n')) {
      const match = entry.match(/^(\d{4}-\d{2}-\d{2}T[\d:]+Z?)/)
      if (match) existing.add(match[1])
    }
  }

  // Append new entries
  let newCount = 0
  const corrections = allResults.filter(r => r.type === 'correction')
  const approvals = allResults.filter(r => r.type === 'approval')

  for (const r of allResults) {
    // Skip if we already have an entry at this exact timestamp
    const tsKey = r.timestamp.slice(0, 20)
    if (existing.has(tsKey)) continue
    existing.add(tsKey)

    const logLine = `\n---\n${r.timestamp} [${r.type}]\n${r.text}\ncwd: ${r.cwd}\nsession: ${r.session}\n`
    fs.appendFileSync(LOG_FILE, logLine)
    newCount++
  }

  console.log(`Scanned ${filesScanned} transcript files`)
  console.log(`Found signals in ${sessionsFound} sessions`)
  console.log(`Total signals: ${allResults.length} (${corrections.length} corrections, ${approvals.length} approvals)`)
  console.log(`New entries added: ${newCount} (deduplicated against existing log)`)
  console.log(`Log file: ${LOG_FILE}`)
}

scanAll().catch(err => {
  console.error('Scan failed:', err.message)
  process.exit(1)
})