hyper-inspector / lib / deep-analyzer.js

// Deep analysis of a corestore - understand WHY it's big
// Maps cores → apps → files → blobs → disk bytes
import RocksDB from 'rocksdb-native'
import b4a from 'b4a'

const CORESTORE_CF = 'corestore'

// Analyze disk usage per core by scanning RocksDB data blocks
// Key format: varint(4) + varint(dataPointer) + varint(DATA_BLOCK=3) + varint(blockIndex)
// This is where the actual data bytes live
export async function analyzeCoreDiskUsage (dbPath, opts = {}) {
  const { readOnly = false, sampleSize = 500000 } = opts

  const db = new RocksDB(dbPath, {
    readOnly,
    columnFamilies: [CORESTORE_CF]
  })
  await db.ready()

  const cs = db.columnFamily(CORESTORE_CF)

  // Scan data blocks (key prefix 0x04 = varint-encoded TL_DATA=4)
  // Within TL_DATA, the next varint is the dataPointer, then the type:
  //   0 = head, 1 = dependency, 2 = hints, 3 = block, 4 = tree, 5 = bitfield, 6 = userData
  const dataPtrStats = new Map()
  let totalScanned = 0
  let totalBytes = 0

  const iter = cs.iterator({ gte: Buffer.from([0x04]), lt: Buffer.from([0x05]), limit: sampleSize })

  await new Promise((resolve, reject) => {
    iter.on('data', ({ key, value }) => {
      totalScanned++
      const valLen = value ? value.length : 0
      totalBytes += valLen

      // Decode dataPointer from key: skip first byte (0x04), then read varint
      let pos = 1
      let dataPtr = 0
      let shift = 0
      while (pos < key.length) {
        const byte = key[pos]
        dataPtr |= (byte & 0x7f) << shift
        pos++
        if ((byte & 0x80) === 0) break
        shift += 7
      }

      // Decode type
      let type = 0
      shift = 0
      while (pos < key.length) {
        const byte = key[pos]
        type |= (byte & 0x7f) << shift
        pos++
        if ((byte & 0x80) === 0) break
        shift += 7
      }

      const stats = dataPtrStats.get(dataPtr) || {
        dataPointer: dataPtr,
        blocks: 0,
        blockBytes: 0,
        trees: 0,
        treeBytes: 0,
        bitfields: 0,
        bitfieldBytes: 0,
        userData: 0,
        userDataBytes: 0,
        heads: 0,
        other: 0,
        otherBytes: 0
      }

      switch (type) {
        case 3: // DATA_BLOCK
          stats.blocks++
          stats.blockBytes += valLen
          break
        case 4: // DATA_TREE
          stats.trees++
          stats.treeBytes += valLen
          break
        case 5: // DATA_BITFIELD
          stats.bitfields++
          stats.bitfieldBytes += valLen
          break
        case 6: // DATA_USER_DATA
          stats.userData++
          stats.userDataBytes += valLen
          break
        case 0: // DATA_HEAD
          stats.heads++
          break
        default:
          stats.other++
          stats.otherBytes += valLen
      }

      dataPtrStats.set(dataPtr, stats)
    })
    iter.on('end', resolve)
    iter.on('error', reject)
  })

  // Also get core info to map dataPointers to cores
  const coreMap = new Map() // dataPointer -> core info
  const coreIter = cs.iterator({ gte: Buffer.from([0x01]), lt: Buffer.from([0x02]), limit: 10000 })

  await new Promise((resolve, reject) => {
    coreIter.on('data', ({ key, value }) => {
      if (!value || key[0] !== 0x01) return
      const discoveryKey = b4a.toString(key.subarray(1, 33), 'hex')

      // Decode @corestore/core { version, corePointer, dataPointer, alias? }
      try {
        let pos = 0
        const readV = (buf, off) => {
          let r = 0, s = 0, p = off
          while (p < buf.length) {
            const b = buf[p]; r |= (b & 0x7f) << s; p++
            if ((b & 0x80) === 0) break; s += 7
          }
          return { value: r, end: p }
        }

        const { value: version, end: e1 } = readV(value, pos)
        const { value: corePointer, end: e2 } = readV(value, e1)
        const { value: dataPointer, end: e3 } = readV(value, e2)
        const { value: flags, end: e4 } = readV(value, e3)

        let alias = null
        if (flags & 1) {
          const { value: nameLen, end: ne } = readV(value, e4)
          const name = b4a.toString(value.subarray(ne, ne + nameLen), 'utf8')
          alias = name
        }

        // Read auth for public key
        let publicKey = null
        const authKey = Buffer.from([0x03])
        // Will be done separately

        coreMap.set(dataPointer, { discoveryKey, corePointer, dataPointer, alias })
      } catch {}
    })
    coreIter.on('end', resolve)
    coreIter.on('error', reject)
  })

  await db.close()

  // Combine: annotate dataPtrStats with core info
  const results = []
  for (const [ptr, stats] of dataPtrStats) {
    const core = coreMap.get(ptr)
    results.push({
      ...stats,
      corePointer: core?.corePointer ?? -1,
      discoveryKey: core?.discoveryKey?.slice(0, 16) ?? 'unknown',
      alias: core?.alias ?? null,
      totalBytes: stats.blockBytes + stats.treeBytes + stats.bitfieldBytes + stats.userDataBytes + stats.otherBytes
    })
  }

  results.sort((a, b) => b.totalBytes - a.totalBytes)

  return {
    totalScanned,
    totalBytes,
    sampleComplete: totalScanned < sampleSize,
    coresAnalyzed: results.length,
    topCores: results
  }
}

export function formatAnalysis (analysis) {
  const lines = []
  lines.push(`Scanned ${analysis.totalScanned} RocksDB entries (${(analysis.totalBytes / 1024 / 1024).toFixed(1)} MB in sample)`)
  lines.push(`${analysis.sampleComplete ? 'Complete scan' : 'Partial scan (hit limit)'}`)
  lines.push(`${analysis.coresAnalyzed} data pointers found\n`)

  lines.push('--- Disk usage by core (top 20) ---\n')

  for (const c of analysis.topCores.slice(0, 20)) {
    const totalMB = (c.totalBytes / 1024 / 1024).toFixed(2)
    const blockMB = (c.blockBytes / 1024 / 1024).toFixed(2)
    const treeMB = (c.treeBytes / 1024 / 1024).toFixed(2)
    const alias = c.alias ? c.alias : c.corePointer >= 0 ? `core #${c.corePointer}` : `data #${c.dataPointer}`

    lines.push(`  ${totalMB.padStart(10)} MB  ${alias} (dkey: ${c.discoveryKey}...)`)
    lines.push(`                    blocks: ${c.blocks} (${blockMB} MB)  tree: ${c.trees} (${treeMB} MB)  bitfield: ${c.bitfields}  userData: ${c.userData}`)
  }

  return lines.join('\n')
}