// du — comprehensive disk usage analysis for all Hyperdrives in a corestore
import RocksDB from 'rocksdb-native'
import b4a from 'b4a'
import { decodeNode, decodeDriveEntry, stripSubEncoder, decodeHeader } from './block-reader.js'
function encodeVarint (n) {
const bytes = []
while (n >= 0x80) { bytes.push((n & 0x7f) | 0x80); n >>>= 7 }
bytes.push(n & 0x7f)
return Buffer.from(bytes)
}
function fmt (bytes) {
if (bytes >= 1024 * 1024 * 1024) return (bytes / 1024 / 1024 / 1024).toFixed(2) + ' GB'
if (bytes >= 1024 * 1024) return (bytes / 1024 / 1024).toFixed(2) + ' MB'
if (bytes >= 1024) return (bytes / 1024).toFixed(1) + ' KB'
return bytes + ' B'
}
// Full disk usage report for a single corestore RocksDB
export async function corestoreDU (dbPath, opts = {}) {
const { readOnly = true, maxBlocksPerDrive = 50000, dirDepth = 2 } = opts
const db = new RocksDB(dbPath, { readOnly, columnFamilies: ['corestore'] })
await db.ready()
const cs = db.columnFamily('corestore')
// Step 1: Enumerate all cores (key prefix 0x01 = TL_CORE_BY_DKEY)
const cores = []
const coreIter = cs.iterator({ gte: Buffer.from([0x01]), lt: Buffer.from([0x02]), limit: 10000 })
await drain(coreIter, ({ key, value }) => {
if (!value || key[0] !== 0x01) return
const discoveryKey = b4a.toString(key.subarray(1, 33), 'hex')
try {
let pos = 0
const r = (buf, off) => { let r = 0, s = 0, p = off; while (p < buf.length) { const b = buf[p]; r |= (b & 0x7f) << s; p++; if ((b & 0x80) === 0) break; s += 7 } return { v: r, e: p } }
const { v: version, e: e1 } = r(value, pos)
const { v: corePointer, e: e2 } = r(value, e1)
const { v: dataPointer, e: e3 } = r(value, e2)
const { v: flags, e: e4 } = r(value, e3)
let alias = null
if (flags & 1) {
const { v: nameLen, e: ne } = r(value, e4)
alias = b4a.toString(value.subarray(ne, ne + nameLen), 'utf8')
}
// Read auth for public key
const authKey = Buffer.concat([encodeVarint(3), encodeVarint(corePointer), encodeVarint(0)])
cores.push({ discoveryKey, corePointer, dataPointer, alias })
} catch {}
})
// Step 2: For each core, check if block 0 is a Hyperbee header
const drives = []
const nonDriveCores = []
for (const core of cores) {
const blockKey = Buffer.concat([encodeVarint(4), encodeVarint(core.dataPointer), encodeVarint(3), encodeVarint(0)])
const blockEnd = Buffer.concat([encodeVarint(4), encodeVarint(core.dataPointer), encodeVarint(3), encodeVarint(1)])
let block0 = null
const b0Iter = cs.iterator({ gte: blockKey, lt: blockEnd, limit: 1 })
await drain(b0Iter, ({ value }) => { block0 = value })
if (!block0) {
nonDriveCores.push({ ...core, type: 'empty' })
continue
}
const hex = b4a.toString(block0.subarray(0, Math.min(12, block0.length)), 'hex')
const isHyperbee = hex.startsWith('0a08687970657262')
if (!isHyperbee) {
nonDriveCores.push({ ...core, type: 'data' })
continue
}
// Parse header for contentFeed
let header = null
try { header = decodeHeader(block0) } catch {}
const contentKey = header?.contentFeed ? b4a.toString(header.contentFeed, 'hex') : null
drives.push({ ...core, header, contentKey, type: 'hyperbee' })
}
// Step 3: For each Hyperbee, read all blocks and decode file entries
const driveReports = []
for (const drive of drives) {
const prefix = Buffer.concat([encodeVarint(4), encodeVarint(drive.dataPointer), encodeVarint(3)])
const endPrefix = Buffer.concat([encodeVarint(4), encodeVarint(drive.dataPointer), encodeVarint(4)])
const files = []
let blocksRead = 0
const iter = cs.iterator({ gte: prefix, lt: endPrefix, limit: maxBlocksPerDrive })
await drain(iter, ({ key, value }) => {
blocksRead++
if (!value) return
// Skip block 0 (header)
let pos = prefix.length
let blockIdx = 0
let shift = 0
while (pos < key.length) {
const byte = key[pos]; blockIdx |= (byte & 0x7f) << shift; pos++
if ((byte & 0x80) === 0) break; shift += 7
}
if (blockIdx === 0) return
try {
const node = decodeNode(value)
if (!node.key || node.key.length === 0) return
const rawKey = stripSubEncoder(node.key)
let filePath = ''
try { filePath = b4a.toString(rawKey, 'utf8') } catch { return }
if (!filePath.startsWith('/')) return
let entry = null
try { entry = decodeDriveEntry(node.value) } catch {}
if (!entry) return
files.push({
path: filePath,
bytes: entry.blob?.byteLength || 0,
blockOffset: entry.blob?.blockOffset,
blockLength: entry.blob?.blockLength,
executable: entry.executable
})
} catch {}
})
// Build directory breakdown
const dirs = buildDirSummary(files, dirDepth)
const totalBytes = files.reduce((s, f) => s + f.bytes, 0)
// Fingerprint the app by unique files
const appName = identifyApp(files)
driveReports.push({
dataPointer: drive.dataPointer,
corePointer: drive.corePointer,
alias: drive.alias,
discoveryKey: drive.discoveryKey,
contentKey: drive.contentKey,
blocksRead,
fileCount: files.length,
totalBytes,
appName,
dirs,
files
})
}
// Step 4: Compute RocksDB disk usage per data pointer (sample)
const diskUsage = new Map()
const diskIter = cs.iterator({ gte: Buffer.from([0x04]), lt: Buffer.from([0x05]), limit: 1000000 })
await drain(diskIter, ({ key, value }) => {
let pos = 1
let dataPtr = 0
let shift = 0
while (pos < key.length) {
const byte = key[pos]; dataPtr |= (byte & 0x7f) << shift; pos++
if ((byte & 0x80) === 0) break; shift += 7
}
const cur = diskUsage.get(dataPtr) || 0
diskUsage.set(dataPtr, cur + (value ? value.length : 0))
})
await db.close()
// Map disk usage to drives
const driveDataPtrs = new Set(driveReports.map(d => d.dataPointer))
let orphanedBytes = 0
let totalDiskBytes = 0
for (const [ptr, bytes] of diskUsage) {
totalDiskBytes += bytes
if (!driveDataPtrs.has(ptr)) {
// Check if this is a content core referenced by a drive
const isContent = driveReports.some(d => {
const contentCore = cores.find(c => c.discoveryKey === d.contentKey?.slice(0, 64))
return contentCore && contentCore.dataPointer === ptr
})
if (!isContent) {
// Check if it's a known non-drive core (odd-numbered = typically content for compat drives)
const isOddPair = driveReports.some(d => d.dataPointer === ptr - 1 || d.dataPointer === ptr + 1)
if (!isOddPair) {
orphanedBytes += bytes
}
}
}
}
return {
driveReports: driveReports.sort((a, b) => b.totalBytes - a.totalBytes),
nonDriveCores,
totalDiskSampled: totalDiskBytes,
orphanedBytes,
coreCount: cores.length,
driveCount: driveReports.length
}
}
function buildDirSummary (files, maxDepth) {
const dirs = new Map()
for (const f of files) {
const parts = f.path.split('/').filter(Boolean)
for (let depth = 1; depth <= Math.min(parts.length, maxDepth); depth++) {
const dirPath = '/' + parts.slice(0, depth).join('/')
const info = dirs.get(dirPath) || { path: dirPath, bytes: 0, count: 0, depth }
info.bytes += f.bytes
info.count++
dirs.set(dirPath, info)
}
// Root
const rootInfo = dirs.get('/') || { path: '/', bytes: 0, count: 0, depth: 0 }
rootInfo.bytes += f.bytes
rootInfo.count++
dirs.set('/', rootInfo)
}
return [...dirs.values()].sort((a, b) => b.bytes - a.bytes)
}
function identifyApp (files) {
// Heuristics based on unique files
for (const f of files) {
if (f.path === '/gateway.js') return 'hyperident-gateway'
if (f.path.includes('/hyper-wallet.bundle.js')) return 'hyper-wallet'
if (f.path === '/ARCHITECTURE.md' && files.some(g => g.path.includes('/by-arch/'))) return 'pear-runtime'
if (f.path.includes('/bare-ffmpeg/') || f.path.includes('/pear-electron/')) return 'keet'
if (f.path.includes('/socks5-server')) return 'hyperident-gateway'
if (f.path.includes('/vendor/pear-docs/')) return 'pear-git'
if (f.path === '/public/app.js') return 'wirecore'
if (f.path.includes('/keet-tui/') || f.path === '/keet-tui.js') return 'keet-tui'
if (f.path.includes('/keet-cli/') || f.path === '/keet-cli.js') return 'keet-cli'
if (f.path.includes('/anproto')) return 'anproto-terminal'
}
// Check top-level files for package.json identification
for (const f of files) {
if (f.path === '/package.json' && f.bytes > 0 && f.bytes < 10000) {
// We can't read the content from here, just note it
return '(unknown app)'
}
}
return files.length > 0 ? '(unknown)' : '(empty)'
}
export function formatDU (result, opts = {}) {
const { dirDepth = 2, minSize = 1024 * 1024 } = opts
const lines = []
lines.push(`${result.coreCount} cores, ${result.driveCount} Hyperdrives found`)
lines.push(`RocksDB sampled: ${fmt(result.totalDiskSampled)}`)
lines.push('')
for (const d of result.driveReports) {
const name = d.appName || d.alias || `core #${d.corePointer}`
lines.push(` ${fmt(d.totalBytes).padStart(10)} ${name} (dptr=${d.dataPointer}, ${d.fileCount} files)`)
// Show top directories
const topDirs = d.dirs
.filter(dir => dir.depth > 0 && dir.depth <= dirDepth && dir.bytes >= minSize)
.slice(0, 15)
for (const dir of topDirs) {
const indent = ' ' + ' '.repeat(dir.depth - 1)
lines.push(`${indent}${fmt(dir.bytes).padStart(10)} ${dir.count.toString().padStart(5)} files ${dir.path}`)
}
lines.push('')
}
if (result.nonDriveCores.length > 0) {
const dataCores = result.nonDriveCores.filter(c => c.type === 'data')
const emptyCores = result.nonDriveCores.filter(c => c.type === 'empty')
if (dataCores.length > 0) {
lines.push(` ${dataCores.length} content/blob cores (paired with Hyperdrives above)`)
}
if (emptyCores.length > 0) {
lines.push(` ${emptyCores.length} empty cores (no data blocks)`)
}
}
return lines.join('\n')
}
// Helper: drain a RocksDB iterator stream
function drain (iter, onData) {
return new Promise((resolve, reject) => {
iter.on('data', onData)
iter.on('end', resolve)
iter.on('error', reject)
})
}