hyper-inspector / lib / drive-extract.js

// Extract (copy out) files from a Hyperdrive stored in a corestore RocksDB.
// Read-only — works on a LOCKED store (running daemon), like `cores`/`usage`.
// Reconstructs each file from raw blob blocks; never writes to the store.
import RocksDB from 'rocksdb-native'
import b4a from 'b4a'
import fs from 'fs'
import path from 'path'
import { enumerateCores } from './core-enumerator.js'
import { listDriveFromDB } from './drive-ls.js'

function encodeVarint (n) {
  const bytes = []
  while (n >= 0x80) { bytes.push((n & 0x7f) | 0x80); n >>>= 7 }
  bytes.push(n & 0x7f)
  return Buffer.from(bytes)
}

// Decode the trailing varint block index from a data-block key.
function blockIndexFromKey (key, prefixLen) {
  let pos = prefixLen, idx = 0, shift = 0
  while (pos < key.length) {
    const b = key[pos++]
    idx |= (b & 0x7f) << shift
    if ((b & 0x80) === 0) break
    shift += 7
  }
  return idx
}

/**
 * Extract files from the Hyperdrive whose metadata (Hyperbee) core is at
 * `dbDataPointer`. Copies file content to `outDir`, preserving paths.
 *
 * @param {string} storagePath  corestore root (the dir with a CORESTORE file; db/ inside)
 * @param {number} dbDataPointer  the metadata core's dataPointer (from `cores`/`usage`)
 * @param {string} outDir  destination directory
 * @param {object} opts  { readOnly, filter, maxBytes, limit, dryRun }
 * @returns {Promise<{written, skipped, totalBytes, blobsDataPointer, files}>}
 */
export async function extractDrive (storagePath, dbDataPointer, outDir, opts = {}) {
  const { readOnly = true, filter = null, maxBytes = Infinity, limit = Infinity, dryRun = false, blobDataPointer = null } = opts
  const dbPath = path.join(storagePath, 'db')

  // 1. file entries (path + blob coords) + (maybe) the content-feed core key
  const { header, files } = await listDriveFromDB(dbPath, dbDataPointer, { readOnly })
  let targets = files.filter(f => f.blob && f.blob.byteLength > 0)
  if (filter) targets = targets.filter(f => f.path.includes(filter))
  targets = targets.filter(f => f.blob.byteLength <= maxBytes).slice(0, limit)

  // 2. resolve the blobs (content) core's dataPointer
  const cores = await enumerateCores(dbPath, { readOnly })
  let blobsDataPointer = null
  let note = null
  if (blobDataPointer != null) {
    blobsDataPointer = blobDataPointer // explicit override
  } else if (header && header.contentFeed) {
    const hex = b4a.toString(header.contentFeed, 'hex')
    const c = cores.find(x => x.publicKey === hex)
    if (c) blobsDataPointer = c.dataPointer
    else note = 'content feed core not found (' + hex.slice(0, 16) + '); pass --blob-ptr'
  }
  if (blobsDataPointer == null) {
    // Fallback: Hyperdrive creates the blobs core paired right after the db
    // core. Find the db core by its dataPointer, take the next corePointer.
    const dbCore = cores.find(x => x.dataPointer === dbDataPointer)
    if (dbCore) {
      const paired = cores.find(x => x.corePointer === dbCore.corePointer + 1)
      if (paired) { blobsDataPointer = paired.dataPointer; note = note || 'blobs core resolved by pairing (corePointer ' + paired.corePointer + ')' }
    }
  }
  if (blobsDataPointer == null) {
    return { written: 0, skipped: targets.length, totalBytes: 0, blobsDataPointer: null, files: [], note: note || 'could not resolve blobs core — pass --blob-ptr <n>' }
  }

  // 3. collect the union of needed block indices, then one pass to read them
  const needed = new Set()
  for (const f of targets) {
    for (let i = f.blob.blockOffset; i < f.blob.blockOffset + f.blob.blockLength; i++) needed.add(i)
  }

  const blockMap = new Map()
  if (needed.size && !dryRun) {
    const db = new RocksDB(dbPath, { readOnly, columnFamilies: ['corestore'] })
    await db.ready()
    const cs = db.columnFamily('corestore')
    const prefix = Buffer.concat([encodeVarint(4), encodeVarint(blobsDataPointer), encodeVarint(3)])
    const end = Buffer.concat([encodeVarint(4), encodeVarint(blobsDataPointer), encodeVarint(4)])
    const iter = cs.iterator({ gte: prefix, lt: end })
    await new Promise((resolve, reject) => {
      iter.on('data', ({ key, value }) => {
        const idx = blockIndexFromKey(key, prefix.length)
        if (value && needed.has(idx)) blockMap.set(idx, Buffer.from(value))
      })
      iter.on('end', resolve)
      iter.on('error', reject)
    })
    await db.close()
  }

  // 4. reassemble each file and write it out
  let written = 0, skipped = 0, totalBytes = 0
  const out = []
  for (const f of targets) {
    const parts = []
    let ok = true
    for (let i = f.blob.blockOffset; i < f.blob.blockOffset + f.blob.blockLength; i++) {
      const b = blockMap.get(i)
      if (!b) { ok = false; break }
      parts.push(b)
    }
    if (!ok) { skipped++; out.push({ path: f.path, bytes: f.blob.byteLength, status: 'missing-blocks' }); continue }
    // hyperblobs stores one blob per block-range; the concatenated blocks ARE
    // the blob (byteOffset/byteLength are cumulative core metadata, NOT a slice).
    const content = Buffer.concat(parts)
    const dest = path.join(outDir, f.path.replace(/^\/+/, ''))
    if (!dryRun) {
      fs.mkdirSync(path.dirname(dest), { recursive: true })
      fs.writeFileSync(dest, content)
    }
    written++; totalBytes += content.length
    out.push({ path: f.path, bytes: content.length, dest, status: dryRun ? 'would-write' : 'written' })
  }

  return { written, skipped, totalBytes, blobsDataPointer, files: out, note }
}