| … | … | @@ -0,0 +1,125 @@ |
| 1 | // Extract (copy out) files from a Hyperdrive stored in a corestore RocksDB. |
| 2 | // Read-only — works on a LOCKED store (running daemon), like `cores`/`usage`. |
| 3 | // Reconstructs each file from raw blob blocks; never writes to the store. |
| 4 | import RocksDB from 'rocksdb-native' |
| 5 | import b4a from 'b4a' |
| 6 | import fs from 'fs' |
| 7 | import path from 'path' |
| 8 | import { enumerateCores } from './core-enumerator.js' |
| 9 | import { listDriveFromDB } from './drive-ls.js' |
| 10 | |
| 11 | function encodeVarint (n) { |
| 12 | const bytes = [] |
| 13 | while (n >= 0x80) { bytes.push((n & 0x7f) | 0x80); n >>>= 7 } |
| 14 | bytes.push(n & 0x7f) |
| 15 | return Buffer.from(bytes) |
| 16 | } |
| 17 | |
| 18 | // Decode the trailing varint block index from a data-block key. |
| 19 | function blockIndexFromKey (key, prefixLen) { |
| 20 | let pos = prefixLen, idx = 0, shift = 0 |
| 21 | while (pos < key.length) { |
| 22 | const b = key[pos++] |
| 23 | idx |= (b & 0x7f) << shift |
| 24 | if ((b & 0x80) === 0) break |
| 25 | shift += 7 |
| 26 | } |
| 27 | return idx |
| 28 | } |
| 29 | |
| 30 | /** |
| 31 | * Extract files from the Hyperdrive whose metadata (Hyperbee) core is at |
| 32 | * `dbDataPointer`. Copies file content to `outDir`, preserving paths. |
| 33 | * |
| 34 | * @param {string} storagePath corestore root (the dir with a CORESTORE file; db/ inside) |
| 35 | * @param {number} dbDataPointer the metadata core's dataPointer (from `cores`/`usage`) |
| 36 | * @param {string} outDir destination directory |
| 37 | * @param {object} opts { readOnly, filter, maxBytes, limit, dryRun } |
| 38 | * @returns {Promise<{written, skipped, totalBytes, blobsDataPointer, files}>} |
| 39 | */ |
| 40 | export async function extractDrive (storagePath, dbDataPointer, outDir, opts = {}) { |
| 41 | const { readOnly = true, filter = null, maxBytes = Infinity, limit = Infinity, dryRun = false, blobDataPointer = null } = opts |
| 42 | const dbPath = path.join(storagePath, 'db') |
| 43 | |
| 44 | // 1. file entries (path + blob coords) + (maybe) the content-feed core key |
| 45 | const { header, files } = await listDriveFromDB(dbPath, dbDataPointer, { readOnly }) |
| 46 | let targets = files.filter(f => f.blob && f.blob.byteLength > 0) |
| 47 | if (filter) targets = targets.filter(f => f.path.includes(filter)) |
| 48 | targets = targets.filter(f => f.blob.byteLength <= maxBytes).slice(0, limit) |
| 49 | |
| 50 | // 2. resolve the blobs (content) core's dataPointer |
| 51 | const cores = await enumerateCores(dbPath, { readOnly }) |
| 52 | let blobsDataPointer = null |
| 53 | let note = null |
| 54 | if (blobDataPointer != null) { |
| 55 | blobsDataPointer = blobDataPointer // explicit override |
| 56 | } else if (header && header.contentFeed) { |
| 57 | const hex = b4a.toString(header.contentFeed, 'hex') |
| 58 | const c = cores.find(x => x.publicKey === hex) |
| 59 | if (c) blobsDataPointer = c.dataPointer |
| 60 | else note = 'content feed core not found (' + hex.slice(0, 16) + '); pass --blob-ptr' |
| 61 | } |
| 62 | if (blobsDataPointer == null) { |
| 63 | // Fallback: Hyperdrive creates the blobs core paired right after the db |
| 64 | // core. Find the db core by its dataPointer, take the next corePointer. |
| 65 | const dbCore = cores.find(x => x.dataPointer === dbDataPointer) |
| 66 | if (dbCore) { |
| 67 | const paired = cores.find(x => x.corePointer === dbCore.corePointer + 1) |
| 68 | if (paired) { blobsDataPointer = paired.dataPointer; note = note || 'blobs core resolved by pairing (corePointer ' + paired.corePointer + ')' } |
| 69 | } |
| 70 | } |
| 71 | if (blobsDataPointer == null) { |
| 72 | return { written: 0, skipped: targets.length, totalBytes: 0, blobsDataPointer: null, files: [], note: note || 'could not resolve blobs core — pass --blob-ptr <n>' } |
| 73 | } |
| 74 | |
| 75 | // 3. collect the union of needed block indices, then one pass to read them |
| 76 | const needed = new Set() |
| 77 | for (const f of targets) { |
| 78 | for (let i = f.blob.blockOffset; i < f.blob.blockOffset + f.blob.blockLength; i++) needed.add(i) |
| 79 | } |
| 80 | |
| 81 | const blockMap = new Map() |
| 82 | if (needed.size && !dryRun) { |
| 83 | const db = new RocksDB(dbPath, { readOnly, columnFamilies: ['corestore'] }) |
| 84 | await db.ready() |
| 85 | const cs = db.columnFamily('corestore') |
| 86 | const prefix = Buffer.concat([encodeVarint(4), encodeVarint(blobsDataPointer), encodeVarint(3)]) |
| 87 | const end = Buffer.concat([encodeVarint(4), encodeVarint(blobsDataPointer), encodeVarint(4)]) |
| 88 | const iter = cs.iterator({ gte: prefix, lt: end }) |
| 89 | await new Promise((resolve, reject) => { |
| 90 | iter.on('data', ({ key, value }) => { |
| 91 | const idx = blockIndexFromKey(key, prefix.length) |
| 92 | if (value && needed.has(idx)) blockMap.set(idx, Buffer.from(value)) |
| 93 | }) |
| 94 | iter.on('end', resolve) |
| 95 | iter.on('error', reject) |
| 96 | }) |
| 97 | await db.close() |
| 98 | } |
| 99 | |
| 100 | // 4. reassemble each file and write it out |
| 101 | let written = 0, skipped = 0, totalBytes = 0 |
| 102 | const out = [] |
| 103 | for (const f of targets) { |
| 104 | const parts = [] |
| 105 | let ok = true |
| 106 | for (let i = f.blob.blockOffset; i < f.blob.blockOffset + f.blob.blockLength; i++) { |
| 107 | const b = blockMap.get(i) |
| 108 | if (!b) { ok = false; break } |
| 109 | parts.push(b) |
| 110 | } |
| 111 | if (!ok) { skipped++; out.push({ path: f.path, bytes: f.blob.byteLength, status: 'missing-blocks' }); continue } |
| 112 | // hyperblobs stores one blob per block-range; the concatenated blocks ARE |
| 113 | // the blob (byteOffset/byteLength are cumulative core metadata, NOT a slice). |
| 114 | const content = Buffer.concat(parts) |
| 115 | const dest = path.join(outDir, f.path.replace(/^\/+/, '')) |
| 116 | if (!dryRun) { |
| 117 | fs.mkdirSync(path.dirname(dest), { recursive: true }) |
| 118 | fs.writeFileSync(dest, content) |
| 119 | } |
| 120 | written++; totalBytes += content.length |
| 121 | out.push({ path: f.path, bytes: content.length, dest, status: dryRun ? 'would-write' : 'written' }) |
| 122 | } |
| 123 | |
| 124 | return { written, skipped, totalBytes, blobsDataPointer, files: out, note } |
| 125 | } |