| … | … | @@ -0,0 +1,67 @@ |
| 1 | // Persist contentHash -> (file, offset, length) beside the drive. | |
| 2 | // | |
| 3 | // A drive stores index -> contentHash. To SERVE it, something must turn a | |
| 4 | // contentHash back into bytes. Rebuilding that by re-chunking every published | |
| 5 | // file costs O(all bytes) on every boot — untenable for a seeder holding a | |
| 6 | // repo's whole release history. | |
| 7 | // | |
| 8 | // So write it once, at publish time, when we already know both halves. Paths | |
| 9 | // are stored RELATIVE to a root so the index survives the tree being moved, | |
| 10 | // and so it never leaks an absolute layout into a file peers could see. | |
| 11 | // | |
| 12 | // This is the same reverse index hyper-overlay is missing (see its issue #140); | |
| 13 | // scoped here to one drive, it is cheap: ~90 bytes per unique chunk as JSON. | |
| 14 | ||
| 15 | const { isBare } = require('which-runtime') | |
| 16 | const fs = isBare ? require('bare-fs') : require('fs') | |
| 17 | const path = isBare ? require('bare-path') : require('path') | |
| 18 | ||
| 19 | const FILE = 'locations.json' | |
| 20 | const VERSION = 1 | |
| 21 | ||
| 22 | function locationsPath (storeDir) { | |
| 23 | return path.join(storeDir, FILE) | |
| 24 | } | |
| 25 | ||
| 26 | /** | |
| 27 | * @param {string} storeDir the drive's store directory | |
| 28 | * @param {Map<string,{file,offset,length}>} locations from ContentStore | |
| 29 | * @param {string} root paths are recorded relative to this | |
| 30 | */ | |
| 31 | function saveLocations (storeDir, locations, root) { | |
| 32 | const entries = [] | |
| 33 | for (const [hash, loc] of locations) { | |
| 34 | const rel = path.relative(root, loc.file) | |
| 35 | // Never record something outside the root — it would not resolve on another | |
| 36 | // machine, and it would bake a local layout into a shared artifact. | |
| 37 | if (rel.startsWith('..') || path.isAbsolute(rel)) continue | |
| 38 | entries.push([hash, rel, loc.offset, loc.length]) | |
| 39 | } | |
| 40 | fs.mkdirSync(storeDir, { recursive: true }) | |
| 41 | const tmp = locationsPath(storeDir) + '.tmp' | |
| 42 | fs.writeFileSync(tmp, JSON.stringify({ version: VERSION, entries })) | |
| 43 | fs.renameSync(tmp, locationsPath(storeDir)) | |
| 44 | return entries.length | |
| 45 | } | |
| 46 | ||
| 47 | /** | |
| 48 | * Merge a saved index into a ContentStore, resolving paths against `root`. | |
| 49 | * Returns how many entries were loaded, or 0 when there is no index. | |
| 50 | */ | |
| 51 | function loadLocations (storeDir, content, root) { | |
| 52 | let raw | |
| 53 | try { | |
| 54 | raw = JSON.parse(fs.readFileSync(locationsPath(storeDir), 'utf8')) | |
| 55 | } catch { | |
| 56 | return 0 | |
| 57 | } | |
| 58 | if (!raw || raw.version !== VERSION || !Array.isArray(raw.entries)) return 0 | |
| 59 | let n = 0 | |
| 60 | for (const [hash, rel, offset, length] of raw.entries) { | |
| 61 | content.add(hash, path.join(root, rel), offset, length) | |
| 62 | n++ | |
| 63 | } | |
| 64 | return n | |
| 65 | } | |
| 66 | ||
| 67 | module.exports = { saveLocations, loadLocations, locationsPath } |
| … | … | @@ -15,6 +15,7 @@ const idEnc = require('hypercore-id-encoding') |
| 15 | 15 | const Corestore = require('corestore') |
| 16 | 16 | const Hyperdrive = require('hyperdrive') |
| 17 | 17 | const { ContentStore, build, stats } = require('./storage.cjs') |
| 18 | const { saveLocations } = require('./locations.cjs') | |
| 18 | 19 | |
| 19 | 20 | function writeChunk (ws, buf) { |
| 20 | 21 | return new Promise((resolve) => { |
| … | … | @@ -28,10 +29,12 @@ function writeChunk (ws, buf) { |
| 28 | 29 | * @param {string} opts.storeDir where the drive lives (pointers only) |
| 29 | 30 | * @param {Array<{name: string, file: string}>} opts.files drive path -> source file on disk |
| 30 | 31 | * @param {function} opts.chunk content-defined chunker |
| 32 | * @param {string} [opts.root] record chunk locations relative to this, | |
| 33 | * so a seeder can resolve them later | |
| 31 | 34 | * @param {object} [opts.corestoreOpts] |
| 32 | 35 | * @returns {Promise<{link, key, blocks, files, bytes, pointerBlocks, literalBlocks}>} |
| 33 | 36 | */ |
| 34 | async function publishTree ({ storeDir, files, chunk, corestoreOpts } = {}) { | |
| 37 | async function publishTree ({ storeDir, files, chunk, root, corestoreOpts } = {}) { | |
| 35 | 38 | if (!storeDir) throw new Error('publishTree requires { storeDir }') |
| 36 | 39 | if (!Array.isArray(files) || files.length === 0) throw new Error('publishTree requires files') |
| 37 | 40 | if (typeof chunk !== 'function') throw new Error('publishTree requires a chunk function') |
| … | … | @@ -67,7 +70,12 @@ async function publishTree ({ storeDir, files, chunk, corestoreOpts } = {}) { |
| 67 | 70 | } |
| 68 | 71 | await drive.getBlobs() |
| 69 | 72 | |
| 73 | // Persist the reverse index so a seeder can serve this drive without | |
| 74 | // re-chunking every published file on boot. | |
| 75 | const locations = root ? saveLocations(storeDir, content.locations, root) : 0 | |
| 76 | ||
| 70 | 77 | return { |
| 78 | locations, | |
| 71 | 79 | link: 'pear://' + idEnc.normalize(drive.key), |
| 72 | 80 | key: drive.key.toString('hex'), |
| 73 | 81 | blocks: drive.blobs.core.length, |
| … | … | @@ -0,0 +1,68 @@ |
| 1 | // Reopen a published drive for SERVING, without re-chunking anything. | |
| 2 | // | |
| 3 | // `pear seed` cannot do this job: it opens the drive by key in PEAR's own | |
| 4 | // corestore and downloads it, which would materialise a full second copy of | |
| 5 | // every byte — the copy this package exists to avoid. It is a re-seeder, not | |
| 6 | // a source. So the origin serves its own drives, out of its own store. | |
| 7 | // | |
| 8 | // The bytes are resolved through the locations index written at publish time, | |
| 9 | // so opening a drive is O(index) rather than O(all published bytes). | |
| 10 | ||
| 11 | const idEnc = require('hypercore-id-encoding') | |
| 12 | const Corestore = require('corestore') | |
| 13 | const Hyperdrive = require('hyperdrive') | |
| 14 | const { ContentStore, build } = require('./storage.cjs') | |
| 15 | const { loadLocations } = require('./locations.cjs') | |
| 16 | ||
| 17 | /** | |
| 18 | * @param {object} opts | |
| 19 | * @param {string} opts.storeDir the drive's store (pointers, tree, metadata) | |
| 20 | * @param {string} opts.root resolves the locations index's relative paths | |
| 21 | * @param {object} [opts.corestoreOpts] | |
| 22 | * @returns {Promise<{drive, store, content, link, key, discoveryKey, blocks, locations}>} | |
| 23 | */ | |
| 24 | async function openDrive ({ storeDir, root, corestoreOpts } = {}) { | |
| 25 | if (!storeDir || !root) throw new Error('openDrive requires { storeDir, root }') | |
| 26 | ||
| 27 | const content = new ContentStore() | |
| 28 | const locations = loadLocations(storeDir, content, root) | |
| 29 | ||
| 30 | const Storage = build(content) | |
| 31 | const store = new Corestore(new Storage(storeDir), corestoreOpts) | |
| 32 | const drive = new Hyperdrive(store) | |
| 33 | await drive.ready() | |
| 34 | await drive.getBlobs() | |
| 35 | ||
| 36 | return { | |
| 37 | drive, | |
| 38 | store, | |
| 39 | content, | |
| 40 | locations, | |
| 41 | link: 'pear://' + idEnc.normalize(drive.key), | |
| 42 | key: drive.key.toString('hex'), | |
| 43 | discoveryKey: drive.discoveryKey, | |
| 44 | blocks: drive.blobs.core.length | |
| 45 | } | |
| 46 | } | |
| 47 | ||
| 48 | /** | |
| 49 | * Announce an already-open drive on a swarm the caller owns. | |
| 50 | * | |
| 51 | * `server: true, client: false` — an origin has everything and needs nothing. | |
| 52 | * A second node holding the same pointers can announce with `client: true` to | |
| 53 | * become an additional source, which is how a pear:// link ends up served by | |
| 54 | * more than one backend. | |
| 55 | */ | |
| 56 | function announce (swarm, opened, { client = false } = {}) { | |
| 57 | const discovery = swarm.join(opened.discoveryKey, { server: true, client }) | |
| 58 | return discovery | |
| 59 | } | |
| 60 | ||
| 61 | async function closeDrive (opened) { | |
| 62 | if (!opened) return | |
| 63 | await opened.drive.close().catch(() => {}) | |
| 64 | await opened.store.close().catch(() => {}) | |
| 65 | if (opened.content && typeof opened.content.close === 'function') opened.content.close() | |
| 66 | } | |
| 67 | ||
| 68 | module.exports = { openDrive, announce, closeDrive } |
| … | … | @@ -10,7 +10,8 @@ |
| 10 | 10 | ".": "./lib/storage.cjs", |
| 11 | 11 | "./overlay": "./lib/overlay-content-store.cjs", |
| 12 | 12 | "./passthrough": "./lib/passthrough.cjs", |
| 13 | "./publish": "./lib/publish.cjs" | |
| 13 | "./publish": "./lib/publish.cjs", | |
| 14 | "./locations": "./lib/locations.cjs" | |
| 14 | 15 | }, |
| 15 | 16 | "scripts": { |
| 16 | 17 | "test:passthrough": "node test/pass-through.mjs ./work/passthrough", |
| … | … | @@ -0,0 +1,73 @@ |
| 1 | // Publish, then reopen for serving WITHOUT the chunker — the locations index | |
| 2 | // must carry everything needed to resolve bytes, and a stock peer must be able | |
| 3 | // to replicate the whole drive out of the reopened one. | |
| 4 | import { createRequire } from 'module' | |
| 5 | const require = createRequire(import.meta.url) | |
| 6 | ||
| 7 | const fs = require('fs') | |
| 8 | const path = require('path') | |
| 9 | const crypto = require('hypercore-crypto') | |
| 10 | const Corestore = require('corestore') | |
| 11 | const Hyperdrive = require('hyperdrive') | |
| 12 | const { publishTree } = require('../lib/publish.cjs') | |
| 13 | const { openDrive, closeDrive } = require('../lib/serve.cjs') | |
| 14 | const { locationsPath } = require('../lib/locations.cjs') | |
| 15 | const { chunk } = await import('../../hyper-overlay/lib/chunker.js') | |
| 16 | ||
| 17 | const ROOT = process.argv[2] | |
| 18 | const releaseDir = path.join(ROOT, 'releases') | |
| 19 | const storeDir = path.join(ROOT, 'drive') | |
| 20 | fs.mkdirSync(path.join(releaseDir, 'v1'), { recursive: true }) | |
| 21 | ||
| 22 | const payload = crypto.randomBytes(6 * 1024 * 1024) | |
| 23 | const small = Buffer.from('{"name":"demo"}') | |
| 24 | fs.writeFileSync(path.join(releaseDir, 'v1', 'app.bin'), payload) | |
| 25 | fs.writeFileSync(path.join(releaseDir, 'v1', 'package.json'), small) | |
| 26 | ||
| 27 | const published = await publishTree({ | |
| 28 | storeDir, | |
| 29 | root: releaseDir, | |
| 30 | chunk, | |
| 31 | files: [ | |
| 32 | { name: '/app.bin', file: path.join(releaseDir, 'v1', 'app.bin') }, | |
| 33 | { name: '/package.json', file: path.join(releaseDir, 'v1', 'package.json') } | |
| 34 | ] | |
| 35 | }) | |
| 36 | ||
| 37 | // Reopen with NO chunker in sight — everything must come from the index. | |
| 38 | const opened = await openDrive({ storeDir, root: releaseDir }) | |
| 39 | ||
| 40 | const gotBin = await opened.drive.get('/app.bin') | |
| 41 | const gotPkg = await opened.drive.get('/package.json') | |
| 42 | ||
| 43 | // And a stock peer must be able to pull it out of the reopened drive. | |
| 44 | const peerStore = new Corestore(path.join(ROOT, 'peer')) | |
| 45 | const peerDrive = new Hyperdrive(peerStore, Buffer.from(published.key, 'hex')) | |
| 46 | await peerDrive.ready() | |
| 47 | const s1 = opened.store.replicate(true) | |
| 48 | const s2 = peerStore.replicate(false) | |
| 49 | s1.pipe(s2).pipe(s1) | |
| 50 | await peerDrive.core.update({ wait: true }) | |
| 51 | const peerBin = await peerDrive.get('/app.bin', { timeout: 15000 }).catch(() => null) | |
| 52 | ||
| 53 | s1.destroy() | |
| 54 | s2.destroy() | |
| 55 | await peerDrive.close() | |
| 56 | await peerStore.close() | |
| 57 | await closeDrive(opened) | |
| 58 | ||
| 59 | console.log( | |
| 60 | JSON.stringify( | |
| 61 | { | |
| 62 | locationsWritten: published.locations, | |
| 63 | locationsLoaded: opened.locations, | |
| 64 | indexBytes: fs.statSync(locationsPath(storeDir)).size, | |
| 65 | sameKey: published.key === opened.key, | |
| 66 | reopenedReadsOk: gotBin !== null && gotBin.equals(payload) && gotPkg.equals(small), | |
| 67 | stockPeerGotBytes: peerBin !== null && peerBin.equals(payload), | |
| 68 | payloadBytes: payload.length | |
| 69 | }, | |
| 70 | null, | |
| 71 | 2 | |
| 72 | ) | |
| 73 | ) |