| … | … | @@ -0,0 +1,86 @@ |
| 1 | // publishTree — write a set of files into a drive that stores no blob bytes. |
| 2 | // |
| 3 | // Lives here rather than in the caller so consumers (pear-git-v3's CI hook, the |
| 4 | // CLI) need only depend on hyper-vdrive: corestore/hyperdrive/id-encoding are |
| 5 | // resolved from this package, not theirs. |
| 6 | // |
| 7 | // The chunker is injected. This package deliberately does not depend on |
| 8 | // hyper-overlay — callers pass its `chunk`, or any function returning |
| 9 | // [{ hash, offset, length, data }]. |
| 10 | |
| 11 | const { isBare } = require('which-runtime') |
| 12 | const fs = isBare ? require('bare-fs') : require('fs') |
| 13 | |
| 14 | const idEnc = require('hypercore-id-encoding') |
| 15 | const Corestore = require('corestore') |
| 16 | const Hyperdrive = require('hyperdrive') |
| 17 | const { ContentStore, build, stats } = require('./storage.cjs') |
| 18 | |
| 19 | function writeChunk (ws, buf) { |
| 20 | return new Promise((resolve) => { |
| 21 | if (ws.write(buf)) return resolve() |
| 22 | ws.once('drain', resolve) |
| 23 | }) |
| 24 | } |
| 25 | |
| 26 | /** |
| 27 | * @param {object} opts |
| 28 | * @param {string} opts.storeDir where the drive lives (pointers only) |
| 29 | * @param {Array<{name: string, file: string}>} opts.files drive path -> source file on disk |
| 30 | * @param {function} opts.chunk content-defined chunker |
| 31 | * @param {object} [opts.corestoreOpts] |
| 32 | * @returns {Promise<{link, key, blocks, files, bytes, pointerBlocks, literalBlocks}>} |
| 33 | */ |
| 34 | async function publishTree ({ storeDir, files, chunk, corestoreOpts } = {}) { |
| 35 | if (!storeDir) throw new Error('publishTree requires { storeDir }') |
| 36 | if (!Array.isArray(files) || files.length === 0) throw new Error('publishTree requires files') |
| 37 | if (typeof chunk !== 'function') throw new Error('publishTree requires a chunk function') |
| 38 | |
| 39 | const content = new ContentStore() |
| 40 | const prepared = [] |
| 41 | let bytes = 0 |
| 42 | |
| 43 | for (const f of files) { |
| 44 | const data = fs.readFileSync(f.file) |
| 45 | const chunks = chunk(data) |
| 46 | for (const c of chunks) content.add(c.hash, f.file, c.offset, c.length) |
| 47 | prepared.push({ name: f.name, chunks, size: data.byteLength }) |
| 48 | bytes += data.byteLength |
| 49 | } |
| 50 | |
| 51 | const before = { redirected: stats.putRedirected, literal: stats.putStored } |
| 52 | const Storage = build(content) |
| 53 | const store = new Corestore(new Storage(storeDir), corestoreOpts) |
| 54 | const drive = new Hyperdrive(store) |
| 55 | |
| 56 | try { |
| 57 | await drive.ready() |
| 58 | for (const f of prepared) { |
| 59 | // 0-byte + dedup is an upstream hyperblobs crash on read |
| 60 | const ws = drive.createWriteStream(f.name, f.size === 0 ? {} : { dedup: true }) |
| 61 | for (const c of f.chunks) await writeChunk(ws, c.data) |
| 62 | await new Promise((resolve, reject) => { |
| 63 | ws.once('error', reject) |
| 64 | ws.once('close', resolve) |
| 65 | ws.end() |
| 66 | }) |
| 67 | } |
| 68 | await drive.getBlobs() |
| 69 | |
| 70 | return { |
| 71 | link: 'pear://' + idEnc.normalize(drive.key), |
| 72 | key: drive.key.toString('hex'), |
| 73 | blocks: drive.blobs.core.length, |
| 74 | files: prepared.length, |
| 75 | bytes, |
| 76 | pointerBlocks: stats.putRedirected - before.redirected, |
| 77 | literalBlocks: stats.putStored - before.literal |
| 78 | } |
| 79 | } finally { |
| 80 | await drive.close().catch(() => {}) |
| 81 | await store.close().catch(() => {}) |
| 82 | content.close() |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | module.exports = { publishTree } |