hyper-vdrive / lib / publish.cjs

// publishTree — write a set of files into a drive that stores no blob bytes.
//
// Lives here rather than in the caller so consumers (pear-git-v3's CI hook, the
// CLI) need only depend on hyper-vdrive: corestore/hyperdrive/id-encoding are
// resolved from this package, not theirs.
//
// The chunker is injected. This package deliberately does not depend on
// hyper-overlay — callers pass its `chunk`, or any function returning
// [{ hash, offset, length, data }].

const { isBare } = require('which-runtime')
const fs = isBare ? require('bare-fs') : require('fs')

const idEnc = require('hypercore-id-encoding')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const CorestoreStorage = require('hypercore-storage')
const { ContentStore, build, stats } = require('./storage.cjs')
const { saveLocations, loadLocations } = require('./locations.cjs')

function writeChunk (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

/**
 * @param {object} opts
 * @param {string} opts.storeDir            where the drive lives (pointers only)
 * @param {Array<{name: string, file: string}>} opts.files  drive path -> source file on disk
 * @param {function} opts.chunk             content-defined chunker
 * @param {string} [opts.root]            record chunk locations relative to this,
 *                                     so a seeder can resolve them later
 * @param {object} [opts.corestoreOpts]
 * @returns {Promise<{link, key, blocks, files, bytes, pointerBlocks, literalBlocks}>}
 */
async function publishTree ({ storeDir, files, chunk, root, compact, corestoreOpts } = {}) {
  let result = null
  if (!storeDir) throw new Error('publishTree requires { storeDir }')
  if (!Array.isArray(files) || files.length === 0) throw new Error('publishTree requires files')
  if (typeof chunk !== 'function') throw new Error('publishTree requires a chunk function')

  const content = new ContentStore()

  // Republishing into an existing drive needs the PREVIOUS version's bytes to be
  // resolvable: `dedup: true` seeds its hash table by reading back the previous
  // entry's blocks, and those reads come through this content store. Without the
  // saved index those chunks resolve to null and the write stalls.
  const priorLocations = root ? loadLocations(storeDir, content, root) : 0

  const prepared = []
  let bytes = 0

  for (const f of files) {
    const data = fs.readFileSync(f.file)
    const chunks = chunk(data)
    for (const c of chunks) content.add(c.hash, f.file, c.offset, c.length)
    prepared.push({ name: f.name, chunks, size: data.byteLength })
    bytes += data.byteLength
  }

  const before = { redirected: stats.putRedirected, literal: stats.putStored }
  const Storage = build(content)
  const store = new Corestore(new Storage(storeDir), corestoreOpts)
  const drive = new Hyperdrive(store)

  try {
    await drive.ready()
    for (const f of prepared) {
      // 0-byte + dedup is an upstream hyperblobs crash on read
      const ws = drive.createWriteStream(f.name, f.size === 0 ? {} : { dedup: true })
      for (const c of f.chunks) await writeChunk(ws, c.data)
      await new Promise((resolve, reject) => {
        ws.once('error', reject)
        ws.once('close', resolve)
        ws.end()
      })
    }
    await drive.getBlobs()

    // Persist the reverse index so a seeder can serve this drive without
    // re-chunking every published file on boot.
    const locations = root ? saveLocations(storeDir, content.locations, root) : 0

    result = {
      locations,
      priorLocations,
      link: 'pear://' + idEnc.normalize(drive.key),
      key: drive.key.toString('hex'),
      blocks: drive.blobs.core.length,
      files: prepared.length,
      bytes,
      pointerBlocks: stats.putRedirected - before.redirected,
      literalBlocks: stats.putStored - before.literal
    }
    return result
  } finally {
    await drive.close().catch(() => {})
    await store.close().catch(() => {})
    content.close()

    // Compact BEFORE anyone else opens the store. RocksDB's footprint carries
    // the write churn of the publish — measured at 76 MB against a 609 KB
    // steady state — and on a daemon the seeder takes the exclusive lock the
    // moment the drive is announced, so this is the only window in which it can
    // be reclaimed without stopping the daemon.
    if (compact !== false) {
      try {
        const s = new CorestoreStorage(storeDir)
        await s.ready()
        await s.compact()
        await s.close()
        if (result) result.compacted = true
      } catch {
        if (result) result.compacted = false
      }
    }
  }
}

module.exports = { publishTree }