hyper-vdrive / test / overlay-binding.mjs

// Publish a vdrive whose bytes are resolved through a REAL hyper-overlay FileIndex,
// from both byte sources: a working tree and a content-addressed spool.
//   node test/overlay-binding.mjs <workDir>
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const path = require('path')
const { execFileSync } = require('child_process')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const CorestoreStorage = require('hypercore-storage')
const { build, stats } = require('../lib/storage.cjs')
const { OverlayContentStore } = require('../lib/overlay-content-store.cjs')

const { FileIndex } = await import('../../hyper-overlay/lib/file-index.js')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]
const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])

const treeDir = ROOT + '/worktree'
const spoolDir = ROOT + '/spool'
fs.mkdirSync(treeDir, { recursive: true })
fs.mkdirSync(spoolDir, { recursive: true })

// a working-tree file and a spool blob (as a mirror node would hold it)
const treeBytes = crypto.randomBytes(5 * 1024 * 1024)
const spoolBytes = crypto.randomBytes(3 * 1024 * 1024)
fs.writeFileSync(path.join(treeDir, 'app.bin'), treeBytes)
const spoolOid = crypto.data(spoolBytes).toString('hex')
fs.writeFileSync(path.join(spoolDir, spoolOid), spoolBytes)

// ── populate a real FileIndex the way the overlay does ──────────────────────
const idxStore = new Corestore(ROOT + '/overlay-store')
const index = new FileIndex(idxStore)
await index.ready()

// the index stores locations only — chunk() also hands back `data` subarrays,
// and putting those in the bee blows past the 15 MiB block cap
const meta = (chunks) => chunks.map((c) => ({ hash: c.hash, offset: c.offset, length: c.length }))

const treeChunks = chunk(treeBytes)
await index.putFile('app.bin', {
  contentHash: crypto.data(treeBytes).toString('hex'),
  size: treeBytes.length,
  mtime: Date.now()
})
await index.putChunkMap('app.bin', meta(treeChunks))

const spoolChunks = chunk(spoolBytes)
await index.putFile('blob.bin', {
  contentHash: spoolOid,
  size: spoolBytes.length,
  mtime: Date.now()
})
await index.putChunkMapByHash(spoolOid, meta(spoolChunks))

// ── resolve through the index, publish a vdrive ─────────────────────────────
const content = new OverlayContentStore(index, { roots: [treeDir], syncBaseDirs: [spoolDir] })
const indexed = await content.indexAll()

const Storage = build(content)
const driveDir = ROOT + '/virtual'
const store = new Corestore(new Storage(driveDir), { primaryKey: Buffer.alloc(32, 7), unsafe: true })
const drive = new Hyperdrive(store)
await drive.ready()

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

for (const [name, chunks] of [['/app.bin', treeChunks], ['/blob.bin', spoolChunks]]) {
  const ws = drive.createWriteStream(name, { dedup: true })
  for (const c of chunks) await write(ws, c.data)
  await new Promise((resolve, reject) => {
    ws.once('error', reject)
    ws.once('close', resolve)
    ws.end()
  })
}

await drive.getBlobs()
const readTree = await drive.get('/app.bin')
const readSpool = await drive.get('/blob.bin')
const driveKey = drive.key.toString('hex')
const blobsLength = drive.blobs.core.length
await drive.close()
await store.close()
await index.close()
await idxStore.close()

const s = new CorestoreStorage(driveDir)
await s.ready()
await s.compact()
await s.close()

content.close()

console.log(
  JSON.stringify(
    {
      indexed,
      driveKey,
      blobsLength,
      workingTreeReadbackOk: readTree !== null && readTree.equals(treeBytes),
      spoolReadbackOk: readSpool !== null && readSpool.equals(spoolBytes),
      redirected: stats.putRedirected,
      literal: stats.putStored,
      hashFailures: stats.hashFailures,
      inputBytes: treeBytes.length + spoolBytes.length,
      driveStoreBytes: du(driveDir)
    },
    null,
    2
  )
)