hyper-vdrive / test / pass-through.mjs

// T2 — does a pass-through storage wrapper survive real hyperdrive use?
// Pass = wrapped and unwrapped runs produce identical drive keys and identical
// bytes, content survives a reopen from disk, and a plain peer can replicate
// the wrapped drive.
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const { WrappedCorestoreStorage, stats } = require('../lib/passthrough.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]
const PRIMARY = Buffer.alloc(32, 7)

const FILES = [
  ['/small.txt', Buffer.from('hello pear\n')],
  ['/empty.bin', Buffer.alloc(0)],
  ['/mid.bin', crypto.randomBytes(300 * 1024)],
  ['/big.bin', crypto.randomBytes(6 * 1024 * 1024)]
]

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

function openStore (dir, wrapped) {
  const storage = wrapped ? new WrappedCorestoreStorage(dir) : dir
  return new Corestore(storage, { primaryKey: PRIMARY, unsafe: true })
}

async function put (drive, name, buf) {
  // NB: 0-byte files must NOT take the dedup path — hyperblobs 2.12.1 writes an
  // empty block map, then seekBlockMap() reads map.blocks === null and throws.
  // Reproduced on a plain store, so it is upstream, not the wrapper.
  const ws = drive.createWriteStream(name, buf.length === 0 ? {} : { dedup: true })
  for (const c of chunk(buf)) await write(ws, c.data)
  await new Promise((resolve, reject) => {
    ws.once('error', reject)
    ws.once('close', resolve)
    ws.end()
  })
}

async function build (dir, wrapped) {
  const store = openStore(dir, wrapped)
  const drive = new Hyperdrive(store)
  await drive.ready()
  for (const [name, buf] of FILES) await put(drive, name, buf)
  await drive.getBlobs()

  const out = {
    driveKey: drive.key.toString('hex'),
    blobsKey: drive.blobs.core.key.toString('hex'),
    coreLength: drive.core.length,
    blobsLength: drive.blobs.core.length,
    rootHash: (await drive.core.treeHash()).toString('hex'),
    digests: {}
  }
  for (const [name, buf] of FILES) {
    const got = await drive.get(name)
    const bytes = got === null ? Buffer.alloc(0) : got
    out.digests[name] = crypto.hash(bytes).toString('hex')
    if (!bytes.equals(buf)) throw new Error('READBACK MISMATCH ' + name + ' (' + (wrapped ? 'wrapped' : 'plain') + ')')
  }
  await drive.close()
  await store.close()
  return out
}

async function reopen (dir, wrapped) {
  const store = openStore(dir, wrapped)
  const drive = new Hyperdrive(store)
  await drive.ready()
  const digests = {}
  for (const [name, buf] of FILES) {
    const got = await drive.get(name)
    const bytes = got === null ? Buffer.alloc(0) : got
    digests[name] = crypto.hash(bytes).toString('hex')
    if (!bytes.equals(buf)) throw new Error('REOPEN MISMATCH ' + name)
  }
  const key = drive.key.toString('hex')
  await drive.close()
  await store.close()
  return { key, digests }
}

// replicate wrapped host -> plain client, in memory
async function replicate (hostDir, clientDir, driveKey) {
  const host = openStore(hostDir, true)
  const hostDrive = new Hyperdrive(host)
  await hostDrive.ready()
  await hostDrive.getBlobs()

  const client = openStore(clientDir, false)
  const clientDrive = new Hyperdrive(client, Buffer.from(driveKey, 'hex'))
  await clientDrive.ready()

  const s1 = host.replicate(true)
  const s2 = client.replicate(false)
  s1.pipe(s2).pipe(s1)

  await clientDrive.core.update({ wait: true })
  const digests = {}
  for (const [name, buf] of FILES) {
    const got = await clientDrive.get(name)
    const bytes = got === null ? Buffer.alloc(0) : got
    digests[name] = crypto.hash(bytes).toString('hex')
    if (!bytes.equals(buf)) throw new Error('REPLICATION MISMATCH ' + name)
  }

  s1.destroy()
  s2.destroy()
  await clientDrive.close()
  await client.close()
  await hostDrive.close()
  await host.close()
  return digests
}

const plain = await build(ROOT + '/plain', false)
const before = { ...stats }
const wrapped = await build(ROOT + '/wrapped', true)
const after = { ...stats }

const same =
  plain.driveKey === wrapped.driveKey &&
  plain.blobsKey === wrapped.blobsKey &&
  plain.rootHash === wrapped.rootHash &&
  JSON.stringify(plain.digests) === JSON.stringify(wrapped.digests)

const re = await reopen(ROOT + '/wrapped', true)
const rep = await replicate(ROOT + '/wrapped', ROOT + '/client', wrapped.driveKey)

console.log(
  JSON.stringify(
    {
      identical: same,
      plainKey: plain.driveKey,
      wrappedKey: wrapped.driveKey,
      rootHashMatch: plain.rootHash === wrapped.rootHash,
      reopenOk: re.key === wrapped.driveKey && JSON.stringify(re.digests) === JSON.stringify(wrapped.digests),
      replicationOk: JSON.stringify(rep) === JSON.stringify(wrapped.digests),
      wrapperTraffic: {
        coreStoragesWrapped: after.wrapped - before.wrapped,
        readBatches: after.read - before.read,
        writeBatches: after.write - before.write,
        getBlock: after.getBlock - before.getBlock,
        putBlock: after.putBlock - before.putBlock
      }
    },
    null,
    2
  )
)