hyper-vdrive / test / redirect.mjs

// T3 — correctness of the block-redirecting drive.
// Pass = same drive key as a stock drive, same bytes on read, survives reopen,
// replicates to a stock peer, and stores no blob bytes.
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const { execFileSync } = require('child_process')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const { ContentStore, build, stats } = require('../lib/storage.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]
const SRC = process.argv.slice(3) // real files on disk = our "overlay"
const PRIMARY = Buffer.alloc(32, 7)
const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

// Index the source files exactly as the overlay does: FastCDC chunks, hash -> location.
const content = new ContentStore()
const sources = []
for (const [i, file] of SRC.entries()) {
  const buf = fs.readFileSync(file)
  const chunks = chunk(buf)
  for (const c of chunks) content.add(c.hash, file, c.offset, c.length)
  // distinct path per source — same basename would overwrite the earlier entry
  sources.push({ file, name: '/v' + i + '-' + file.split('/').slice(-1)[0], buf, chunks })
}

async function makeDrive (dir, redirect) {
  const pointers = null
  const Storage = redirect ? build(content) : null
  const store = new Corestore(redirect ? new Storage(dir) : dir, {
    primaryKey: PRIMARY,
    unsafe: true
  })
  const drive = new Hyperdrive(store)
  await drive.ready()
  return { store, drive, pointers }
}

async function publish (dir, redirect) {
  const { store, drive, pointers } = await makeDrive(dir, redirect)
  for (const s of sources) {
    const ws = drive.createWriteStream(s.name, { dedup: true })
    for (const c of s.chunks) await write(ws, c.data)
    await new Promise((resolve, reject) => {
      ws.once('error', reject)
      ws.once('close', resolve)
      ws.end()
    })
  }
  await drive.getBlobs()
  const out = {
    driveKey: drive.key.toString('hex'),
    blobsKey: drive.blobs.core.key.toString('hex'),
    rootHash: (await drive.core.treeHash()).toString('hex'),
    blobsLength: drive.blobs.core.length
  }
  await drive.close()
  await store.close()
  return out
}

async function verifyReads (dir, redirect) {
  const { store, drive } = await makeDrive(dir, redirect)
  for (const s of sources) {
    const got = await drive.get(s.name)
    if (got === null || !got.equals(s.buf)) throw new Error('MISMATCH ' + s.name)
  }
  await drive.close()
  await store.close()
  return true
}

async function replicateToStockPeer (hostDir, clientDir, driveKey) {
  const { store: host, drive: hostDrive } = await makeDrive(hostDir, true)
  await hostDrive.getBlobs()

  const client = new Corestore(clientDir, { primaryKey: Buffer.alloc(32, 9), unsafe: true })
  const clientDrive = new Hyperdrive(client, Buffer.from(driveKey, 'hex'))
  await clientDrive.ready()

  const s1 = host.replicate(true)
  const s2 = client.replicate(false)
  s1.pipe(s2).pipe(s1)

  await clientDrive.core.update({ wait: true })
  for (const s of sources) {
    const got = await clientDrive.get(s.name)
    if (got === null || !got.equals(s.buf)) throw new Error('REPLICATION MISMATCH ' + s.name)
  }

  s1.destroy()
  s2.destroy()
  await clientDrive.close()
  await client.close()
  await hostDrive.close()
  await host.close()
  return true
}

const stock = await publish(ROOT + '/stock', false)
const stockBytes = du(ROOT + '/stock')

const virt = await publish(ROOT + '/virtual', true)
const virtBytes = du(ROOT + '/virtual')
const pointerBytes = 0

const reopenOk = await verifyReads(ROOT + '/virtual', true)
const replicationOk = await replicateToStockPeer(ROOT + '/virtual', ROOT + '/peer', virt.driveKey)
const peerBytes = du(ROOT + '/peer')
content.close()

console.log(
  JSON.stringify(
    {
      sameDriveKey: stock.driveKey === virt.driveKey,
      sameBlobsKey: stock.blobsKey === virt.blobsKey,
      sameRootHash: stock.rootHash === virt.rootHash,
      blobsLength: { stock: stock.blobsLength, virtual: virt.blobsLength },
      reopenOk,
      replicationOk,
      inputBytes: sources.reduce((a, s) => a + s.buf.length, 0),
      stockStoreBytes: stockBytes,
      virtualStoreBytes: virtBytes,
      pointerFileBytes: pointerBytes,
      peerStoreBytes: peerBytes,
      stats
    },
    null,
    2
  )
)