hyper-vdrive / test / small-files.mjs

// Many small files — an app tree, not one big artifact.
// With blobs virtualised, what remains is bee metadata + one blob id per file.
//   node test/small-files.mjs <workDir> <treeDir>
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const path = require('path')
const { execFileSync } = require('child_process')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const { ContentStore, build, stats } = require('../lib/storage.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]
const TREE = process.argv[3]
const PRIMARY = Buffer.alloc(32, 7)
const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])

function walk (dir, base = dir, out = []) {
  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
    const full = path.join(dir, e.name)
    if (e.isDirectory()) walk(full, base, out)
    else if (e.isFile()) out.push({ file: full, name: '/' + path.relative(base, full) })
  }
  return out
}

const files = walk(TREE)
// force coverage of the 0-byte case even if the tree has none
const emptyDir = ROOT + '/empties'
fs.mkdirSync(emptyDir, { recursive: true })
for (const n of ['a.stamp', 'b.keep', 'c.empty']) {
  const f = path.join(emptyDir, n)
  fs.writeFileSync(f, '')
  files.push({ file: f, name: '/__empty__/' + n })
}

const content = new ContentStore()
let inputBytes = 0
let chunkCount = 0
for (const f of files) {
  f.bytes = fs.readFileSync(f.file)
  f.chunks = chunk(f.bytes)
  inputBytes += f.bytes.length
  chunkCount += f.chunks.length
  for (const c of f.chunks) content.add(c.hash, f.file, c.offset, c.length)
}

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

async function publish (dir, virtual) {
  const Storage = virtual ? build(content) : null
  const store = new Corestore(virtual ? new Storage(dir) : dir, {
    primaryKey: PRIMARY,
    unsafe: true
  })
  const drive = new Hyperdrive(store)
  await drive.ready()

  for (const f of files) {
    // 0-byte + dedup is an upstream crash on read (hyperblobs seekBlockMap, null map)
    const ws = drive.createWriteStream(f.name, f.bytes.length === 0 ? {} : { dedup: true })
    for (const c of f.chunks) await write(ws, c.data)
    await new Promise((resolve, reject) => {
      ws.once('error', reject)
      ws.once('close', resolve)
      ws.end()
    })
  }

  await drive.getBlobs()
  const out = {
    driveKey: drive.key.toString('hex'),
    metaLength: drive.core.length,
    blobsLength: drive.blobs.core.length
  }

  let mismatches = 0
  for (const f of files) {
    const got = await drive.get(f.name)
    const bytes = got === null ? Buffer.alloc(0) : got
    if (!bytes.equals(f.bytes)) mismatches++
  }
  out.mismatches = mismatches

  await drive.close()
  await store.close()
  out.storeBytes = du(dir)
  return out
}

const t0 = Date.now()
const stock = await publish(ROOT + '/stock', false)
const tStock = Date.now() - t0

const t1 = Date.now()
const virt = await publish(ROOT + '/virtual', true)
const tVirt = Date.now() - t1
content.close()

console.log(
  JSON.stringify(
    {
      files: files.length,
      inputBytes,
      chunks: chunkCount,
      sameDriveKey: stock.driveKey === virt.driveKey,
      metaLength: { stock: stock.metaLength, virtual: virt.metaLength },
      blobsLength: { stock: stock.blobsLength, virtual: virt.blobsLength },
      mismatches: { stock: stock.mismatches, virtual: virt.mismatches },
      storeBytes: { stock: stock.storeBytes, virtual: virt.storeBytes },
      ratio: +(stock.storeBytes / virt.storeBytes).toFixed(1),
      bytesPerFileVirtual: Math.round(virt.storeBytes / files.length),
      elapsedMs: { stock: tStock, virtual: tVirt },
      stats
    },
    null,
    2
  )
)