hyper-overlay / bench / put-stream-vs-file.js

/**
 * putStream vs putFile — measures the cost of the tmpfile spool.
 *
 * Run: bare bench/put-stream-vs-file.js [source-file]
 *
 * Defaults to /tmp/stress-2gb.bin. Source file must already exist; the
 * intent is to compare the same input through both code paths.
 *
 * For each method we record:
 *   - elapsed wall time
 *   - peak RSS Δ vs. start
 *   - peak corestore size on disk
 *   - peak spool size on disk (os.tmpdir()/hyper-overlay-upload — only
 *     populated by putStream)
 *   - final corestore size
 *   - returned oid (must match between the two for the same source)
 *
 * Disk poll is `du`-style recursive readdir+stat at 100ms cadence; that's
 * itself I/O so absolute numbers may be slightly inflated, but the
 * comparison between methods is fair (both pay the same cost).
 */

import fs from 'bare-fs'
import path from 'bare-path'
import os from 'bare-os'
import process from 'bare-process'
import hrtime from 'bare-hrtime'
import Corestore from 'corestore'

import { HyperOverlay } from '../index.js'

const SRC = Bare.argv[2] || '/tmp/stress-2gb.bin'

if (!fs.existsSync(SRC)) {
  console.error('source file not found: ' + SRC)
  Bare.exit(1)
}

const SPOOL_DIR = path.join(os.tmpdir(), 'hyper-overlay-upload')

function du (dir) {
  let total = 0
  let entries
  try { entries = fs.readdirSync(dir, { withFileTypes: true }) } catch { return 0 }
  for (const entry of entries) {
    const p = path.join(dir, entry.name)
    if (entry.isDirectory()) total += du(p)
    else if (entry.isFile()) {
      try { total += fs.statSync(p).size } catch {}
    }
  }
  return total
}

function fmtMB (b) { return (b / 1024 / 1024).toFixed(0).padStart(5) + ' MB' }
function fmtMs (ms) {
  if (ms < 1000) return ms.toFixed(0).padStart(5) + ' ms'
  return (ms / 1000).toFixed(2).padStart(5) + ' s '
}

async function bench (label, runFn) {
  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-bench-'))
  const storeDir = path.join(tmpDir, 'store')

  // Make sure the spool dir is empty before this run so we attribute
  // any growth to this method only.
  try { fs.rmSync(SPOOL_DIR, { recursive: true, force: true }) } catch {}

  const corestore = new Corestore(storeDir)
  const overlay = new HyperOverlay(corestore)
  await overlay.ready()

  const baselineRSS = process.memoryUsage().rss
  let peakRSS = baselineRSS
  let peakStore = 0
  let peakSpool = 0

  let stop = false
  const sampler = (async () => {
    while (!stop) {
      const r = process.memoryUsage().rss
      if (r > peakRSS) peakRSS = r
      const s = du(storeDir)
      if (s > peakStore) peakStore = s
      const t = du(SPOOL_DIR)
      if (t > peakSpool) peakSpool = t
      await new Promise(res => setTimeout(res, 100))
    }
  })()

  const t0 = Number(hrtime.bigint()) / 1e6
  const result = await runFn(overlay)
  const elapsed = Number(hrtime.bigint()) / 1e6 - t0

  stop = true
  await sampler

  const finalStore = du(storeDir)

  await overlay.close()
  await corestore.close()
  fs.rmSync(tmpDir, { recursive: true, force: true })
  try { fs.rmSync(SPOOL_DIR, { recursive: true, force: true }) } catch {}

  return {
    label,
    elapsed,
    peakRSS: peakRSS - baselineRSS,
    peakStore,
    peakSpool,
    finalStore,
    oid: result.oid,
    size: result.size
  }
}

const srcSize = fs.statSync(SRC).size
console.log('source: ' + SRC + '  (' + (srcSize / 1024 / 1024).toFixed(0) + ' MB)')
console.log()

const r1 = await bench('putStream', overlay =>
  overlay.putStream('big.bin', fs.createReadStream(SRC))
)
const r2 = await bench('putFile', overlay =>
  overlay.putFile('big.bin', SRC)
)

const header  = '              elapsed     RSS Δ   spool peak   store peak   store final'
const sep     = '              -------     -----   ----------   ----------   -----------'
function row (r) {
  return r.label.padEnd(12) + '  ' +
    fmtMs(r.elapsed) + '   ' +
    fmtMB(r.peakRSS) + '   ' +
    fmtMB(r.peakSpool) + '   ' +
    fmtMB(r.peakStore) + '   ' +
    fmtMB(r.finalStore)
}

console.log(header)
console.log(sep)
console.log(row(r1))
console.log(row(r2))
console.log()
console.log('oid match: ' + (r1.oid === r2.oid ? 'yes ✓' : 'NO  ' + r1.oid + ' vs ' + r2.oid))
console.log('size:      ' + r1.size + ' bytes (both)')