hyper-overlay / test / conformance / verify-chunker.mjs

/**
 * Verifies the FastCDC chunker fixtures against THIS build of lib/chunker.js.
 *
 * Doubles as the specification a native port must satisfy: every assertion here
 * is one the C++ implementation has to pass with the same fixture file.
 *
 * A failure here is NOT a test to regenerate past. Chunk boundaries are content
 * addresses — changing them means every peer on the old version shares nothing
 * with every peer on the new one, and existing overlay content is orphaned. That
 * is a migration, not a test update.
 *
 * Run: node test/conformance/verify-chunker.mjs
 */

import { readFileSync } from 'fs'
import { fileURLToPath } from 'url'
import { dirname, join } from 'path'
import {
  chunk, hashChunk, selectTier, getTierParams, TIERS, MIN_SIZE, AVG_SIZE, MAX_SIZE
} from '../../lib/chunker.js'

const __dir = dirname(fileURLToPath(import.meta.url))
const v = JSON.parse(readFileSync(join(__dir, 'chunker-vectors.json'), 'utf8'))

let pass = 0, fail = 0
const failures = []
const ok = (cond, what) => { if (cond) pass++; else { fail++; failures.push(what) } }

function lcg32 (seed, length) {
  const out = Buffer.alloc(length)
  let s = seed >>> 0
  for (let i = 0; i < length; i++) {
    s = (Math.imul(s, 1664525) + 1013904223) >>> 0
    out[i] = (s >>> 24) & 0xff
  }
  return out
}

function materialise (spec) {
  if (spec.kind === 'hex') return Buffer.from(spec.hex, 'hex')
  if (spec.kind === 'lcg32') return lcg32(spec.seed, spec.length)
  if (spec.kind === 'const') return Buffer.alloc(spec.length, spec.byte)
  throw new Error('unknown input spec: ' + spec.kind)
}

// ── constants ────────────────────────────────────────────────────────────────
ok(v.constants.MIN_SIZE === MIN_SIZE && v.constants.AVG_SIZE === AVG_SIZE &&
   v.constants.MAX_SIZE === MAX_SIZE, 'default chunk-size constants changed')
ok(JSON.stringify(v.constants.TIERS) === JSON.stringify(TIERS), 'TIERS table changed')

// ── mask degeneracy ──────────────────────────────────────────────────────────
// Recorded rather than assumed: every tier's avgSize is a power of two, so the
// two-phase normalisation collapses to a single mask. A port may implement one
// mask, but only because THIS holds — if a future tier breaks it, both the port
// and this assertion must change together.
for (const m of v.masks) {
  const t = TIERS[m.tier]
  const large = (1 << Math.ceil(Math.log2(t.avgSize))) - 1
  const small = (1 << Math.floor(Math.log2(t.avgSize))) - 1
  ok(large === m.large && small === m.small, `tier ${m.tier}: mask values changed`)
  ok((large === small) === m.equal, `tier ${m.tier}: mask degeneracy changed`)
}

// ── tier selection ───────────────────────────────────────────────────────────
for (const t of v.tierSelection) {
  ok(selectTier(t.size) === t.tier, `selectTier(${t.size}) => ${selectTier(t.size)}, want ${t.tier}`)
  ok(JSON.stringify(getTierParams(t.tier)) === JSON.stringify(t.params),
     `getTierParams(${t.tier}) changed`)
}

// ── hashChunk: the leaf construction ─────────────────────────────────────────
for (const h of v.hashChunk) {
  const got = hashChunk(Buffer.from(h.hex, 'hex'))
  ok(got === h.hash, `hashChunk[${h.label}] => ${got}, want ${h.hash}`)
}

// ── chunk boundaries ─────────────────────────────────────────────────────────
for (const t of v.chunkCases) {
  const data = materialise(t.input)
  const res = chunk(data, t.tier !== null ? { tier: t.tier } : undefined)
  if (res.length !== t.chunkCount) {
    fail++
    failures.push(`${t.label}: ${res.length} chunks, want ${t.chunkCount}`)
    continue
  }
  let bad = null
  for (let i = 0; i < res.length; i++) {
    const a = res[i], b = t.chunks[i]
    if (a.offset !== b.offset || a.length !== b.length || a.hash !== b.hash) {
      bad = `chunk ${i}: got {off:${a.offset},len:${a.length},hash:${a.hash.slice(0, 16)}} ` +
            `want {off:${b.offset},len:${b.length},hash:${b.hash.slice(0, 16)}}`
      break
    }
  }
  // Tiling is re-checked here, not just at generation: a port could match every
  // recorded boundary and still emit chunks that do not reconstruct the file.
  const total = res.reduce((n, c) => n + c.length, 0)
  if (!bad && total !== data.length) bad = `chunks cover ${total} of ${data.length} bytes`
  ok(!bad, `${t.label}: ${bad}`)
}

console.log(`hyper-overlay chunker conformance @ ${v.overlayVersion}`)
console.log(`  ${pass} assertions pass, ${fail} fail`)
console.log(`  ${v.chunkCases.length} chunk cases, ${v.hashChunk.length} hash cases, ${v.tierSelection.length} tier cases`)
for (const f of failures) console.log(`  FAIL  ${f}`)
console.log(fail === 0 ? 'PASS' : 'FAIL')
process.exit(fail === 0 ? 0 : 1)