/**
* Conformance vectors for the FastCDC chunker.
*
* WHY THIS MATTERS MORE THAN THE MESSAGE VECTORS. A message divergence fails
* loudly — the frame does not decode. A CHUNKING divergence fails SILENTLY: both
* implementations work, produce valid chunks, verify their own hashes, and simply
* never share a single chunk with each other. Deduplication quietly drops to
* zero and nothing errors.
*
* Run: node test/conformance/generate-chunker.mjs > test/conformance/chunker-vectors.json
* Check: node test/conformance/verify-chunker.mjs
*
* THE FIVE THINGS A PORT MUST MATCH EXACTLY
*
* 1. THE CHUNK HASH IS NOT PLAIN BLAKE2B. hashChunk is hypercore-crypto's
* `data()`: blake2b-256(0x00 || uint64_LE(byteLength) || data). A port that
* hashes the bytes alone gets different content hashes for identical bytes and
* shares nothing. Pinned by the hashChunk cases below.
*
* 2. THE ROLLING HASH IS A VARIANT. `hash = ((hash << 1) + GEAR[byte]) >>> 0` —
* shift LEFT, 32-bit unsigned. Canonical FastCDC shifts right. The hash is NOT
* reset between the two mask phases, and the first minSize bytes are never
* hashed at all (i starts at minSize).
*
* 3. THE CUT IS EXCLUSIVE. On a mask match the loop breaks WITHOUT incrementing,
* so the chunk is data[offset, offset+i) and the byte that triggered the cut
* belongs to the NEXT chunk. A one-off here still produces plausible chunks.
*
* 4. THE FINAL-CHUNK TEST IS `remaining <= minSize`, not `<`. Exactly-minSize
* remaining is a final chunk, not a search.
*
* 5. TIER MASKS DEGENERATE. large = (1<<ceil(log2(avg)))-1, small =
* (1<<floor(log2(avg)))-1 — and every tier's avgSize is a power of two, so
* large === small for all four tiers and the two-phase normalisation collapses
* to one mask. That is asserted below so a port neither "fixes" it nor relies
* on it accidentally.
*
* INPUTS are given either as literal hex (small cases) or as a deterministic PRNG
* spec, so a port reproduces multi-MB inputs without this file carrying them:
*
* lcg32: s = (s * 1664525 + 1013904223) mod 2^32 ; byte = (s >>> 24) & 0xff
*
* with `s` starting at `seed`, one byte per iteration.
*/
import { readFileSync } from 'fs'
import { fileURLToPath } from 'url'
import { dirname, join } from 'path'
import {
chunk, hashChunk, selectTier, getTierParams, TIERS, MIN_SIZE, AVG_SIZE, MAX_SIZE
} from '../../lib/chunker.js'
const __dir = dirname(fileURLToPath(import.meta.url))
const pkg = JSON.parse(readFileSync(join(__dir, '../../package.json'), 'utf8'))
function lcg32 (seed, length) {
const out = Buffer.alloc(length)
let s = seed >>> 0
for (let i = 0; i < length; i++) {
s = (Math.imul(s, 1664525) + 1013904223) >>> 0
out[i] = (s >>> 24) & 0xff
}
return out
}
const cases = []
function addChunkCase (label, input, opts = {}) {
const data = input.buf
const res = chunk(data, opts.tier !== undefined ? { tier: opts.tier } : undefined)
// Chunks must tile the input exactly — a generator that emits a gap or an
// overlap would bake a broken expectation into every port that follows it.
let acc = 0
for (const ch of res) {
if (ch.offset !== acc) throw new Error(`${label}: chunk offset gap at ${ch.offset}, expected ${acc}`)
acc += ch.length
}
if (acc !== data.length) throw new Error(`${label}: chunks cover ${acc} of ${data.length} bytes`)
cases.push({
label,
note: opts.note || null,
input: input.spec,
tier: opts.tier !== undefined ? opts.tier : null,
autoTier: opts.tier === undefined ? selectTier(data.length) : null,
chunkCount: res.length,
chunks: res.map(ch => ({ offset: ch.offset, length: ch.length, hash: ch.hash }))
})
}
const hexInput = (hex) => ({ buf: Buffer.from(hex, 'hex'), spec: { kind: 'hex', hex } })
const prngInput = (seed, length) => ({
buf: lcg32(seed, length),
spec: { kind: 'lcg32', seed, length }
})
// ── degenerate + boundary sizes ──────────────────────────────────────────────
addChunkCase('empty', hexInput(''), { note: 'returns NO chunks, not one empty chunk' })
addChunkCase('single byte', hexInput('00'))
addChunkCase('minSize-1', prngInput(1, MIN_SIZE - 1))
addChunkCase('exactly minSize', prngInput(2, MIN_SIZE),
{ note: 'final-chunk test is `remaining <= minSize`, so this is ONE chunk with no search' })
addChunkCase('minSize+1', prngInput(3, MIN_SIZE + 1),
{ note: 'first size that enters the cut-point search' })
// ── crossing avg and max ─────────────────────────────────────────────────────
addChunkCase('avgSize', prngInput(4, AVG_SIZE))
addChunkCase('between avg and max', prngInput(5, (AVG_SIZE + MAX_SIZE) >> 1))
addChunkCase('exactly maxSize', prngInput(6, MAX_SIZE))
addChunkCase('several chunks', prngInput(7, MAX_SIZE * 4 + 1234))
// ── degenerate content: the mask may never match ─────────────────────────────
addChunkCase('all zero bytes', { buf: Buffer.alloc(MAX_SIZE * 2 + 7, 0x00), spec: { kind: 'const', byte: 0, length: MAX_SIZE * 2 + 7 } },
{ note: 'constant input drives the gear hash deterministically; exercises the maxSize cutoff' })
addChunkCase('all 0xff bytes', { buf: Buffer.alloc(MAX_SIZE + MIN_SIZE, 0xff), spec: { kind: 'const', byte: 255, length: MAX_SIZE + MIN_SIZE } })
// ── forced tiers on identical input ──────────────────────────────────────────
// The SAME bytes chunked under each tier must give different boundaries. If a
// port ignores the tier parameter these four collapse to one answer.
for (let t = 0; t < TIERS.length; t++) {
addChunkCase(`forced tier ${t}`, prngInput(99, 1048576), { tier: t })
}
// ── auto tier selection across a threshold ───────────────────────────────────
addChunkCase('auto tier under 1MB', prngInput(11, 1048575))
// ── hashChunk: pins the leaf construction ────────────────────────────────────
const hashCases = [
{ label: 'empty', hex: '' },
{ label: 'single zero byte', hex: '00' },
{ label: 'abc', hex: Buffer.from('abc').toString('hex') },
{ label: '32 zero bytes', hex: '00'.repeat(32) },
{ label: '255 bytes 0x00..0xfe', hex: Buffer.from(Array.from({ length: 255 }, (_, i) => i)).toString('hex') }
].map(h => ({
...h,
hash: hashChunk(Buffer.from(h.hex, 'hex'))
}))
// ── selectTier / getTierParams ───────────────────────────────────────────────
const tierCases = [0, 1, 1048575, 1048576, 104857599, 104857600, 10737418239, 10737418240, 99999999999]
.map(size => ({ size, tier: selectTier(size), params: getTierParams(selectTier(size)) }))
// ── mask degeneracy, asserted rather than assumed ────────────────────────────
const maskCases = TIERS.map((t, i) => {
const large = (1 << Math.ceil(Math.log2(t.avgSize))) - 1
const small = (1 << Math.floor(Math.log2(t.avgSize))) - 1
return { tier: i, avgSize: t.avgSize, large, small, equal: large === small }
})
process.stdout.write(JSON.stringify({
generator: 'hyper-overlay/test/conformance/generate-chunker.mjs',
overlayVersion: pkg.version,
prng: 'lcg32: s = (s * 1664525 + 1013904223) mod 2^32; byte = (s >>> 24) & 0xff; one byte per step, s starts at seed',
hashConstruction: 'blake2b-256(0x00 || uint64_LE(byteLength) || data) — hypercore-crypto crypto.data, NOT plain blake2b',
constants: { MIN_SIZE, AVG_SIZE, MAX_SIZE, TIERS },
masks: maskCases,
tierSelection: tierCases,
hashChunk: hashCases,
chunkCases: cases
}, null, 2) + '\n')