hyper-overlay / test / chunker.test.js

import test from 'brittle'
import { chunk, hashChunk, MIN_SIZE, AVG_SIZE, MAX_SIZE, selectTier, getTierParams, TIERS } from '../lib/chunker.js'
import crypto from 'hypercore-crypto'

test('empty buffer produces zero chunks', (t) => {
  const result = chunk(Buffer.alloc(0))
  t.is(result.length, 0)
})

test('buffer smaller than MIN_SIZE produces one chunk', (t) => {
  const data = crypto.randomBytes(100)
  const result = chunk(data)
  t.is(result.length, 1)
  t.is(result[0].length, 100)
  t.is(result[0].offset, 0)
})

test('buffer exactly MIN_SIZE produces one chunk', (t) => {
  const data = crypto.randomBytes(MIN_SIZE)
  const result = chunk(data)
  t.is(result.length, 1)
  t.is(result[0].length, MIN_SIZE)
})

test('large buffer produces multiple chunks', (t) => {
  const data = crypto.randomBytes(256 * 1024) // 256KB
  const result = chunk(data)
  t.ok(result.length > 1, `got ${result.length} chunks`)

  // Verify all chunks are within size bounds
  for (const c of result) {
    // Last chunk can be smaller than MIN_SIZE
    if (c !== result[result.length - 1]) {
      t.ok(c.length >= MIN_SIZE, `chunk at ${c.offset} is ${c.length} bytes (min: ${MIN_SIZE})`)
    }
    t.ok(c.length <= MAX_SIZE, `chunk at ${c.offset} is ${c.length} bytes (max: ${MAX_SIZE})`)
  }
})

test('chunks cover entire buffer without gaps or overlap', (t) => {
  const data = crypto.randomBytes(128 * 1024) // 128KB
  const result = chunk(data)

  let totalBytes = 0
  let expectedOffset = 0
  for (const c of result) {
    t.is(c.offset, expectedOffset, `chunk offset matches`)
    expectedOffset += c.length
    totalBytes += c.length
  }
  t.is(totalBytes, data.length, 'total bytes matches input')
})

test('deterministic — same input always produces same chunks', (t) => {
  const data = crypto.randomBytes(64 * 1024)
  const result1 = chunk(data)
  const result2 = chunk(data)

  t.is(result1.length, result2.length, 'same chunk count')
  for (let i = 0; i < result1.length; i++) {
    t.is(result1[i].hash, result2[i].hash, `chunk ${i} hash matches`)
    t.is(result1[i].offset, result2[i].offset, `chunk ${i} offset matches`)
    t.is(result1[i].length, result2[i].length, `chunk ${i} length matches`)
  }
})

test('CDC stability — edit in middle changes only ~2 chunks', (t) => {
  const data = Buffer.alloc(128 * 1024)
  crypto.randomBytes(128 * 1024).copy(data)

  const original = chunk(data)

  // Edit 100 bytes in the middle
  const modified = Buffer.from(data)
  crypto.randomBytes(100).copy(modified, 64 * 1024)

  const edited = chunk(modified)

  // Count chunks with different hashes
  const origHashes = new Set(original.map(c => c.hash))
  const editHashes = new Set(edited.map(c => c.hash))

  let shared = 0
  for (const h of editHashes) {
    if (origHashes.has(h)) shared++
  }

  const changedChunks = edited.length - shared
  t.ok(changedChunks <= 4, `only ${changedChunks} chunks changed (expected ≤4)`)
  t.ok(shared > edited.length / 2, `${shared}/${edited.length} chunks shared`)
})

test('average chunk size approaches tier AVG_SIZE', (t) => {
  const data = crypto.randomBytes(1024 * 1024) // 1MB
  const tier = selectTier(data.length)
  const params = getTierParams(tier)
  const result = chunk(data)

  const avgSize = data.length / result.length
  // Allow 50% tolerance — CDC averages are approximate
  t.ok(avgSize > params.avgSize * 0.5, `avg ${avgSize} > ${params.avgSize * 0.5}`)
  t.ok(avgSize < params.avgSize * 2.0, `avg ${avgSize} < ${params.avgSize * 2.0}`)
})

// ── Adaptive tier tests ───────────────────────────────────────

test('selectTier returns correct tier for file sizes', (t) => {
  t.is(selectTier(100), 0, '100 bytes = tier 0')
  t.is(selectTier(500000), 0, '500KB = tier 0')
  t.is(selectTier(1048575), 0, 'just under 1MB = tier 0')
  t.is(selectTier(1048576), 1, 'exactly 1MB = tier 1')
  t.is(selectTier(50000000), 1, '50MB = tier 1')
  t.is(selectTier(104857600), 2, '100MB = tier 2')
  t.is(selectTier(5000000000), 2, '5GB = tier 2')
  t.is(selectTier(10737418240), 3, '10GB = tier 3')
  t.is(selectTier(200000000000), 3, '200GB = tier 3')
})

test('tier 0 uses small chunks', (t) => {
  const data = crypto.randomBytes(256 * 1024) // 256KB = tier 0
  const result = chunk(data)

  for (const c of result) {
    t.ok(c.length <= 65536, `chunk ${c.length} <= 64KB max`)
  }
})

test('tier 1 uses larger chunks', (t) => {
  const data = crypto.randomBytes(2 * 1024 * 1024) // 2MB = tier 1
  const result = chunk(data)

  // Tier 1: max 256KB. Should have fewer, larger chunks than tier 0
  for (const c of result) {
    t.ok(c.length <= 262144, `chunk ${c.length} <= 256KB max`)
  }
  // 2MB at ~64KB avg should produce ~30 chunks (not ~125 like tier 0 would)
  t.ok(result.length < 60, `${result.length} chunks (tier 1 should produce fewer than tier 0)`)
})

test('forced tier overrides auto-selection', (t) => {
  const data = crypto.randomBytes(2 * 1024 * 1024) // 2MB = auto tier 1

  const autoResult = chunk(data)
  const forcedResult = chunk(data, { tier: 0 })

  // Forcing tier 0 on a 2MB file should produce more, smaller chunks
  t.ok(forcedResult.length > autoResult.length,
    `tier 0 forced: ${forcedResult.length} chunks vs auto tier 1: ${autoResult.length}`)
})

test('same tier produces identical chunks', (t) => {
  const data = crypto.randomBytes(2 * 1024 * 1024)
  const a = chunk(data, { tier: 1 })
  const b = chunk(data, { tier: 1 })

  t.is(a.length, b.length)
  for (let i = 0; i < a.length; i++) {
    t.is(a[i].hash, b[i].hash, `chunk ${i} hash matches`)
  }
})

test('hashChunk produces consistent hex string', (t) => {
  const data = Buffer.from('hello world')
  const h1 = hashChunk(data)
  const h2 = hashChunk(data)
  t.is(h1, h2, 'same hash')
  t.is(h1.length, 64, '32 bytes = 64 hex chars')
  t.ok(/^[0-9a-f]{64}$/.test(h1), 'valid hex')
})

test('different data produces different hashes', (t) => {
  const h1 = hashChunk(Buffer.from('hello'))
  const h2 = hashChunk(Buffer.from('world'))
  t.not(h1, h2, 'different hashes')
})

import { Readable } from 'streamx'
import { chunkStream, createStreamingHasher } from '../lib/chunker.js'

function bufToStream (buf, partSize) {
  const parts = []
  for (let i = 0; i < buf.length; i += partSize) {
    parts.push(buf.subarray(i, Math.min(i + partSize, buf.length)))
  }
  return Readable.from(parts)
}

async function collect (asyncIter) {
  const out = []
  for await (const v of asyncIter) out.push(v)
  return out
}

test('chunkStream produces same chunk hashes as chunk() on same data', async (t) => {
  const data = crypto.randomBytes(2 * 1024 * 1024) // 2 MB → tier 1 boundary
  const tier = selectTier(data.length)
  const sync = chunk(data, { tier })
  const streamed = await collect(chunkStream(bufToStream(data, 8192), { tier }))

  t.is(streamed.length, sync.length, 'same chunk count')
  for (let i = 0; i < sync.length; i++) {
    t.is(streamed[i].hash, sync[i].hash, 'chunk ' + i + ' hash matches')
    t.is(streamed[i].offset, sync[i].offset, 'chunk ' + i + ' offset matches')
    t.is(streamed[i].length, sync[i].length, 'chunk ' + i + ' length matches')
  }
})

test('chunkStream is byte-exact for small files (< MIN_SIZE)', async (t) => {
  const data = crypto.randomBytes(1024)
  const out = await collect(chunkStream(bufToStream(data, 256), { tier: 0 }))
  t.is(out.length, 1, 'single chunk for sub-MIN_SIZE input')
  t.is(out[0].length, data.length)
  t.is(out[0].hash, hashChunk(data))
})

test('chunkStream chunk.data is independent of reader buffers', async (t) => {
  const data = crypto.randomBytes(64 * 1024)
  const out = await collect(chunkStream(bufToStream(data, 4096), { tier: 0 }))
  // verify each chunk's bytes match the original at its offset
  for (const c of out) {
    const expected = data.subarray(c.offset, c.offset + c.length)
    t.is(Buffer.compare(c.data, expected), 0, 'chunk bytes match source range')
  }
})

test('createStreamingHasher digest matches hashChunk when size is provided', (t) => {
  const data = crypto.randomBytes(123456)
  const ref = hashChunk(data)

  const h = createStreamingHasher({ size: data.length })
  // Feed in 8K slices to simulate a stream
  for (let i = 0; i < data.length; i += 8192) {
    h.update(data.subarray(i, Math.min(i + 8192, data.length)))
  }
  t.is(h.digest(), ref, 'streaming hash equals one-shot hash')
  t.is(h.bytes, data.length, 'byte counter matches input length')
})

test('createStreamingHasher without size produces a different (plain blake2b) digest', (t) => {
  const data = crypto.randomBytes(1024)
  const h = createStreamingHasher()
  h.update(data)
  const without = h.digest()
  const withSize = createStreamingHasher({ size: data.length })
  withSize.update(data)
  t.not(without, withSize.digest(), 'no-size digest differs from sized digest')
  t.not(without, hashChunk(data), 'no-size digest also differs from hashChunk')
})