// Runs the C++ chunker against the fixtures generated from the JS reference
// (hyper-overlay/test/conformance/chunker-vectors.json).
//
// Build:
// g++ -O2 -std=c++17 -I liboverlay/include $(pkg-config --cflags libsodium) \
// -o /tmp/ho_cdc liboverlay/test/test_chunker.cpp $(pkg-config --libs libsodium)
// Run:
// /tmp/ho_cdc <path-to-chunker-vectors.json>
//
// A chunking divergence is the worst kind of bug this port can have: both
// implementations produce valid chunks, verify their own hashes, and simply never
// share one. Nothing errors; dedup silently goes to zero. So this checks the
// boundaries themselves, not just that chunking "works".
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <map>
#include <sstream>
#include <string>
#include <vector>
#include "overlay/chunker.h"
using namespace hyperoverlay;
// ---- minimal JSON (test-only; the library stays dependency-free) ------------
struct Json {
enum Type { NUL, BOOL, NUM, STR, ARR, OBJ } type = NUL;
bool b = false; double num = 0; std::string str;
std::vector<Json> arr; std::map<std::string, Json> obj;
const Json *find(const std::string &k) const {
auto it = obj.find(k); return it == obj.end() ? nullptr : &it->second;
}
double n() const { return num; }
};
struct JP {
const char *p;
void ws() { while (*p==' '||*p=='\n'||*p=='\t'||*p=='\r') p++; }
Json parse() {
ws();
if (*p=='{') return obj(); if (*p=='[') return arr();
if (*p=='"') { Json j; j.type=Json::STR; j.str=str(); return j; }
if (!strncmp(p,"true",4)) { p+=4; Json j; j.type=Json::BOOL; j.b=true; return j; }
if (!strncmp(p,"false",5)) { p+=5; Json j; j.type=Json::BOOL; j.b=false; return j; }
if (!strncmp(p,"null",4)) { p+=4; return Json(); }
Json j; j.type=Json::NUM; j.num=strtod(p,(char**)&p); return j;
}
std::string str() {
p++; std::string o;
while (*p && *p!='"') {
if (*p=='\\') { p++; switch(*p){case 'n':o+='\n';break;case 't':o+='\t';break;
case 'r':o+='\r';break;default:o+=*p;} p++; }
else o+=*p++;
}
p++; return o;
}
Json obj() { Json j; j.type=Json::OBJ; p++; ws(); if (*p=='}'){p++;return j;}
while (*p) { ws(); std::string k=str(); ws(); p++; j.obj[k]=parse(); ws();
if (*p==','){p++;continue;} if (*p=='}'){p++;break;} break; } return j; }
Json arr() { Json j; j.type=Json::ARR; p++; ws(); if (*p==']'){p++;return j;}
while (*p) { j.arr.push_back(parse()); ws();
if (*p==','){p++;continue;} if (*p==']'){p++;break;} break; } return j; }
};
static int passed = 0, failed = 0;
static void check(bool c, const std::string &what) {
if (c) passed++; else { failed++; printf(" FAIL %s\n", what.c_str()); }
}
static std::string hex32(const uint8_t *h) {
static const char *d = "0123456789abcdef";
std::string s; for (int i = 0; i < 32; i++) { s += d[h[i]>>4]; s += d[h[i]&0xf]; }
return s;
}
static std::vector<uint8_t> from_hex(const std::string &h) {
std::vector<uint8_t> o;
for (size_t i = 0; i + 1 < h.size(); i += 2)
o.push_back((uint8_t)strtoul(h.substr(i,2).c_str(), nullptr, 16));
return o;
}
// Must match the generator exactly, or every multi-MB case compares against
// different bytes and the failures are meaningless.
// s = (s * 1664525 + 1013904223) mod 2^32 ; byte = (s >>> 24) & 0xff
static std::vector<uint8_t> lcg32(uint32_t seed, size_t len) {
std::vector<uint8_t> o(len);
uint32_t s = seed;
for (size_t i = 0; i < len; i++) {
s = (uint32_t)(s * 1664525u + 1013904223u);
o[i] = (uint8_t)((s >> 24) & 0xff);
}
return o;
}
static std::vector<uint8_t> materialise(const Json &spec) {
const std::string kind = spec.find("kind")->str;
if (kind == "hex") return from_hex(spec.find("hex")->str);
if (kind == "lcg32") return lcg32((uint32_t)spec.find("seed")->n(), (size_t)spec.find("length")->n());
if (kind == "const") return std::vector<uint8_t>((size_t)spec.find("length")->n(),
(uint8_t)spec.find("byte")->n());
printf(" FAIL unknown input spec kind: %s\n", kind.c_str());
failed++;
return {};
}
int main(int argc, char **argv) {
const char *path = argc > 1 ? argv[1]
: "../../hypercore-kb/hyper-fleeky/hyper-overlay/test/conformance/chunker-vectors.json";
std::ifstream f(path);
if (!f) { printf("cannot open %s\n", path); return 2; }
std::stringstream ss; ss << f.rdbuf();
const std::string text = ss.str();
JP jp{ text.c_str() };
const Json root = jp.parse();
printf("hyper-overlay C++ chunker conformance (fixtures @ %s)\n",
root.find("overlayVersion") ? root.find("overlayVersion")->str.c_str() : "?");
// ---- constants ----------------------------------------------------------
const Json *k = root.find("constants");
check((uint32_t)k->find("MIN_SIZE")->n() == cdc::MIN_SIZE &&
(uint32_t)k->find("AVG_SIZE")->n() == cdc::AVG_SIZE &&
(uint32_t)k->find("MAX_SIZE")->n() == cdc::MAX_SIZE, "default chunk-size constants");
const Json *jt = k->find("TIERS");
for (size_t i = 0; i < jt->arr.size(); i++) {
const Json &e = jt->arr[i];
check((uint32_t)e.find("minSize")->n() == cdc::TIERS[i].minSize &&
(uint32_t)e.find("avgSize")->n() == cdc::TIERS[i].avgSize &&
(uint32_t)e.find("maxSize")->n() == cdc::TIERS[i].maxSize,
"TIERS[" + std::to_string(i) + "] params");
}
// ---- masks (including the degeneracy) -----------------------------------
for (const Json &m : root.find("masks")->arr) {
const int t = (int)m.find("tier")->n();
const uint32_t large = cdc::mask_large(cdc::TIERS[t].avgSize);
const uint32_t small = cdc::mask_small(cdc::TIERS[t].avgSize);
check(large == (uint32_t)m.find("large")->n(), "tier " + std::to_string(t) + " large mask");
check(small == (uint32_t)m.find("small")->n(), "tier " + std::to_string(t) + " small mask");
check((large == small) == m.find("equal")->b, "tier " + std::to_string(t) + " mask degeneracy");
}
// ---- tier selection -----------------------------------------------------
for (const Json &t : root.find("tierSelection")->arr) {
const uint64_t size = (uint64_t)t.find("size")->n();
check(cdc::select_tier(size) == (int)t.find("tier")->n(),
"select_tier(" + std::to_string(size) + ")");
}
// ---- hashChunk: the leaf construction -----------------------------------
for (const Json &h : root.find("hashChunk")->arr) {
const std::vector<uint8_t> in = from_hex(h.find("hex")->str);
uint8_t out[32];
cdc::hash_chunk(in.data(), in.size(), out);
check(hex32(out) == h.find("hash")->str,
"hashChunk[" + h.find("label")->str + "] got " + hex32(out));
}
// ---- chunk boundaries ---------------------------------------------------
for (const Json &c : root.find("chunkCases")->arr) {
const std::string label = c.find("label")->str;
const std::vector<uint8_t> data = materialise(*c.find("input"));
const Json *jtier = c.find("tier");
const int tier = (jtier && jtier->type == Json::NUM) ? (int)jtier->n() : -1;
const std::vector<cdc::Chunk> got = cdc::chunk(data.data(), data.size(), tier);
const Json *want = c.find("chunks");
if (got.size() != want->arr.size()) {
check(false, label + ": " + std::to_string(got.size()) + " chunks, want " +
std::to_string(want->arr.size()));
continue;
}
std::string bad;
for (size_t i = 0; i < got.size() && bad.empty(); i++) {
const Json &w = want->arr[i];
if (got[i].offset != (uint64_t)w.find("offset")->n() ||
got[i].length != (uint64_t)w.find("length")->n() ||
hex32(got[i].hash) != w.find("hash")->str) {
bad = "chunk " + std::to_string(i) +
": got {off:" + std::to_string(got[i].offset) +
",len:" + std::to_string(got[i].length) +
",hash:" + hex32(got[i].hash).substr(0,16) + "}" +
" want {off:" + std::to_string((uint64_t)w.find("offset")->n()) +
",len:" + std::to_string((uint64_t)w.find("length")->n()) +
",hash:" + w.find("hash")->str.substr(0,16) + "}";
}
}
// Tiling checked independently: matching every recorded boundary and still
// not reconstructing the input is a real failure mode.
if (bad.empty()) {
uint64_t total = 0;
for (const cdc::Chunk &ch : got) total += ch.length;
if (total != data.size())
bad = "chunks cover " + std::to_string(total) + " of " + std::to_string(data.size());
}
check(bad.empty(), label + ": " + bad);
}
// ---- streaming produces the SAME boundaries as buffered -----------------
//
// A holder that chunks a multi-GB file in blocks and one that chunks a small
// file in memory must agree, or they produce different content addresses for
// identical bytes and share nothing. The reference states this as a property
// ("regardless of input block size"), so it is checked as one — deliberately
// with block sizes that are not multiples of anything, since a bug here would
// hide behind tidy alignment.
{
printf("\n-- streaming vs buffered\n");
for (int tier = 0; tier <= 1; tier++) {
const size_t n = tier == 0 ? 300 * 1024 : 3 * 1024 * 1024;
std::vector<uint8_t> data(n);
uint32_t s = 99 + tier;
for (size_t i = 0; i < n; i++) { s = s * 1664525u + 1013904223u; data[i] = (uint8_t)(s >> 24); }
const std::vector<cdc::Chunk> want = cdc::chunk(data.data(), data.size(), tier);
for (size_t block : { (size_t)1, (size_t)7, (size_t)4093, (size_t)65536, n }) {
cdc::StreamChunker sc(tier);
std::vector<cdc::Chunk> got;
auto emit = [&](uint64_t off, const uint8_t *d, size_t len) {
cdc::Chunk c; c.offset = off; c.length = len;
cdc::hash_chunk(d, len, c.hash);
got.push_back(c);
};
for (size_t off = 0; off < n; off += block)
sc.push(data.data() + off, std::min(block, n - off), emit);
sc.finish(emit);
bool same = got.size() == want.size();
for (size_t i = 0; same && i < got.size(); i++)
same = got[i].offset == want[i].offset && got[i].length == want[i].length &&
memcmp(got[i].hash, want[i].hash, 32) == 0;
check(same, "tier " + std::to_string(tier) + ", " + std::to_string(block) +
"-byte blocks: " + std::to_string(got.size()) + " chunks, boundaries identical to buffered");
}
}
}
// ---- the incremental hash equals the one-shot hash ---------------------
//
// The receive path never holds a whole file, so it verifies with this. If it
// disagreed with hash_chunk, every completed transfer would fail its final
// check — or worse, pass one that should not.
{
printf("\n-- streaming hasher\n");
std::vector<uint8_t> data(777 * 1024);
uint32_t s = 4242;
for (size_t i = 0; i < data.size(); i++) { s = s * 1664525u + 1013904223u; data[i] = (uint8_t)(s >> 24); }
uint8_t one_shot[32];
cdc::hash_chunk(data.data(), data.size(), one_shot);
cdc::StreamingHasher h;
h.init(data.size());
for (size_t off = 0; off < data.size(); off += 3571)
h.update(data.data() + off, std::min((size_t)3571, data.size() - off));
check(h.digest() == to_hex(one_shot, 32), "fed in 3571-byte pieces, digest matches hash_chunk");
check(h.bytes() == data.size(), "and it counts the bytes it consumed");
// The size is baked into the leaf prefix, so a hasher opened for the wrong
// size cannot agree even on identical bytes. That is the property the
// journal's resume check leans on later.
cdc::StreamingHasher wrong;
wrong.init(data.size() + 1);
wrong.update(data.data(), data.size());
check(wrong.digest() != to_hex(one_shot, 32), "a hasher opened at the wrong size cannot match");
// digest() must not consume the state — finalize asks, then may keep going.
cdc::StreamingHasher twice;
twice.init(data.size());
twice.update(data.data(), data.size());
const std::string d1 = twice.digest();
check(d1 == twice.digest() && d1 == to_hex(one_shot, 32), "digest() is repeatable, not destructive");
}
printf("\n %d assertions pass, %d fail\n", passed, failed);
printf("%s\n", failed == 0 ? "PASS" : "FAIL");
return failed == 0 ? 0 : 1;
}