// The receive path: a partial on disk, verified as it fills, renamed into place
// only when the whole file hashes to what was asked for.
//
// Everything here is invisible from the wire — the frames are identical whether a
// requester assembles in RAM or on disk — so none of it can be caught by a
// fixture or a transcript. That is exactly why it gets its own suite.
//
// Build:
// g++ -O2 -std=c++17 -I liboverlay/include $(pkg-config --cflags libsodium) \
// -o /tmp/ho_xfer liboverlay/test/test_transfer.cpp $(pkg-config --libs libsodium)
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>
#include <sys/stat.h>
#include "overlay/transfer.h"
using namespace hyperoverlay;
static int passed = 0, failed = 0;
static void check(bool c, const std::string &what) {
if (c) { passed++; printf(" ok %s\n", what.c_str()); }
else { failed++; printf(" FAIL %s\n", what.c_str()); }
}
// Feeding a chunk is not itself an assertion — there are hundreds of them — but a
// rejection where one is not expected must not pass silently either.
static void check_ok(const xfer::Transfer::WriteResult &r) {
if (!r.ok) { failed++; printf(" FAIL unexpected write rejection: %s\n", r.error.c_str()); }
}
static const char *TMP = "/tmp/claude-1000/";
static std::string tmp_path(const std::string &n) { return std::string(TMP) + n; }
static std::vector<uint8_t> make_bytes(size_t n, uint32_t seed) {
std::vector<uint8_t> v(n);
uint32_t s = seed;
for (size_t i = 0; i < n; i++) { s = s * 1664525u + 1013904223u; v[i] = (uint8_t)(s >> 24); }
return v;
}
static std::vector<uint8_t> read_file(const std::string &path) {
std::vector<uint8_t> out;
FILE *f = std::fopen(path.c_str(), "rb");
if (!f) return out;
std::fseek(f, 0, SEEK_END);
const long n = std::ftell(f);
std::fseek(f, 0, SEEK_SET);
if (n > 0) { out.resize((size_t)n); if (std::fread(out.data(), 1, out.size(), f) != out.size()) out.clear(); }
std::fclose(f);
return out;
}
// Build the meta a holder would advertise for these bytes.
static xfer::Meta meta_for(const std::vector<uint8_t> &content) {
xfer::Meta m;
m.size = content.size();
uint8_t whole[32];
cdc::hash_chunk(content.data(), content.size(), whole);
m.contentHash = to_hex(whole, 32);
for (const cdc::Chunk &c : cdc::chunk(content.data(), content.size())) {
xfer::ChunkMeta cm;
std::memcpy(cm.hash, c.hash, 32);
cm.offset = c.offset;
cm.length = c.length;
m.chunks.push_back(cm);
}
return m;
}
static void clean(const std::string &dest) {
std::remove(dest.c_str());
std::remove(xfer::partial_path_for(dest).c_str());
}
// Feed chunk `i` of `content` into the transfer.
static xfer::Transfer::WriteResult feed(xfer::Transfer &t, const xfer::Meta &m,
const std::vector<uint8_t> &content, size_t i) {
const xfer::ChunkMeta &c = m.chunks[i];
return t.write_chunk(i, content.data() + c.offset, (size_t)c.length);
}
int main() {
printf("hyper-overlay C++ transfer (partial-file receive)\n");
const std::vector<uint8_t> content = make_bytes(300 * 1024, 5);
const xfer::Meta meta = meta_for(content);
// ---- the partial is a separate file, preallocated -----------------------
{
printf("\n-- the partial\n");
const std::string dest = tmp_path("ho_x_1.bin");
clean(dest);
xfer::Transfer t;
check(t.start(dest, meta), "start opens a transfer");
check(io::file_exists(xfer::partial_path_for(dest)), "a partial appears beside the destination");
check(!io::file_exists(dest), "and the destination does not exist yet");
uint64_t sz = 0;
io::file_size(xfer::partial_path_for(dest), sz);
// Preallocated, so chunks can be written at their offsets from the first
// one rather than the file growing in arrival order.
check(sz == content.size(), "the partial is preallocated to the full size");
// The suffix is deliberately not a plain ".partial": host apps sweep that.
check(xfer::partial_path_for(dest).find(".overlay-partial") != std::string::npos,
"the suffix is .overlay-partial, which host sweeps do not match");
t.cancel();
}
// ---- out-of-order arrival still assembles ------------------------------
//
// The whole-file hash consumes bytes IN ORDER, but chunks do not arrive in
// order the moment more than one peer serves. The frontier is what reconciles
// those, so feeding backwards is the case worth testing.
{
printf("\n-- out-of-order arrival\n");
const std::string dest = tmp_path("ho_x_2.bin");
clean(dest);
xfer::Transfer t;
t.start(dest, meta);
check(t.total() == meta.chunks.size() && t.total() > 4,
"the file is several chunks (" + std::to_string(t.total()) + ")");
for (size_t i = meta.chunks.size(); i-- > 0;) check_ok(feed(t, meta, content, i));
check(t.complete(), "every chunk accepted, in reverse order");
check(t.hash_frontier() == t.total(), "the hash frontier still reached the end");
const xfer::Transfer::FinalizeResult f = t.finalize();
check(f.ok, "finalize succeeds: " + f.error);
check(read_file(dest) == content, "the assembled file is byte-identical");
check(!io::file_exists(xfer::partial_path_for(dest)), "the partial is gone");
}
// ---- a chunk that does not match is refused before it reaches disk ------
{
printf("\n-- bad chunks\n");
const std::string dest = tmp_path("ho_x_3.bin");
clean(dest);
xfer::Transfer t;
t.start(dest, meta);
std::vector<uint8_t> wrong((size_t)meta.chunks[0].length, 0xAB);
xfer::Transfer::WriteResult r = t.write_chunk(0, wrong.data(), wrong.size());
check(!r.ok && r.error.find("hash mismatch") != std::string::npos, "wrong bytes are refused");
check(!r.fatal, "and it is not fatal — another holder may serve it correctly");
check(t.received() == 0, "nothing was recorded as received");
// Right bytes, wrong length: caught before hashing, since a length that
// disagrees with the advertisement means the chunk list and the stream
// disagree about the file's shape.
r = t.write_chunk(0, content.data(), (size_t)meta.chunks[0].length - 1);
check(!r.ok && r.error.find("length mismatch") != std::string::npos, "a short chunk is refused");
r = t.write_chunk((uint64_t)meta.chunks.size() + 99, content.data(), 1);
check(!r.ok && r.error.find("out of range") != std::string::npos, "an out-of-range index is refused");
t.cancel();
}
// ---- a holder that lies consistently is caught at the whole-file hash ---
//
// Per-chunk hashes only prove the holder served what it ADVERTISED. If the
// advertisement itself is for other content, every chunk passes and the file
// is still wrong — which is why there are two checks and not one.
{
printf("\n-- the advertised file is not the requested one\n");
const std::string dest = tmp_path("ho_x_4.bin");
clean(dest);
const std::vector<uint8_t> other = make_bytes(300 * 1024, 6);
xfer::Meta lying = meta_for(other); // honest chunk list for OTHER bytes...
lying.contentHash = meta.contentHash; // ...announced as the hash we asked for
xfer::Transfer t;
t.start(dest, lying);
for (size_t i = 0; i < lying.chunks.size(); i++) check_ok(feed(t, lying, other, i));
check(t.complete(), "every chunk matched its advertised hash");
const xfer::Transfer::FinalizeResult f = t.finalize();
check(!f.ok && f.hash_mismatch, "but the whole-file hash rejects it");
check(!io::file_exists(dest), "nothing was written under the destination name");
check(!io::file_exists(xfer::partial_path_for(dest)),
"and the partial is dropped — resuming it would rebuild the same wrong file");
}
// ---- pause keeps the partial, cancel destroys it -----------------------
{
printf("\n-- pause vs cancel\n");
const std::string dest = tmp_path("ho_x_5.bin");
clean(dest);
xfer::Transfer t;
t.start(dest, meta);
for (size_t i = 0; i < 3; i++) check_ok(feed(t, meta, content, i));
t.pause();
check(io::file_exists(xfer::partial_path_for(dest)), "pause leaves the partial on disk");
xfer::Transfer t2;
t2.start(dest, meta);
for (size_t i = 0; i < 3; i++) check_ok(feed(t2, meta, content, i));
t2.cancel();
check(!io::file_exists(xfer::partial_path_for(dest)), "cancel removes it");
}
// ---- resume: a partial's good chunks are kept, and only the rest asked for
{
printf("\n-- resume\n");
const std::string dest = tmp_path("ho_x_6.bin");
clean(dest);
const size_t half = meta.chunks.size() / 2;
{
xfer::Transfer t;
t.start(dest, meta);
for (size_t i = 0; i < half; i++) check_ok(feed(t, meta, content, i));
t.pause();
}
xfer::Transfer t2;
double last_progress = -1;
check(t2.start(dest, meta, xfer::Options(), [&](double p) { last_progress = p; }),
"a second transfer opens the existing partial");
check(t2.received() == half, "it recovers exactly the chunks already written (" +
std::to_string(t2.received()) + "/" + std::to_string(half) + ")");
check(t2.missing().size() == meta.chunks.size() - half,
"and asks only for the ones it lacks");
check(last_progress == 1.0, "the re-verification reports progress to completion");
// Without a journal this scan is O(file). That cost is the entire argument
// for step 3, so it is asserted rather than assumed.
check(t2.hash_frontier() == half, "the whole-file hash resumed at the recovered frontier");
for (size_t i = half; i < meta.chunks.size(); i++) check_ok(feed(t2, meta, content, i));
const xfer::Transfer::FinalizeResult f = t2.finalize();
check(f.ok, "the resumed transfer finalizes: " + f.error);
check(read_file(dest) == content, "and the file is byte-identical to the original");
}
// ---- a same-size partial of GARBAGE recovers nothing --------------------
//
// Size alone must never be taken as evidence. Every chunk is re-hashed where
// it should be, so a stale partial for different content contributes nothing
// rather than being merged into the result.
{
printf("\n-- a foreign partial\n");
const std::string dest = tmp_path("ho_x_7.bin");
clean(dest);
const std::vector<uint8_t> junk = make_bytes(content.size(), 999);
FILE *f = std::fopen(xfer::partial_path_for(dest).c_str(), "wb");
std::fwrite(junk.data(), 1, junk.size(), f);
std::fclose(f);
xfer::Transfer t;
t.start(dest, meta);
check(t.received() == 0, "a same-size partial for other content recovers no chunks");
check(t.missing().size() == meta.chunks.size(), "so everything is still wanted");
for (size_t i = 0; i < meta.chunks.size(); i++) check_ok(feed(t, meta, content, i));
check(t.finalize().ok, "and the transfer overwrites it correctly");
check(read_file(dest) == content, "with the right bytes");
}
// ---- the memory cap is a refusal, not a hint ---------------------------
//
// The stash is what keeps the hash pump off the disk it just wrote to. Its cap
// is the only thing between a fast link and unbounded growth, so the
// zero-budget case must still be correct — just slower.
{
printf("\n-- the stash cap\n");
const std::string dest = tmp_path("ho_x_8.bin");
clean(dest);
xfer::Options generous; generous.mem_stash_bytes = 64u * 1024u * 1024u;
xfer::Transfer big;
big.start(dest, meta, generous);
for (size_t i = 0; i < meta.chunks.size(); i++) check_ok(feed(big, meta, content, i));
check(big.finalize().ok, "with a generous budget the transfer completes");
check(big.stash_hits() > 0 && big.readbacks() == 0,
"and the hash pump never touches the disk (" + std::to_string(big.stash_hits()) +
" stashed, " + std::to_string(big.readbacks()) + " read back)");
clean(dest);
xfer::Options none; none.mem_stash_bytes = 0;
xfer::Transfer lean;
lean.start(dest, meta, none);
for (size_t i = 0; i < meta.chunks.size(); i++) check_ok(feed(lean, meta, content, i));
check(lean.finalize().ok, "with NO budget it still completes");
check(read_file(dest) == content, "and produces identical bytes");
check(lean.stash_hits() == 0 && lean.readbacks() > 0,
"having read every chunk back instead (" + std::to_string(lean.readbacks()) + ")");
}
// ---- a zero-byte file ---------------------------------------------------
//
// Empty files are common (.keep, __init__.py, build stamps) and they broke the
// reference: a 0-chunk transfer never streamed, so it never completed and the
// requester hung. There is no chunk to trigger completion here either, so the
// path has to work by construction.
{
printf("\n-- a zero-byte file\n");
const std::string dest = tmp_path("ho_x_9.bin");
clean(dest);
const std::vector<uint8_t> empty;
const xfer::Meta m = meta_for(empty);
check(m.chunks.empty(), "an empty file has no chunks at all");
xfer::Transfer t;
check(t.start(dest, m), "a zero-chunk transfer opens");
check(t.complete(), "and is complete on arrival");
const xfer::Transfer::FinalizeResult f = t.finalize();
check(f.ok, "it finalizes: " + f.error);
check(io::file_exists(dest) && read_file(dest).empty(), "leaving an empty file in place");
}
// =======================================================================
// The journal
// =======================================================================
const std::string JDIR = tmp_path("ho_journals");
::mkdir(JDIR.c_str(), 0755);
auto journal_of = [&](const std::string &dest) {
return JDIR + "/" + xfer::journal_name_for(dest);
};
xfer::Options journaled;
journaled.journal_dir = JDIR;
// ---- resume becomes O(1), and is provably not a re-read ----------------
{
printf("\n-- journaled resume\n");
const std::string dest = tmp_path("ho_x_10.bin");
clean(dest);
std::remove(journal_of(dest).c_str());
const size_t half = meta.chunks.size() / 2;
{
xfer::Transfer t;
t.start(dest, meta, journaled);
for (size_t i = 0; i < half; i++) check_ok(feed(t, meta, content, i));
t.pause();
}
check(io::file_exists(journal_of(dest)),
"pausing writes a journal, in the journal dir and not beside the file");
xfer::Transfer t2;
bool verified = false;
t2.start(dest, meta, journaled, [&](double) { verified = true; });
check(t2.resumed_from_journal(), "the resume came from the journal");
check(!verified, "so no re-verification pass ran at all");
check(t2.received() == half, "it knows exactly which chunks are present");
check(t2.missing().size() == meta.chunks.size() - half, "and asks only for the rest");
check(t2.hash_frontier() == half, "the running whole-file digest was restored, not recomputed");
for (size_t i = half; i < meta.chunks.size(); i++) check_ok(feed(t2, meta, content, i));
const xfer::Transfer::FinalizeResult f = t2.finalize();
check(f.ok, "the journal-resumed transfer finalizes: " + f.error);
check(read_file(dest) == content, "with bytes identical to the original");
check(!io::file_exists(journal_of(dest)), "and its journal is cleaned up");
}
// ---- F1: a journal that cannot be whole-file-verified is not trusted ----
//
// THE failure this mechanism exists to prevent, and the reason a journal is not
// simply believed. A stale partial for DIFFERENT content can have the same size
// and the same chunk count; its journal then passes every cheap check. With no
// content hash there is no whole-file verification at finalize either, so the
// merged old+new bytes would finalize as success and nothing would ever notice.
{
printf("\n-- F1: a journal with nothing to verify against\n");
const std::string dest = tmp_path("ho_x_11.bin");
clean(dest);
std::remove(journal_of(dest).c_str());
// The decoy has to actually COLLIDE, or this scenario tests nothing. Same
// size is easy; the same chunk count is not, because the boundaries are
// content-defined — a first attempt here used a fixed seed and produced a
// decoy with a different chunk count, i.e. one the cheap checks would have
// rejected anyway. So search for a seed that genuinely collides, and fail
// loudly rather than quietly testing a weaker case if none does.
std::vector<uint8_t> other;
xfer::Meta meta_b;
bool collided = false;
for (uint32_t seed = 1; seed < 4000 && !collided; seed++) {
other = make_bytes(content.size(), seed);
if (other == content) continue;
meta_b = meta_for(other);
collided = meta_b.chunks.size() == meta.chunks.size();
}
check(collided && meta_b.size == meta.size && meta_b.contentHash != meta.contentHash,
"the decoy genuinely collides: same size, same chunk count (" +
std::to_string(meta_b.chunks.size()) + "), different content");
// Write a partial + journal for B, as an abandoned earlier download would.
{
xfer::Transfer t;
t.start(dest, meta_b, journaled);
for (size_t i = 0; i < meta_b.chunks.size(); i++) check_ok(feed(t, meta_b, other, i));
t.pause();
}
check(io::file_exists(journal_of(dest)), "the decoy left a complete journal behind");
// Now fetch A into the same destination, with NO content hash — the case
// where nothing downstream could catch a bad resume.
xfer::Meta meta_a_nohash = meta;
meta_a_nohash.contentHash.clear();
xfer::Transfer t2;
bool verified = false;
t2.start(dest, meta_a_nohash, journaled, [&](double) { verified = true; });
// The WRITE side of the same rule: a transfer with nothing to verify
// against must not leave a journal for someone else to trust either.
check(!t2.flush_journal(), "a transfer with no content hash writes no journal");
check(!t2.resumed_from_journal(), "the journal is REFUSED when there is no hash to verify against");
check(verified, "so it falls back to re-verifying every chunk");
check(t2.received() == 0, "which finds none of the decoy's chunks acceptable");
check(t2.missing().size() == meta.chunks.size(), "and the whole file is still wanted");
for (size_t i = 0; i < meta.chunks.size(); i++) check_ok(feed(t2, meta, content, i));
check(t2.finalize().ok, "the transfer then completes");
check(read_file(dest) == content, "with the requested bytes, not a merge of both");
}
// ---- a journal for another file is refused on every dimension ----------
{
printf("\n-- a journal that does not describe this transfer\n");
const std::string dest = tmp_path("ho_x_12.bin");
// The decoy must COLLIDE on size and chunk count, or the cheap checks reject
// it and the content-hash binding is never exercised — the mistake that made
// the first version of this suite pass with the binding deleted.
std::vector<uint8_t> other;
xfer::Meta meta_b;
bool collided = false;
for (uint32_t seed = 1; seed < 4000 && !collided; seed++) {
other = make_bytes(content.size(), seed);
if (other == content) continue;
meta_b = meta_for(other);
collided = meta_b.chunks.size() == meta.chunks.size();
}
check(collided, "the decoy collides on size and chunk count, so only the hash can reject it");
clean(dest);
std::remove(journal_of(dest).c_str());
{
xfer::Transfer t;
t.start(dest, meta_b, journaled);
for (size_t i = 0; i < meta_b.chunks.size(); i++) check_ok(feed(t, meta_b, other, i));
t.pause();
}
xfer::Transfer t2;
t2.start(dest, meta, journaled);
check(!t2.resumed_from_journal(), "a journal written for another content hash is refused");
check(t2.received() == 0, "and re-verification keeps none of its chunks");
// A corrupt journal must degrade to re-verification, never to a bad resume.
clean(dest);
{
xfer::Transfer t;
t.start(dest, meta, journaled);
for (size_t i = 0; i < 4; i++) check_ok(feed(t, meta, content, i));
t.pause();
}
{
// Guarded: if journaling ever stops writing, this must report that
// rather than dereference a null FILE* and take the suite down with a
// segfault that looks like a library fault.
FILE *f = std::fopen(journal_of(dest).c_str(), "r+b");
check(f != nullptr, "a journal exists to corrupt");
if (f) {
std::fseek(f, 1, SEEK_SET);
const uint8_t junk = 0xFF;
std::fwrite(&junk, 1, 1, f); // break the magic
std::fclose(f);
}
}
xfer::Transfer t3;
t3.start(dest, meta, journaled);
check(!t3.resumed_from_journal(), "a corrupt journal is refused");
check(t3.received() == 4, "and re-verification recovers the real chunks anyway");
}
// ---- a fresh partial invalidates a stale journal ------------------------
{
printf("\n-- journal lifecycle\n");
const std::string dest = tmp_path("ho_x_13.bin");
clean(dest);
std::remove(journal_of(dest).c_str());
{
xfer::Transfer t;
t.start(dest, meta, journaled);
for (size_t i = 0; i < 5; i++) check_ok(feed(t, meta, content, i));
t.pause();
}
check(io::file_exists(journal_of(dest)), "a paused transfer has a journal");
// Deleting the partial without the journal is exactly what a user clearing
// a download directory does.
std::remove(xfer::partial_path_for(dest).c_str());
xfer::Transfer t2;
t2.start(dest, meta, journaled);
check(!t2.resumed_from_journal() && t2.received() == 0,
"with the partial gone the transfer starts clean");
t2.cancel();
check(!io::file_exists(journal_of(dest)), "cancel takes the journal with it");
}
// ---- journaling is off by default --------------------------------------
{
printf("\n-- no journal dir\n");
const std::string dest = tmp_path("ho_x_14.bin");
clean(dest);
xfer::Transfer t;
t.start(dest, meta); // default Options: no journal dir
for (size_t i = 0; i < 4; i++) check_ok(feed(t, meta, content, i));
check(!t.flush_journal(), "flushing without a journal dir is a no-op, not a crash");
t.pause();
xfer::Transfer t2;
bool verified = false;
t2.start(dest, meta, xfer::Options(), [&](double) { verified = true; });
check(!t2.resumed_from_journal() && verified && t2.received() == 4,
"and resume still works, by re-verification");
}
printf("\n %d assertions pass, %d fail\n", passed, failed);
printf("%s\n", failed == 0 ? "PASS" : "FAIL");
return failed == 0 ? 0 : 1;
}