// Three peers joining one topic, WITH overlays attached — the standalone repro
// for the double free.
//
// WHY THIS EXISTS, AND WHY IT IS NOT A GODOT TEST.
//
// Layer 13 reproduces a `free(): double free detected in tcache 2` reliably: two
// peers mesh and transfer, then a THIRD peer joins the same topic and the process
// aborts. That repro is unusable as a diagnostic, because neither sanitizer can
// see it:
//
// * ASan refuses outright — Godot dlopen()s GDExtensions with RTLD_DEEPBIND,
// which is incompatible with the sanitizer runtime.
// * valgrind can attach, but Godot under memcheck does not finish booting in
// ten minutes, and every window in the harness (10s DHT, 25s mesh, 5s overlay
// channel) is wall-clock — the run bails long before it reaches the third
// peer.
//
// The FIRST version of this file drove only UdxTransport/SwarmManager and passed
// 25/25 clean, which was itself the finding: the swarm path alone does not do it.
// What layer 13 has and that version lacked is an ATTACHED OVERLAY. HyperOverlay
// is a Godot Node, but the two pieces that matter — hyperoverlay::Protocol and
// OverlayTransport — are plain C++, and set_mux_callbacks takes plain
// std::function. So this mirrors HyperOverlay::udx_add_peer/udx_remove_peer
// directly, with no Godot in the process.
//
// It also closes a real coverage hole: NOTHING in test/ drove SwarmManager before
// this file — the phase and interop tests stop at DhtConnector.
//
// Build/run:
// node ../integration/layer2_testnet.js & # writes /tmp/layer_bootstrap.txt
// SAN=address ./build_tests.sh
// ASAN_OPTIONS=detect_leaks=0 ./test_three_peer_swarm [host:port] [rounds]
//
// A clean exit does NOT prove the bug is gone — it is timing-sensitive, so the
// round count matters. A crash under ASan prints the two stacks that the glibc
// abort never gave us.
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <unistd.h>
#include <sodium.h>
#include "noise_ik.h"
#include "overlay/file_index.h"
#include "overlay/fileio.h"
#include "overlay/protocol.h"
#include "overlay_transport.h"
#include "udx_transport.h"
static int g_pass = 0;
static int g_fail = 0;
static void check(bool ok, const char *name) {
if (ok) {
g_pass++;
printf(" PASS: %s\n", name);
} else {
g_fail++;
printf(" FAIL: %s\n", name);
}
}
static void sleep_ms(int ms) {
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
}
static uint64_t now_ms() {
return (uint64_t)std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
// One peer: transport + overlay, wired the way HyperOverlay wires them.
struct Node {
UdxTransport t;
hyperoverlay::index::FileIndex ix;
std::unique_ptr<hyperoverlay::Protocol> proto;
// on_mux_ready / on_mux_gone fire on the UDX THREAD, and the main thread
// tears the node down, so this map needs the lock. HyperOverlay does not need
// one because every touch is marshalled onto the loop; here the test thread
// is a second toucher.
std::mutex m;
std::map<int32_t, hyperswarm::OverlayTransport *> peers;
std::string label;
bool open_overlay(const std::string &dir) {
if (!ix.open(dir + "/overlay-index.log")) return false;
proto.reset(new hyperoverlay::Protocol(&ix,
[this](const std::string &hash, hyperoverlay::ContentLocation &out) {
const hyperoverlay::index::FileMeta *fm = ix.get_file(
hyperoverlay::index::FileIndex::content_register_key(hash));
if (!fm || fm->linkname.empty()) return false;
out.path = fm->linkname;
out.size = fm->size;
return true;
}));
proto->tick(now_ms());
// EXACTLY HyperOverlay::udx_add_peer / udx_remove_peer, minus Godot.
t.set_mux_callbacks(
[this](int32_t id, Protomux *mux) {
if (!proto || !mux) return;
std::lock_guard<std::mutex> lk(m);
if (peers.count(id)) return;
auto *ot = new hyperswarm::OverlayTransport(mux, proto.get(),
label + "/" + std::to_string(id),
[this](hyperswarm::OverlayTransport *x) {
if (proto) proto->add_peer(x);
});
if (ot->slot_mismatch()) { proto->remove_peer(ot); delete ot; return; }
peers[id] = ot;
},
[this](int32_t id) {
std::lock_guard<std::mutex> lk(m);
auto it = peers.find(id);
if (it == peers.end()) return;
if (proto) proto->remove_peer(it->second);
delete it->second; // <-- the deletion the pair() callback outlives
peers.erase(it);
});
return true;
}
int overlay_peers() {
std::lock_guard<std::mutex> lk(m);
return (int)peers.size();
}
// Mirrors HyperOverlay::content_hash_of — the leaf construction is NOT a plain
// digest, so anything else yields a hash the overlay will reject.
static std::string hash_file(const std::string &path) {
uint64_t size = 0;
if (!hyperoverlay::io::file_size(path, size)) return "";
hyperoverlay::io::File f;
if (!f.open_read(path)) return "";
hyperoverlay::cdc::StreamingHasher h;
h.init(size);
std::vector<uint8_t> buf(1u << 20);
uint64_t off = 0;
while (off < size) {
const size_t want = (size_t)std::min<uint64_t>(buf.size(), size - off);
if (!f.read_at(off, buf.data(), want)) return "";
h.update(buf.data(), want);
off += want;
}
return h.digest();
}
// Mirrors HyperOverlay::serve: register by content hash with the disk path in
// linkname, then precompute the chunk map.
std::string serve_file(const std::string &path) {
uint64_t sz = 0;
if (!hyperoverlay::io::file_size(path, sz)) return "";
const std::string hash = hash_file(path);
if (hash.empty()) return "";
hyperoverlay::index::FileMeta meta;
meta.contentHash = hash;
meta.linkname = path;
meta.size = sz;
ix.put_file(hyperoverlay::index::FileIndex::content_register_key(hash), meta);
if (!ix.has_chunk_map_by_hash(hash)) {
const std::vector<hyperoverlay::index::ChunkRef> map =
hyperoverlay::Protocol::build_chunk_map(path, sz);
if (!map.empty()) ix.put_chunk_map_by_hash(hash, map);
}
return hash;
}
void request(const std::string &hash, const std::string &dest) {
if (proto) proto->request(hash, dest);
}
void tick() {
if (proto) proto->tick(now_ms());
}
void close_overlay() {
std::lock_guard<std::mutex> lk(m);
for (auto &kv : peers) {
if (proto) proto->remove_peer(kv.second);
delete kv.second;
}
peers.clear();
proto.reset();
ix.close();
}
};
static bool wait_dht(UdxTransport **ts, int n, int timeout_ms) {
int waited = 0;
while (waited < timeout_ms) {
bool all = true;
for (int i = 0; i < n; i++)
if (!ts[i]->is_dht_active()) all = false;
if (all) return true;
sleep_ms(50);
waited += 50;
}
return false;
}
static bool wait_connected(UdxTransport **ts, int n, int want, int timeout_ms, Node **nodes, int nn) {
int waited = 0;
while (waited < timeout_ms) {
bool all = true;
for (int i = 0; i < n; i++)
if (ts[i]->get_swarm_status().peers_connected < want) all = false;
for (int i = 0; i < nn; i++) nodes[i]->tick();
if (all) return true;
sleep_ms(50);
waited += 50;
}
return false;
}
static void drain(UdxTransport *t) {
SwarmEvent se;
while (t->pop_swarm_event(se)) { }
PeerEvent pe;
while (t->pop_peer_event(pe)) { }
}
int main(int argc, char **argv) {
if (sodium_init() < 0) {
printf("sodium_init failed\n");
return 1;
}
std::string bootstrap;
if (argc > 1) {
bootstrap = argv[1];
} else {
std::ifstream f("/tmp/layer_bootstrap.txt");
if (f) std::getline(f, bootstrap);
}
while (!bootstrap.empty() && (bootstrap.back() == '\n' || bootstrap.back() == '\r' || bootstrap.back() == ' '))
bootstrap.pop_back();
size_t comma = bootstrap.find(',');
if (comma != std::string::npos) bootstrap = bootstrap.substr(0, comma);
if (bootstrap.empty()) {
printf("FAIL: no bootstrap. Start the testnet first:\n");
printf(" node ../integration/layer2_testnet.js &\n");
return 1;
}
printf("=== Three-peer swarm join + overlay (bootstrap %s) ===\n", bootstrap.c_str());
const int ROUNDS = (argc > 2) ? atoi(argv[2]) : 5;
for (int round = 1; round <= ROUNDS; round++) {
printf("\n--- round %d/%d ---\n", round, ROUNDS);
// A FRESH TOPIC PER ROUND, and this is not incidental. Every round joins
// with new ephemeral DHT keys, and an announce lingers on the DHT for
// ~30 minutes after the process that made it is gone. Reusing one topic
// therefore poisons every round after the first: the new peers try to
// handshake the previous rounds' dead keys, holepunch replies get matched
// to dead slots, and the mesh never forms — which is exactly the
// stale-announce failure next-fixes.md records as A1, reproduced here by
// accident. Rounds 2-5 failed to mesh before this was per-round.
uint8_t topic[32];
char seed[80];
snprintf(seed, sizeof(seed), "three-peer-swarm/%d/%d", (int)getpid(), round);
crypto_generichash(topic, sizeof(topic),
(const uint8_t *)seed, strlen(seed), nullptr, 0);
const char *bs[1] = { bootstrap.c_str() };
Node a, b, c;
a.label = "A"; b.label = "B"; c.label = "C";
char dir[128];
for (int i = 0; i < 3; i++) {
snprintf(dir, sizeof(dir), "/tmp/hsp3/%d-%d-%d", (int)getpid(), round, i);
std::string cmd = std::string("mkdir -p ") + dir;
if (system(cmd.c_str()) != 0) { }
}
snprintf(dir, sizeof(dir), "/tmp/hsp3/%d-%d-0", (int)getpid(), round);
check(a.open_overlay(dir), "A overlay opens");
snprintf(dir, sizeof(dir), "/tmp/hsp3/%d-%d-1", (int)getpid(), round);
check(b.open_overlay(dir), "B overlay opens");
snprintf(dir, sizeof(dir), "/tmp/hsp3/%d-%d-2", (int)getpid(), round);
check(c.open_overlay(dir), "C overlay opens");
Ed25519KeyPair ka = NoiseIK::generate_keypair();
Ed25519KeyPair kb = NoiseIK::generate_keypair();
Ed25519KeyPair kc = NoiseIK::generate_keypair();
a.t.start_dht(ka, bs, 1);
b.t.start_dht(kb, bs, 1);
UdxTransport *pair[2] = { &a.t, &b.t };
Node *pair_n[2] = { &a, &b };
if (!wait_dht(pair, 2, 10000)) {
check(false, "A and B DHTs active");
continue;
}
check(true, "A and B DHTs active");
a.t.swarm_join(topic, true, true);
b.t.swarm_join(topic, true, true);
bool meshed = wait_connected(pair, 2, 1, 25000, pair_n, 2);
check(meshed, "A and B meshed");
if (!meshed) {
a.close_overlay(); b.close_overlay(); c.close_overlay();
a.t.stop(); b.t.stop();
continue;
}
// The overlay channel is what layer 13 has and the first version of this
// harness did not.
int waited = 0;
while (waited < 5000 && (a.overlay_peers() == 0 || b.overlay_peers() == 0)) {
a.tick(); b.tick();
sleep_ms(50);
waited += 50;
}
check(a.overlay_peers() > 0 && b.overlay_peers() > 0, "A and B overlay transports attached");
drain(&a.t);
drain(&b.t);
// ---- A REAL TRANSFER BEFORE C ARRIVES ----
// This is the last structural difference from layer 13, and it is not
// decoration: Protocol::add_peer re-sends outstanding requests to a peer
// that joins late, so it walks download state. When C arrives, that state
// exists and has already COMPLETED — which the previous version of this
// harness, passing 45/45, never exercised.
{
char src[128];
snprintf(src, sizeof(src), "/tmp/hsp3/%d-%d-payload.bin", (int)getpid(), round);
FILE *fp = fopen(src, "wb");
if (fp) {
// A few hundred KB of deterministic noise: several chunks, so the
// scheduler runs rather than a single-chunk special case.
std::vector<uint8_t> data(400 * 1024);
uint32_t x = 987654321u;
for (size_t i = 0; i < data.size(); i++) {
x = x * 1103515245u + 12345u;
data[i] = (uint8_t)((x >> 16) & 0xff);
}
fwrite(data.data(), 1, data.size(), fp);
fclose(fp);
}
const std::string hash = a.serve_file(src);
check(!hash.empty(), "A serves the payload");
bool done = false;
b.proto->on_complete = [&done](const std::string &, const std::string &) { done = true; };
char dest[160];
snprintf(dest, sizeof(dest), "/tmp/hsp3/%d-%d-1/fetched.bin", (int)getpid(), round);
b.request(hash, dest);
int t_waited = 0;
while (t_waited < 30000 && !done) {
a.tick(); b.tick();
sleep_ms(50);
t_waited += 50;
}
check(done, "B fetched the payload from A");
// Detach the callback: `done` is a stack local and the Protocol
// outlives this scope.
b.proto->on_complete = nullptr;
}
sleep_ms(1000);
// ---- the third peer: the step that aborts in layer 13 ----
c.t.start_dht(kc, bs, 1);
UdxTransport *one[1] = { &c.t };
Node *one_n[1] = { &c };
bool c_up = wait_dht(one, 1, 10000);
check(c_up, "C DHT active");
if (c_up) {
c.t.swarm_join(topic, true, true);
bool c_meshed = wait_connected(one, 1, 1, 25000, one_n, 1);
check(c_meshed, "C meshed with the existing pair");
// Let every side build its OverlayTransport for the new connections —
// A and B each gain one for C, and C gains two.
for (int i = 0; i < 60; i++) { a.tick(); b.tick(); c.tick(); sleep_ms(50); }
printf(" [overlay peers: A=%d B=%d C=%d]\n",
a.overlay_peers(), b.overlay_peers(), c.overlay_peers());
drain(&a.t); drain(&b.t); drain(&c.t);
}
// Teardown drives udx_remove_peer on every side, which is where the
// OverlayTransports are deleted while their muxes may still be alive.
c.t.stop();
b.t.stop();
a.t.stop();
sleep_ms(500);
c.close_overlay();
b.close_overlay();
a.close_overlay();
sleep_ms(300);
check(true, "round completed without abort");
}
printf("\n=== Summary: %d/%d passed ===\n", g_pass, g_pass + g_fail);
return g_fail == 0 ? 0 : 1;
}