#ifndef HYPEROVERLAY_FILE_INDEX_H
#define HYPEROVERLAY_FILE_INDEX_H
// Local index: what this peer holds, and the chunk map for each of it.
//
// NO CONFORMANCE SURFACE. The JS keeps this in a Hyperbee over a Hypercore, but
// the overlay protocol never replicates cores — the wire only ever carries the
// 15 messages — so the storage format is a purely local choice and this does NOT
// need Hyperbee, Hypercore or a Corestore. That is the single biggest reason a
// native port is tractable at all.
//
// WHY AN APPEND-ONLY LOG RATHER THAN LevelDB/RocksDB/SQLite. The working set is a
// few thousand entries read at startup and appended to on registration. A
// embedded KV would add a heavyweight dependency, a build-system entanglement and
// an extraction blocker, to solve a problem this does not have. The log is
// rebuilt into an in-memory map on open; lookups are hash-map lookups.
//
// CRASH SAFETY is the part that actually matters, because the process is a game
// that gets killed. Every record carries its length and an 8-byte checksum, so a
// half-written tail record from a kill mid-append is DETECTED and truncated on
// open rather than being read as garbage. Tested directly — see
// test_file_index.cpp, which corrupts a tail on purpose.
//
// Mirrors the JS FileIndex API (lib/file-index.js) so the protocol driver reads
// the same either side:
// put_file / get_file / has_file / del_file / list_files
// put_chunk_map / get_chunk_map / has_chunk_map / del_chunk_map
// put_chunk_map_by_hash / get_chunk_map_by_hash / ...
// put_index_meta / get_index_meta
// evict_content
//
// Dependencies: cenc.h and libsodium (checksum only).
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
#include <sodium.h>
#include "cenc.h"
namespace hyperoverlay {
namespace index {
// Files registered by CONTENT hash live under this prefix, matching the JS, so
// "do I hold content X" is a single lookup rather than a scan.
static constexpr const char *CONTENT_REGISTER_PREFIX = "/mir/";
struct FileMeta {
std::string contentHash; // empty == null
uint64_t size = 0;
uint64_t mtime = 0;
bool hashing = false;
bool executable = false;
std::string linkname; // empty == null
};
struct ChunkRef {
std::string hash; // hex
uint64_t offset = 0;
uint64_t length = 0;
};
class FileIndex {
public:
// Opening REPLAYS the log. A record whose length runs past EOF, or whose
// checksum does not match, ends the replay and truncates the file there —
// that is a kill mid-append, and everything before it is still good.
bool open(const std::string &path) {
close();
path_ = path;
files_.clear();
chunk_maps_.clear();
meta_.clear();
std::vector<uint8_t> buf;
if (!read_all(path, buf)) {
// No file yet is not an error; the first append creates it.
fp_ = std::fopen(path.c_str(), "ab");
return fp_ != nullptr;
}
size_t off = 0;
size_t good = 0;
while (off + 12 <= buf.size()) {
uint32_t len = 0;
std::memcpy(&len, buf.data() + off, 4);
if (off + 12 + len > buf.size()) break; // torn tail
uint8_t sum[8];
checksum(buf.data() + off + 12, len, sum);
if (std::memcmp(sum, buf.data() + off + 4, 8) != 0) break; // corrupt tail
apply(buf.data() + off + 12, len);
off += 12 + len;
good = off;
}
if (good != buf.size()) truncate(path, good); // drop the bad tail
truncated_bytes_ = buf.size() - good;
fp_ = std::fopen(path.c_str(), "ab");
return fp_ != nullptr;
}
void close() {
if (fp_) { std::fclose(fp_); fp_ = nullptr; }
}
~FileIndex() { close(); }
// Bytes discarded from a torn tail on the last open(). Non-zero means the
// process was killed mid-append; surfaced rather than hidden so a caller can
// log it instead of wondering why an entry vanished.
size_t truncated_bytes() const { return truncated_bytes_; }
// ---- files -------------------------------------------------------------
bool put_file(const std::string &p, const FileMeta &m) {
files_[p] = m;
return append(REC_FILE, [&](cenc::State &s, bool pre) {
str(s, pre, p); str(s, pre, m.contentHash);
num(s, pre, m.size); num(s, pre, m.mtime);
byte(s, pre, m.hashing ? 1 : 0); byte(s, pre, m.executable ? 1 : 0);
str(s, pre, m.linkname);
});
}
const FileMeta *get_file(const std::string &p) const {
auto it = files_.find(p);
return it == files_.end() ? nullptr : &it->second;
}
bool has_file(const std::string &p) const { return files_.count(p) != 0; }
bool del_file(const std::string &p) {
files_.erase(p);
chunk_maps_.erase(chunk_key(p));
return append(REC_DEL_FILE, [&](cenc::State &s, bool pre) { str(s, pre, p); });
}
std::vector<std::string> list_files(const std::string &prefix = "") const {
std::vector<std::string> out;
for (const auto &kv : files_)
if (prefix.empty() || kv.first.compare(0, prefix.size(), prefix) == 0)
out.push_back(kv.first);
return out;
}
// ---- chunk maps --------------------------------------------------------
bool put_chunk_map(const std::string &p, const std::vector<ChunkRef> &c) {
return put_map(chunk_key(p), c);
}
const std::vector<ChunkRef> *get_chunk_map(const std::string &p) const {
return get_map(chunk_key(p));
}
bool has_chunk_map(const std::string &p) const { return get_map(chunk_key(p)) != nullptr; }
bool del_chunk_map(const std::string &p) { return del_map(chunk_key(p)); }
// By CONTENT hash — the lookup the serve path uses, since a contentRequest
// names a hash and not a path.
bool put_chunk_map_by_hash(const std::string &h, const std::vector<ChunkRef> &c) {
return put_map(hash_key(h), c);
}
const std::vector<ChunkRef> *get_chunk_map_by_hash(const std::string &h) const {
return get_map(hash_key(h));
}
bool has_chunk_map_by_hash(const std::string &h) const { return get_map(hash_key(h)) != nullptr; }
bool del_chunk_map_by_hash(const std::string &h) { return del_map(hash_key(h)); }
// ---- index meta --------------------------------------------------------
bool put_index_meta(const std::string &k, const std::string &v) {
meta_[k] = v;
return append(REC_META, [&](cenc::State &s, bool pre) { str(s, pre, k); str(s, pre, v); });
}
const std::string *get_index_meta(const std::string &k) const {
auto it = meta_.find(k);
return it == meta_.end() ? nullptr : &it->second;
}
// ---- content lifecycle -------------------------------------------------
static std::string content_register_key(const std::string &contentHash) {
return std::string(CONTENT_REGISTER_PREFIX) + contentHash;
}
// Drop everything derived from one content hash: its by-hash chunk map and
// its register entry. Mirrors the JS evictContent.
bool evict_content(const std::string &contentHash) {
bool ok = del_chunk_map_by_hash(contentHash);
ok = del_file(content_register_key(contentHash)) && ok;
return ok;
}
size_t file_count() const { return files_.size(); }
size_t chunk_map_count() const { return chunk_maps_.size(); }
// Rewrite the log with only live state. An append-only log grows with
// churn — without this, a peer that re-registers the same paths for months
// replays an ever-longer file at every start.
bool compact() {
if (path_.empty()) return false;
const std::string tmp = path_ + ".compact";
close();
{
FILE *out = std::fopen(tmp.c_str(), "wb");
if (!out) { fp_ = std::fopen(path_.c_str(), "ab"); return false; }
FILE *prev = fp_;
fp_ = out;
for (const auto &kv : files_) {
const FileMeta m = kv.second;
const std::string p = kv.first;
append(REC_FILE, [&](cenc::State &s, bool pre) {
str(s, pre, p); str(s, pre, m.contentHash);
num(s, pre, m.size); num(s, pre, m.mtime);
byte(s, pre, m.hashing ? 1 : 0); byte(s, pre, m.executable ? 1 : 0);
str(s, pre, m.linkname);
});
}
for (const auto &kv : chunk_maps_) {
const std::string k = kv.first;
const std::vector<ChunkRef> v = kv.second;
append(REC_CHUNKMAP, [&](cenc::State &s, bool pre) { str(s, pre, k); chunks(s, pre, v); });
}
for (const auto &kv : meta_) {
const std::string k = kv.first, v = kv.second;
append(REC_META, [&](cenc::State &s, bool pre) { str(s, pre, k); str(s, pre, v); });
}
std::fclose(fp_);
fp_ = prev;
}
close();
std::remove(path_.c_str());
std::rename(tmp.c_str(), path_.c_str());
fp_ = std::fopen(path_.c_str(), "ab");
return fp_ != nullptr;
}
private:
enum RecType : uint8_t { REC_FILE = 1, REC_DEL_FILE = 2, REC_CHUNKMAP = 3, REC_DEL_CHUNKMAP = 4, REC_META = 5 };
static std::string chunk_key(const std::string &p) { return "p:" + p; }
static std::string hash_key(const std::string &h) { return "h:" + h; }
// Two-pass encode helpers so a record is sized then written, mirroring cenc.
static void str(cenc::State &s, bool pre, const std::string &v) {
if (pre) cenc::string_preencode(s, v); else cenc::string_encode(s, v);
}
static void num(cenc::State &s, bool pre, uint64_t v) {
if (pre) cenc::uint_preencode(s, v); else cenc::uint_encode(s, v);
}
static void byte(cenc::State &s, bool pre, uint8_t v) {
if (pre) cenc::uint8_preencode(s); else cenc::uint8_encode(s, v);
}
static void chunks(cenc::State &s, bool pre, const std::vector<ChunkRef> &c) {
num(s, pre, c.size());
for (const ChunkRef &r : c) { str(s, pre, r.hash); num(s, pre, r.offset); num(s, pre, r.length); }
}
static void checksum(const uint8_t *d, size_t n, uint8_t out[8]) {
crypto_generichash(out, 8, d, n, nullptr, 0);
}
template <typename Fn>
bool append(RecType t, Fn &&fn) {
if (!fp_) return false;
cenc::State pre;
cenc::uint8_preencode(pre);
fn(pre, true);
std::vector<uint8_t> body(pre.end);
cenc::State enc;
enc.buffer = body.data(); enc.start = 0; enc.end = body.size();
cenc::uint8_encode(enc, (uint8_t)t);
fn(enc, false);
uint8_t hdr[12];
const uint32_t len = (uint32_t)body.size();
std::memcpy(hdr, &len, 4);
checksum(body.data(), body.size(), hdr + 4);
if (std::fwrite(hdr, 1, 12, fp_) != 12) return false;
if (std::fwrite(body.data(), 1, body.size(), fp_) != body.size()) return false;
std::fflush(fp_);
return true;
}
void apply(const uint8_t *d, size_t n) {
cenc::State s;
s.buffer = const_cast<uint8_t *>(d); s.start = 0; s.end = n;
const uint8_t t = cenc::uint8_decode(s);
if (t == REC_FILE) {
const std::string p = cenc::string_decode(s);
FileMeta m;
m.contentHash = cenc::string_decode(s);
m.size = cenc::uint_decode(s);
m.mtime = cenc::uint_decode(s);
m.hashing = cenc::uint8_decode(s) != 0;
m.executable = cenc::uint8_decode(s) != 0;
m.linkname = cenc::string_decode(s);
files_[p] = m;
} else if (t == REC_DEL_FILE) {
const std::string p = cenc::string_decode(s);
files_.erase(p);
chunk_maps_.erase(chunk_key(p));
} else if (t == REC_CHUNKMAP) {
const std::string k = cenc::string_decode(s);
const uint64_t cnt = cenc::uint_decode(s);
std::vector<ChunkRef> v;
for (uint64_t i = 0; i < cnt && cenc::has_more(s); i++) {
ChunkRef r;
r.hash = cenc::string_decode(s);
r.offset = cenc::uint_decode(s);
r.length = cenc::uint_decode(s);
v.push_back(r);
}
chunk_maps_[k] = v;
} else if (t == REC_DEL_CHUNKMAP) {
chunk_maps_.erase(cenc::string_decode(s));
} else if (t == REC_META) {
const std::string k = cenc::string_decode(s);
meta_[k] = cenc::string_decode(s);
}
}
bool put_map(const std::string &k, const std::vector<ChunkRef> &c) {
chunk_maps_[k] = c;
return append(REC_CHUNKMAP, [&](cenc::State &s, bool pre) { str(s, pre, k); chunks(s, pre, c); });
}
const std::vector<ChunkRef> *get_map(const std::string &k) const {
auto it = chunk_maps_.find(k);
return it == chunk_maps_.end() ? nullptr : &it->second;
}
bool del_map(const std::string &k) {
chunk_maps_.erase(k);
return append(REC_DEL_CHUNKMAP, [&](cenc::State &s, bool pre) { str(s, pre, k); });
}
static bool read_all(const std::string &p, std::vector<uint8_t> &out) {
FILE *f = std::fopen(p.c_str(), "rb");
if (!f) return false;
std::fseek(f, 0, SEEK_END);
const long n = std::ftell(f);
std::fseek(f, 0, SEEK_SET);
out.resize(n > 0 ? (size_t)n : 0);
if (n > 0 && std::fread(out.data(), 1, (size_t)n, f) != (size_t)n) { std::fclose(f); return false; }
std::fclose(f);
return true;
}
static void truncate(const std::string &p, size_t n) {
std::vector<uint8_t> buf;
if (!read_all(p, buf)) return;
FILE *f = std::fopen(p.c_str(), "wb");
if (!f) return;
if (n) std::fwrite(buf.data(), 1, n, f);
std::fclose(f);
}
std::string path_;
FILE *fp_ = nullptr;
size_t truncated_bytes_ = 0;
std::map<std::string, FileMeta> files_; // ordered: list_files is prefix-friendly
std::unordered_map<std::string, std::vector<ChunkRef>> chunk_maps_;
std::unordered_map<std::string, std::string> meta_;
};
} // namespace index
} // namespace hyperoverlay
#endif // HYPEROVERLAY_FILE_INDEX_H