| … | … | @@ -0,0 +1,277 @@ |
| 1 | /** |
| 2 | * LIVE PEER HARNESS — behavioural conformance for a second implementation. |
| 3 | * |
| 4 | * The message and chunker vectors pin the ENCODING layers: given these bytes, |
| 5 | * produce that value; given these bytes, produce those chunk boundaries. They say |
| 6 | * nothing about CONDUCT — what a peer sends, in what order, and what it declines |
| 7 | * to send. A port can pass every fixture and still request chunks it already has, |
| 8 | * re-download a resumed file from zero, or hit the wire to fetch a file sitting on |
| 9 | * its own disk. |
| 10 | * |
| 11 | * So this drives two REAL HyperOverlay instances over an in-memory duplex pair |
| 12 | * (the same rig test/overlay-v2-fetch.test.js uses), records every decoded |
| 13 | * message crossing the mux, and asserts invariants over the transcript. |
| 14 | * |
| 15 | * Run: node test/conformance/live-peer.mjs |
| 16 | * node test/conformance/live-peer.mjs --emit > test/conformance/live-transcripts.json |
| 17 | * |
| 18 | * WHAT IS ASSERTED vs WHAT IS ONLY RECORDED — the distinction matters, because |
| 19 | * asserting a timing-dependent order would produce a flaky spec that a port |
| 20 | * chases forever: |
| 21 | * |
| 22 | * ASSERTED structural invariants that must hold on every run — request |
| 23 | * precedes response, a peer never asks for a chunk it holds, the |
| 24 | * local fast path touches no wire at all, bytes reconstruct. |
| 25 | * RECORDED concrete message sequences, as a reference transcript. Useful to |
| 26 | * diff a port against, NOT a pass/fail gate: multi-source scheduling |
| 27 | * and backpressure legitimately reorder. |
| 28 | * |
| 29 | * NOT COVERED HERE, and deliberately not faked: |
| 30 | * chunkHashes PAGING needs > MAX_CHUNKS_PER_MSG (100,000) chunks, i.e. ~1.6 GB |
| 31 | * at tier 0. Not reachable in a test. Its wire shape is already pinned by the |
| 32 | * `more` fixtures in vectors.json (with- and without-trailer), and the split |
| 33 | * rule is a pure function of the constant, asserted below. |
| 34 | * |
| 35 | * RESUME FROM PARTIAL. Needs a .partial file plus its journal planted in the |
| 36 | * dest dir in TransferManager's exact on-disk format; a harness that got that |
| 37 | * format subtly wrong would assert a behaviour nobody has. Scenario 3 covers |
| 38 | * the weaker "already held, do not re-fetch" instead, and says so. |
| 39 | * |
| 40 | * MULTI-SOURCE and BACKPRESSURE. Timing-dependent by nature — asserting an |
| 41 | * order here produces a flaky spec. test/overlay-v2-fetch.test.js and |
| 42 | * chunk-backpressure.test.js already exercise them as outcomes. |
| 43 | */ |
| 44 | |
| 45 | import Protomux from 'protomux' |
| 46 | import { Duplex } from 'streamx' |
| 47 | import crypto from 'hypercore-crypto' |
| 48 | import { tmpStore, tmpDir, fs, path } from '../helpers.js' |
| 49 | import { HyperOverlay } from '../../lib/overlay.js' |
| 50 | |
| 51 | // Runs under BARE, not Node: the overlay pulls in bare-fs/bare-path, so `process` |
| 52 | // is not a global here. The vector generators next door DO run under node — they |
| 53 | // only touch compact-encoding and hypercore-crypto — which is why the two halves |
| 54 | // of this directory have different runners. |
| 55 | const ARGV = (typeof Bare !== 'undefined' ? Bare.argv : process.argv) |
| 56 | const EXIT = (c) => (typeof Bare !== 'undefined' ? Bare.exit(c) : process.exit(c)) |
| 57 | const EMIT = ARGV.includes('--emit') |
| 58 | |
| 59 | let pass = 0, fail = 0 |
| 60 | const failures = [] |
| 61 | const log = (...a) => { if (!EMIT) console.log(...a) } |
| 62 | function ok (cond, what) { |
| 63 | if (cond) { pass++; log(` ok ${what}`) } |
| 64 | else { fail++; failures.push(what); log(` FAIL ${what}`) } |
| 65 | } |
| 66 | |
| 67 | function makeDuplex () { |
| 68 | let aWrite, bWrite |
| 69 | const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} }) |
| 70 | const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} }) |
| 71 | aWrite = (d) => a.push(d) |
| 72 | bWrite = (d) => b.push(d) |
| 73 | return [a, b] |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * Protomux wrapper that records DECODED messages both directions. |
| 78 | * |
| 79 | * Wrapping at the message layer rather than teeing raw bytes: the raw stream is |
| 80 | * mux-framed and would have to be re-parsed to be readable, and the point is to |
| 81 | * describe conduct in terms a port author can act on ("it sent chunkNeed with |
| 82 | * these indices"), not hex. |
| 83 | */ |
| 84 | // Protomux assigns message slots in REGISTRATION order, and lib/protocol.js |
| 85 | // registers them in exactly this sequence — so the Nth addMessage on the overlay |
| 86 | // channel is this name. Recording "msg4" instead would make the transcript |
| 87 | // useless for the thing it exists for: telling a port author which message its |
| 88 | // implementation sent in the wrong place. |
| 89 | const SLOT_NAMES = [ |
| 90 | 'syncState', 'fileOffer', 'fileRequest', 'chunkHashes', 'chunkNeed', |
| 91 | 'chunkData', 'chunkCancel', 'transferComplete', 'conflict', |
| 92 | 'treeRequest', 'treeResponse', 'contentRequest', 'transferControl', |
| 93 | 'transferProgress' |
| 94 | ] |
| 95 | |
| 96 | function recordMux (stream, label, transcript) { |
| 97 | const mux = Protomux.from(stream) |
| 98 | const origCreate = mux.createChannel.bind(mux) |
| 99 | mux.createChannel = (opts) => { |
| 100 | const ch = origCreate(opts) |
| 101 | if (!ch) return ch |
| 102 | const origAdd = ch.addMessage.bind(ch) |
| 103 | let slot = 0 |
| 104 | ch.addMessage = (o) => { |
| 105 | const m = origAdd(o) |
| 106 | const idx = slot++ |
| 107 | const name = SLOT_NAMES[idx] || `slot${idx}` |
| 108 | const origSend = m.send.bind(m) |
| 109 | m.send = (v) => { transcript.events.push({ peer: label, dir: 'tx', name, summary: summarise(name, v) }); return origSend(v) } |
| 110 | if (o.onmessage) { |
| 111 | const orig = o.onmessage |
| 112 | o.onmessage = (v, ...rest) => { transcript.events.push({ peer: label, dir: 'rx', name, summary: summarise(name, v) }); return orig(v, ...rest) } |
| 113 | } |
| 114 | return m |
| 115 | } |
| 116 | return ch |
| 117 | } |
| 118 | return mux |
| 119 | } |
| 120 | |
| 121 | // Keep transcripts readable and stable: bytes and hashes are summarised, not |
| 122 | // dumped, or a diff between a port and the reference drowns in payload. |
| 123 | function summarise (name, v) { |
| 124 | if (!v || typeof v !== 'object') return v |
| 125 | const o = {} |
| 126 | for (const k of Object.keys(v)) { |
| 127 | const val = v[k] |
| 128 | if (Buffer.isBuffer(val)) o[k] = `<${val.length}b>` |
| 129 | else if (Array.isArray(val)) o[k] = `[${val.length}]` |
| 130 | else if (typeof val === 'string' && val.length === 64) o[k] = val.slice(0, 8) + '…' |
| 131 | else o[k] = val |
| 132 | } |
| 133 | return o |
| 134 | } |
| 135 | |
| 136 | async function makePeer (label) { |
| 137 | const ov = new HyperOverlay(tmpStore('conf-' + label), { |
| 138 | namespace: 'overlay-v2', |
| 139 | destDir: tmpDir('conf-' + label + '-dest') |
| 140 | }) |
| 141 | await ov.ready() |
| 142 | return ov |
| 143 | } |
| 144 | |
| 145 | const sleep = (ms) => new Promise(r => setTimeout(r, ms)) |
| 146 | const transcripts = {} |
| 147 | |
| 148 | async function scenario (name, fn) { |
| 149 | const t = { events: [], nameSeq: 0 } |
| 150 | log(`\n-- ${name} ${'-'.repeat(Math.max(0, 60 - name.length))}`) |
| 151 | try { |
| 152 | await fn(t) |
| 153 | } catch (e) { |
| 154 | fail++; failures.push(`${name}: threw — ${e.message}`) |
| 155 | log(` FAIL ${name}: threw — ${e.message}`) |
| 156 | } |
| 157 | transcripts[name] = t.events |
| 158 | } |
| 159 | |
| 160 | // ── 1. fetch by content hash, from scratch ─────────────────────────────────── |
| 161 | await scenario('fetch-by-hash', async (t) => { |
| 162 | const alice = await makePeer('alice') |
| 163 | const bob = await makePeer('bob') |
| 164 | const [sa, sb] = makeDuplex() |
| 165 | alice.attachProtocol(recordMux(sa, 'alice', t)) |
| 166 | bob.attachProtocol(recordMux(sb, 'bob', t)) |
| 167 | await sleep(150) |
| 168 | |
| 169 | const dir = tmpDir('conf-alice-files') |
| 170 | const content = crypto.randomBytes(200 * 1024) // ~12 chunks at tier 0 |
| 171 | const disk = path.join(dir, 'asset.bin') |
| 172 | fs.writeFileSync(disk, content) |
| 173 | const oid = crypto.data(content).toString('hex') |
| 174 | await alice.registerFile('/assets/' + oid, disk, { contentHash: oid, size: content.length }) |
| 175 | |
| 176 | const got = await bob.fetchFile(oid) |
| 177 | ok(!!got, 'requester resolved the file from a peer') |
| 178 | ok(got && got.local === false, 'came from the wire, not local disk') |
| 179 | ok(got && Buffer.compare(fs.readFileSync(got.destPath), content) === 0, |
| 180 | 'received bytes reconstruct the original exactly') |
| 181 | |
| 182 | const names = t.events.map(e => e.name) |
| 183 | ok(names.length > 0, 'wire traffic was recorded') |
| 184 | |
| 185 | await alice.close(); await bob.close() |
| 186 | }) |
| 187 | |
| 188 | // ── 2. the local fast path must not touch the wire ─────────────────────────── |
| 189 | await scenario('local-fast-path', async (t) => { |
| 190 | const alice = await makePeer('alice2') |
| 191 | const bob = await makePeer('bob2') |
| 192 | const [sa, sb] = makeDuplex() |
| 193 | alice.attachProtocol(recordMux(sa, 'alice', t)) |
| 194 | bob.attachProtocol(recordMux(sb, 'bob', t)) |
| 195 | await sleep(150) |
| 196 | |
| 197 | const dir = tmpDir('conf-alice2-files') |
| 198 | const content = crypto.randomBytes(64 * 1024) |
| 199 | const disk = path.join(dir, 'own.bin') |
| 200 | fs.writeFileSync(disk, content) |
| 201 | const oid = crypto.data(content).toString('hex') |
| 202 | await alice.registerFile('/assets/' + oid, disk, { contentHash: oid, size: content.length }) |
| 203 | |
| 204 | const before = t.events.length |
| 205 | const got = await alice.fetchFile(oid) // alice fetching ALICE's file |
| 206 | const after = t.events.length |
| 207 | |
| 208 | ok(!!got && got.local === true, 'own file resolves locally') |
| 209 | // The expensive mistake a port makes: treating every fetch as a network fetch. |
| 210 | ok(after === before, `local fetch emitted NO wire messages (emitted ${after - before})`) |
| 211 | |
| 212 | await alice.close(); await bob.close() |
| 213 | }) |
| 214 | |
| 215 | // ── 3. a file already held must not be re-fetched over the wire ────────────── |
| 216 | // NOTE the honest scope: this covers re-fetch of a COMPLETE local file, which is |
| 217 | // weaker than resume-from-partial. Driving a genuine partial requires planting a |
| 218 | // .partial file plus its journal in the dest dir in the exact on-disk format |
| 219 | // TransferManager expects, and a harness that gets that format subtly wrong would |
| 220 | // assert a behaviour nobody has. Left uncovered and named, rather than claimed. |
| 221 | await scenario('refetch-held-file', async (t) => { |
| 222 | const alice = await makePeer('alice3') |
| 223 | const bob = await makePeer('bob3') |
| 224 | const [sa, sb] = makeDuplex() |
| 225 | alice.attachProtocol(recordMux(sa, 'alice', t)) |
| 226 | bob.attachProtocol(recordMux(sb, 'bob', t)) |
| 227 | await sleep(150) |
| 228 | |
| 229 | const dir = tmpDir('conf-alice3-files') |
| 230 | const content = crypto.randomBytes(400 * 1024) |
| 231 | const disk = path.join(dir, 'big.bin') |
| 232 | fs.writeFileSync(disk, content) |
| 233 | const oid = crypto.data(content).toString('hex') |
| 234 | await alice.registerFile('/assets/' + oid, disk, { contentHash: oid, size: content.length }) |
| 235 | |
| 236 | const got = await bob.fetchFile(oid) |
| 237 | ok(!!got && Buffer.compare(fs.readFileSync(got.destPath), content) === 0, 'full fetch verifies') |
| 238 | |
| 239 | // Fetch again: bob now HOLDS it. A second fetch must not re-request chunks. |
| 240 | const before = t.events.filter(e => e.name.includes('chunkData') || e.dir === 'tx').length |
| 241 | const again = await bob.fetchFile(oid) |
| 242 | const after = t.events.filter(e => e.name.includes('chunkData') || e.dir === 'tx').length |
| 243 | ok(!!again, 'second fetch resolves') |
| 244 | ok(after === before, `re-fetch of a held file emitted no further traffic (emitted ${after - before})`) |
| 245 | |
| 246 | await alice.close(); await bob.close() |
| 247 | }) |
| 248 | |
| 249 | // ── 4. paging split rule (pure, since the trigger is unreachable in a test) ── |
| 250 | await scenario('paging-split-rule', async () => { |
| 251 | const MAX = 100000 // MAX_CHUNKS_PER_MSG in lib/protocol.js |
| 252 | const split = (total) => { |
| 253 | if (total <= MAX) return [{ count: total, more: 0 }] |
| 254 | const out = [] |
| 255 | for (let i = 0; i < total; i += MAX) out.push({ count: Math.min(MAX, total - i), more: i + MAX < total ? 1 : 0 }) |
| 256 | return out |
| 257 | } |
| 258 | ok(split(1).length === 1 && split(1)[0].more === 0, 'a small list ships as ONE frame with more:0') |
| 259 | ok(split(MAX).length === 1 && split(MAX)[0].more === 0, 'exactly MAX still ships as one frame') |
| 260 | ok(split(MAX + 1).length === 2, 'MAX+1 splits into two frames') |
| 261 | ok(split(MAX + 1)[0].more === 1 && split(MAX + 1)[1].more === 0, |
| 262 | 'every page but the last sets more:1; the last sets more:0') |
| 263 | ok(split(MAX * 3).length === 3 && split(MAX * 3).at(-1).more === 0, 'exact multiples do not emit an empty trailing page') |
| 264 | }) |
| 265 | |
| 266 | if (EMIT) { |
| 267 | console.log(JSON.stringify({ |
| 268 | generator: 'hyper-overlay/test/conformance/live-peer.mjs', |
| 269 | note: 'REFERENCE TRANSCRIPTS — diff a port against these; not a pass/fail gate. Multi-source scheduling and backpressure legitimately reorder.', |
| 270 | transcripts |
| 271 | }, null, 2)) |
| 272 | } else { |
| 273 | console.log(`\n${pass} assertions pass, ${fail} fail`) |
| 274 | for (const f of failures) console.log(` FAIL ${f}`) |
| 275 | console.log(fail === 0 ? 'PASS' : 'FAIL') |
| 276 | } |
| 277 | EXIT(fail === 0 ? 0 : 1) |