| … | … | @@ -0,0 +1,121 @@ |
| 1 | // DidPeer.getRepo({ since }) — the PARTIAL CAR, over a real replicated firehose (org #34). |
| 2 | // |
| 3 | // The membrane accepted `since` and threw it away: pds/lib/server.js called front.getRepo() with no |
| 4 | // arguments, so it never reached the peer and the identical full CAR came back, byte for byte. The trap |
| 5 | // is not the wasted bytes — it is that a caller passing `since` got correct results BY ACCIDENT and |
| 6 | // would have been silently wrong the day partial CARs landed. This is that day, so the contract has to |
| 7 | // be nailed down here rather than inferred at the membrane. |
| 8 | // |
| 9 | // This runs over a real swarm + a real replicated firehose (not a hand-built one), because the whole |
| 10 | // mechanism is "the front already holds the frames" — a fixture that hands the frames over directly |
| 11 | // would not test the claim that makes this cheap. |
| 12 | // |
| 13 | // Run: node test/get-repo-since.test.mjs |
| 14 | import { mkdtempSync, rmSync } from 'node:fs' |
| 15 | import { join } from 'node:path' |
| 16 | import { tmpdir } from 'node:os' |
| 17 | import createTestnet from '@hyperswarm/testnet' |
| 18 | import Hyperswarm from 'hyperswarm' |
| 19 | import { RepoStore } from 'atproto-store/lib/repo-store.js' |
| 20 | import { RepoManager } from 'atproto-store/lib/repo-manager.js' |
| 21 | import { FirehoseSequencer } from 'atproto-store/lib/sequencer.js' |
| 22 | import { openCorestore } from 'atproto-store/lib/corestore.js' |
| 23 | import { createAtprotoIdentityFromKey } from 'atproto-identity/lib/atproto-identity.js' |
| 24 | import { discoveryTopic } from 'atproto-identity/lib/discovery.js' |
| 25 | import { readCar, listRecords, loadCommit, writeCar } from 'atproto-repo/lib/repo.js' |
| 26 | import { DidPeer } from '../index.js' |
| 27 | |
| 28 | const assert = (c, m) => { if (!c) { console.error('FAIL:', m); process.exit(1) } console.log(' ok -', m) } |
| 29 | const id = createAtprotoIdentityFromKey(new Uint8Array(32).fill(9), { handle: 'since.test', pdsUrl: 'https://pds.local' }) |
| 30 | const testnet = await createTestnet(3) |
| 31 | |
| 32 | // ---- holder: a real repo with a real sequenced firehose --------------------------------------------- |
| 33 | const hDir = mkdtempSync(join(tmpdir(), 'since-holder-')) |
| 34 | const hStore = openCorestore(hDir) |
| 35 | const repo = await new RepoStore(hStore, { name: `atproto-repo:${id.did}` }).ready() |
| 36 | let micros = 1_700_000_000_000_000n |
| 37 | const mgr = await new RepoManager(repo, { did: id.did, privateKey: id.privateKey, clock: () => (micros += 1_000_000n) }).init() |
| 38 | const seq = await new FirehoseSequencer(hStore.get({ name: 'firehose' })).ready() |
| 39 | |
| 40 | let t0 = Date.parse('2026-08-07T00:00:00.000Z') |
| 41 | const write = async (rkey, text) => { |
| 42 | const w = await mgr.applyWrite({ action: 'create', collection: 'app.bsky.feed.post', rkey, record: { $type: 'app.bsky.feed.post', text, createdAt: new Date(t0 += 1000).toISOString() } }) |
| 43 | await seq.sequenceCommit(w, { time: new Date(t0).toISOString() }) |
| 44 | return w |
| 45 | } |
| 46 | const w1 = await write('a1', 'first') |
| 47 | // Snapshot what a caller would hold at w1, NOW — before the repo advances. getRepoFromCommit rebuilds |
| 48 | // the MST from the store's CURRENT records and refuses if it does not match the commit handed in, so |
| 49 | // asking for w1's repo after two more writes correctly throws. (It did, and the refusal was right.) |
| 50 | const priorCar = await readCar((await (await import('atproto-store/lib/read-view.js')).getRepoFromCommit(repo, w1.signedCommit)).carBytes) |
| 51 | const w2 = await write('a2', 'second') |
| 52 | const w3 = await write('a3', 'third') |
| 53 | |
| 54 | const holder = new DidPeer(hStore, { did: id.did, verifyKey: id.publicKey }) |
| 55 | await holder.provide({ repoKey: repo.core.key, firehoseKey: seq.core.key }) |
| 56 | const hSwarm = new Hyperswarm({ bootstrap: testnet.bootstrap }) |
| 57 | hSwarm.on('connection', (c) => holder.attach(c)) |
| 58 | await hSwarm.join(discoveryTopic(id.did), { server: true, client: false }).flushed() |
| 59 | |
| 60 | // ---- reader: a keyless front, exactly as the membrane fronts one ------------------------------------ |
| 61 | const rStore = openCorestore(mkdtempSync(join(tmpdir(), 'since-reader-'))) |
| 62 | const rSwarm = new Hyperswarm({ bootstrap: testnet.bootstrap }) |
| 63 | const front = new DidPeer(rStore, { did: id.did, verifyKey: id.publicKey }) |
| 64 | rSwarm.on('connection', (c) => front.attach(c)) |
| 65 | await rSwarm.join(discoveryTopic(id.did), { server: false, client: true }).flushed() |
| 66 | assert(!!(await front.ready({ timeout: 20000 })), 'front reached a verified head over the swarm') |
| 67 | |
| 68 | // Wait for the firehose to replicate all three frames — the diff is assembled FROM them, so a front that |
| 69 | // has only the head frame would legitimately refuse and the test would prove nothing. |
| 70 | for (let i = 0; i < 100 && front.seq.core.length < 3; i++) await new Promise((r) => setTimeout(r, 100)) |
| 71 | assert(front.seq.core.length >= 3, 'all three commit frames replicated to the front (' + front.seq.core.length + ')') |
| 72 | |
| 73 | // ---- 1. no `since` -> the FULL CAR, unchanged -------------------------------------------------------- |
| 74 | // The regression that matters most: every existing caller passes nothing, and must be byte-identical. |
| 75 | const full = await front.getRepo() |
| 76 | const fullCar = await readCar(full.carBytes) |
| 77 | assert(listRecords(fullCar).length === 3, 'no-since getRepo still returns the complete repo (3 records)') |
| 78 | assert(full.partial === false, 'and declares itself NOT partial') |
| 79 | assert(full.sinceHonoured === null, 'sinceHonoured is null when nothing was asked — not false, which would imply a refusal') |
| 80 | |
| 81 | // ---- 2. `since` -> a real partial CAR ---------------------------------------------------------------- |
| 82 | const d = await front.getRepo({ since: w1.rev }) |
| 83 | assert(d.partial === true, 'getRepo({since}) returns a PARTIAL CAR') |
| 84 | assert(d.sinceHonoured === true, 'and says the parameter was honoured') |
| 85 | assert(d.fromRev === w1.rev && d.toRev === w3.rev, 'reporting the range it covers: ' + d.fromRev + ' -> ' + d.toRev) |
| 86 | |
| 87 | const diffCar = await readCar(d.carBytes) |
| 88 | assert(loadCommit(diffCar).cid.equals(loadCommit(fullCar).cid), 'rooted at the SAME head as the full CAR') |
| 89 | |
| 90 | // The defining property: it is not self-contained. If this ever stops throwing, we are serving a full |
| 91 | // CAR again and have silently regressed to accept-and-ignore. |
| 92 | let walked = null |
| 93 | try { listRecords(diffCar); walked = 'walked' } catch { walked = 'threw' } |
| 94 | assert(walked === 'threw', 'the partial CAR is NOT self-contained — walking it alone throws') |
| 95 | |
| 96 | // ---- 3. it COMPLETES a caller prior state ------------------------------------------------------------ |
| 97 | // The only check that proves the omission was sufficient rather than merely plausible. |
| 98 | const union = new Map([...priorCar.blocks, ...diffCar.blocks]) |
| 99 | const merged = await readCar(await writeCar(loadCommit(fullCar).cid, [...union.values()])) |
| 100 | const texts = listRecords(merged).map((r) => r.value.text).sort() |
| 101 | assert(JSON.stringify(texts) === JSON.stringify(['first', 'second', 'third']), |
| 102 | 'prior state + diff walks to the COMPLETE record set: ' + texts.join(',')) |
| 103 | |
| 104 | // ---- 4. an unanswerable `since` falls back to the FULL CAR and says so ------------------------------- |
| 105 | // Never a 500, never a silent full CAR pretending to be a diff. '2222222222222' is below every real rev |
| 106 | // (the TID alphabet starts at '2', not 'a'), so the firehose provably cannot reach back to it. |
| 107 | const fb = await front.getRepo({ since: '2222222222222' }) |
| 108 | assert(fb.partial === false, 'an unanswerable since falls back to a full CAR') |
| 109 | assert(fb.sinceHonoured === false, 'and reports that it was NOT honoured — the anti-silence property') |
| 110 | assert(typeof fb.reason === 'string' && fb.reason.length > 0, 'with a reason: ' + fb.reason) |
| 111 | assert(listRecords(await readCar(fb.carBytes)).length === 3, 'and the fallback CAR is genuinely complete') |
| 112 | |
| 113 | // ---- 5. since == head -> an empty diff --------------------------------------------------------------- |
| 114 | const empty = await front.getRepo({ since: w3.rev }) |
| 115 | assert(empty.partial === true && empty.empty === true, 'since == head gives an EMPTY diff, not an error') |
| 116 | assert((await readCar(empty.carBytes)).blocks.size === 1, 'carrying the commit block alone') |
| 117 | |
| 118 | console.log('\nGET-REPO-SINCE OK: partial CARs are real, self-declaring, and refuse rather than mislead') |
| 119 | await hSwarm.destroy(); await rSwarm.destroy(); await testnet.destroy() |
| 120 | try { rmSync(hDir, { recursive: true, force: true }) } catch {} |
| 121 | process.exit(0) |