OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-store commits / 8963c5d

0.0.23 (org #34): refuse a from-genesis diff — it is ALWAYS bigger than the full repo
Found by e2e against live prod, not by a test. wj62's real firehose starts with a frame declaring no prior state (its repo was re-authored), which reached a branch NO fixture could: RepoManager.init() mints an unsequenced genesis commit, so a synthetic repo's first frame always carries a real `since`. Measured on pds.7rnx.net 2026-08-07, same 168 records: no since (full repo) 46,106 bytes 224 blocks since=<one commit back> 2,439 bytes 8 blocks <- the real win, 19x since=<before genesis> 202,262 bytes 719 blocks <- 4.4x LARGER That branch served the 202 KB answer. It was CORRECT — the CAR walked to all 168 records — and useless: strictly more bytes than just asking for the repo. Structural, not a fluke. The union from genesis holds every block the repo has EVER carried, including superseded MST nodes and deleted records, so it is always a superset of the current repo and can never beat the full CAR. Refusing hands the caller the smaller, simpler, complete answer they actually wanted, and the reason says it is a SIZE decision rather than a correctness one. The 2,439-byte case is the point of the feature and is unaffected. Test names the fixture gap explicitly and synthesises the null-`since` frame, since the producer cannot emit one. 80/80, 243 asserts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
8963c5d978f29b3648432234747316288e5b7d10
by plankatron <fleeky@gmail.com> · 2026-08-07T18:52:21.000Z

Changed files (3) +67 -6

lib/repo-diff.js +25 -5
……@@ -65,9 +65,23 @@ export async function collectFramesSince ({ getFrame, length, since }) {
6565 if (!covered && newerFrames.length) {
6666 const base = newerFrames[0].since
6767 if (base === null || base === undefined) {
68 // Defensive: another producer may genuinely emit a first frame with no prior state.
69 covered = true
70 anchorRev = null
68 // ⛔ FROM-GENESIS IS REFUSED, and this is a size argument rather than a correctness one.
69 //
70 // The oldest frame we hold declares NO prior state, so the union of every frame is complete and
71 // walkable on its own — this branch used to serve it. Measured on prod 2026-08-07 (wj62, whose
72 // firehose genuinely starts with a null `since` after its re-authoring): that answer was
73 // **202,262 bytes / 719 blocks** against **46,106 bytes / 224 blocks** for the full repo. 4.4x
74 // LARGER, for the same 168 records.
75 //
76 // That is not a fixture artefact, it is structural: the union from genesis contains every block
77 // the repo has EVER held — every superseded MST node and every deleted record — so it is
78 // ALWAYS a superset of the current repo. A from-genesis diff can therefore never beat the full
79 // CAR, and refusing hands the caller the smaller, simpler, complete answer they actually wanted.
80 //
81 // Deliberately NOT `covered = true`. My unit fixture never reached this branch (RepoManager.init
82 // mints an unsequenced genesis commit, so a synthetic repo's first frame carries a real `since`)
83 // and only a live repo exposed it. See the test named for exactly that.
84 return { frames: newerFrames, covered: false, anchorRev: null, reason: 'from-genesis' }
7185 } else if (!newer(base, since)) {
7286 // The oldest frame we hold builds on a state at or BEFORE what the caller has. There is no commit
7387 // between the two, so nothing is missing from the middle.
……@@ -137,9 +151,15 @@ export async function findCommitBlock ({ getFrame, length, cid, scanLimit = 64 }
137151export async function repoDiffSince ({ getFrame, length, since, headCommitCid, headCommitBytes = null, headRev }) {
138152 if (!since) return { ok: false, reason: 'no since given' }
139153 if (!length) return { ok: false, reason: 'firehose is empty — no history to diff from' }
140 const { frames, covered, anchorRev } = await collectFramesSince({ getFrame, length, since })
154 const c = await collectFramesSince({ getFrame, length, since })
155 const { frames, covered, anchorRev } = c
141156 if (!covered) {
142 return { ok: false, reason: 'firehose does not reach back to rev ' + since + ' — cannot prove the diff is complete' }
157 return {
158 ok: false,
159 reason: c.reason === 'from-genesis'
160 ? 'since predates the firehose start — a from-genesis diff is always a superset of the full repo, so the full CAR is smaller and complete'
161 : 'firehose does not reach back to rev ' + since + ' — cannot prove the diff is complete'
162 }
143163 }
144164 // The head block, located in the firehose rather than supplied. `headCommitBytes` remains accepted so a
145165 // caller that legitimately holds the bytes (a test, or a producer assembling its own commit) can pass
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "atproto-store",
3 "version": "0.0.22",
3 "version": "0.0.23",
44 "description": "Holepunch-native atproto repo store — records in a classic Hyperbee (collection/rkey -> DAG-CBOR). The P2P home; the atproto-repo module is the bridge serializer over it.",
55 "type": "module",
66 "main": "lib/repo-store.js",
test/repo-diff.test.js +41 -0
……@@ -288,3 +288,44 @@ test('no since -> no diff (the caller must not get a partial CAR by accident)',
288288 const d = await repoDiffSince({ ...frameReader(r.seq), since: null, ...headBlock(w1) })
289289 t.absent(d.ok, 'a missing `since` is never treated as "from the beginning"')
290290})
291
292// A from-genesis diff is refused BECAUSE IT IS BIGGER, not because it is wrong.
293//
294// ⛔ THIS BRANCH WAS UNREACHABLE IN FIXTURES AND ONLY A LIVE REPO EXPOSED IT. RepoManager.init() mints
295// an unsequenced genesis commit, so a synthetic repo's first frame always carries a real `since` — every
296// test above takes the other path. wj62's real firehose starts with a null `since` (its repo was
297// re-authored), and on prod 2026-08-07 this branch served 202,262 bytes / 719 blocks where the full repo
298// is 46,106 / 224. Same 168 records, 4.4x the bytes.
299//
300// Structural, not a fluke: the union from genesis holds every block the repo has EVER carried — every
301// superseded MST node, every deleted record — so it is ALWAYS a superset of the current repo and can
302// never beat the full CAR.
303test('a from-genesis diff is refused — it is always a superset of the full repo', async (t) => {
304 const r = await newRepo(t)
305 const w1 = await r.write('a1', 'first')
306 const w2 = await r.write('a2', 'second')
307
308 // Synthesise what a real re-authored repo emits: an oldest frame declaring no prior state. Rewriting
309 // the frame body is the only way to reach this branch, precisely because the producer cannot make it.
310 const frames = []
311 for (let i = 0; i < r.seq.core.length; i++) frames.push(await r.seq.core.get(i))
312 const HDR = dagCbor.encode({ t: '#commit', op: 1 })
313 const patched = frames.map((f, i) => {
314 if (i !== 0) return f
315 const body = dagCbor.decode(f.subarray(HDR.length))
316 body.since = null
317 const bb = dagCbor.encode(body)
318 const out = new Uint8Array(HDR.length + bb.length)
319 out.set(HDR, 0); out.set(bb, HDR.length)
320 return out
321 })
322
323 const got = await collectFramesSince({ getFrame: (i) => patched[i], length: patched.length, since: BEFORE_ALL })
324 t.absent(got.covered, 'not treated as covered')
325 t.is(got.reason, 'from-genesis', 'and the reason is named, so the caller can say something specific')
326
327 const d = await repoDiffSince({ getFrame: (i) => patched[i], length: patched.length, since: BEFORE_ALL, headCommitCid: w2.commitCid, headRev: w2.rev })
328 t.absent(d.ok, 'refused')
329 t.ok(/superset of the full repo/.test(d.reason), 'explaining it is a SIZE decision, not a correctness one: ' + d.reason)
330 t.ok(w1.rev < w2.rev, 'sanity')
331})