// com.atproto.sync.getRepo?since — the PARTIAL CAR, assembled from the firehose.
//
// ⛔ WHAT `since` ACTUALLY MEANS. The lexicon calls it "the revision of the repo to create a diff from",
// and the reference PDS answers it by streaming every block whose creation rev is greater than `since`,
// rooted at the CURRENT commit. That is NOT a self-contained repo: the MST nodes that did not change are
// deliberately ABSENT, so the tree cannot be walked from the root without the blocks the caller is
// assumed to already hold. A consumer with a block store applies the diff and re-walks; a consumer
// without one CANNOT USE IT AT ALL. This distinction is the whole hazard in the feature and every
// surface here states which side it is on.
//
// WHY THE FIREHOSE AND NOT A BLOCK-REV INDEX. Our record store holds the CURRENT records and rebuilds
// the MST on demand (read-view.js:assembleRepo) — there is no per-block "created at rev" tag to filter
// on, and no historical MST to diff against. But every `#commit` frame ALREADY carries exactly the right
// bytes: `blocks` is a CAR of the blocks that were new at that commit (firehose.js:commitFrame). The
// union of those CARs over the frames newer than `since` IS the diff, by construction, with no new
// storage and no second source of truth. This is the "firehose range" spec/convergence.md §7 named.
//
// ⚠ A DIFF IS NOT NECESSARILY SMALLER, and the intuition that it must be is wrong here. A diff spanning
// N commits carries every INTERMEDIATE MST root; the full repo carries only the final one. On a small
// repo — favela's wj62 is 45 KB — a multi-commit diff can be the LARGER of the two. The saving appears
// only once the untouched part of the tree dwarfs the churn, which is precisely why spec/convergence.md
// §7 ranked this last. Nothing here should be sold as a size optimisation for a small repo.
//
// ⚠ COVERAGE IS PROVEN, NEVER ASSUMED. A firehose that does not reach back to `since` cannot produce a
// complete diff, and a diff missing its middle is far worse than no diff — the caller would apply it,
// see a valid-looking commit, and hold a repo with a hole. So `covered` is only ever true when we have
// POSITIVE evidence: either a frame at or older than `since` (proof we hold that history), or a genesis
// frame (proof there is no earlier history to hold). Anything else falls back to the full CAR and says
// so, out loud, rather than serving a diff it cannot justify.
import { readCar, writeCar } from 'atproto-repo/lib/repo.js'
import { decodeFrame } from './firehose.js'
// TIDs are lexicographically ordered by construction, so string comparison IS rev comparison. Used
// throughout this stack (favela-web's head check does the same) — stated here so nobody "fixes" it into
// a numeric parse.
const newer = (a, b) => String(a) > String(b)
// Read frames BACKWARD from the end of a firehose until we have both (a) every #commit newer than
// `since` and (b) the proof of coverage described above. Bounded by the number of commits since that
// rev plus one — never a full-history scan, which matters on a repo with thousands of frames.
//
// getFrame(i) -> frame bytes for the 0-based index i. length = the number of frames.
export async function collectFramesSince ({ getFrame, length, since }) {
const newerFrames = [] // #commit bodies with rev > since, OLDEST FIRST
let covered = false
let anchorRev = null // the rev we are diffing FROM, once proven
for (let i = length - 1; i >= 0; i--) {
let d = null
try { d = decodeFrame(await getFrame(i)) } catch { d = null }
if (!d || d.t !== '#commit' || !d.body || !d.body.rev) continue
if (newer(d.body.rev, since)) { newerFrames.unshift(d.body); continue }
// A frame at or older than `since`: we demonstrably hold history reaching back that far.
covered = true
anchorRev = d.body.rev
break
}
// The scan ran off the front of the firehose without finding an older frame. The oldest frame we DO
// have still carries the rev it built on, in its own `since` field — and that is enough to decide.
//
// ⚠ There is no "first frame has no since" case to lean on here, though it reads like the obvious one.
// RepoManager.init() mints an EMPTY genesis commit and persists it as the head WITHOUT sequencing it,
// so the first frame a repo ever emits already has a real `since` pointing at that unsequenced commit.
// A genesis heuristic written against the lexicon rather than against this producer is dead code.
if (!covered && newerFrames.length) {
const base = newerFrames[0].since
if (base === null || base === undefined) {
// ⛔ FROM-GENESIS IS REFUSED, and this is a size argument rather than a correctness one.
//
// The oldest frame we hold declares NO prior state, so the union of every frame is complete and
// walkable on its own — this branch used to serve it. Measured on prod 2026-08-07 (wj62, whose
// firehose genuinely starts with a null `since` after its re-authoring): that answer was
// **202,262 bytes / 719 blocks** against **46,106 bytes / 224 blocks** for the full repo. 4.4x
// LARGER, for the same 168 records.
//
// That is not a fixture artefact, it is structural: the union from genesis contains every block
// the repo has EVER held — every superseded MST node and every deleted record — so it is
// ALWAYS a superset of the current repo. A from-genesis diff can therefore never beat the full
// CAR, and refusing hands the caller the smaller, simpler, complete answer they actually wanted.
//
// Deliberately NOT `covered = true`. My unit fixture never reached this branch (RepoManager.init
// mints an unsequenced genesis commit, so a synthetic repo's first frame carries a real `since`)
// and only a live repo exposed it. See the test named for exactly that.
return { frames: newerFrames, covered: false, anchorRev: null, reason: 'from-genesis' }
} else if (!newer(base, since)) {
// The oldest frame we hold builds on a state at or BEFORE what the caller has. There is no commit
// between the two, so nothing is missing from the middle.
covered = true
anchorRev = base
}
// Otherwise `base` is NEWER than the caller's rev: at least one commit sits between their state and
// our oldest frame, and we do not have it. Refuse — the caller gets the full CAR instead. This is
// also what a `since` from before the repo existed hits, which is correct: we cannot prove our
// firehose reaches the repo's first write rather than having been truncated, and the full repo is
// the right answer to "give me everything" anyway.
}
return { frames: newerFrames, covered, anchorRev }
}
// Union the `blocks` CARs of the given #commit frames into ONE CAR rooted at `headCommit`.
//
// Deduped by CID: a block touched by three commits appears once. Returns null when the head block is
// neither in the frames nor supplied — the caller must then LOCATE it (findCommitBlock) or refuse,
// because a CAR rooted at a commit nobody carries cannot be walked by anyone.
export async function unionDiffCar ({ frames, headCommitCid, headCommitBytes = null }) {
const blocks = new Map() // cidStr -> { cid, bytes }
for (const body of frames) {
if (!body || !body.blocks) continue
let car = null
try { car = await readCar(body.blocks) } catch { continue }
for (const [cidStr, block] of car.blocks) if (!blocks.has(cidStr)) blocks.set(cidStr, block)
}
const headStr = headCommitCid.toString()
if (!blocks.has(headStr)) {
if (!headCommitBytes) return null // caller must locate it; see findCommitBlock
blocks.set(headStr, { cid: headCommitCid, bytes: headCommitBytes })
}
const carBytes = await writeCar(headCommitCid, [...blocks.values()])
return { carBytes, blocks: blocks.size }
}
// Find a commit block BY CID inside the firehose, newest frame first.
//
// ⛔ THE HEAD BLOCK IS NEVER SYNTHESISED. It would be easy to hand `dagCbor.encode(head.signed)` in from
// the front and root the CAR at that — and it would also mean serving a CAR rooted at a commit the
// firehose does not actually carry, which cannot be walked even by a caller holding everything else. So
// the block is LOCATED in the frames instead. That the search can fail is the point: a head the firehose
// never carried is a refusal, not a block we invent. It also removes any need for a dag-cbor dependency
// in the front.
export async function findCommitBlock ({ getFrame, length, cid, scanLimit = 64 }) {
const want = cid.toString()
const stop = Math.max(0, length - scanLimit)
for (let i = length - 1; i >= stop; i--) {
let d = null
try { d = decodeFrame(await getFrame(i)) } catch { continue }
if (!d || d.t !== '#commit' || !d.body || !d.body.blocks) continue
try {
const car = await readCar(d.body.blocks)
const hit = car.blocks.get(want)
if (hit) return hit
} catch { continue }
}
return null
}
// The whole operation: firehose + a verified head -> a partial CAR, or an honest refusal.
//
// Returns { ok, carBytes, blocks, fromRev, toRev, reason }. `ok: false` means DO NOT SERVE A DIFF — the
// caller must fall back to the full CAR and report that `since` was not honoured. It never throws for a
// coverage problem, because "I cannot answer that" is a normal answer here, not an error.
export async function repoDiffSince ({ getFrame, length, since, headCommitCid, headCommitBytes = null, headRev }) {
if (!since) return { ok: false, reason: 'no since given' }
if (!length) return { ok: false, reason: 'firehose is empty — no history to diff from' }
const c = await collectFramesSince({ getFrame, length, since })
const { frames, covered, anchorRev } = c
if (!covered) {
return {
ok: false,
reason: c.reason === 'from-genesis'
? 'since predates the firehose start — a from-genesis diff is always a superset of the full repo, so the full CAR is smaller and complete'
: 'firehose does not reach back to rev ' + since + ' — cannot prove the diff is complete'
}
}
// The head block, located in the firehose rather than supplied. `headCommitBytes` remains accepted so a
// caller that legitimately holds the bytes (a test, or a producer assembling its own commit) can pass
// them, but the normal path finds them.
const locate = async () => headCommitBytes
? { cid: headCommitCid, bytes: headCommitBytes }
: await findCommitBlock({ getFrame, length, cid: headCommitCid })
// `since` is at or ahead of our head: nothing has changed that the caller does not have. The correct
// answer is a CAR carrying the commit alone — an EMPTY diff, which is a real answer and not an error.
if (!frames.length) {
const head = await locate()
if (!head) return { ok: false, reason: 'the verified head commit is not carried by this firehose' }
const car = await unionDiffCar({ frames: [], headCommitCid, headCommitBytes: head.bytes })
return { ok: true, ...car, fromRev: anchorRev, toRev: headRev || null, empty: true }
}
// The head we serve must be the head these frames build up to. If the front's verified head is NEWER
// than the last frame we have, the firehose we are reading is behind it and the diff would be rooted
// at a commit whose blocks are not in the union — a CAR that cannot be walked even WITH the caller's
// prior state. Refuse rather than emit it.
const lastRev = frames[frames.length - 1].rev
if (headRev && newer(headRev, lastRev)) {
return { ok: false, reason: 'verified head (' + headRev + ') is ahead of the firehose tail (' + lastRev + ')' }
}
let car = await unionDiffCar({ frames, headCommitCid, headCommitBytes })
if (!car) {
// The frames did not carry the head block. Look for it before giving up — then refuse rather than
// synthesise one, because a CAR rooted at a commit this firehose never carried is unwalkable for
// everyone, including a caller holding every other block.
const head = await locate()
if (!head) return { ok: false, reason: 'the verified head commit is not carried by this firehose' }
car = await unionDiffCar({ frames, headCommitCid, headCommitBytes: head.bytes })
}
return { ok: true, ...car, fromRev: anchorRev, toRev: lastRev, empty: false }
}