// repo-sync — the convergence loop: compare our copy to the authoritative repo, pull what is missing.
//
// The real repo CAR is built here with atproto-repo's own createRepo, so parseRepo is tested against
// bytes produced by the same codec the network serves — not a hand-rolled fixture that could agree with
// a wrong implementation.
//
// What is under test is the POLICY, because that is what can hurt:
// - an unverifiable repo is refused ENTIRELY, never partially believed
// - the three drift states stay DISTINCT (collapsing them is how "140/140 bytes" read as healthy on a
// node missing 26 records outright)
// - extra-local records are reported and NEVER deleted
// - a dry run touches nothing
//
// Run: node test/repo-sync.test.mjs
import { createRepo } from 'atproto-repo/lib/repo.js'
import { parseRepo, diffRepo, applyBackfill } from '../lib/repo-sync.js'
const assert = (cond, msg) => { if (!cond) throw new Error('FAIL: ' + msg); console.log(' ok -', msg) }
const DID = 'did:plc:wj62ebbusln6gxo5azo6ogzo'
const COLL = 'app.bsky.feed.post'
// A real 3-record repo, signed with a fixed valid scalar. No key GENERATION here on purpose: the signing
// primitive lives in atproto-repo's own nested @noble/curves, and a test that reaches through a sibling's
// node_modules breaks the first time the dep layout changes. The signature-failure branch is exercised
// with a bogus key instead, which is all that branch needs.
const PRIV = new Uint8Array(32).fill(1)
const mk = (rkey, text) => ({ collection: COLL, rkey, record: { $type: COLL, text, createdAt: '2026-08-0' + rkey.length + 'T00:00:00.000Z', langs: ['en'] } })
const built = await createRepo({ records: [mk('aaa', 'one'), mk('bbbb', 'two'), mk('ccccc', 'three')], did: DID, rev: '3msaaaaaaaa22', privateKey: PRIV })
console.log('# parseRepo enumerates a real repo CAR')
{
// pubkey null = skip the signature step; the DID, block-fidelity and MST checks all still run.
const r = await parseRepo({ did: DID, carBytes: built.carBytes, pubkey: null })
assert(r && r.ok, 'a well-formed repo parses')
assert(r.records.length === 3, 'every record is enumerated from the MST (3)')
assert(r.records.every((x) => x.bytes && x.bytes.length), 'each record carries its ORIGINAL bytes')
assert(r.rev === '3msaaaaaaaa22', 'the commit rev is reported — this is the staleness answer')
assert(r.records.some((x) => x.rkey === 'ccccc'), 'including the one our fake node never indexed')
}
console.log('\n# ⛔ the wrong DID is refused, however well signed')
{
const r = await parseRepo({ did: 'did:plc:someoneelse', carBytes: built.carBytes, pubkey: null })
assert(r && r.ok === false && r.reason === 'did-mismatch', 'no cross-repo substitution')
assert(r.records === undefined, 'and NO records are handed back to be indexed')
}
console.log('\n# ⛔ a bad signature is refused ENTIRELY — not partially believed')
{
const bogus = new Uint8Array(33); bogus[0] = 2; bogus[1] = 7
const r = await parseRepo({ did: DID, carBytes: built.carBytes, pubkey: bogus })
assert(r && r.ok === false && r.reason === 'bad-signature', 'the key comes from the DID document, so a mismatch is fatal')
assert(r.records === undefined, 'and NO records are handed back to be indexed')
}
console.log('\n# garbage is null, never a throw')
{
assert((await parseRepo({ did: DID, carBytes: new Uint8Array([1, 2, 3]), pubkey: null })) === null, 'malformed CAR -> null')
}
const remote = await parseRepo({ did: DID, carBytes: built.carBytes, pubkey: null })
console.log('\n# diff keeps the three drift states DISTINCT')
{
const TRACKED = new Map([[COLL, { bytes: true }]])
const local = { [COLL]: ['aaa', 'bbbb', 'ghost'] }
const listRefsFor = async function * (_d, c) { for (const rkey of local[c] || []) yield { rkey } }
const hasBytes = async (_did, _c, rkey) => rkey === 'aaa'
const d = await diffRepo({ did: DID, remote, tracked: TRACKED, listRefsFor, hasBytes })
assert(d.missingFromIndex.length === 1 && d.missingFromIndex[0].rkey === 'ccccc', "missingFromIndex names ccccc")
assert(d.indexedWithoutBytes.length === 1 && d.indexedWithoutBytes[0].rkey === 'bbbb', 'indexedWithoutBytes names bbbb')
assert(d.extraLocal.length === 1 && d.extraLocal[0].rkey === 'ghost', 'extraLocal names ghost')
assert(d.inSync === false, 'and it is not in sync')
}
console.log('\n# ⛔ ACCEPTANCE (#31): ONE missing follow is named')
{
// The blind spot this closed: every check compared the POST index, so a follow that never arrived was
// invisible and `diff` printed a clean bill of health.
const FOLLOW = 'app.bsky.graph.follow'
const remoteWithGraph = {
...remote,
records: [
...remote.records,
{ collection: FOLLOW, rkey: 'f1', bytes: new Uint8Array([1]), record: {} },
{ collection: FOLLOW, rkey: 'f2', bytes: new Uint8Array([1]), record: {} }
]
}
// Likes/follows keep NO bytes locally — asking would report a gap on every one ever indexed.
const TRACKED = new Map([[COLL, { bytes: true }], [FOLLOW, { bytes: false }]])
const local = { [COLL]: remote.records.map((r) => r.rkey), [FOLLOW]: ['f1'] } // f2 never arrived
const listRefsFor = async function * (_d, c) { for (const rkey of local[c] || []) yield { rkey } }
const d = await diffRepo({ did: DID, remote: remoteWithGraph, tracked: TRACKED, listRefsFor, hasBytes: async () => true })
assert(d.inSync === false, 'DRIFTED — a missing follow is no longer invisible')
assert(d.missingFromIndex.length === 1, 'exactly one gap')
assert(d.missingFromIndex[0].collection === FOLLOW && d.missingFromIndex[0].rkey === 'f2', 'and it is NAMED: app.bsky.graph.follow/f2')
assert(d.perCollection[FOLLOW].authority === 2 && d.perCollection[FOLLOW].indexed === 1, 'the per-collection line shows 2 vs 1')
assert(d.perCollection[COLL].authority === d.perCollection[COLL].indexed, 'while posts are reported level')
}
console.log('\n# bytes are only demanded where this node stores them')
{
const FOLLOW = 'app.bsky.graph.follow'
const remoteF = { ...remote, records: [{ collection: FOLLOW, rkey: 'f1', bytes: new Uint8Array([1]), record: {} }] }
const TRACKED = new Map([[FOLLOW, { bytes: false }]])
const listRefsFor = async function * (_d, c) { if (c === FOLLOW) yield { rkey: 'f1' } }
let asked = false
const d = await diffRepo({ did: DID, remote: remoteF, tracked: TRACKED, listRefsFor, hasBytes: async () => { asked = true; return false } })
assert(asked === false, 'hasBytes is never called for a bytes:false collection')
assert(d.inSync === true, 'so an indexed follow with no local bytes is NOT a false gap')
}
console.log('\n# ignored-by-policy and UNKNOWN are different answers')
{
const TRACKED = new Map([[COLL, { bytes: true }]])
const remoteX = {
...remote,
records: [
...remote.records,
{ collection: 'net.favela.test', rkey: 't1', bytes: new Uint8Array([1]), record: {} },
{ collection: 'com.example.brandnew', rkey: 'x1', bytes: new Uint8Array([1]), record: {} }
]
}
const listRefsFor = async function * (_d, c) { if (c === COLL) for (const r of remote.records) yield { rkey: r.rkey } }
const d = await diffRepo({
did: DID, remote: remoteX, tracked: TRACKED, listRefsFor, hasBytes: async () => true,
isIgnored: (c) => c === 'net.favela.test'
})
assert(d.ignored === 1 && d.ignoredByCollection['net.favela.test'] === 1, 'the ignore-listed collection is counted as ignored')
assert(d.unknown === 1 && d.unknownByCollection['com.example.brandnew'] === 1, 'an UNCLASSIFIED lexicon is reported as UNKNOWN, not dropped')
assert(d.inSync === true, 'neither blocks in-sync — they are statements about coverage, not drift')
}
console.log('\n# backfill: DRY RUN touches nothing')
{
const listRefsFor = async function * (_d, c) { if (c === COLL) yield { rkey: 'bbbb' } }
const d = await diffRepo({ did: DID, remote, tracked: new Map([[COLL, { bytes: true }]]), listRefsFor, hasBytes: async () => false })
const ops = []; const bytes = []
const r = await applyBackfill({
did: DID, remote, diff: d,
applyOps: async (_d, o) => ops.push(...o), putBytes: async (...a) => bytes.push(a), write: false
})
assert(ops.length === 0 && bytes.length === 0, 'a dry run writes nothing at all')
assert(r.indexed === 0 && r.bytesWritten === 0, 'and reports nothing written')
}
console.log('\n# backfill routes each state to the RIGHT write')
{
const listRefsFor = async function * (_d, c) { if (c === COLL) yield { rkey: 'bbbb' } }
const d = await diffRepo({ did: DID, remote, tracked: new Map([[COLL, { bytes: true }]]), listRefsFor, hasBytes: async () => false })
const ops = []; const bytes = []
const r = await applyBackfill({
did: DID, remote, diff: d,
applyOps: async (_d, o) => ops.push(...o), putBytes: async (...a) => bytes.push(a), write: true
})
// 'aaa' and 'ccccc' were never indexed -> applyCommit (indexes AND authors bytes through the gates).
assert(r.indexed === 2, 'unindexed records go through applyOps (2)')
assert(ops.every((o) => o.action === 'create'), 'as create ops')
// 'bbbb' is indexed already -> the appview dedup gate would return early, so bytes are written direct.
assert(r.bytesWritten === 1 && bytes[0][2] === 'bbbb', 'the already-indexed record gets a DIRECT byte write')
}
console.log('\n# ⛔ extra-local records are reported, never deleted')
{
const listRefsFor = async function * (_d, c) { if (c === COLL) yield { rkey: 'ghost' } }
const d = await diffRepo({ did: DID, remote, tracked: new Map([[COLL, { bytes: true }]]), listRefsFor, hasBytes: async () => true })
assert(d.extraLocal.length === 1, 'the ghost is reported')
const r = await applyBackfill({ did: DID, remote, diff: d, applyOps: async () => {}, putBytes: async () => {}, write: true })
assert(r.skippedExtraLocal === 1, 'and counted as SKIPPED — this cannot tell a stale local copy from an unpublished one')
}
console.log('\n# --limit bounds the work')
{
const listRefsFor = async function * () {}
const d = await diffRepo({ did: DID, remote, tracked: new Map([[COLL, { bytes: true }]]), listRefsFor, hasBytes: async () => false })
const ops = []
const r = await applyBackfill({ did: DID, remote, diff: d, applyOps: async (_x, o) => ops.push(...o), putBytes: async () => {}, write: true, limit: 2 })
assert(r.indexed === 2, 'stopped after 2 — a large repo can be drained in bounded passes')
}
// ---- `since` / partial CARs (org #34) ---------------------------------------------------------------
//
// The membrane now HONOURS `since` (atproto-bridge 0.9.45) and returns a genuine diff. Favela cannot
// apply one: a partial CAR omits the unchanged MST nodes, so it needs a BLOCK store to union against and
// this node holds RECORDS. Sending `since` "worked" before only because the membrane ignored it — this
// is the silent regression the issue predicted, closed before it could happen.
console.log('\n# ⛔ `since` is refused here — favela has no block store to apply a diff against')
{
const { createFetchRepo } = await import('../lib/repo-sync.js')
let called = 0
const fetchRepo = createFetchRepo({
doFetch: async () => { called++; return { ok: true, arrayBuffer: async () => new Uint8Array([1]).buffer, headers: { get: () => null } } },
resolveIdentity: async () => ({ pds: 'https://pds.test', pubkey: new Uint8Array(33) })
})
let threw = null
try { await fetchRepo({ did: DID, since: '3ke6kg5tl6222' }) } catch (e) { threw = e }
assert(threw !== null, '`since` throws rather than quietly fetching something unusable')
assert(/block store/.test(threw.message), 'and the message says WHY, not just that it is unsupported')
assert(called === 0, 'and no request was made — refused before the wire, not after')
}
console.log('\n# a partial CAR is named as such, never reported as corruption')
{
// verifyRepo on a partial CAR fails on a missing MST block and would report `malformed-repo`, sending
// the reader hunting for corruption that is not there. The two problems have different fixes.
const r = await parseRepo({ did: DID, carBytes: new Uint8Array([1, 2, 3]), pubkey: null, partial: true })
assert(r && r.ok === false, 'refused')
assert(r.reason === 'partial-car', 'with its OWN reason, distinct from malformed-repo')
}
console.log('\n# the full-repo path still reads the partial header and defaults to false')
{
const { createFetchRepo } = await import('../lib/repo-sync.js')
const mk = (hdr) => createFetchRepo({
doFetch: async () => ({ ok: true, arrayBuffer: async () => new Uint8Array([9, 9]).buffer, headers: { get: (k) => (k === 'atproto-repo-partial' ? hdr : null) } }),
resolveIdentity: async () => ({ pds: 'https://pds.test', pubkey: new Uint8Array(33) })
})
assert((await mk('true')({ did: DID })).partial === true, 'a membrane that hands us a partial CAR unasked is detected')
assert((await mk(null)({ did: DID })).partial === false, 'and the ordinary full CAR reports partial:false')
}
console.log('\nall repo-sync assertions passed')