// repair-bytes — re-author record BYTES this node indexed but never stored.
//
// ⛔ THE STATE THIS EXISTS TO CLEAR. Until 0.3.37 the appview's create-gate asked the ROSTER instead of
// the local hpp (see lib/hosted-dids.js), so a node that rostered a DID it did not co-host refused to
// author every one of that DID's records. The refusal is by design — hpp may hold a pointer to bytes
// that differ from our re-encode — but on such a node NO local hpp would ever write them.
//
// The miss is PERMANENT without this, and that is the part worth understanding: the record was still
// INDEXED (appview-no-clobber pins that trade — "we refuse the byte authorship, not the indexing"), so
// on every later replay the dedup gate at appview.js:377 sees a matching `sig` and returns before the
// write is retried. The index says the post exists; the store has no bytes; _resolveRecord returns null;
// the post 404s forever. ashatron sat like this from 2026-07-31 to 2026-08-06.
//
// WHERE THE BYTES COME FROM. `com.atproto.sync.getRecord` — a CAR carrying the signed commit and the MST
// blocks proving the record's inclusion — NOT `repo.getRecord`, whose hydrated JSON cannot be verified.
// The four checks in lib/record-fetch.js (commit is for this DID, signature validates against the DID
// DOCUMENT's key, the CID is provably at <collection>/<rkey> under the commit root, and the bytes hash
// to that CID) mean the served bytes are canonical no matter WHO served them. That is what makes this
// safe to run against our own membrane, which fronts the peers that actually hold the repo.
//
// ⛔ AND THE RE-ENCODE IS STILL GUARDED. OverlayRepoStore.put() takes a DECODED record and re-encodes it,
// but what we verified was the ORIGINAL bytes. If decode->encode is not byte-identical we REFUSE, exactly
// as lib/retention.js does — writing bytes that no longer match their CID is the 2026-07-30 outage, and
// a repair that re-introduced it would be worse than the gap it closes.
//
// NEVER OVERWRITES. Only absences are filled. A record whose bytes are already here is left untouched,
// so this cannot clobber hpp's canonical files no matter how it is invoked.
//
// DRY RUN BY DEFAULT. `write: false` reports what it WOULD do and touches nothing.
import * as dagCbor from '@ipld/dag-cbor'
import b4a from 'b4a'
export function emptyReport () {
return {
scanned: 0, // index entries examined
present: 0, // bytes already on disk — the healthy majority
missing: 0, // indexed, no bytes: the defect's footprint
written: 0, // bytes fetched, verified and stored
unfetchable: 0, // no peer/PDS could produce them (deleted, or genuinely lost)
unverified: 0, // served, but failed one of the four proofs — never written
reencodeRefused: 0, // canonical bytes that do not survive decode->encode — never written
storeErrors: 0,
samples: [] // a bounded list of what is missing, so a dry run is actionable
}
}
// listRefs(did) -> async iterable of { did, rkey, collection }
// openStore(did) -> OverlayRepoStore (ready)
// fetchRecord(pointer) -> { bytes, commit, identity, ... } | null (lib/record-fetch.js)
// verifyRecord(fetched, pointer) -> boolean (lib/record-fetch.js)
export async function repairBytes ({
dids = [],
listRefs,
openStore,
fetchRecord,
verifyRecord,
write = false,
limit = 0, // 0 = no cap; otherwise stop after this many REPAIR ATTEMPTS (dry run: after this many finds)
sampleLimit = 25,
onEvent = null
} = {}) {
if (typeof listRefs !== 'function') throw new Error('repairBytes requires listRefs')
if (typeof openStore !== 'function') throw new Error('repairBytes requires openStore')
if (write && typeof fetchRecord !== 'function') throw new Error('repairBytes --write requires fetchRecord')
if (write && typeof verifyRecord !== 'function') throw new Error('repairBytes --write requires verifyRecord')
const report = emptyReport()
const emit = (ev, detail) => { if (onEvent) onEvent(ev, detail) }
let attempts = 0
for (const did of dids) {
let store
try {
store = await openStore(did)
} catch (e) {
report.storeErrors++
emit('store-failed', { did, error: (e && e.message) || String(e) })
continue
}
if (!store) { report.storeErrors++; emit('store-failed', { did, error: 'no store' }); continue }
for await (const ref of listRefs(did)) {
if (limit && attempts >= limit) return report
const collection = ref && ref.collection
const rkey = ref && ref.rkey
// An index entry written before `collection` was recorded cannot be located in the store or asked
// for over XRPC. Counted, not guessed at.
if (!collection || !rkey) { report.scanned++; report.storeErrors++; continue }
report.scanned++
const pointer = { did, collection, rkey }
let existing = null
try {
existing = await store.getRaw(collection, rkey)
} catch (e) {
report.storeErrors++
emit('read-failed', { pointer, error: (e && e.message) || String(e) })
continue
}
if (existing != null) { report.present++; continue }
report.missing++
if (report.samples.length < sampleLimit) report.samples.push(pointer)
emit('missing', { pointer })
attempts++
if (!write) continue // dry run: counted and sampled, nothing touched
const fetched = await fetchRecord(pointer)
if (!fetched || !fetched.bytes) { report.unfetchable++; emit('unfetchable', { pointer }); continue }
let ok = false
try { ok = await verifyRecord(fetched, pointer) } catch { ok = false }
if (!ok) { report.unverified++; emit('unverified', { pointer }); continue }
// The guard from lib/retention.js, for the same reason: put() re-encodes, and we verified the
// ORIGINAL bytes. If the round trip is not byte-identical the store would persist bytes that no
// longer match the CID we just proved. Refuse, loudly.
let value
try {
value = dagCbor.decode(fetched.bytes)
} catch {
report.reencodeRefused++
emit('decode-failed', { pointer })
continue
}
let reencoded
try {
reencoded = dagCbor.encode(value)
} catch {
report.reencodeRefused++
emit('reencode-failed', { pointer })
continue
}
if (!b4a.equals(b4a.from(reencoded), b4a.from(fetched.bytes))) {
report.reencodeRefused++
emit('reencode-differs', { pointer, verified: fetched.bytes.length, reencoded: reencoded.length })
continue
}
try {
await store.put(collection, rkey, value)
report.written++
emit('written', { pointer, bytes: fetched.bytes.length })
} catch (e) {
report.storeErrors++
emit('write-failed', { pointer, error: (e && e.message) || String(e) })
}
}
}
return report
}