// Every blob a repo's records REFER to — spec/convergence.md §6.
//
// Records are not the whole repo. Blobs (images, avatars, banners, video) live OUTSIDE the MST and are
// addressed by CID from inside records, so a node can hold a byte-perfect, fully verified record set and
// still be unable to render a single image. Nothing checked this until now.
//
// ⛔ WHY A GENERIC WALK AND NOT A LIST OF PATHS. The obvious implementation reads the paths we know:
// profile.avatar, profile.banner, embed.images[].image, embed.video, embed.external.thumb,
// recordWithMedia.media.*. That list is wrong the day a lexicon adds a field, and wrong SILENTLY — a
// blob nobody enumerates is a blob nobody notices is missing, which is precisely the failure this is
// built to catch. So walk the decoded record and take anything shaped like a blob, wherever it sits.
//
// A blob ref is `{$type:'blob', ref:<cid>, mimeType, size}` in the current lexicon. The pre-lexicon
// "legacy blob" form is `{cid:'<string>', mimeType}` with no $type; wj62's repo predates the cutover in
// places, so both are taken. `ref` itself arrives in four shapes (multiformats CID, {$link}, {'/'} or a
// bare string) — blobCid() in render.js already normalises all four and is reused rather than reinvented.
import { blobCid } from './render.js'
// Depth cap: records are small and self-describing, but this walks UNTRUSTED decoded input. A cycle is
// impossible out of dag-cbor (no back-references) yet a deeply nested record should still cost bounded
// work rather than blowing the stack.
const MAX_DEPTH = 12
function pushRef (out, seen, node, where) {
const cid = blobCid(node.ref !== undefined ? node.ref : node.cid)
if (typeof cid !== 'string' || !cid) return
if (seen.has(cid)) return // the same blob referenced twice is ONE blob
seen.add(cid)
out.push({
cid,
mimeType: typeof node.mimeType === 'string' ? node.mimeType : null,
size: Number.isFinite(node.size) ? node.size : null,
where // a breadcrumb, so a report can say WHICH field is missing its bytes
})
}
function walk (node, out, seen, depth, where) {
if (!node || depth > MAX_DEPTH) return
if (Array.isArray(node)) {
for (let i = 0; i < node.length; i++) walk(node[i], out, seen, depth + 1, `${where}[${i}]`)
return
}
if (typeof node !== 'object') return
// A multiformats CID is an object, but it is a LEAF — descending into its internals would yield
// garbage fields, so stop here. It is only a blob when it sat under a `ref`/`cid` key, handled below.
if (node.asCID === node) return
const isBlob = node.$type === 'blob' ||
(node.ref !== undefined && typeof node.mimeType === 'string') ||
(typeof node.cid === 'string' && typeof node.mimeType === 'string' && node.$type === undefined)
if (isBlob) { pushRef(out, seen, node, where); return }
for (const k of Object.keys(node)) walk(node[k], out, seen, depth + 1, where ? `${where}.${k}` : k)
}
// One record -> [{ cid, mimeType, size, where }], deduped within the record.
export function collectBlobRefs (record) {
const out = []
walk(record, out, new Set(), 0, '')
return out
}
// Every blob the whole repo refers to -> Map<cid, { cid, mimeType, size, refs:[{collection,rkey,where}] }>.
// Keyed by CID because blobs are content-addressed: the same image embedded in two posts is ONE blob,
// and reporting it twice would overstate a gap.
export function collectRepoBlobRefs (records) {
const byCid = new Map()
for (const r of records || []) {
if (!r || !r.record) continue
for (const b of collectBlobRefs(r.record)) {
let e = byCid.get(b.cid)
if (!e) { e = { cid: b.cid, mimeType: b.mimeType, size: b.size, refs: [] }; byCid.set(b.cid, e) }
e.refs.push({ collection: r.collection, rkey: r.rkey, where: b.where })
}
}
return byCid
}
// Which referenced blobs are absent locally. hasBlob(cid) -> boolean.
//
// This answers the REFERENCED question only. An ORPHAN — bytes uploaded and referenced by no record — is
// invisible to any walk of records, by construction, and needs com.atproto.sync.listBlobs, which the
// membrane does not implement (spec §6). That is the class that lost the 2026-07-03 image, so the gap is
// named in the result rather than left for the reader to infer from its absence.
// listAuthorityBlobs() -> { cids: [...] } | null (com.atproto.sync.listBlobs, optional)
//
// Two questions, and they need different sources:
// MISSING — the records refer to it, we do not have the bytes. (records, always available)
// ORPHAN — the authority HAS it, no record refers to it. (needs the authority's inventory)
//
// An orphan is invisible to any walk of records by construction, which is exactly why the 2026-07-03
// image could be lost without any surface noticing. When no inventory is available the result says
// orphansChecked:false rather than reporting zero orphans — "we did not look" and "there are none" are
// different answers, and conflating them is the habit this whole spec exists to break.
export async function diffBlobs ({ records, hasBlob, listAuthorityBlobs = null }) {
const byCid = collectRepoBlobRefs(records)
const missing = []
let present = 0
for (const e of byCid.values()) {
let here = false
try { here = await hasBlob(e.cid) } catch { here = false }
if (here) present++
else missing.push(e)
}
let orphansChecked = false
let orphans = []
let inventory = null
if (typeof listAuthorityBlobs === 'function') {
let inv = null
try { inv = await listAuthorityBlobs() } catch { inv = null }
if (inv && Array.isArray(inv.cids)) {
orphansChecked = true
inventory = inv.cids.length
orphans = inv.cids.filter((cid) => !byCid.has(cid)).map((cid) => ({ cid }))
}
}
return {
referenced: byCid.size,
present,
missing,
inventory, // null when the authority could not be asked
orphansChecked,
orphans,
orphanNote: orphansChecked
? 'orphans = blobs the authority holds that no record references'
: 'orphan blobs (uploaded, referenced by no record) are NOT checked — needs com.atproto.sync.listBlobs (spec/convergence.md §6)'
}
}
// ---- content-address verification -------------------------------------------------------------------
//
// ⛔ NEVER write fetched bytes under a CID without checking them. The CID *is* the hash (raw codec,
// sha-256), so verifying is free and refusing is absolute: a server that returns the wrong body cannot
// get it stored under the right name. This is the blob analogue of the record-side rule that bytes must
// re-encode to their own CID — the whole reason it is safe to fetch from our own membrane, or anyone.
//
// cidForBlobSync is atproto-repo's own raw-codec derivation (the same one atproto-store's
// OverlayBlobStore uses to name these files), so this cannot drift from how they were written.
export async function verifyBlobCid (bytes, cid) {
if (!bytes || !bytes.length || typeof cid !== 'string' || !cid) return false
try {
const { cidForBlobSync } = await import('atproto-repo/lib/cid.js')
return String(cidForBlobSync(bytes)) === cid
} catch {
return false // cannot verify -> do not write. Never "assume it is fine".
}
}