// BlobPointerStore — the blob half of the "pointers in the autobase, bytes in the overlay" design.
// This is to blobs exactly what PointerSubstrate is to records, and it exists because the records
// side got this right and the blobs side did not.
//
// THE GAP IT CLOSES. OverlayBlobStore keeps a per-blob metadata record { mimeType, size, oid } in a
// Hyperbee, and that bee is NODE-LOCAL — in the live daemon it lives in the membrane corestore
// (device-node.js: `overlay-blobs:<did>` on membraneStore), which does not replicate. Its get() reads
// that bee FIRST and returns null when the entry is missing, BEFORE it ever looks at the file or the
// overlay. So a blob uploaded on node A produces two artifacts with very different fates:
//
// the FILE + overlay registration -> content-addressed, fetchable by `oid` from any peer
// the bee entry { mimeType,size,oid } -> stays on node A, forever
//
// Node B therefore cannot serve the blob even when the bytes are reachable, because the one thing it
// is missing is the cid -> oid mapping needed to ask for them. Since the atproto blob CID (raw codec,
// cidForBlobSync) and the overlay oid (blake2b) are DIFFERENT hashes, B cannot derive one from the
// other — the mapping has to be replicated, not recomputed. Symptom in production: an avatar that
// serves or 404s depending on which co-owner the bridge happens to front.
//
// THE FIX is the pattern PointerSubstrate already proves for records: put a tiny pointer in the SHARED
// autobase so every co-owner converges on which blobs exist and the oid to fetch each one by, while the
// bytes stay out of the append-only log. Pointer key is `__blob__/<cid>`, in the RESERVED "__"
// namespace — RepoBase.splitKey and PointerSubstrate both return null for a collection starting with
// "__", so these entries can never be mistaken for records or materialized into a signed commit.
//
// Drop-in: same public surface as BlobStore / OverlayBlobStore (ready / put / get / has / close), so it
// composes under DualBlobStore unchanged.
const BLOB_PREFIX = '__blob__/'
export class BlobPointerStore {
// statebase: the shared StateBase (set/get). blobs: an OverlayBlobStore (the local bytes + bee).
constructor (statebase, blobs, { did = null } = {}) {
if (!statebase || typeof statebase.set !== 'function' || typeof statebase.get !== 'function') {
throw new Error('BlobPointerStore: a StateBase (set/get) is required for the shared pointer set')
}
if (!blobs || typeof blobs.put !== 'function' || typeof blobs.adoptMeta !== 'function') {
throw new Error('BlobPointerStore: an OverlayBlobStore (put/get/adoptMeta) is required for the bytes')
}
this.sb = statebase
this.blobs = blobs
this._did = did
}
static key (cidStr) { return BLOB_PREFIX + cidStr }
async ready () { return this } // the caller readies the underlying OverlayBlobStore
// Write the bytes locally (file + overlay registration + local bee), then publish the SHARED pointer
// so any co-owner can resolve this blob by oid.
//
// A failed pointer write does NOT fail the upload. The blob is already durably stored and servable
// from this node, so throwing here would turn a cross-node limitation into a total upload failure —
// strictly worse than today. But it is logged LOUDLY rather than swallowed, because the resulting
// state (blob local-only) is exactly the silent condition this module exists to end. The common
// cause is a non-writable device: sb.set appends to the autobase, so only an admitted writer can
// publish a pointer.
async put (bytes, mimeType) {
const r = await this.blobs.put(bytes, mimeType)
const cidStr = r.cid.toString()
if (r.oid) {
try {
await this.sb.set(BlobPointerStore.key(cidStr), { $oid: r.oid, size: r.size, mimeType: r.mimeType })
} catch (err) {
this._warn(cidStr, 'pointer write failed — blob is stored LOCAL-ONLY and will not serve from other co-owners', err && err.message)
}
} else {
this._warn(cidStr, 'no overlay oid — blob cannot be published as a shared pointer (is the overlay attached?)', null)
}
return r
}
// Local first (bee hit -> file, or fetch-by-oid). On a miss, consult the SHARED pointer: adopt its
// metadata into the local bee, then re-enter the ordinary get() path, which now finds the mapping and
// fetches the bytes by oid. Reusing get() rather than fetching here keeps ONE byte-resolution path.
async get (cidStr) {
const cid = String(cidStr)
const local = await this.blobs.get(cid)
if (local) return local
const ptr = await this._pointer(cid)
if (!ptr || !ptr.$oid) return null
await this.blobs.adoptMeta(cid, { mimeType: ptr.mimeType, size: ptr.size, oid: ptr.$oid })
return this.blobs.get(cid)
}
// Known here OR known to the swarm. A shared pointer means the bytes are fetchable even if this node
// has never seen them, so reporting only the local bee would understate availability.
async has (cidStr) {
const cid = String(cidStr)
if (await this.blobs.has(cid)) return true
return !!(await this._pointer(cid))
}
// MIGRATION: publish a shared pointer for every blob this node already holds but has never announced.
// The counterpart of PointerSubstrate.backfill for records, and the reason blobs uploaded BEFORE this
// module existed are not stranded: their bytes and metadata are already here, only the announcement is
// missing. Idempotent and convergence-safe — SKIPS any cid that already has a pointer (a co-owner may
// have published a newer one; never clobber it) and never deletes.
//
// WRITABLE-ONLY: sb.set appends to the autobase, so the caller must gate on base.writable.
// `unpublishable` counts blobs with no local oid — nothing to point at; those need reindexOverlay()
// first to re-register the file and recover an oid.
async backfill () {
if (typeof this.blobs.entries !== 'function') {
throw new Error('BlobPointerStore.backfill: the blob store must enumerate entries() to migrate blobs → pointers')
}
let written = 0
let skipped = 0
let unpublishable = 0
let total = 0
for await (const e of this.blobs.entries()) {
total++
if (!e.oid) { unpublishable++; continue }
const key = BlobPointerStore.key(e.cid)
if ((await this.sb.get(key)) != null) { skipped++; continue }
await this.sb.set(key, { $oid: e.oid, size: e.size, mimeType: e.mimeType })
written++
}
return { written, skipped, unpublishable, total }
}
// PROACTIVE blob convergence — the half that was missing.
//
// Publishing `__blob__/<cid>` makes a blob FETCHABLE by any co-owner, but nothing was walking those
// pointers, so blobs only ever moved LAZILY: when something happened to call get(). Two peers that
// never request each other's blobs therefore never exchanged them — images resolved to bsky (the
// bridge relays get-blob by RPC to whoever holds the bytes) while silently never replicating between
// nodes. This is the blob counterpart of SwarmDevice.reconcileBytes, which deliberately skips
// `__`-prefixed keys and so could never cover them.
//
// PULL-ONLY: it never writes a pointer, so it is safe on a NON-WRITABLE replica — exactly the peer
// that needs to catch up. Presence is tested with hasLocalFile (a stat), not get(), so a converged
// node does no reads at all. Bounded per pass with `more`, so a large backlog cannot stall the caller.
// Every blob CID this DID has PUBLISHED a pointer for — the inventory behind com.atproto.sync.listBlobs.
//
// ⛔ THIS IS THE ONLY WAY TO SEE AN ORPHAN. A blob referenced by some record can be found by walking
// records (favela-web/lib/blob-refs.js does exactly that, and needs no server support). A blob that is
// uploaded and referenced by NOTHING is invisible to any such walk, by construction — and that is the
// class that permanently lost the 2026-07-03 image. The pointer set is the authority's own inventory,
// so it is the only place the difference shows up.
//
// Reads the SHARED state, so it lists what the DID has published across all its co-owners, not merely
// what this node happens to hold bytes for. Sorted for a stable cursor; `since` is deliberately NOT
// supported (pointers carry no rev, so a rev-filter would silently return the wrong set — better to
// not offer it than to answer wrongly).
async list ({ limit = 500, cursor = null } = {}) {
let map
try {
map = await ((typeof this.sb.listNow === 'function') ? this.sb.listNow(BLOB_PREFIX) : this.sb.list(BLOB_PREFIX))
} catch (e) {
return { cids: [], cursor: null, error: e.message }
}
const all = Object.keys(map || {})
.filter((k) => k.startsWith(BLOB_PREFIX))
.map((k) => k.slice(BLOB_PREFIX.length))
.filter(Boolean)
.sort()
const start = cursor ? all.findIndex((c) => c > String(cursor)) : 0
const from = start < 0 ? all.length : start
const n = Math.max(1, Math.min(Number(limit) || 500, 1000))
const page = all.slice(from, from + n)
const next = (from + n) < all.length ? page[page.length - 1] : null
return { cids: page, cursor: next, total: all.length }
}
async reconcile ({ limit = 8 } = {}) {
if (typeof this.blobs.hasLocalFile !== 'function') return { skipped: 'not-supported' }
let map
try {
map = await ((typeof this.sb.listNow === 'function') ? this.sb.listNow(BLOB_PREFIX) : this.sb.list(BLOB_PREFIX))
} catch (e) { return { skipped: 'list-failed', error: e.message } }
let checked = 0; let missing = 0; let fetched = 0; let failed = 0; let more = false
for (const key of Object.keys(map)) {
if (!key.startsWith(BLOB_PREFIX)) continue
const cid = key.slice(BLOB_PREFIX.length)
if (!cid) continue
checked++
if (this.blobs.hasLocalFile(cid)) continue // bytes already here — a stat, not a read
missing++
if (fetched + failed >= limit) { more = true; continue }
let got = null
try { got = await this.get(cid) } catch {}
if (got) fetched++; else failed++
}
const sig = checked + ':' + missing + ':' + fetched + ':' + failed
if ((fetched || missing) && sig !== this._lastReconcileSig) {
try { console.log(JSON.stringify({ msg: 'blob reconcile: pulling shared-pointer blobs', did: this._did, checked, missing, fetched, failed, more })) } catch {}
}
this._lastReconcileSig = sig
return { checked, missing, fetched, failed, more }
}
// Pass through to the byte layer's repair (re-register blob files so peers can fetch them by oid).
async reindexOverlay () {
if (typeof this.blobs.reindexOverlay !== 'function') return { skipped: 'not-supported' }
return this.blobs.reindexOverlay()
}
async _pointer (cidStr) {
try { return await this.sb.get(BlobPointerStore.key(cidStr)) } catch { return null }
}
_warn (cid, msg, error) {
try { console.error(JSON.stringify({ msg: 'blob pointer: ' + msg, did: this._did, cid, error })) } catch {}
}
async close () { try { await this.blobs.close() } catch {} }
}
export { BLOB_PREFIX }