// PointerSubstrate — the RepoBase KV that restores SYMMETRIC multi-writer convergence to the record
// set while keeping the bytes off the append-only log. A tiny { $oid, size } POINTER per record lives
// in the shared StateBase autobase (so every attested device converges on WHICH records exist, exactly
// like the default statebase path), and the record BYTES live as deduped files in the overlay,
// content-addressed + served by `oid` (blake2b). This is the "pointers in the autobase + bytes in the
// overlay" design: the autobase carries ~50 bytes/record instead of the full record (+ the firehose's
// inline copy), and cross-device convergence is the autobase — no bespoke overlay-to-overlay sync
// (RepoMirror is retired).
//
// RepoBase sees the SAME interface + shape as any substrate: set(key,record) / del(key) /
// get(key)->record|null / list(prefix)->{key:record} / listNow. The pointer indirection is internal:
// list/get RESOLVE each pointer to its record bytes via the overlay store (a local file hit, or a
// fetch-by-oid from a connected peer when this node — e.g. a just-promoted signer — holds only the
// converged pointer). Reserved "__" keys (firehose/admission) never reach here (swarm-device writes
// those to the StateBase directly); we defensively skip any that appear in a raw list.
const SEP = '/'
// The record-layer DELETE TOMBSTONE convention (owned by atproto-room's RepoBase; duplicated as a
// predicate here because the dependency direction is atproto-room -> atproto-store, never back).
// A deleted record keeps a pointer-shaped { $deleted:true } entry instead of vanishing, so a peer
// can tell "deleted" from "not replicated to me yet" — see RepoBase.deleteRecord.
const isTombstone = (v) => !!(v && typeof v === 'object' && v.$deleted === true)
function split (key) {
const k = String(key)
const i = k.indexOf(SEP)
if (i < 0) throw new Error('PointerSubstrate: expected a "collection/rkey" record key, got ' + JSON.stringify(key))
return { collection: k.slice(0, i), rkey: k.slice(i + 1) }
}
// Like RepoBase.splitKey: null for a reserved/non-record key (single-segment, or a "__"-prefixed
// collection such as __commit__/<rev>), so a raw statebase.list('') never resolves control data.
function recSplit (key) {
const i = key.indexOf(SEP)
if (i < 0) return null
const collection = key.slice(0, i)
if (collection.startsWith('__')) return null
return { collection, rkey: key.slice(i + 1) }
}
export class PointerSubstrate {
constructor (statebase, store) {
if (!statebase || typeof statebase.set !== 'function' || typeof statebase.list !== 'function') {
throw new Error('PointerSubstrate: a StateBase (set/get/del/list) is required for the pointer set')
}
if (!store || typeof store.put !== 'function' || typeof store.ensureLocal !== 'function') {
throw new Error('PointerSubstrate: an OverlayRepoStore (put/ensureLocal) is required for the bytes')
}
this.statebase = statebase
this.store = store
}
// Write the bytes to the overlay (dedup + servable by oid), then the tiny pointer to the autobase.
// A TOMBSTONE value is not a record: it carries no bytes, so it is stored as the pointer itself and
// the record's bytes are dropped. (RepoBase.deleteRecord routes deletes through set(), so this is
// the delete path for the pointer design.)
async set (key, record) {
const { collection, rkey } = split(key)
if (isTombstone(record)) {
await this.statebase.set(key, { $deleted: true }) // converges symmetrically, like any pointer
try { await this.store.del(collection, rkey) } catch {}
return
}
const { oid, size } = await this.store.put(collection, rkey, record)
await this.statebase.set(key, { $oid: oid, size })
}
// Legacy hard-delete. Retained for callers that genuinely want the key GONE; record deletion now
// goes through set(TOMBSTONE) so absence stays unambiguous.
async del (key) {
const { collection, rkey } = split(key)
await this.statebase.del(key)
try { await this.store.del(collection, rkey) } catch {}
}
// Resolve the pointer to the record bytes (local file, or fetch-by-oid from a peer).
async get (key) {
const ptr = await this.statebase.get(key)
if (!ptr || isTombstone(ptr)) return null // deleted reads as absent
const { collection, rkey } = split(key)
return this.store.ensureLocal(collection, rkey, ptr && ptr.$oid)
}
// The converged pointer set (authoritative) resolved to { key: record }. Skips reserved keys.
async list (prefix = '') { return this._resolve(await this.statebase.list(prefix)) }
async listNow (prefix = '') {
const raw = (typeof this.statebase.listNow === 'function') ? await this.statebase.listNow(prefix) : await this.statebase.list(prefix)
return this._resolve(raw)
}
async _resolve (ptrMap) {
const out = {}
for (const key of Object.keys(ptrMap)) {
const s = recSplit(key)
if (!s) continue // reserved/control key — not a record
const ptr = ptrMap[key]
if (isTombstone(ptr)) continue // deleted — never resolve bytes for a tombstone
const rec = await this.store.ensureLocal(s.collection, s.rkey, ptr && ptr.$oid)
if (rec != null) out[key] = rec
}
return out
}
// ── Step 2a MIGRATION: files-only → pointers (idempotent) ──────────────────────────────────────
// A repo previously written by OverlayRepoSubstrate has its record BYTES on disk as overlay files
// but NO pointers in the shared autobase. Under this substrate, listRecords() reads the pointer set
// → ZERO, so the first sign() would materialize an EMPTY repo and wipe every record off the relay
// (the wj62-wipe hazard). backfill() closes that gap: for every record file that has no pointer yet,
// it writes the tiny {$oid,size} pointer into the autobase — so the converged pointer set reflects
// the real record set before any sign runs.
//
// Idempotent + convergence-safe: SKIPS any key that already has a pointer (a peer may have converged
// a newer put OR a delete tombstone — never resurrect/clobber it), and never deletes. Re-registering
// the byte-identical file via store.put is harmless (the determinism contract: same bytes → same oid)
// and yields the oid without trusting stale overlay index state. WRITABLE-ONLY: statebase.set appends
// to the autobase, so only an admitted writer (the creator) may run this — the caller gates on
// base.writable. Returns { written, skipped, total }.
// ── MIGRATION: INLINE RECORD → POINTER (idempotent) ────────────────────────────────────────────
// The inverse gap to backfill(). backfill covers "bytes on disk, no entry in the autobase"; this
// covers "an entry in the autobase that is a full RECORD instead of a { $oid, size } pointer" —
// written through the LEGACY substrate against a DID that now runs the pointer design.
//
// Such a repo reads perfectly on the node that holds it and is UNFETCHABLE BY EVERY OTHER NODE:
// _resolve passes ptr.$oid (undefined) to ensureLocal, which falls back to the local file, so the
// records resolve locally and no peer has an oid to fetch by. A co-owner would converge N keys it can
// never turn into bytes. Live on loser-club: three records, all inline, so that DID cannot be
// co-owned at all — the config and roster work is wasted until this is fixed.
//
// backfill() cannot do it: it SKIPS any key that already has a value (`if (existing != null)`), which
// is correct for its job — never clobber a converged pointer or tombstone — but an inline record IS
// such a value.
//
// CONTENT IS PRESERVED, NOT CHOSEN. What the repo currently publishes is the LOCAL FILE (that is what
// ensureLocal returns today), not the inline copy in the log — so the pointer must address the file's
// bytes or the signed repo would change underneath us. We therefore re-put the record read back from
// disk, which registers it with the overlay (making it fetchable, the whole point) and returns its
// oid; DAG-CBOR is canonical so the rewrite is byte-identical. That is ASSERTED, not assumed: if the
// resulting oid differs from what the file already hashed to, the key is skipped and counted rather
// than silently republishing different content. Only when there is no local file at all do we fall
// back to the inline record — then it is the only copy in existence.
//
// Nothing is deleted (statebase.set appends; the old value stays in oplog history) and files are
// never modified beyond a byte-identical rewrite. WRITABLE-ONLY: the caller gates on base.writable.
// Returns { converted, alreadyPointer, tombstones, fromInline, mismatched, failed, total }.
// TWO PASSES, and the order is a SAFETY property, not tidiness.
//
// PASS 1 MATERIALIZES every inline record to a file. PASS 2 publishes the pointers. Interleaving them
// (put/set per record, as this did originally) leaves the set only partly protected while it runs.
//
// Why that matters: a legacy repo being flipped to this substrate reads as an EMPTY record set (_resolve
// has no $oid and no file to fall back to), and sign()'s anti-wipe guard aborts only when record FILES
// exist. With no files it does not fire, and the first sign publishes a delete-every-record commit.
// Materializing the whole set FIRST arms that guard before a single autobase entry is touched — so if
// pass 2 never runs (crash, lost writability, autobase error), the repo is protected rather than wiped.
//
// Pass 1 is also purely local: content-addressed files, nothing published, no writability needed, and
// nothing to roll back. Pass 2 is the only part that appends.
async migrateInline () {
const raw = await this.statebase.list('')
let converted = 0; let alreadyPointer = 0; let tombstones = 0; let fromInline = 0; let mismatched = 0; let failed = 0; let total = 0
const staged = []
for (const key of Object.keys(raw)) {
const s = recSplit(key)
if (!s) continue // reserved/control key — never part of the record pointer set
total++
const v = raw[key]
if (isTombstone(v)) { tombstones++; continue }
if (v && v.$oid) { alreadyPointer++; continue } // already the right shape — idempotent
if (!v || typeof v !== 'object') { failed++; continue }
try {
const before = (typeof this.store.localOid === 'function') ? await this.store.localOid(s.collection, s.rkey) : null
const onDisk = await this.store.get(s.collection, s.rkey)
const record = onDisk != null ? onDisk : v // the file is what resolves today; the inline copy is the fallback
if (onDisk == null) fromInline++
const { oid, size } = await this.store.put(s.collection, s.rkey, record)
if (!oid) { failed++; continue } // no overlay → no oid to publish; leave the entry alone
if (before && oid !== before.oid) { mismatched++; continue } // would change published content — refuse
staged.push({ key, oid, size })
} catch { failed++ }
}
const materialized = staged.length
for (const { key, oid, size } of staged) {
try { await this.statebase.set(key, { $oid: oid, size }); converted++ } catch { failed++ }
}
return { converted, materialized, alreadyPointer, tombstones, fromInline, mismatched, failed, total }
}
async backfill () {
if (typeof this.store.records !== 'function') {
throw new Error('PointerSubstrate.backfill: the store must enumerate records() to migrate files → pointers')
}
let written = 0
let skipped = 0
const files = await this.store.records() // [{ key, bytes, record }] — the on-disk record files
for (const { key, record } of files) {
const s = recSplit(key)
if (!s) continue // never migrate a reserved/control key into the record pointer set
const existing = await this.statebase.get(key)
if (existing != null) { skipped++; continue } // pointer already converged (put or tombstone) — leave it
const { oid, size } = await this.store.put(s.collection, s.rkey, record)
await this.statebase.set(key, { $oid: oid, size })
written++
}
return { written, skipped, total: files.length }
}
}