// retention hooks — retain / knownRev / evict, and the §7.6 byte budget.
// Run: node test/retention.test.mjs
//
// The store, appview, index and clock are injected, so the tiering, skip-test, LRU order and the
// byte-exactness refusal are all tested without a Corestore or disk. The refusal case is the important
// one: OverlayRepoStore re-encodes what it is given, and an encoding divergence federates as corruption
// rather than failing loudly, so it has to be caught here.
import * as dagCbor from '@ipld/dag-cbor'
import { cidForBytesSync } from 'atproto-repo/lib/cid.js'
import { createRetentionHooks, createMemoryIndex, retentionKey } from '../lib/retention.js'
let pass = 0
const ok = (c, m) => { if (!c) { console.error('FAIL:', m); process.exit(1) } console.log(' ok -', m); pass++ }
const DID = 'did:plc:replier'
const ptr = (over = {}) => ({ did: DID, collection: 'app.bsky.feed.post', rkey: '3kdef', rev: '3lbbb', ...over })
// A verified record, exactly as record-fetch.js hands it over.
function verified (value, over = {}) {
const bytes = dagCbor.encode(value)
return { bytes, recordCid: cidForBytesSync(bytes), ...over }
}
const POST = { $type: 'app.bsky.feed.post', text: 'hello', createdAt: '2026-07-30T00:00:00Z' }
// Fakes.
function fakeStore () {
const files = new Map()
const s = {
puts: 0,
dels: 0,
async put (c, r, v) { s.puts++; files.set(`${c}/${r}`, v) },
// Models OverlayRepoStore.getRaw — { bytes, record } or null. The no-lossy-overwrite guard reads it.
async getRaw (c, r) { const v = files.get(`${c}/${r}`); return v ? { bytes: dagCbor.encode(v), record: v } : null },
async del (c, r) { s.dels++; return files.delete(`${c}/${r}`) },
get (c, r) { return files.get(`${c}/${r}`) || null },
get count () { return files.size }
}
return s
}
function fakeAppview () {
const ops = []
return { ops, async applyCommit (did, o) { ops.push({ did, ops: o }); return o.length } }
}
// ── retain: the happy path ───────────────────────────────────────────────────────────────────────
{
const store = fakeStore(); const appview = fakeAppview(); const index = createMemoryIndex()
const h = createRetentionHooks({ openStore: async () => store, appview, index })
ok((await h.retainRecord(verified(POST), ptr())) === true, 'a verified record is retained')
ok(store.puts === 1 && store.get('app.bsky.feed.post', '3kdef').text === 'hello',
'the DECODED record goes to the per-DID OverlayRepoStore — files, not cores')
ok(appview.ops.length === 1 && appview.ops[0].ops[0].action === 'create',
'and is folded into the appview as a synthesised commit op, reusing the existing ingest path rather than a second route')
ok(appview.ops[0].did === DID, 'attributed to the FOREIGN author, not to us')
ok(h.stats.retained === 1 && h.stats.bytes > 0, 'byte accounting is live')
}
// ── knownRev: per-record, and it feeds the skip-test ─────────────────────────────────────────────
{
const index = createMemoryIndex()
const h = createRetentionHooks({ openStore: async () => fakeStore(), index })
ok((await h.knownRev(ptr())) === null, 'nothing known before retaining')
await h.retainRecord(verified(POST), ptr({ rev: '3lbbb' }))
ok((await h.knownRev(ptr())) === '3lbbb', 'after retaining, the rev is known — this is what lets the drain skip before the network')
ok((await h.knownRev(ptr({ rkey: 'other' }))) === null,
'the rev is per-RECORD, not per-repo: Tier B is a partial cache, so a repo-level head would claim completeness we do not have')
}
{
// A backfilled pointer carries rev: null. Retaining it must not ERASE a rev we already knew.
const index = createMemoryIndex()
const h = createRetentionHooks({ openStore: async () => fakeStore(), index })
await h.retainRecord(verified(POST), ptr({ rev: '3lbbb' }))
await h.retainRecord(verified(POST), ptr({ rev: null }))
ok((await h.knownRev(ptr())) === '3lbbb',
'a null-rev backfill does not downgrade a known rev — otherwise the skip-test would forget state and re-fetch forever')
}
// ── byte-exactness: the refusal ──────────────────────────────────────────────────────────────────
{
const store = fakeStore(); const events = []
const h = createRetentionHooks({ openStore: async () => store, index: createMemoryIndex(), onEvent: (e, d) => events.push(e) })
// Bytes whose CID does not match what a re-encode produces — i.e. the store would persist something
// other than what was verified.
const bad = { bytes: dagCbor.encode(POST), recordCid: cidForBytesSync(dagCbor.encode({ different: true })) }
ok((await h.retainRecord(bad, ptr())) === false,
'a record that would not round-trip byte-exactly is REFUSED — the store re-encodes, and a divergence federates as corruption rather than failing loudly')
ok(store.puts === 0, 'and nothing is written')
ok(events.includes('reencode-mismatch'), 'the refusal is reported, not silent')
ok(h.stats.reencodeRefused === 1, 'and counted')
}
{
const h = createRetentionHooks({ openStore: async () => fakeStore(), index: createMemoryIndex() })
ok((await h.retainRecord({ bytes: new Uint8Array([0xff, 0xff, 0xff]), recordCid: { toString: () => 'x' } }, ptr())) === false,
'undecodable bytes are refused, not thrown')
ok((await h.retainRecord(null, ptr())) === false, 'a null record is refused')
}
// ── a failed index fold must not unwind the write ────────────────────────────────────────────────
{
const store = fakeStore(); const events = []
const h = createRetentionHooks({
openStore: async () => store,
appview: { async applyCommit () { throw new Error('index down') } },
index: createMemoryIndex(),
onEvent: (e) => events.push(e)
})
ok((await h.retainRecord(verified(POST), ptr())) === true, 'a failed appview fold still counts as retained')
ok(store.count === 1, 'because the bytes are safely on disk — a render gap heals on re-index, data loss does not')
ok(events.includes('index-failed'), 'and the gap is reported')
}
// ── a failed store write DOES fail the retain ────────────────────────────────────────────────────
{
const h = createRetentionHooks({
openStore: async () => ({ async put () { throw new Error('disk full') } }),
index: createMemoryIndex()
})
ok((await h.retainRecord(verified(POST), ptr())) === false, 'if the bytes cannot be stored, retain fails')
ok(h.stats.storeErrors === 1, 'and is counted')
}
// ── ⛔ NEVER write into a repo hpp authors (the 2026-07-31 wedge) ─────────────────────────────────
{
const store = fakeStore(); const events = []
const h = createRetentionHooks({
openStore: async () => store,
index: createMemoryIndex(),
isForeign: (did) => did !== 'did:plc:ours',
onEvent: (e) => events.push(e)
})
ok((await h.retainRecord(verified(POST), ptr({ did: 'did:plc:ours' }))) === false,
'a record authored by a DID hpp owns is REFUSED — Constellation reports our own replies to our own posts, and re-encoding those over hpp canonical files wedged the sign gate')
ok(store.puts === 0, 'and nothing is written into hpp\'s store')
ok(h.stats.skippedOwn === 1 && events.includes('skipped-own-did'), 'counted and reported, not silent')
ok((await h.retainRecord(verified(POST), ptr({ did: 'did:plc:someoneelse' }))) === true, 'a genuinely foreign record still retains')
}
{
const h = createRetentionHooks({ openStore: async () => fakeStore(), index: createMemoryIndex() })
ok((await h.retainRecord(verified(POST), ptr())) === true,
'the DEFAULT treats everything as foreign — an unconfigured caller must not silently clobber, so the daemon has to opt IN to ownership')
}
// ── ⛔ NO LOSSY OVERWRITE — the guard putPost got in 0.2.50 and this path bypassed ────────────────
{
// hpp wrote the record WITH langs; ours is the same record WITHOUT it. That exact 392->375 byte drop
// is what invalidated the converged pointer and stopped wj62's head advancing.
const canonical = { ...POST, langs: ['en'] }
const store = fakeStore(); const events = []
store.getRaw = async () => ({ bytes: dagCbor.encode(canonical), record: canonical })
const h = createRetentionHooks({ openStore: async () => store, index: createMemoryIndex(), onEvent: (e) => events.push(e) })
ok((await h.retainRecord(verified(POST), ptr())) === false,
'a copy MISSING a field the canonical bytes have is REFUSED — this is the exact shape that wedged the sign gate')
ok(store.puts === 0, 'the canonical bytes are left untouched')
ok(h.stats.lossyRefused === 1 && events.includes('lossy-refused'), 'and the divergence is LOUD')
}
{
// The rule is NO LOSSY OVERWRITE, not NO OVERWRITE — freezing at first-seen version is its own bug.
const existing = { ...POST, text: 'older text' }
const store = fakeStore()
store.getRaw = async () => ({ bytes: dagCbor.encode(existing), record: existing })
const h = createRetentionHooks({ openStore: async () => store, index: createMemoryIndex() })
ok((await h.retainRecord(verified(POST), ptr())) === true, 'a CHANGED value still writes')
ok(store.puts === 1, 'because refusing every overwrite would freeze ingested records forever')
}
{
const added = { ...POST, langs: ['en'] }
const store = fakeStore()
store.getRaw = async () => ({ bytes: dagCbor.encode(POST), record: POST })
const h = createRetentionHooks({ openStore: async () => store, index: createMemoryIndex() })
ok((await h.retainRecord(verified(added), ptr())) === true, 'ADDING a field still writes — only dropping one is refused')
}
{
const store = fakeStore()
store.getRaw = async () => null
const h = createRetentionHooks({ openStore: async () => store, index: createMemoryIndex() })
ok((await h.retainRecord(verified(POST), ptr())) === true, 'no existing record means nothing to clobber')
}
// ── evict ────────────────────────────────────────────────────────────────────────────────────────
{
const store = fakeStore(); const appview = fakeAppview(); const index = createMemoryIndex()
const h = createRetentionHooks({ openStore: async () => store, appview, index })
await h.retainRecord(verified(POST), ptr())
await h.evictRecord(ptr())
ok(store.dels === 1 && store.count === 0, 'evict removes the file')
ok(appview.ops[1].ops[0].action === 'delete', 'and un-indexes it so it stops rendering')
ok((await index.get(retentionKey(ptr()))) === null, 'and drops the bookkeeping row')
ok(h.stats.evicted === 1 && h.stats.bytes === 0, 'byte accounting returns to zero')
}
{
// A store that has already lost the file must not leave a permanently un-evictable row.
const index = createMemoryIndex()
const h = createRetentionHooks({
openStore: async () => ({ async put () {}, async del () { throw new Error('gone') } }),
index
})
await h.retainRecord(verified(POST), ptr())
await h.evictRecord(ptr())
ok((await index.get(retentionKey(ptr()))) === null,
'the index row is dropped even when the store del fails — otherwise a lost file would inflate the budget forever')
}
// ── the §7.6 byte budget: LRU eviction ───────────────────────────────────────────────────────────
{
const store = fakeStore(); const index = createMemoryIndex()
let clock = 1000
const one = dagCbor.encode(POST).length
const h = createRetentionHooks({
openStore: async () => store,
index,
budgetBytes: one * 3 + 1, // room for three
now: () => clock++
})
for (let i = 0; i < 5; i++) await h.retainRecord(verified(POST), ptr({ rkey: `rk${i}` }))
ok(index.size === 3, 'the budget caps how many records are retained')
ok(h.stats.evictedForBudget === 2, 'and the overflow is evicted, counted rather than silent')
ok((await index.get(retentionKey(ptr({ rkey: 'rk0' })))) === null, 'the OLDEST-touched record goes first (LRU)')
ok((await index.get(retentionKey(ptr({ rkey: 'rk4' })))) !== null, 'the newest survives')
ok((await h.totalBytes()) <= one * 3 + 1, 'and the total is back under budget')
}
{
// touch() moves a record to the front of the LRU, so eviction reflects RENDER activity.
const index = createMemoryIndex()
let clock = 1000
const one = dagCbor.encode(POST).length
const h = createRetentionHooks({ openStore: async () => fakeStore(), index, budgetBytes: one * 2 + 1, now: () => clock++ })
await h.retainRecord(verified(POST), ptr({ rkey: 'a' }))
await h.retainRecord(verified(POST), ptr({ rkey: 'b' }))
await h.touch(ptr({ rkey: 'a' })) // 'a' is now the most recently used
await h.retainRecord(verified(POST), ptr({ rkey: 'c' }))
ok((await index.get(retentionKey(ptr({ rkey: 'b' })))) === null, 'the least-recently-TOUCHED is evicted, not the least-recently-written')
ok((await index.get(retentionKey(ptr({ rkey: 'a' })))) !== null, 'a touched record survives')
ok((await h.touch(ptr({ rkey: 'nope' }))) === false, 'touching an unknown record is a no-op')
}
{
const index = createMemoryIndex()
const h = createRetentionHooks({ openStore: async () => fakeStore(), index, budgetBytes: null })
for (let i = 0; i < 20; i++) await h.retainRecord(verified(POST), ptr({ rkey: `rk${i}` }))
ok(index.size === 20 && (await h.enforceBudget()) === 0,
'a null budget is unbounded — explicit, so the caller is choosing the risk rather than inheriting it')
}
// ── construction guards ──────────────────────────────────────────────────────────────────────────
{
let threw = false
try { createRetentionHooks({ index: createMemoryIndex() }) } catch { threw = true }
ok(threw, 'openStore is required')
threw = false
try { createRetentionHooks({ openStore: async () => {} }) } catch { threw = true }
ok(threw, 'an index is required')
}
console.log(`\nRETENTION OK: ${pass} assertions`)