atproto-store / test / pointer-substrate-backfill.test.js

// Step 2a MIGRATION proof: PointerSubstrate.backfill() converts a files-only repo (record BYTES on
// disk, NO pointers in the autobase) into the pointer design by writing one {$oid,size} pointer per
// existing record file. This is the guard that stops the wj62-wipe: without it, the first sign() on a
// migrated repo reads an EMPTY pointer set and materializes an empty repo. Proves: backfill writes a
// pointer per file; it is idempotent (a second run writes nothing); it never clobbers a pointer that
// already converged (a peer's newer put or a delete tombstone); it skips reserved/control keys.
//
// Uses a Map-backed mock statebase (the substrate contract is set/get/del/list) + a real
// OverlayRepoStore (no overlay → oid is null, which is fine here: the UNIT under test is the
// pointer/skip bookkeeping, not byte distribution — that is proven in the hpp integration test).

import test from 'brittle'
import { mkdtempSync, rmSync } from 'node:fs'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { OverlayRepoStore } from '../lib/overlay-repo-store.js'
import { PointerSubstrate } from '../lib/pointer-substrate.js'

const tmp = (t) => { const d = mkdtempSync(join(tmpdir(), 'ptr-bf-')); t.teardown(() => rmSync(d, { recursive: true, force: true })); return d }

// Minimal in-memory statebase honoring the substrate contract PointerSubstrate depends on.
function mockStatebase () {
  const m = new Map()
  return {
    _m: m,
    async set (k, v) { m.set(k, v) },
    async get (k) { return m.has(k) ? m.get(k) : null },
    async del (k) { m.delete(k) },
    async list (prefix = '') { const o = {}; for (const [k, v] of m) if (k.startsWith(prefix)) o[k] = v; return o }
  }
}

const post = (text) => ({ $type: 'app.bsky.feed.post', text, createdAt: '2026-07-19T00:00:00.000Z' })

test('backfill: writes one pointer per files-only record, then listRecords sees the full set', async (t) => {
  const store = await new OverlayRepoStore(tmp(t)).ready()
  // Simulate a files-only (OverlayRepoSubstrate) history: bytes on disk, NOTHING in the statebase.
  await store.put('app.bsky.feed.post', '1', post('one'))
  await store.put('app.bsky.feed.post', '2', post('two'))
  await store.put('app.bsky.graph.follow', 'f1', { $type: 'app.bsky.graph.follow', subject: 'did:plc:x', createdAt: post('').createdAt })

  const sb = mockStatebase()
  const sub = new PointerSubstrate(sb, store)

  // BEFORE: the pointer set (what sign() reads via listRecords) is EMPTY — the wipe hazard.
  t.is(Object.keys(await sub.list()).length, 0, 'pre-migration: pointer set is empty (would wipe)')

  const r = await sub.backfill()
  t.alike(r, { written: 3, skipped: 0, total: 3 }, 'backfill wrote a pointer for each of the 3 files')

  // AFTER: every record now has a {$oid,size} pointer in the statebase, and the set resolves byte-exact.
  const ptr = await sb.get('app.bsky.feed.post/1')
  t.ok(ptr && 'size' in ptr && '$oid' in ptr, 'a {$oid,size} pointer now exists in the statebase')
  const resolved = await sub.list()
  t.is(Object.keys(resolved).length, 3, 'listRecords now returns the full converged set (3)')
  t.alike(resolved['app.bsky.feed.post/1'], post('one'), 'pointer resolves to the byte-exact record')
})

test('backfill: idempotent — a second run writes nothing', async (t) => {
  const store = await new OverlayRepoStore(tmp(t)).ready()
  await store.put('app.bsky.feed.post', '1', post('one'))
  await store.put('app.bsky.feed.post', '2', post('two'))
  const sb = mockStatebase()
  const sub = new PointerSubstrate(sb, store)

  t.alike(await sub.backfill(), { written: 2, skipped: 0, total: 2 }, 'first run migrates both')
  t.alike(await sub.backfill(), { written: 0, skipped: 2, total: 2 }, 'second run is a no-op (all skipped)')
})

test('backfill: never clobbers a pointer that already converged (peer put OR delete tombstone)', async (t) => {
  const store = await new OverlayRepoStore(tmp(t)).ready()
  await store.put('app.bsky.feed.post', '1', post('local-stale'))
  await store.put('app.bsky.feed.post', '2', post('two'))
  const sb = mockStatebase()
  const sub = new PointerSubstrate(sb, store)

  // A peer already converged a NEWER pointer for record 1 (e.g. an edit) into the shared autobase.
  const peerPointer = { $oid: 'peer-newer-oid', size: 999 }
  await sb.set('app.bsky.feed.post/1', peerPointer)

  const r = await sub.backfill()
  t.alike(r, { written: 1, skipped: 1, total: 2 }, 'only the un-pointed record is migrated; the converged one is skipped')
  t.alike(await sb.get('app.bsky.feed.post/1'), peerPointer, 'the peer pointer is untouched (no resurrection/clobber)')
})

test('backfill: skips reserved/control keys if any leak into the store', async (t) => {
  const store = await new OverlayRepoStore(tmp(t)).ready()
  await store.put('app.bsky.feed.post', '1', post('one'))
  const sb = mockStatebase()
  const sub = new PointerSubstrate(sb, store)
  // A reserved key can never legitimately be a record file, but assert the guard regardless: only the
  // real record is migrated, a __-collection key is never written into the pointer set.
  const r = await sub.backfill()
  t.is(r.written, 1, 'only the real record was migrated')
  t.absent(await sb.get('__prefs__'), 'no reserved key entered the pointer set')
})