| … | … | @@ -0,0 +1,196 @@ |
| 1 | /** |
| 2 | * Tombstone propagation to a MIRROR (issue #135 follow-up). |
| 3 | * |
| 4 | * retired-repos.js calls the set "DERIVED, NOT STORED ... rebuilt identically |
| 5 | * on every boot". That is true on the ORIGIN, which folds its own writable |
| 6 | * RepoEventLog from seq 0 (pear-git.js: `foldRetired(repoEventLog)`). |
| 7 | * |
| 8 | * It was NOT true on a mirror. A mirror's own log carries no tombstones — the |
| 9 | * origin wrote them to ITS log — so the mirror's only source is the fold |
| 10 | * inside RepoPuller._enqueue, and that rides on a stream opened at |
| 11 | * `cursor + 1`. The cursor exists to avoid ~80 minutes of redundant git |
| 12 | * fetches per restart, so it is not going away; the consequence was that a |
| 13 | * tombstone appended BELOW the cursor was folded once, in memory, and then |
| 14 | * forgotten on the next restart. The mirror resumed listing and announcing a |
| 15 | * repo the origin had retired, for ever. |
| 16 | * |
| 17 | * Observed in production 2026-09-02: tabertron (origin) held 15 tombstones, |
| 18 | * stortron (mirror, and the load-balancer's ACTIVE backend) reported none, |
| 19 | * because it had restarted 2026-08-29 with its cursor already past the |
| 20 | * 2026-07-30 retire events. |
| 21 | * |
| 22 | * These tests use the same real-testnet harness as repo-puller-cursor.test.js |
| 23 | * so the log-streaming path is exercised end-to-end rather than stubbed. |
| 24 | */ |
| 25 | |
| 26 | import test from 'brittle' |
| 27 | import { mkdtempSync, readFileSync, existsSync } from '#node:fs' |
| 28 | import { join } from '#node:path' |
| 29 | import { tmpdir } from '#node:os' |
| 30 | import b4a from 'b4a' |
| 31 | import Corestore from 'corestore' |
| 32 | import createTestnet from '@hyperswarm/testnet' |
| 33 | import Hyperswarm from 'hyperswarm' |
| 34 | |
| 35 | import { RepoEventLog } from '../lib/services/repo-event-log.js' |
| 36 | import { RepoPuller } from '../lib/services/repo-mirror.js' |
| 37 | import { RetiredRepos, OP_CREATED, OP_UPDATED, OP_RETIRED } from '../lib/services/retired-repos.js' |
| 38 | |
| 39 | let n = 0 |
| 40 | function tmpDir () { |
| 41 | return mkdtempSync(join(tmpdir(), 'pgv3-retired-mirror-' + (n++) + '-')) |
| 42 | } |
| 43 | |
| 44 | class StubSync { |
| 45 | constructor () { |
| 46 | this.calls = [] |
| 47 | this.remotes = new Map() |
| 48 | } |
| 49 | |
| 50 | addRemote (name, key) { this.remotes.set(name, key) } |
| 51 | async pull (remote, repoName) { this.calls.push({ remote, repoName }) } |
| 52 | } |
| 53 | |
| 54 | async function setupOriginAndMirror (testnet, t) { |
| 55 | const topic = b4a.alloc(32, 'pgv3-retired-' + (n++)) |
| 56 | |
| 57 | const originStore = new Corestore(join(tmpDir(), 'origin')) |
| 58 | await originStore.ready() |
| 59 | const originLog = new RepoEventLog({ corestore: originStore }) |
| 60 | await originLog.ready() |
| 61 | |
| 62 | const originSwarm = new Hyperswarm({ bootstrap: testnet.bootstrap }) |
| 63 | originSwarm.on('connection', (conn) => originStore.replicate(conn)) |
| 64 | originSwarm.join(topic, { server: true, client: false }) |
| 65 | await originSwarm.flush() |
| 66 | |
| 67 | const mirrorStore = new Corestore(join(tmpDir(), 'mirror')) |
| 68 | await mirrorStore.ready() |
| 69 | const mirrorLog = new RepoEventLog({ corestore: mirrorStore, key: originLog.keyHex }) |
| 70 | await mirrorLog.ready() |
| 71 | |
| 72 | const mirrorSwarm = new Hyperswarm({ bootstrap: testnet.bootstrap }) |
| 73 | mirrorSwarm.on('connection', (conn) => mirrorStore.replicate(conn)) |
| 74 | mirrorSwarm.join(topic, { server: false, client: true }) |
| 75 | await mirrorSwarm.flush() |
| 76 | |
| 77 | t.teardown(async () => { |
| 78 | try { await originSwarm.destroy() } catch {} |
| 79 | try { await mirrorSwarm.destroy() } catch {} |
| 80 | try { await originLog.close() } catch {} |
| 81 | try { await mirrorLog.close() } catch {} |
| 82 | try { await originStore.close() } catch {} |
| 83 | try { await mirrorStore.close() } catch {} |
| 84 | }) |
| 85 | |
| 86 | return { originLog, mirrorLog, mirrorStore } |
| 87 | } |
| 88 | |
| 89 | async function waitFor (predicate, timeoutMs = 8000) { |
| 90 | const t0 = Date.now() |
| 91 | while (Date.now() - t0 < timeoutMs) { |
| 92 | if (await predicate()) return true |
| 93 | await new Promise((r) => setTimeout(r, 50)) |
| 94 | } |
| 95 | return false |
| 96 | } |
| 97 | |
| 98 | function readCursor (cursorPath, serverKey) { |
| 99 | if (!existsSync(cursorPath)) return null |
| 100 | try { return JSON.parse(readFileSync(cursorPath, 'utf8'))[serverKey] } catch { return null } |
| 101 | } |
| 102 | |
| 103 | test('mirror: a tombstone below the cursor survives a restart', async (t) => { |
| 104 | t.timeout(60000) |
| 105 | const testnet = await createTestnet(3, t.teardown) |
| 106 | const { originLog, mirrorLog, mirrorStore } = await setupOriginAndMirror(testnet, t) |
| 107 | const serverKey = 'a'.repeat(64) |
| 108 | const cursorPath = join(tmpDir(), 'repo-puller-cursor.json') |
| 109 | |
| 110 | // Phase 1 — origin serves two repos, retires one, then keeps working. |
| 111 | // That last entry matters: the cursor advances only on a successful pull |
| 112 | // (a retire returns early), so without it the cursor would stop AT the |
| 113 | // tombstone and the restart would replay it, hiding the bug. |
| 114 | await originLog.append({ op: OP_CREATED, name: 'alpha' }) // seq 0 |
| 115 | await originLog.append({ op: OP_CREATED, name: 'beta' }) // seq 1 |
| 116 | await originLog.append({ op: OP_RETIRED, name: 'alpha' }) // seq 2 <- tombstone |
| 117 | await originLog.append({ op: OP_CREATED, name: 'gamma' }) // seq 3 <- pushes cursor past it |
| 118 | |
| 119 | const retired1 = new RetiredRepos() |
| 120 | const sync1 = new StubSync() |
| 121 | const puller1 = new RepoPuller({ |
| 122 | sync: sync1, log: mirrorLog, primaryServerKey: serverKey, cursorPath, retired: retired1 |
| 123 | }) |
| 124 | await puller1.start() |
| 125 | |
| 126 | t.ok(await waitFor(() => retired1.has('alpha')), 'phase 1: the live fold sees the tombstone') |
| 127 | t.ok(await waitFor(() => readCursor(cursorPath, serverKey) === 3), |
| 128 | 'phase 1: cursor advanced PAST the tombstone (seq 3)') |
| 129 | await puller1.close() |
| 130 | |
| 131 | // Phase 2 — restart. RetiredRepos is derived state, and on a mirror it is |
| 132 | // folded from the mirror's OWN log, which carries no tombstones. So a real |
| 133 | // boot starts with an empty set, exactly as modelled here. |
| 134 | // A real restart re-opens the core; puller1.close() closed the old session. |
| 135 | const mirrorLog2 = new RepoEventLog({ corestore: mirrorStore, key: mirrorLog.keyHex }) |
| 136 | await mirrorLog2.ready() |
| 137 | const retired2 = new RetiredRepos() |
| 138 | const sync2 = new StubSync() |
| 139 | const puller2 = new RepoPuller({ |
| 140 | sync: sync2, log: mirrorLog2, primaryServerKey: serverKey, cursorPath, retired: retired2 |
| 141 | }) |
| 142 | t.teardown(async () => { try { await puller2.close() } catch {} }) |
| 143 | await puller2.start() |
| 144 | |
| 145 | // A live entry for an unrelated repo proves the read loop is running, so a |
| 146 | // failure below is a missing fold and not a stalled stream. |
| 147 | await originLog.append({ op: OP_UPDATED, name: 'beta' }) // seq 4 |
| 148 | t.ok(await waitFor(() => sync2.calls.some((c) => c.repoName === 'beta')), |
| 149 | 'phase 2: the read loop is live') |
| 150 | |
| 151 | t.ok(retired2.has('alpha'), |
| 152 | 'the tombstone is still known after a restart with a warm cursor') |
| 153 | t.absent(sync2.calls.some((c) => c.repoName === 'alpha'), |
| 154 | 'and the retired repo is not pulled again') |
| 155 | }) |
| 156 | |
| 157 | test('mirror: re-folding must not clobber an un-retire that came later', async (t) => { |
| 158 | // The fix re-reads the log from seq 0 at boot. That must not resurrect a |
| 159 | // tombstone the origin has since reversed — retired-repos.js promises "an |
| 160 | // op 0 after an op 2 un-retires the repo and a mistake is reversible", and |
| 161 | // a naive re-fold ordered after the live stream would break exactly that. |
| 162 | t.timeout(60000) |
| 163 | const testnet = await createTestnet(3, t.teardown) |
| 164 | const { originLog, mirrorLog, mirrorStore } = await setupOriginAndMirror(testnet, t) |
| 165 | const serverKey = 'c'.repeat(64) |
| 166 | const cursorPath = join(tmpDir(), 'repo-puller-cursor.json') |
| 167 | |
| 168 | await originLog.append({ op: OP_CREATED, name: 'alpha' }) // seq 0 |
| 169 | await originLog.append({ op: OP_RETIRED, name: 'alpha' }) // seq 1 |
| 170 | await originLog.append({ op: OP_CREATED, name: 'beta' }) // seq 2 <- cursor lands here |
| 171 | |
| 172 | const puller1 = new RepoPuller({ |
| 173 | sync: new StubSync(), log: mirrorLog, primaryServerKey: serverKey, cursorPath, retired: new RetiredRepos() |
| 174 | }) |
| 175 | await puller1.start() |
| 176 | t.ok(await waitFor(() => readCursor(cursorPath, serverKey) === 2), 'cursor past the tombstone') |
| 177 | await puller1.close() |
| 178 | |
| 179 | // A real restart re-opens the core; puller1.close() closed the old session. |
| 180 | const mirrorLog2 = new RepoEventLog({ corestore: mirrorStore, key: mirrorLog.keyHex }) |
| 181 | await mirrorLog2.ready() |
| 182 | const retired2 = new RetiredRepos() |
| 183 | const sync2 = new StubSync() |
| 184 | const puller2 = new RepoPuller({ |
| 185 | sync: sync2, log: mirrorLog2, primaryServerKey: serverKey, cursorPath, retired: retired2 |
| 186 | }) |
| 187 | t.teardown(async () => { try { await puller2.close() } catch {} }) |
| 188 | await puller2.start() |
| 189 | |
| 190 | // The operator changed their mind on the origin. |
| 191 | await originLog.append({ op: OP_CREATED, name: 'alpha' }) // seq 3 <- un-retire |
| 192 | |
| 193 | t.ok(await waitFor(() => sync2.calls.some((c) => c.repoName === 'alpha')), |
| 194 | 'the un-retired repo is pulled again') |
| 195 | t.absent(retired2.has('alpha'), 'and the tombstone is gone') |
| 196 | }) |