// seq-guard — the policy that stops a relay parking us forever. Run: brittle-node pds/test/seq-guard.test.js
//
// Pins the arithmetic that caused a 20-hour silent federation outage on 2026-07-31: our firehose held 79
// frames while bsky's relay sat at cursor 9834, so the relay could only ever ask for frames we cannot
// produce. Everything local reported health.
import test from 'brittle'
import { decideLift, decideHeal, hostStatusUrl, runSeqGuard } from '../lib/seq-guard.js'
test('decideLift: the real outage numbers', (t) => {
const d = decideLift({ emitted: 79, cursor: 9834 })
t.ok(d.lift, 'emitted 79 vs cursor 9834 → must lift')
t.is(d.to, 9835, 'lifts to cursor + 1 so our next frame lands strictly above it')
t.is(d.reason, 'behind-relay-cursor')
})
test('decideLift: no lift when we are ahead — the healthy steady state', (t) => {
t.absent(decideLift({ emitted: 9900, cursor: 9834 }).lift, 'ahead of the cursor → nothing to do')
})
// REGRESSION (2026-08-05). This asserted the OPPOSITE — "equal is still wedged, the relay asks for
// > cursor" — which reads `emitted` as the NEXT seq. It is the LAST one (FleetRelay.length = seqBase +
// internal), so at equality our next frame is cursor + 1 and the relay does request it. Equality is the
// healthy fully-caught-up state.
test('decideLift: equality is CAUGHT UP, not wedged', (t) => {
const d = decideLift({ emitted: 9834, cursor: 9834 })
t.absent(d.lift, 'emitted === cursor → the relay has read everything we emitted; our next frame is cursor + 1')
t.is(d.reason, 'at-or-ahead-of-cursor')
})
// The off-by-one was not a harmless extra lift: it was a self-sustaining loop. Lifting to cursor + 1 adds
// the aggregate's whole internal length to emitted, so FleetRelay.subscribe's `cursor - seqBase` clamps to
// 0 and the relay re-ingests every frame renumbered; its cursor then climbs to the new emitted and the
// guard fires again. These are the four live lifts observed on stortron in one 4-day process — every one
// of them at exact equality, never below. See hyper-social org issue #25.
test('decideLift: the observed runaway ladder no longer lifts', (t) => {
for (const [emitted, cursor] of [[14713, 14713], [19575, 19575], [24438, 24438], [29302, 29302]]) {
t.absent(decideLift({ emitted, cursor }).lift, `emitted ${emitted} === cursor ${cursor} → caught up, no lift`)
}
})
// The wedge is strictly emitted < cursor — including one short, which is the tightest real case: our next
// frame would be exactly `cursor`, and the relay only ever asks for > cursor.
test('decideLift: one short of the cursor is still wedged', (t) => {
const d = decideLift({ emitted: 9833, cursor: 9834 })
t.ok(d.lift, 'next frame would be 9834, which the relay never asks for')
t.is(d.to, 9835)
})
test('decideLift: refuses to act on nonsense rather than guessing', (t) => {
t.absent(decideLift({ emitted: 10, cursor: 0 }).lift, 'a relay with no cursor for us is not a parked relay')
t.absent(decideLift({ emitted: undefined, cursor: 5 }).lift, 'unknown emitted → no action')
t.absent(decideLift({ emitted: 5, cursor: null }).lift, 'unknown cursor → no action')
})
test('hostStatusUrl encodes the hostname and tolerates a trailing slash', (t) => {
t.is(hostStatusUrl('https://bsky.network/', 'pds.7rnx.net'),
'https://bsky.network/xrpc/com.atproto.sync.getHostStatus?hostname=pds.7rnx.net')
})
// A fake relay with just the surface the guard touches.
const fakeRelay = (length) => ({
length,
_base: 0,
async setSeqBase (n) { if (n > this._base) { this._base = n; this.length = n } ; return this._base }
})
const fakeFetch = (body, ok = true) => async () => ({ ok, status: ok ? 200 : 500, json: async () => body })
test('runSeqGuard lifts and crawls when parked', async (t) => {
const relay = fakeRelay(79)
let crawled = false
const r = await runSeqGuard({
relay,
relayBase: 'https://bsky.network',
hostname: 'pds.7rnx.net',
fetchImpl: fakeFetch({ hostname: 'pds.7rnx.net', seq: 9834, status: 'offline' }),
requestCrawl: async () => { crawled = true }
})
t.ok(r.lifted, 'lifted')
t.is(r.seqBase, 9835, 'floor set above the relay cursor')
t.ok(crawled, 'and asked the relay to crawl — lifting alone leaves it parked')
})
test('runSeqGuard is read-only on a healthy pass', async (t) => {
const relay = fakeRelay(9900)
let crawled = false
const r = await runSeqGuard({
relay,
relayBase: 'https://bsky.network',
hostname: 'pds.7rnx.net',
fetchImpl: fakeFetch({ seq: 9834, status: 'active' }),
requestCrawl: async () => { crawled = true }
})
t.absent(r.lifted, 'nothing lifted')
t.absent(crawled, 'and NO outward call made — a guard must be silent when there is nothing wrong')
t.is(relay._base, 0, 'the floor is untouched')
})
test('runSeqGuard never throws — an unreachable relay is not our emergency', async (t) => {
const r = await runSeqGuard({
relay: fakeRelay(79),
relayBase: 'https://bsky.network',
hostname: 'pds.7rnx.net',
fetchImpl: async () => { throw new Error('ECONNREFUSED') }
})
t.absent(r.ok, 'reports failure')
t.ok(r.error, 'with the reason, rather than taking the membrane down')
})
test('runSeqGuard stays inert when unconfigured — outward-facing means opt-in', async (t) => {
const r = await runSeqGuard({ relay: fakeRelay(79) })
t.is(r.skipped, 'not-configured')
})
// ── self-heal: a relay that marked us OFFLINE while we are caught up (2026-10-13) ───────────────────
//
// Measured live: getHostStatus said status "offline", seq 40262, while we were caught up — the guard's
// arithmetic was healthy, so it did nothing, and bsky's relay would never have reconnected on its own
// (reference_bsky-relay-offline-requestcrawl). One requestCrawl brought it to "active" in under 10 s.
// So: offline + not behind → crawl, at most once per interval.
test('decideHeal: offline while caught up → crawl', (t) => {
const d = decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: 0, now: 1e9 })
t.ok(d.heal, 'offline and nothing else will reconnect it')
})
test('decideHeal: active → nothing', (t) => {
t.absent(decideHeal({ relayStatus: 'active', lifted: false, lastHealAt: 0, now: 1e9 }).heal)
})
test('decideHeal: a lift already crawled → no second crawl in the same pass', (t) => {
t.absent(decideHeal({ relayStatus: 'offline', lifted: true, lastHealAt: 0, now: 1e9 }).heal)
})
test('decideHeal: at most once per interval — a relay that stays offline is not hammered', (t) => {
const now = 1e9
t.absent(decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: now - 5 * 60e3, now }).heal, '5 min after the last heal: wait')
t.ok(decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: now - 16 * 60e3, now }).heal, '16 min after: crawl again')
})
test('runSeqGuard heals an offline-but-caught-up relay with ONE crawl, and reports it', async (t) => {
const relay = fakeRelay(40300)
let crawls = 0
const r = await runSeqGuard({
relay,
relayBase: 'https://bsky.network',
hostname: 'pds.7rnx.net',
fetchImpl: fakeFetch({ hostname: 'pds.7rnx.net', seq: 40262, status: 'offline' }),
requestCrawl: async () => { crawls++ },
lastHealAt: 0,
now: 1e9
})
t.absent(r.lifted, 'caught up: nothing to lift')
t.is(crawls, 1, 'one requestCrawl')
t.ok(r.healed, 'reported as a heal')
t.is(r.healedAt, 1e9, 'with the time, so the next pass can rate-limit')
t.is(r.relayStatus, 'offline', 'and what it saw')
t.is(relay._base, 0, 'the floor is untouched')
})
test('runSeqGuard does not heal without a configured crawl (outward-facing stays opt-in)', async (t) => {
const r = await runSeqGuard({
relay: fakeRelay(40300),
relayBase: 'https://bsky.network',
hostname: 'pds.7rnx.net',
fetchImpl: fakeFetch({ seq: 40262, status: 'offline' })
})
t.absent(r.healed)
})