// seq-guard — detect that a downstream relay has parked us, and lift our emitted-seq floor past it.
//
// THE FAILURE. Our emitted `seq` is a Hypercore length (FleetSequencer/FirehoseSequencer): the block
// index IS the seq. A storage rebuild resets that core to 0, so our emitted sequence restarts near zero.
// atproto relays (indigo) keep a STRICTLY FORWARD per-host cursor and never roll it back: once our seq
// is below their stored cursor they treat us as a future-cursor, park the host, and never read again.
//
// Measured 2026-07-31 on pds.7rnx.net: our firehose held 79 frames while bsky's relay sat at cursor 9834
// with status "offline". Federation had been dead for ~20 hours. NOTHING on our side reported a problem —
// the membrane was healthy, the peers were healthy, the repo signed correctly. The only symptom was that
// the outside world stopped seeing us, which no local surface can observe.
//
// THE MECHANISM TO FIX IT ALREADY EXISTED and was never invoked: FleetRelay.setSeqBase(n) raises the
// emitted floor monotonically and persists it (pds-serve's `relay-seq-base` file, kept BESIDE the
// front-store so it survives a corestore reset). control.js exposes it as `set-seq-base`. What was
// missing is the part that NOTICES — recovery required a human to know to read getHostStatus and hand-call
// a UDS RPC. So it stayed parked until someone went looking.
//
// This closes that: read the relay's own view of us, and if it can never advance, lift the floor above its
// cursor and ask it to crawl. Read-only on every healthy pass.
//
// ⚠ OUTWARD-FACING and therefore OPT-IN. It talks to bsky's relay, so it stays inert unless crawlRelay +
// hostname are configured — the same gating the feed-registry reactor uses for requestCrawl.
// Pure decision, so the policy is testable without a network or a relay.
//
// emitted — our current emitted seq (relay.length = seqBase + internal)
// cursor — the relay's stored cursor for our host (getHostStatus.seq)
//
// Lift when `emitted < cursor`: the relay will only ever ask for frames after `cursor`, and our next
// frame can only be `emitted + 1`, so if that still lands at or below `cursor` we are wedged by
// arithmetic regardless of what we sign. Lifting to `cursor + margin` makes our next emitted frame land
// strictly above its cursor.
//
// ⛔ EQUALITY IS NOT WEDGED — corrected 2026-08-05 from live evidence. This originally lifted on
// `emitted <= cursor`, justified as "the relay asks for seq > cursor, so emitted === cursor can never
// advance it". That reads `emitted` as the NEXT seq; it is the LAST one (FleetRelay.length = seqBase +
// internal). At equality our next frame is `cursor + 1`, which the relay does request — equality is the
// healthy FULLY-CAUGHT-UP state, the terminal state of a working firehose.
//
// The cost of the off-by-one was a self-sustaining loop, not a no-op. setSeqBase(cursor + 1) adds the
// whole internal length to emitted, so FleetRelay.subscribe's `cursor - seqBase` goes negative and clamps
// to 0 — the relay re-ingests every frame renumbered, its cursor climbs to the new emitted, and the guard
// fires again. Measured on stortron over one 4-day process: four lifts, all with ourEmitted EXACTLY equal
// to relayCursor, never below; seqBase ladder 9855 -> 14714 -> 19576 -> 24439 -> 29303, each step ~4860 =
// the aggregate's internal length. See hyper-social org issue #25.
//
// Deliberately NOT keyed on status === 'offline'. A relay can report active while still holding a cursor
// we can never reach (it reconnects, asks for >cursor, gets nothing, idles). The arithmetic is the real
// condition; the status is a symptom of it.
export function decideLift ({ emitted, cursor, margin = 1 } = {}) {
const e = Number(emitted)
const c = Number(cursor)
if (!Number.isFinite(e) || !Number.isFinite(c)) return { lift: false, reason: 'unknown-seq' }
if (c <= 0) return { lift: false, reason: 'relay-has-no-cursor' }
if (e >= c) return { lift: false, reason: 'at-or-ahead-of-cursor' }
return { lift: true, to: c + Math.max(1, Number(margin) || 1), reason: 'behind-relay-cursor', emitted: e, cursor: c }
}
// ⛔ SELF-HEAL (2026-10-13). The arithmetic above is the condition for a LIFT, and it is right not to key
// on `offline`. But "offline while caught up" is a second, separate failure: measured live, getHostStatus
// said status "offline", seq 40262, our emitted seq at or above it — so decideLift said healthy and the
// guard did nothing — and bsky's relay NEVER reconnects on its own once it has marked a host offline
// (reference_bsky-relay-offline-requestcrawl). One requestCrawl returned it to "active" in under 10 s.
// Every OTA restart drops the relay's subscribeRepos connection and can cause it.
//
// So: crawl when the relay says offline and the lift branch did not already crawl, at most once per
// `minIntervalMs`, so a relay that stays offline for its own reasons is asked, not hammered.
export const HEAL_INTERVAL_MS = 15 * 60 * 1000
export function decideHeal ({ relayStatus, lifted = false, lastHealAt = 0, now = Date.now(), minIntervalMs = HEAL_INTERVAL_MS } = {}) {
if (relayStatus !== 'offline') return { heal: false, reason: 'relay-not-offline' }
if (lifted) return { heal: false, reason: 'lift-already-crawled' }
if (lastHealAt && now - lastHealAt < minIntervalMs) return { heal: false, reason: 'healed-recently' }
return { heal: true, reason: 'offline-while-caught-up' }
}
export function hostStatusUrl (relayBase, hostname) {
return `${String(relayBase).replace(/\/$/, '')}/xrpc/com.atproto.sync.getHostStatus?hostname=${encodeURIComponent(hostname)}`
}
// One pass. Returns a result object describing what it saw and did — never throws: a guard that can take
// the membrane down is worse than the parked state it repairs.
export async function runSeqGuard ({
relay,
relayBase,
hostname,
fetchImpl = null,
margin = 1,
requestCrawl = null,
log = null,
// Self-heal rate limit: the caller keeps the last heal time across passes and hands it back in.
lastHealAt = 0,
now = Date.now()
} = {}) {
const emit = (o) => { if (log) { try { log(o) } catch {} } return o }
if (!relay || !relayBase || !hostname) return { ok: false, skipped: 'not-configured' }
const doFetch = fetchImpl || (typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : null)
if (!doFetch) return { ok: false, skipped: 'no-fetch' }
let status
try {
const res = await doFetch(hostStatusUrl(relayBase, hostname))
if (!res || !res.ok) return { ok: false, error: 'getHostStatus ' + (res && res.status) }
status = await res.json()
} catch (e) {
// The relay being unreachable is not our emergency — try again next pass.
return { ok: false, error: (e && e.message) || 'fetch failed' }
}
const emitted = relay.length
const d = decideLift({ emitted, cursor: status && status.seq, margin })
if (!d.lift) {
const out = { ok: true, lifted: false, emitted, cursor: status && status.seq, relayStatus: status && status.status, reason: d.reason }
const h = requestCrawl ? decideHeal({ relayStatus: out.relayStatus, lifted: false, lastHealAt, now }) : { heal: false }
if (!h.heal) return out
// LOUD, same reason as the lift: this repairs a state no local surface can see.
emit({ msg: 'seq-guard: relay has us OFFLINE while we are caught up — asking it to crawl', hostname, relayCursor: out.cursor, ourEmitted: emitted })
try {
await requestCrawl()
emit({ msg: 'seq-guard: requestCrawl sent (self-heal)', hostname })
return { ...out, healed: true, healedAt: now }
} catch (e) {
emit({ msg: 'seq-guard: self-heal requestCrawl failed', error: e && e.message })
// Counted as an attempt, so a failing crawl is retried at the interval rather than every pass.
return { ...out, healed: false, healError: (e && e.message) || String(e), healedAt: now }
}
}
// LOUD: this is a real repair of a state nothing else reports. The 20-hour outage was invisible
// precisely because no line like this existed.
emit({
msg: 'seq-guard: relay cursor is AHEAD of our emitted seq — we can never satisfy it; lifting the floor',
hostname,
relayStatus: status.status,
relayCursor: status.seq,
ourEmitted: emitted,
liftingTo: d.to
})
const seqBase = await relay.setSeqBase(d.to)
let crawled = null
if (requestCrawl) {
try { await requestCrawl(); crawled = true } catch (e) { crawled = false; emit({ msg: 'seq-guard: requestCrawl failed', error: e && e.message }) }
}
emit({ msg: 'seq-guard: floor lifted', seqBase, emitted: relay.length, crawled })
return { ok: true, lifted: true, seqBase, emitted: relay.length, cursor: status.seq, crawled }
}