OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-bridge commits / 69ff03c

0.9.53: the relay guard heals "offline while caught up", and remembers what it saw
Measured 2026-10-13: bsky's relay had pds.7rnx.net "offline" at seq 40262 while we were caught up. decideLift (rightly) saw nothing to lift, so the guard did nothing, and the relay never reconnects on its own. One requestCrawl -> "active" in under 10 s. Every OTA restart can cause it. - seq-guard: decideHeal({ relayStatus, lifted, lastHealAt, now }) — crawl when the relay says offline, the lift branch did not already crawl, and the last heal is older than 15 min. runSeqGuard returns healed/healedAt (a failed crawl still counts, so it retries at the interval, not every pass) and logs it loudly. Still opt-in: no requestCrawl configured, no heal. - membrane: carries lastHealAt across passes; keeps the last pass result and heal count behind guard() for metrics; adds a pass 2 min after boot (the relay marks us offline only after its reconnects fail — after the boot pass saw "active"); the stop step clears both timers. Tests first: 6 new seq-guard tests red on the missing export; suites Bare 17/17, Node 79/79. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69ff03cc1d0515192555d07370e7bf0c8a4638a1
by plankatron <fleeky@gmail.com> · 2026-10-13T22:33:05.000Z

Changed files (4) +114 -6

package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "atproto-bridge",
3 "version": "0.9.52",
3 "version": "0.9.53",
44 "description": "Stateless keyless atproto MEMBRANE \u2014 fronts self-custody hyper-proto-peers to bluesky over HTTP/WS, holds no keys and never signs. Bare-runnable. Deploy root / flat closure over the identity+repo+store+pds packages.",
55 "type": "module",
66 "main": "index.js",
pds/lib/membrane.js +19 -2
……@@ -178,6 +178,9 @@ export async function startMembrane (cfg, { log = defaultLog, bootstrap = null }
178178 // ⛔ The timer is CLEARED by a stop step: `unref` alone let it outlive a stopped membrane, and a
179179 // restart in the same process would then run two guards.
180180 let seqGuardTimer = null
181 let seqGuardSoon = null
182 // What the last pass saw and did, for `metrics`; and the last self-heal, for its rate limit.
183 const guard = { last: null, lastAt: null, lastHealAt: 0, heals: 0, configured: !!(cfg.crawlRelay && cfg.crawlHostname) }
181184 if (cfg.crawlRelay && cfg.crawlHostname) {
182185 const { runSeqGuard } = await import('./seq-guard.js')
183186 const crawl = async () => {
……@@ -191,9 +194,21 @@ export async function startMembrane (cfg, { log = defaultLog, bootstrap = null }
191194 relayBase: cfg.crawlRelay,
192195 hostname: cfg.crawlHostname,
193196 requestCrawl: crawl,
194 log
197 log,
198 lastHealAt: guard.lastHealAt,
199 now: Date.now()
200 }).then((r) => {
201 guard.last = r
202 guard.lastAt = Date.now()
203 if (r && r.healedAt) guard.lastHealAt = r.healedAt
204 if (r && r.healed) guard.heals++
195205 }).catch((e) => log({ msg: 'seq-guard pass failed', error: e && e.message }))
196206 pass() // boot pass, backgrounded — must never delay listen or the readiness probe
207 // ⛔ A SECOND PASS SHORTLY AFTER BOOT. A restart drops the relay's subscribeRepos connection, and the
208 // relay only marks us offline after its reconnects fail — after the boot pass has already seen
209 // "active". Without this an OTA could leave bsky disconnected for a full interval.
210 seqGuardSoon = setTimeout(pass, 2 * 60 * 1000)
211 if (seqGuardSoon.unref) seqGuardSoon.unref()
197212 seqGuardTimer = setInterval(pass, 15 * 60 * 1000)
198213 if (seqGuardTimer.unref) seqGuardTimer.unref()
199214 } else {
……@@ -256,10 +271,12 @@ export async function startMembrane (cfg, { log = defaultLog, bootstrap = null }
256271 relay,
257272 handlers: membraneHandlers({ peers }),
258273 status: () => ({ ok: true, membrane: true, port, fronted: peers.list() }),
274 // The relay guard's last pass and heals, for `metrics`. Read-only.
275 guard: () => ({ ...guard }),
259276 // The stop sequence, unchanged in order from pds-serve.js (control is the caller's, so it is not
260277 // here), plus the seq-guard timer.
261278 steps: [
262 ['seq-guard', () => { if (seqGuardTimer) clearInterval(seqGuardTimer) }],
279 ['seq-guard', () => { if (seqGuardTimer) clearInterval(seqGuardTimer); if (seqGuardSoon) clearTimeout(seqGuardSoon) }],
263280 ['peers', () => peers.close()],
264281 ['relay', () => relay.close()],
265282 ['swarm', () => swarm.destroy()],
pds/lib/seq-guard.js +37 -2
……@@ -57,6 +57,23 @@ export function decideLift ({ emitted, cursor, margin = 1 } = {}) {
5757 return { lift: true, to: c + Math.max(1, Number(margin) || 1), reason: 'behind-relay-cursor', emitted: e, cursor: c }
5858}
5959
60// ⛔ SELF-HEAL (2026-10-13). The arithmetic above is the condition for a LIFT, and it is right not to key
61// on `offline`. But "offline while caught up" is a second, separate failure: measured live, getHostStatus
62// said status "offline", seq 40262, our emitted seq at or above it — so decideLift said healthy and the
63// guard did nothing — and bsky's relay NEVER reconnects on its own once it has marked a host offline
64// (reference_bsky-relay-offline-requestcrawl). One requestCrawl returned it to "active" in under 10 s.
65// Every OTA restart drops the relay's subscribeRepos connection and can cause it.
66//
67// So: crawl when the relay says offline and the lift branch did not already crawl, at most once per
68// `minIntervalMs`, so a relay that stays offline for its own reasons is asked, not hammered.
69export const HEAL_INTERVAL_MS = 15 * 60 * 1000
70export function decideHeal ({ relayStatus, lifted = false, lastHealAt = 0, now = Date.now(), minIntervalMs = HEAL_INTERVAL_MS } = {}) {
71 if (relayStatus !== 'offline') return { heal: false, reason: 'relay-not-offline' }
72 if (lifted) return { heal: false, reason: 'lift-already-crawled' }
73 if (lastHealAt && now - lastHealAt < minIntervalMs) return { heal: false, reason: 'healed-recently' }
74 return { heal: true, reason: 'offline-while-caught-up' }
75}
76
6077export function hostStatusUrl (relayBase, hostname) {
6178 return `${String(relayBase).replace(/\/$/, '')}/xrpc/com.atproto.sync.getHostStatus?hostname=${encodeURIComponent(hostname)}`
6279}
……@@ -70,7 +87,10 @@ export async function runSeqGuard ({
7087 fetchImpl = null,
7188 margin = 1,
7289 requestCrawl = null,
73 log = null
90 log = null,
91 // Self-heal rate limit: the caller keeps the last heal time across passes and hands it back in.
92 lastHealAt = 0,
93 now = Date.now()
7494} = {}) {
7595 const emit = (o) => { if (log) { try { log(o) } catch {} } return o }
7696 if (!relay || !relayBase || !hostname) return { ok: false, skipped: 'not-configured' }
……@@ -89,7 +109,22 @@ export async function runSeqGuard ({
89109
90110 const emitted = relay.length
91111 const d = decideLift({ emitted, cursor: status && status.seq, margin })
92 if (!d.lift) return { ok: true, lifted: false, emitted, cursor: status && status.seq, relayStatus: status && status.status, reason: d.reason }
112 if (!d.lift) {
113 const out = { ok: true, lifted: false, emitted, cursor: status && status.seq, relayStatus: status && status.status, reason: d.reason }
114 const h = requestCrawl ? decideHeal({ relayStatus: out.relayStatus, lifted: false, lastHealAt, now }) : { heal: false }
115 if (!h.heal) return out
116 // LOUD, same reason as the lift: this repairs a state no local surface can see.
117 emit({ msg: 'seq-guard: relay has us OFFLINE while we are caught up — asking it to crawl', hostname, relayCursor: out.cursor, ourEmitted: emitted })
118 try {
119 await requestCrawl()
120 emit({ msg: 'seq-guard: requestCrawl sent (self-heal)', hostname })
121 return { ...out, healed: true, healedAt: now }
122 } catch (e) {
123 emit({ msg: 'seq-guard: self-heal requestCrawl failed', error: e && e.message })
124 // Counted as an attempt, so a failing crawl is retried at the interval rather than every pass.
125 return { ...out, healed: false, healError: (e && e.message) || String(e), healedAt: now }
126 }
127 }
93128
94129 // LOUD: this is a real repair of a state nothing else reports. The 20-hour outage was invisible
95130 // precisely because no line like this existed.
pds/test/seq-guard.test.js +57 -1
……@@ -4,7 +4,7 @@
44// frames while bsky's relay sat at cursor 9834, so the relay could only ever ask for frames we cannot
55// produce. Everything local reported health.
66import test from 'brittle'
7import { decideLift, hostStatusUrl, runSeqGuard } from '../lib/seq-guard.js'
7import { decideLift, decideHeal, hostStatusUrl, runSeqGuard } from '../lib/seq-guard.js'
88
99test('decideLift: the real outage numbers', (t) => {
1010 const d = decideLift({ emitted: 79, cursor: 9834 })
……@@ -110,3 +110,59 @@ test('runSeqGuard stays inert when unconfigured — outward-facing means opt-in'
110110 const r = await runSeqGuard({ relay: fakeRelay(79) })
111111 t.is(r.skipped, 'not-configured')
112112})
113
114// ── self-heal: a relay that marked us OFFLINE while we are caught up (2026-10-13) ───────────────────
115//
116// Measured live: getHostStatus said status "offline", seq 40262, while we were caught up — the guard's
117// arithmetic was healthy, so it did nothing, and bsky's relay would never have reconnected on its own
118// (reference_bsky-relay-offline-requestcrawl). One requestCrawl brought it to "active" in under 10 s.
119// So: offline + not behind → crawl, at most once per interval.
120
121test('decideHeal: offline while caught up → crawl', (t) => {
122 const d = decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: 0, now: 1e9 })
123 t.ok(d.heal, 'offline and nothing else will reconnect it')
124})
125
126test('decideHeal: active → nothing', (t) => {
127 t.absent(decideHeal({ relayStatus: 'active', lifted: false, lastHealAt: 0, now: 1e9 }).heal)
128})
129
130test('decideHeal: a lift already crawled → no second crawl in the same pass', (t) => {
131 t.absent(decideHeal({ relayStatus: 'offline', lifted: true, lastHealAt: 0, now: 1e9 }).heal)
132})
133
134test('decideHeal: at most once per interval — a relay that stays offline is not hammered', (t) => {
135 const now = 1e9
136 t.absent(decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: now - 5 * 60e3, now }).heal, '5 min after the last heal: wait')
137 t.ok(decideHeal({ relayStatus: 'offline', lifted: false, lastHealAt: now - 16 * 60e3, now }).heal, '16 min after: crawl again')
138})
139
140test('runSeqGuard heals an offline-but-caught-up relay with ONE crawl, and reports it', async (t) => {
141 const relay = fakeRelay(40300)
142 let crawls = 0
143 const r = await runSeqGuard({
144 relay,
145 relayBase: 'https://bsky.network',
146 hostname: 'pds.7rnx.net',
147 fetchImpl: fakeFetch({ hostname: 'pds.7rnx.net', seq: 40262, status: 'offline' }),
148 requestCrawl: async () => { crawls++ },
149 lastHealAt: 0,
150 now: 1e9
151 })
152 t.absent(r.lifted, 'caught up: nothing to lift')
153 t.is(crawls, 1, 'one requestCrawl')
154 t.ok(r.healed, 'reported as a heal')
155 t.is(r.healedAt, 1e9, 'with the time, so the next pass can rate-limit')
156 t.is(r.relayStatus, 'offline', 'and what it saw')
157 t.is(relay._base, 0, 'the floor is untouched')
158})
159
160test('runSeqGuard does not heal without a configured crawl (outward-facing stays opt-in)', async (t) => {
161 const r = await runSeqGuard({
162 relay: fakeRelay(40300),
163 relayBase: 'https://bsky.network',
164 hostname: 'pds.7rnx.net',
165 fetchImpl: fakeFetch({ seq: 40262, status: 'offline' })
166 })
167 t.absent(r.healed)
168})