OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-bridge commits / b3fa49d

0.9.51: bounded, timed shutdown that exits 75 on OTA
The 0.9.50 OTA took about 50 s to exit while pds.7rnx.net answered 503, and it exited 0: shutdown() called process.exit(0) itself, so release-ready's exit 75 never ran. The supervisor treated that as a crash, applying backoff instead of an immediate OTA respawn. Nothing recorded which step was slow. pds/lib/shutdown.js runs the same steps in the same order, with three changes: - the caller's exit code wins - every step logs its duration - a 15 s deadline exits anyway and names the step it was stuck on A test measures that an in-flight bodyless GET does not hold server.close (bare-http1 marks it idle once the request stream closes), so the holder was elsewhere. Verified on the real entrypoint with a throwaway store: - release-ready over the control socket: exit 75 in 806 ms - SIGTERM: exit 0 in 798 ms - swarm.destroy was the slow step, at 714-720 ms Bare suite 14/14, Node suite 73/73. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
b3fa49d92fc49796189ad7baa593e4d14d890f3c
by plankatron <fleeky@gmail.com> · 2026-10-13T01:03:42.000Z

Changed files (4) +204 -17

package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "atproto-bridge",
3 "version": "0.9.50",
3 "version": "0.9.51",
44 "description": "Stateless keyless atproto MEMBRANE \u2014 fronts self-custody hyper-proto-peers to bluesky over HTTP/WS, holds no keys and never signs. Bare-runnable. Deploy root / flat closure over the identity+repo+store+pds packages.",
55 "type": "module",
66 "main": "index.js",
pds/bin/pds-serve.js +18 -16
……@@ -17,6 +17,7 @@ import { openCorestore } from 'atproto-store/lib/corestore.js'
1717import { loadConfig } from '../lib/config.js'
1818import { loadOrCreateSwarmKeyPair } from '../lib/swarm-keypair.js'
1919import { startControl } from '../lib/control.js'
20import { createShutdown } from '../lib/shutdown.js'
2021import { FleetRelay } from '../lib/fleet-relay.js'
2122import { DidPeerRouter } from 'atproto-did-peer'
2223import { resolvePlcVerifyKey } from '../lib/verify-key.js'
……@@ -159,7 +160,7 @@ if (control && control.svc && !control.svc.hasHandler('release-ready')) {
159160 control.svc.handle('release-ready', async (ctx, params = {}) => {
160161 const ver = (params && params.version) || '?'
161162 console.log(JSON.stringify({ msg: 'ota: release-ready — sidecar installed; exiting 75 for supervisor respawn', version: ver }))
162 setImmediate(() => { shutdown().catch(() => {}).then(() => process.exit(75)).catch(() => process.exit(75)) })
163 setImmediate(() => { shutdown(75) })
163164 return { ok: true, willRespawn: true }
164165 })
165166 console.log(JSON.stringify({ msg: 'ota: release-ready handler armed (push-based; sidecar installs, we respawn)' }))
……@@ -257,18 +258,19 @@ console.log(JSON.stringify({
257258 feedRegistries: cfg.feedRegistries
258259}))
259260
260let shuttingDown = false
261async function shutdown () {
262 if (shuttingDown) return
263 shuttingDown = true
264 try { await control.close() } catch {}
265 try { await peers.close() } catch {}
266 try { await relay.close() } catch {}
267 try { await swarm.destroy() } catch {}
268 try { await new Promise((r) => server.close(r)) } catch {}
269 // Release the front-store Corestore fd-lock last so a redeploy can reopen.
270 try { await frontStore.close() } catch {}
271 process.exit(0)
272}
273process.on('SIGINT', shutdown)
274process.on('SIGTERM', shutdown)
261// Bounded and timed (lib/shutdown.js): the caller's exit code wins, every step logs its duration, and
262// a deadline exits anyway, naming the step it was stuck on. 0.9.50's OTA took ~50 s and exited 0.
263const shutdown = createShutdown({
264 exit: (code) => process.exit(code),
265 steps: [
266 ['control', () => control.close()],
267 ['peers', () => peers.close()],
268 ['relay', () => relay.close()],
269 ['swarm', () => swarm.destroy()],
270 ['server', () => new Promise((resolve) => server.close(resolve))],
271 // Release the front-store Corestore fd-lock last so a redeploy can reopen.
272 ['frontStore', () => frontStore.close()]
273 ]
274})
275process.on('SIGINT', () => shutdown(0))
276process.on('SIGTERM', () => shutdown(0))
pds/lib/shutdown.js +63 -0
……@@ -0,0 +1,63 @@
1// createShutdown — the membrane's stop sequence, as a testable unit.
2//
3// ⛔ THE INCIDENT (0.9.50 OTA, 2026-10-12). release-ready logged "exiting 75"; the process then took
4// ~50 s to exit while pds.7rnx.net answered 503, and it exited 0. Two defects:
5// 1. shutdown() called process.exit(0) itself, so release-ready's `.then(exit 75)` never ran. The
6// supervisor happened to respawn on 0 too, which is the only reason the OTA landed at all.
7// 2. Nothing bounded or timed the steps, so a slow one stretched the outage and left no record of
8// which it was. (Not server.close: test/bare/shutdown.test.js measures that an in-flight GET does
9// not hold it.)
10// So: the caller names the exit code, every step is timed, and a deadline exits anyway and names the
11// step it was stuck on. The next slow OTA then says where the time went.
12
13export const DEFAULT_SHUTDOWN_DEADLINE_MS = 15000
14
15/**
16 * @param {object} opts
17 * @param {Array<[string, Function]>} opts.steps run in order; a throw is logged and skipped past
18 * @param {Function} opts.exit called exactly once, with the first caller's code
19 * @param {number} [opts.deadlineMs] overall budget; past it, exit is called with the steps unfinished
20 * @param {Function} [opts.log] receives one object per line
21 * @returns {(code: number) => Promise<void>}
22 */
23export function createShutdown ({ steps, exit, deadlineMs = DEFAULT_SHUTDOWN_DEADLINE_MS, log = defaultLog }) {
24 let started = null
25
26 return function shutdown (code = 0) {
27 // A second caller (SIGTERM arriving mid-OTA) joins the first: the steps run once, its code wins.
28 if (started) return started
29 started = run(code)
30 return started
31 }
32
33 async function run (code) {
34 const t0 = Date.now()
35 let current = null
36 let exited = false
37 const finish = () => { if (exited) return; exited = true; clearTimeout(timer); exit(code) }
38
39 // NOT unref'd: an unref'd deadline is skipped exactly when nothing else holds the loop open.
40 const timer = setTimeout(() => {
41 log({ msg: 'shutdown: deadline reached, exiting with steps unfinished', stuck: current, deadlineMs, code })
42 finish()
43 }, deadlineMs)
44
45 for (const [name, fn] of steps) {
46 if (exited) return
47 current = name
48 const s = Date.now()
49 let error = null
50 try { await fn() } catch (err) { error = err && err.message ? err.message : String(err) }
51 const line = { msg: 'shutdown: step', step: name, ms: Date.now() - s }
52 if (error) line.error = error
53 log(line)
54 }
55 current = null
56 log({ msg: 'shutdown: done', ms: Date.now() - t0, code })
57 finish()
58 }
59}
60
61function defaultLog (o) { console.log(JSON.stringify(o)) }
62
63export default createShutdown
pds/test/bare/shutdown.test.js +122 -0
……@@ -0,0 +1,122 @@
1// The membrane's shutdown must END, so an OTA respawn always happens.
2//
3// ⛔ THE INCIDENT (0.9.50 OTA, 2026-10-12). release-ready logged "exiting 75", then the process took
4// ~50 s to exit while pds.7rnx.net answered 503, and it exited 0, not 75. The exit code was wrong
5// because shutdown() called process.exit(0) itself before release-ready's `.then(exit 75)` could run.
6// Which await was slow was never recorded.
7//
8// So the sequence must (1) exit with the code its caller asked for, (2) time every step, and (3) give
9// up at a deadline, naming the step it was stuck on, so the next slow OTA says where the time went.
10
11import '../../lib/bare-globals.js' // FIRST: cborg touches TextEncoder at module load
12import test from 'brittle'
13import tcp from 'bare-tcp'
14import { createPdsServer } from '../../lib/server-bare.js'
15import { createShutdown } from '../../lib/shutdown.js'
16
17const never = () => new Promise(() => {})
18
19// Records exit() instead of exiting, and resolves when it is called.
20function fakeExit () {
21 let resolve = null
22 const exited = new Promise((r) => { resolve = r })
23 const exit = (code) => { exit.code = code; resolve(code) }
24 exit.code = null
25 exit.exited = exited
26 return exit
27}
28
29const DID = 'did:plc:shutdownhold'
30
31// A fronted DID whose front never resolves: every request for it stays in flight.
32function hangingPeers () {
33 let called = null
34 const entered = new Promise((resolve) => { called = resolve })
35 return {
36 entered,
37 isFronted: (did) => did === DID,
38 resolveHandle: () => null,
39 front: () => { called(); return new Promise(() => {}) }
40 }
41}
42
43async function listening (peers) {
44 const server = createPdsServer({ peers })
45 await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
46 return server
47}
48
49// A raw request, so no client-side agent or keep-alive policy decides anything.
50function holdRequest (port) {
51 const socket = tcp.createConnection(port, '127.0.0.1')
52 socket.on('error', () => {})
53 socket.write(`GET /xrpc/com.atproto.sync.getLatestCommit?did=${DID} HTTP/1.1\r\nHost: x\r\n\r\n`)
54 return socket
55}
56
57const within = (p, ms) => Promise.race([p.then(() => true), new Promise((resolve) => setTimeout(() => resolve(false), ms))])
58
59test('⛔ an OTA shutdown exits 75, not 0', async (t) => {
60 const exit = fakeExit()
61 const shutdown = createShutdown({ steps: [['a', async () => {}]], exit, log: () => {} })
62 await shutdown(75)
63 t.is(await exit.exited, 75, 'the code the caller asked for — 0.9.50 exited 0 on release-ready')
64})
65
66test('⛔ a step that never settles cannot hold the exit past the deadline', async (t) => {
67 const exit = fakeExit()
68 const lines = []
69 const shutdown = createShutdown({
70 steps: [['control', async () => {}], ['swarm', never], ['server', async () => {}]],
71 deadlineMs: 300,
72 exit,
73 log: (o) => lines.push(o)
74 })
75 shutdown(75)
76 t.ok(await within(exit.exited, 2000), 'exit is called although a step is still pending')
77 t.is(exit.code, 75, 'with the requested code')
78 t.ok(lines.some((o) => o.stuck === 'swarm'), 'and the log names the step it was stuck on')
79})
80
81test('every step is timed, and a throwing step does not stop the ones after it', async (t) => {
82 const exit = fakeExit()
83 const ran = []
84 const lines = []
85 const shutdown = createShutdown({
86 steps: [['a', async () => { ran.push('a'); throw new Error('boom') }], ['b', async () => { ran.push('b') }]],
87 exit,
88 log: (o) => lines.push(o)
89 })
90 await shutdown(0)
91 await exit.exited
92 t.alike(ran, ['a', 'b'], 'b still runs after a throws')
93 const steps = lines.filter((o) => o.step).map((o) => o.step)
94 t.alike(steps, ['a', 'b'], 'one timing line per step')
95 t.ok(lines.every((o) => !o.step || Number.isFinite(o.ms)), 'each with its duration in ms')
96})
97
98test('a second call (SIGTERM during an OTA) does not run the steps again', async (t) => {
99 const exit = fakeExit()
100 let runs = 0
101 const shutdown = createShutdown({ steps: [['a', async () => { runs++ }]], exit, log: () => {} })
102 await Promise.all([shutdown(75), shutdown(0)])
103 await exit.exited
104 t.is(runs, 1, 'steps run once')
105 t.is(exit.code, 75, 'and the first caller\'s code wins')
106})
107
108// MEASURED, and kept as a guard on the reading above: bare-http1 4.5.7 marks a connection idle as soon
109// as its REQUEST stream closes (server-connection.js `_req.on('close')`), which for a bodyless GET is
110// immediate, so close() destroys it even while the handler is still awaiting. An in-flight GET is not
111// what held 0.9.50's exit.
112test('server.close() is not held by an in-flight bodyless GET', async (t) => {
113 const peers = hangingPeers()
114 const server = await listening(peers)
115 const client = holdRequest(server.address().port)
116 await peers.entered
117
118 const closed = within(new Promise((resolve) => server.close(resolve)), 1500)
119 t.ok(await closed, 'server.close() completes with a request in flight')
120
121 client.destroy()
122})