| … | … | @@ -9,7 +9,16 @@ |
| 9 | 9 | // the next increment — the render layer already takes a `viewer`. |
| 10 | 10 | import '../lib/bare-globals.js' |
| 11 | 11 | import http from 'bare-http1' |
| 12 | import bareFetch from 'bare-fetch' | |
| 12 | // Every outbound HTTP call in this daemon goes through the BOUNDED fetch, never bare-fetch directly. | |
| 13 | // A raw bare-fetch to an HTTPS host that accepts TCP and then goes silent holds its socket for ever | |
| 14 | // (bare-tls does not forward 'timeout' up to the wrapper bare-https puts around the socket, so the | |
| 15 | // agent's own reaper cannot fire). That is what put 47,191 sockets and 24.3 GB into this process and | |
| 16 | // OOM-killed stortron twice, on 2026-08-27 and 2026-08-29. See lib/bounded-fetch.js for the full | |
| 17 | // mechanism and test/bounded-fetch.test.mjs for the drill that proves both the leak and the fix. | |
| 18 | import { createBoundedFetch } from '../lib/bounded-fetch.js' | |
| 19 | const bareFetch = createBoundedFetch({ | |
| 20 | onTimeout: ({ url, ms }) => console.error(JSON.stringify({ msg: 'fetch deadline exceeded — socket reaped', url, ms })) | |
| 21 | }) | |
| 13 | 22 | import { readFile, writeFile, mkdir, rename } from 'bare-fs/promises' |
| 14 | 23 | import fs from 'bare-fs' |
| 15 | 24 | import Hyperswarm from 'hyperswarm' |
| … | … | @@ -0,0 +1,106 @@ |
| 1 | // Bounded fetch — the fix for the 2026-08-30 stortron socket leak. | |
| 2 | // | |
| 3 | // WHY THIS EXISTS. favela-web reached 24.3 GB RSS holding 47,191 leaked sockets (46,857 of them | |
| 4 | // outbound :443) and OOM-killed the box twice, because an HTTPS fetch to a host that accepts TCP but | |
| 5 | // never completes the TLS handshake holds its socket FOR EVER. Three faults compound, and no one of | |
| 6 | // them is sufficient to explain it: | |
| 7 | // | |
| 8 | // 1. bare-https WRAPS the raw socket — `HTTPSAgent.createConnection` returns | |
| 9 | // `new HTTPSSocket(super.createConnection(opts), opts)`. `HTTPAgent.addRequest` then attaches | |
| 10 | // `.on('timeout', () => socket.destroy())` to the WRAPPER, while bare-tcp arms `setTimeout` on the | |
| 11 | // INNER socket. bare-tls forwards data/drain/end/close/error upward but NOT 'timeout'. So on HTTPS | |
| 12 | // the agent's own reaper can never fire. Plain HTTP is unaffected — nothing wraps there. | |
| 13 | // 2. bare-fetch silently drops `timeout`: its Request destructures only body/method/headers/signal/ | |
| 14 | // agent. Every call site passing `timeout: 30000` was a no-op. | |
| 15 | // 3. bare-fetch's `signal` rejects the promise but never destroys the socket — its abort() is | |
| 16 | // registered before the ClientRequest exists and only cancels bodies. And AbortController is not a | |
| 17 | // global in Bare (verified, v1.28.7), so reaching for one crashes the daemon outright. | |
| 18 | // | |
| 19 | // Measured against a blackhole listener (accept, never reply), 5 concurrent fetches: | |
| 20 | // stock agent -> 5 client sockets still ESTABLISHED at t+20s, no promise settled, held for ever | |
| 21 | // this module -> 0 client sockets at t+20s | |
| 22 | // | |
| 23 | // Hence two mechanisms, BOTH required: | |
| 24 | // - ReapingHTTPSAgent forwards the inner socket's 'timeout' up to the wrapper, which makes the | |
| 25 | // agent's EXISTING destroy path work as designed. This is what reclaims the fd and the memory. | |
| 26 | // - a promise deadline settles the caller, because destroying the socket underneath bare-fetch does | |
| 27 | // not reject its promise. | |
| 28 | // | |
| 29 | // On the timer: bare-tcp's timeout is an IDLE timeout (`_timer.refresh()` on both read and write), not | |
| 30 | // an absolute one, so a slow-but-live transfer is never cut off. It is a liveness deadline on a foreign | |
| 31 | // socket, not a poll — the failure it exists to catch is a counterparty that emits NOTHING, ever, and | |
| 32 | // silence is only observable against a clock. That is why it is a timer and not an event subscription. | |
| 33 | import http from 'bare-http1' | |
| 34 | import https from 'bare-https' | |
| 35 | import bareFetch from 'bare-fetch' | |
| 36 | ||
| 37 | // Idle, not absolute. Generous enough that a slow upstream that is still talking survives; any host | |
| 38 | // that goes silent is reaped this long after its last byte instead of never. | |
| 39 | export const DEFAULT_SOCKET_IDLE_MS = 30000 | |
| 40 | // Whole-request ceiling, so a caller is never left pending for ever. Matches the most generous value | |
| 41 | // already passed by an existing call site (bin/favela-web.js used timeout: 60000). | |
| 42 | export const DEFAULT_DEADLINE_MS = 60000 | |
| 43 | ||
| 44 | class ReapingHTTPSAgent extends https.Agent { | |
| 45 | createConnection (opts) { | |
| 46 | const socket = super.createConnection(opts) // an HTTPSSocket wrapping a bare-tcp socket | |
| 47 | const inner = socket && socket.socket | |
| 48 | // The one line the whole leak turned on: let the wrapper hear its own socket time out, so | |
| 49 | // HTTPAgent.addRequest's ontimeout -> socket.destroy() can run. | |
| 50 | if (inner && typeof inner.on === 'function') { | |
| 51 | inner.on('timeout', () => socket.emit('timeout')) | |
| 52 | } | |
| 53 | return socket | |
| 54 | } | |
| 55 | } | |
| 56 | ||
| 57 | function protocolOf (input) { | |
| 58 | try { | |
| 59 | if (typeof input === 'string') return new URL(input).protocol | |
| 60 | if (input && input.url) return new URL(input.url).protocol | |
| 61 | if (input && input.protocol) return input.protocol | |
| 62 | } catch {} | |
| 63 | return null | |
| 64 | } | |
| 65 | ||
| 66 | export function createBoundedFetch (opts = {}) { | |
| 67 | const socketIdleMs = opts.socketIdleMs || DEFAULT_SOCKET_IDLE_MS | |
| 68 | const deadlineMs = opts.deadlineMs || DEFAULT_DEADLINE_MS | |
| 69 | const onTimeout = typeof opts.onTimeout === 'function' ? opts.onTimeout : null | |
| 70 | ||
| 71 | const httpsAgent = new ReapingHTTPSAgent({ keepAlive: 1000, timeout: socketIdleMs }) | |
| 72 | const httpAgent = new http.Agent({ keepAlive: 1000, timeout: socketIdleMs }) | |
| 73 | ||
| 74 | function boundedFetch (input, init = {}) { | |
| 75 | const proto = protocolOf(input) | |
| 76 | // Only override when we can tell — passing an https agent to an http URL would be worse than | |
| 77 | // leaving bare-fetch to choose its own global. | |
| 78 | const agent = init.agent || (proto === 'https:' ? httpsAgent : proto === 'http:' ? httpAgent : null) | |
| 79 | // Honour the `timeout` callers already pass and believe in; it has been inert until now. | |
| 80 | const ms = init.timeout || deadlineMs | |
| 81 | ||
| 82 | const request = bareFetch(input, agent ? { ...init, agent } : init) | |
| 83 | ||
| 84 | let timer = null | |
| 85 | const deadline = new Promise((resolve, reject) => { | |
| 86 | timer = setTimeout(() => { | |
| 87 | if (onTimeout) { try { onTimeout({ url: String(input), ms }) } catch {} } | |
| 88 | // The socket is already being reaped by the agent's idle timeout; this only frees the caller. | |
| 89 | reject(new Error(`fetch deadline exceeded after ${ms}ms: ${String(input)}`)) | |
| 90 | }, ms) | |
| 91 | if (typeof timer.unref === 'function') timer.unref() | |
| 92 | }) | |
| 93 | ||
| 94 | return Promise.race([request, deadline]).finally(() => { if (timer) clearTimeout(timer) }) | |
| 95 | } | |
| 96 | ||
| 97 | // Exposed so the leak is OBSERVABLE rather than inferred: these are the only sockets this fetch owns, | |
| 98 | // and a growing count here is the signature that cost 24.3 GB before anyone could see it. | |
| 99 | boundedFetch.agents = { https: httpsAgent, http: httpAgent } | |
| 100 | boundedFetch.socketCount = () => | |
| 101 | [...(httpsAgent.sockets || [])].length + [...(httpAgent.sockets || [])].length | |
| 102 | ||
| 103 | return boundedFetch | |
| 104 | } | |
| 105 | ||
| 106 | export default createBoundedFetch |
| … | … | @@ -1,7 +1,7 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "favela-web", |
| 3 | "version": "0.3.87", | |
| 4 | "description": "favela-web — the web/SSR surface of Favela, the native P2P-local, community-owned social client + appview over the self-custody atproto stack (hyper-proto-peer). One of the Favela surfaces (alongside favela-cli and favela-native). bsky is an optional POSSE export, not the home. See DESIGN.md.", | |
| 3 | "version": "0.3.88", | |
| 4 | "description": "favela-web \u2014 the web/SSR surface of Favela, the native P2P-local, community-owned social client + appview over the self-custody atproto stack (hyper-proto-peer). One of the Favela surfaces (alongside favela-cli and favela-native). bsky is an optional POSSE export, not the home. See DESIGN.md.", | |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "bin/favela-web.js", |
| 7 | 7 | "bin": { |
| … | … | @@ -57,7 +57,7 @@ |
| 57 | 57 | "web": "bare bin/favela-web.js", |
| 58 | 58 | "test": "node test/render.test.mjs && node test/static-assets.test.mjs && node test/resize-plan.test.mjs && node test/notifications.test.mjs && node test/ap-followers.test.mjs && node test/reach.test.mjs && node test/strip-vectors.test.mjs && node test/edit.test.mjs && node test/facets.test.mjs && node test/mutes.test.mjs && node test/search.test.mjs && node test/threadgate.test.mjs && node test/bookmarks.test.mjs && node test/postgate.test.mjs && node test/lists.test.mjs && node test/cli-reachable.test.mjs && node test/strongref.test.mjs && node test/lens-import.test.mjs && node test/lens-thread-extract.test.mjs && node test/fronts-diag.test.mjs && node test/gate-policy.test.mjs && node test/sessions.test.mjs && node test/check.test.mjs && node test/auth.test.mjs && node test/browser-auth-client.test.mjs && node test/communities.test.mjs && node test/feedgen.test.mjs && node test/feed-reactor.test.mjs && node test/labeler-export.test.mjs && node test/label-firehose.test.mjs && node test/pairing.test.mjs && node test/writer.test.mjs && node test/blob-serve.test.mjs && node test/blobs.test.mjs && node test/spacedust.test.mjs && node test/spacedust-failclosed.test.mjs && node test/link-drain.test.mjs && node test/record-verify.test.mjs && node test/constellation.test.mjs && node test/retention.test.mjs && node test/hosted-dids.test.mjs && node test/repair-bytes.test.mjs && node test/repo-sync.test.mjs && node test/blob-refs.test.mjs && node test/inbound.test.mjs && node test/followers.test.mjs && node test/repair-strongrefs.test.mjs && node test/conformance.test.mjs && node test/dns-txt.test.mjs", |
| 59 | 59 | "test:ws": "bare test/subscribe-labels-ws.test.mjs", |
| 60 | "test:bare": "bare test/appview.test.mjs && bare test/appview-ingest.test.mjs && bare test/appview-dedup.test.mjs && bare test/appview-no-clobber.test.mjs && bare test/appview-blocks.test.mjs && bare test/appview-thread.test.mjs && bare test/appview-author-filter.test.mjs && bare test/appview-collections.test.mjs", | |
| 60 | "test:bare": "bare test/appview.test.mjs && bare test/appview-ingest.test.mjs && bare test/appview-dedup.test.mjs && bare test/appview-no-clobber.test.mjs && bare test/appview-blocks.test.mjs && bare test/appview-thread.test.mjs && bare test/appview-author-filter.test.mjs && bare test/appview-collections.test.mjs && bare test/bounded-fetch.test.mjs", | |
| 61 | 61 | "spike:pack": "bare spike/mobile-pack-probe.mjs", |
| 62 | 62 | "gen:static": "node scripts/gen-static-assets.mjs" |
| 63 | 63 | }, |
| … | … | @@ -0,0 +1,73 @@ |
| 1 | // BOUNDED FETCH — proves the 2026-08-30 stortron socket leak and its fix with the SAME drill. | |
| 2 | // | |
| 3 | // The leak: favela-web held 47,191 sockets / 24.3 GB and OOM-killed stortron twice (Aug 27, Aug 29). An | |
| 4 | // HTTPS fetch to a host that accepts TCP but never completes the TLS handshake keeps its socket FOR | |
| 5 | // EVER, because bare-https wraps the raw socket in an HTTPSSocket while bare-tcp arms the idle timer on | |
| 6 | // the INNER socket, and bare-tls never forwards 'timeout' upward — so HTTPAgent's | |
| 7 | // ontimeout -> socket.destroy() can never fire on HTTPS. Plain HTTP is unaffected (nothing wraps). | |
| 8 | // | |
| 9 | // Ground truth is the PEER, not the agent's bookkeeping: HTTPAgent.ontimeout deletes only from | |
| 10 | // _freeSockets, so `agent.sockets` still lists sockets it has destroyed. The blackhole server sees a FIN | |
| 11 | // when — and only when — the client socket is really gone, so that is what this asserts. | |
| 12 | // | |
| 13 | // Asserts, against a listener that accepts and never replies: | |
| 14 | // (1) RED — stock fetches never settle and the peer still sees every connection open; | |
| 15 | // (2) GREEN — bounded fetches settle, and the peer sees every connection closed; | |
| 16 | // (3) the deadline is a SEPARATE, required mechanism: destroying the socket under bare-fetch does not | |
| 17 | // reject its promise, and AbortController does not exist in Bare to do it for us. | |
| 18 | // Run: `bare test/bounded-fetch.test.mjs` | |
| 19 | import tcp from 'bare-tcp' | |
| 20 | import bareFetch from 'bare-fetch' | |
| 21 | import { createBoundedFetch } from '../lib/bounded-fetch.js' | |
| 22 | ||
| 23 | const assert = (cond, msg) => { if (!cond) throw new Error('FAIL: ' + msg); console.log(' ok -', msg) } | |
| 24 | const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) | |
| 25 | ||
| 26 | const IDLE = 1500 // short so the test is quick; production default is 30s | |
| 27 | const DEADLINE = 3000 | |
| 28 | const N = 4 | |
| 29 | ||
| 30 | // A blackhole: accept, then say nothing, for ever. `live` counts peers that have not hung up. | |
| 31 | let live = 0 | |
| 32 | let accepted = 0 | |
| 33 | const sockets = [] | |
| 34 | const server = tcp.createServer((sock) => { | |
| 35 | live++ | |
| 36 | accepted++ | |
| 37 | sockets.push(sock) | |
| 38 | const drop = () => { if (!sock._counted) { sock._counted = true; live-- } } | |
| 39 | sock.on('end', drop).on('close', drop).on('error', drop) | |
| 40 | sock.resume() // a paused stream never emits 'end', so the FIN would be invisible | |
| 41 | }) | |
| 42 | await new Promise((r) => server.listen(0, '127.0.0.1', r)) | |
| 43 | const PORT = server.address().port | |
| 44 | const URL_ = `https://127.0.0.1:${PORT}/probe` | |
| 45 | ||
| 46 | // ---------- (1) RED ---------- | |
| 47 | let stockSettled = 0 | |
| 48 | for (let i = 0; i < N; i++) bareFetch(URL_).then(() => stockSettled++, () => stockSettled++) | |
| 49 | await sleep(IDLE * 2 + 1000) | |
| 50 | assert(stockSettled === 0, `stock fetch never settles, long past the idle timeout (settled=${stockSettled})`) | |
| 51 | assert(live === N, `THE LEAK: the peer still sees all ${N} stock connections open (live=${live})`) | |
| 52 | ||
| 53 | const leakedForContrast = live | |
| 54 | ||
| 55 | // ---------- (2)+(3) GREEN ---------- | |
| 56 | const timedOut = [] | |
| 57 | const boundedFetch = createBoundedFetch({ socketIdleMs: IDLE, deadlineMs: DEADLINE, onTimeout: (d) => timedOut.push(d) }) | |
| 58 | const results = await Promise.allSettled(Array.from({ length: N }, () => boundedFetch(URL_))) | |
| 59 | ||
| 60 | assert(results.every((r) => r.status === 'rejected'), 'every bounded fetch settles (rejected) instead of hanging') | |
| 61 | assert(results.every((r) => /deadline exceeded/.test(r.reason.message)), 'each rejection names the deadline that fired') | |
| 62 | assert(timedOut.length === N, `onTimeout reported all ${N} deadlines, so the leak is observable (got ${timedOut.length})`) | |
| 63 | ||
| 64 | // Give the agent's idle timer room to reap past the deadline. | |
| 65 | await sleep(IDLE + 1500) | |
| 66 | const stillLive = live - leakedForContrast | |
| 67 | assert(accepted === N * 2, `the ${N} bounded connections were actually opened (accepted=${accepted} of ${N * 2})`) | |
| 68 | assert(stillLive === 0, `the peer saw every bounded connection CLOSE (still open=${stillLive})`) | |
| 69 | assert(live === leakedForContrast, 'and the stock leak is still open for contrast — the fix is the agent, not timing') | |
| 70 | ||
| 71 | for (const s of sockets) { try { s.destroy() } catch {} } | |
| 72 | server.close() | |
| 73 | console.log('\nbounded-fetch: all assertions passed') |