OverviewFilesCommitsIssuesDocsReleasesPeers

favela-web / test / bounded-fetch.test.mjs

100 lines · javascript ref:
// BOUNDED FETCH — proves the 2026-08-30 stortron socket leak and its fix with the SAME drill.
//
// The leak: favela-web held 47,191 sockets / 24.3 GB and OOM-killed stortron twice (Aug 27, Aug 29). An
// HTTPS fetch to a host that accepts TCP but never completes the TLS handshake keeps its socket FOR
// EVER, because bare-https wraps the raw socket in an HTTPSSocket while bare-tcp arms the idle timer on
// the INNER socket, and bare-tls never forwards 'timeout' upward — so HTTPAgent's
// ontimeout -> socket.destroy() can never fire on HTTPS. Plain HTTP is unaffected (nothing wraps).
//
// Ground truth for the FD is the PEER: the blackhole server sees a FIN when — and only when — the client
// socket is really gone. But a closed fd is not a freed socket. 2026-09-29 stortron was back at 20.3 GB
// with only ~1.2k fds open: bare-tls never settles a pending _open when destroyed mid-handshake, so the
// wrapper never emits 'close', HTTPAgent's onremove never runs, and `agent.sockets` keeps every wrapper
// (~150 KB each) for ever. So this ALSO asserts the agent lets go of them — (4).
//
// Asserts, against a listener that accepts and never replies:
//   (1) RED   — stock fetches never settle and the peer still sees every connection open;
//   (2) GREEN — bounded fetches settle, and the peer sees every connection closed;
//   (3) the deadline is a SEPARATE, required mechanism: it is what settles a caller whose deadline is
//       SHORTER than the idle reap (handles.js: 5s vs 30s), and AbortController does not exist in Bare;
//   (4) the agent RELEASES every reaped socket object — the fd closing is not enough;
//   (5) when the idle reap fires FIRST (deadline > idle), the reap itself now settles the caller —
//       completing the stuck TLS open lets bare-fetch's request error out instead of hanging for ever.
// Run: `bare test/bounded-fetch.test.mjs`
import tcp from 'bare-tcp'
import bareFetch from 'bare-fetch'
import { createBoundedFetch } from '../lib/bounded-fetch.js'

const assert = (cond, msg) => { if (!cond) throw new Error('FAIL: ' + msg); console.log('  ok -', msg) }
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))

const IDLE = 1500      // short so the test is quick; production default is 30s
const DEADLINE = 1000  // < IDLE: the production ordering on the leaking path (handles.js 5s vs 30s)
const LONG_DEADLINE = 3000 // > IDLE: the reap fires first (callers passing timeout above 30s)
const N = 4

// A blackhole: accept, then say nothing, for ever. `live` counts peers that have not hung up.
let live = 0
let accepted = 0
const sockets = []
const server = tcp.createServer((sock) => {
  live++
  accepted++
  sockets.push(sock)
  const drop = () => { if (!sock._counted) { sock._counted = true; live-- } }
  sock.on('end', drop).on('close', drop).on('error', drop)
  sock.resume() // a paused stream never emits 'end', so the FIN would be invisible
})
await new Promise((r) => server.listen(0, '127.0.0.1', r))
const PORT = server.address().port
const URL_ = `https://127.0.0.1:${PORT}/probe`

// ---------- (1) RED ----------
let stockSettled = 0
for (let i = 0; i < N; i++) bareFetch(URL_).then(() => stockSettled++, () => stockSettled++)
await sleep(IDLE * 2 + 1000)
assert(stockSettled === 0, `stock fetch never settles, long past the idle timeout (settled=${stockSettled})`)
assert(live === N, `THE LEAK: the peer still sees all ${N} stock connections open (live=${live})`)

const leakedForContrast = live

// ---------- (2)+(3) GREEN ----------
const timedOut = []
const boundedFetch = createBoundedFetch({ socketIdleMs: IDLE, deadlineMs: DEADLINE, onTimeout: (d) => timedOut.push(d) })
const results = await Promise.allSettled(Array.from({ length: N }, () => boundedFetch(URL_)))

assert(results.every((r) => r.status === 'rejected'), 'every bounded fetch settles (rejected) instead of hanging')
assert(results.every((r) => /deadline exceeded/.test(r.reason.message)), 'each rejection names the deadline that fired')
assert(timedOut.length === N, `onTimeout reported all ${N} deadlines, so the leak is observable (got ${timedOut.length})`)

// Give the agent's idle timer room to reap past the deadline.
await sleep(IDLE + 1500)
const stillLive = live - leakedForContrast
assert(accepted === N * 2, `the ${N} bounded connections were actually opened (accepted=${accepted} of ${N * 2})`)
assert(stillLive === 0, `the peer saw every bounded connection CLOSE (still open=${stillLive})`)
assert(live === leakedForContrast, 'and the stock leak is still open for contrast — the fix is the agent, not timing')

// ---------- (4) the socket OBJECTS are freed, not just the fds ----------
const held = [...boundedFetch.agents.https.sockets].length
assert(held === 0, `the agent released every reaped socket object (still held=${held} of ${N})`)

// ---------- (5) reap before deadline ----------
const lateTimedOut = []
const lateFetch = createBoundedFetch({ socketIdleMs: IDLE, deadlineMs: LONG_DEADLINE, onTimeout: (d) => lateTimedOut.push(d) })
const t0 = Date.now()
const late = await Promise.allSettled(Array.from({ length: N }, () => lateFetch(URL_)))
const took = Date.now() - t0
assert(late.every((r) => r.status === 'rejected'), 'reap-first: every fetch settles (rejected)')
// bare-fetch wraps any socket failure as NETWORK_ERROR, so the reap shows up under that code, not the
// message bounded-fetch passes to the stuck open.
assert(late.every((r) => r.reason.code === 'NETWORK_ERROR' && !/deadline exceeded/.test(r.reason.message)), 'reap-first: the rejection is the reap (NETWORK_ERROR), not the deadline')
assert(took < LONG_DEADLINE && lateTimedOut.length === 0, `reap-first: settled at the idle reap, before the deadline (${took}ms, deadlines fired=${lateTimedOut.length})`)
await sleep(500)
const lateHeld = [...lateFetch.agents.https.sockets].length
assert(lateHeld === 0, `reap-first: the agent released every socket object (still held=${lateHeld} of ${N})`)
assert(live === leakedForContrast, `reap-first: the peer saw every connection CLOSE (still open=${live - leakedForContrast})`)

for (const s of sockets) { try { s.destroy() } catch {} }
server.close()
console.log('\nbounded-fetch: all assertions passed')