OverviewFilesCommitsIssuesDocsReleasesPeers

favela-web / lib / bounded-fetch.js

119 lines · javascript ref:
// Bounded fetch — the fix for the 2026-08-30 stortron socket leak.
//
// WHY THIS EXISTS. favela-web reached 24.3 GB RSS holding 47,191 leaked sockets (46,857 of them
// outbound :443) and OOM-killed the box twice, because an HTTPS fetch to a host that accepts TCP but
// never completes the TLS handshake holds its socket FOR EVER. Three faults compound, and no one of
// them is sufficient to explain it:
//
//  1. bare-https WRAPS the raw socket — `HTTPSAgent.createConnection` returns
//     `new HTTPSSocket(super.createConnection(opts), opts)`. `HTTPAgent.addRequest` then attaches
//     `.on('timeout', () => socket.destroy())` to the WRAPPER, while bare-tcp arms `setTimeout` on the
//     INNER socket. bare-tls forwards data/drain/end/close/error upward but NOT 'timeout'. So on HTTPS
//     the agent's own reaper can never fire. Plain HTTP is unaffected — nothing wraps there.
//  2. bare-fetch silently drops `timeout`: its Request destructures only body/method/headers/signal/
//     agent. Every call site passing `timeout: 30000` was a no-op.
//  3. bare-fetch's `signal` rejects the promise but never destroys the socket — its abort() is
//     registered before the ClientRequest exists and only cancels bodies. And AbortController is not a
//     global in Bare (verified, v1.28.7), so reaching for one crashes the daemon outright.
//
// Measured against a blackhole listener (accept, never reply), 5 concurrent fetches:
//   stock agent  -> 5 client sockets still ESTABLISHED at t+20s, no promise settled, held for ever
//   this module  -> 0 client sockets at t+20s
//
// Hence two mechanisms, BOTH required:
//   - ReapingHTTPSAgent forwards the inner socket's 'timeout' up to the wrapper, which makes the
//     agent's EXISTING destroy path work as designed. This is what reclaims the fd and the memory.
//   - a promise deadline settles the caller, because destroying the socket underneath bare-fetch does
//     not reject its promise.
//
// On the timer: bare-tcp's timeout is an IDLE timeout (`_timer.refresh()` on both read and write), not
// an absolute one, so a slow-but-live transfer is never cut off. It is a liveness deadline on a foreign
// socket, not a poll — the failure it exists to catch is a counterparty that emits NOTHING, ever, and
// silence is only observable against a clock. That is why it is a timer and not an event subscription.
import http from 'bare-http1'
import https from 'bare-https'
import bareFetch from 'bare-fetch'

// Idle, not absolute. Generous enough that a slow upstream that is still talking survives; any host
// that goes silent is reaped this long after its last byte instead of never.
export const DEFAULT_SOCKET_IDLE_MS = 30000
// Whole-request ceiling, so a caller is never left pending for ever. Matches the most generous value
// already passed by an existing call site (bin/favela-web.js used timeout: 60000).
export const DEFAULT_DEADLINE_MS = 60000

class ReapingHTTPSAgent extends https.Agent {
  createConnection (opts) {
    const socket = super.createConnection(opts) // an HTTPSSocket wrapping a bare-tcp socket
    const inner = socket && socket.socket
    // The one line the whole leak turned on: let the wrapper hear its own socket time out, so
    // HTTPAgent.addRequest's ontimeout -> socket.destroy() can run.
    if (inner && typeof inner.on === 'function') {
      inner.on('timeout', () => socket.emit('timeout'))
      // The second line, found 2026-09-29 when favela-web was back at 20.3 GB with only ~1.2k fds open.
      // Destroying the wrapper mid-handshake closes the inner socket, but bare-tls (<= 3.1.11) settles
      // only a pending DESTROY on inner close, never a pending OPEN. streamx will not run _destroy until
      // _open calls back, so the wrapper never emits 'close', HTTPAgent's onremove never runs, and
      // agent._sockets holds the wrapper (~150 KB: 64 KB read buffer, TLS state, fetch closures) for
      // ever. Settle the open with an error so the destroy the agent asked for can finish.
      inner.on('close', () => {
        const cb = socket._pendingOpen
        if (typeof cb !== 'function') return
        socket._pendingOpen = null
        cb(new Error('socket closed during TLS handshake'))
      })
    }
    return socket
  }
}

function protocolOf (input) {
  try {
    if (typeof input === 'string') return new URL(input).protocol
    if (input && input.url) return new URL(input.url).protocol
    if (input && input.protocol) return input.protocol
  } catch {}
  return null
}

export function createBoundedFetch (opts = {}) {
  const socketIdleMs = opts.socketIdleMs || DEFAULT_SOCKET_IDLE_MS
  const deadlineMs = opts.deadlineMs || DEFAULT_DEADLINE_MS
  const onTimeout = typeof opts.onTimeout === 'function' ? opts.onTimeout : null

  const httpsAgent = new ReapingHTTPSAgent({ keepAlive: 1000, timeout: socketIdleMs })
  const httpAgent = new http.Agent({ keepAlive: 1000, timeout: socketIdleMs })

  function boundedFetch (input, init = {}) {
    const proto = protocolOf(input)
    // Only override when we can tell — passing an https agent to an http URL would be worse than
    // leaving bare-fetch to choose its own global.
    const agent = init.agent || (proto === 'https:' ? httpsAgent : proto === 'http:' ? httpAgent : null)
    // Honour the `timeout` callers already pass and believe in; it has been inert until now.
    const ms = init.timeout || deadlineMs

    const request = bareFetch(input, agent ? { ...init, agent } : init)

    let timer = null
    const deadline = new Promise((resolve, reject) => {
      timer = setTimeout(() => {
        if (onTimeout) { try { onTimeout({ url: String(input), ms }) } catch {} }
        // The socket is already being reaped by the agent's idle timeout; this only frees the caller.
        reject(new Error(`fetch deadline exceeded after ${ms}ms: ${String(input)}`))
      }, ms)
      if (typeof timer.unref === 'function') timer.unref()
    })

    return Promise.race([request, deadline]).finally(() => { if (timer) clearTimeout(timer) })
  }

  // Exposed so the leak is OBSERVABLE rather than inferred: these are the only sockets this fetch owns,
  // and a growing count here is the signature that cost 24.3 GB before anyone could see it.
  boundedFetch.agents = { https: httpsAgent, http: httpAgent }
  boundedFetch.socketCount = () =>
    [...(httpsAgent.sockets || [])].length + [...(httpAgent.sockets || [])].length

  return boundedFetch
}

export default createBoundedFetch