| … | … | @@ -3,6 +3,26 @@ |
| 3 | 3 | // registry's input validation — a malformed value must never reach the config |
| 4 | 4 | // that gets deployed, especially now that deploy() rolls back on configtest |
| 5 | 5 | // failure rather than silently shipping garbage. |
| 6 | // Per-route request timeout in seconds for PLAIN HTTP proxying. |
| 7 | // |
| 8 | // ⛔ NOT the same thing as the vhost-level `ProxyTimeout 3600` below, and the split is the point. |
| 9 | // 3600 exists for WEBSOCKETS — a firehose or spacedust subscription is idle for long stretches and must |
| 10 | // not be severed. But it also applied to ordinary HTTP, so a backend that accepted a connection and |
| 11 | // never answered held an Apache worker for up to an HOUR per request. |
| 12 | // |
| 13 | // On 2026-08-24 that took the whole proxy down: both favela-web backends restarted at once (autoupdate |
| 14 | // on both), every request to them hung, and Apache stopped serving EVERY vhost — including ones whose |
| 15 | // backends were perfectly healthy. Recovering it required a restart. |
| 16 | // |
| 17 | // WebSocket upgrades are routed by a `RewriteRule [P]`, which is NOT affected by a ProxyPass timeout, |
| 18 | // so bounding the ProxyPass costs WS nothing. |
| 19 | // |
| 20 | // 300s, matching what the balancer path already used: 5s was tried and was too aggressive — a data |
| 21 | // stall on a large transfer (the pear-git bootstrap tarball) tripped failover into a 502, issue #101. |
| 22 | // Liveness is a separate concern, handled by the TCP health checks. Override per host/service with |
| 23 | // `timeout`. |
| 24 | const DEFAULT_PROXY_TIMEOUT = 300 |
| 25 | |
| 6 | 26 | function assertSafe (value, kind) { |
| 7 | 27 | if (typeof value !== 'string' || value.length === 0 || /[\s"'<>\n\r]/.test(value)) { |
| 8 | 28 | throw new Error(`unsafe ${kind} for Apache config: ${JSON.stringify(value)}`) |
| … | … | @@ -90,13 +110,15 @@ export function generateVhosts (registry) { |
| 90 | 110 | lines.push(' SSLEngine on') |
| 91 | 111 | lines.push(' # mod_md supplies the *.7rnx.net certificate for this host.') |
| 92 | 112 | lines.push(' ProxyPreserveHost On') |
| 113 | // Governs the WEBSOCKET path only, now that the ProxyPass below carries its own timeout: a WS |
| 114 | // upgrade goes through the RewriteRule [P] and must be allowed to idle. |
| 93 | 115 | lines.push(' ProxyTimeout 3600') |
| 94 | 116 | lines.push(' RequestHeader set X-Forwarded-Proto "https"') |
| 95 | 117 | lines.push(' RewriteEngine On') |
| 96 | 118 | // WebSocket (e.g. com.atproto.sync.subscribeRepos firehose) — must precede ProxyPass. |
| 97 | 119 | lines.push(' RewriteCond %{HTTP:Upgrade} =websocket [NC]') |
| 98 | 120 | lines.push(` RewriteRule ^/(.*)$ ws://${node.ip}:${h.port}/$1 [P,L]`) |
| 99 | | lines.push(` ProxyPass / ${backend}/`) |
| 121 | lines.push(` ProxyPass / ${backend}/ timeout=${h.timeout ? Number(h.timeout) : DEFAULT_PROXY_TIMEOUT}`) |
| 100 | 122 | lines.push(` ProxyPassReverse / ${backend}/`) |
| 101 | 123 | lines.push(` ErrorLog \${APACHE_LOG_DIR}/7rnx-${h.name}-error.log`) |
| 102 | 124 | lines.push(` CustomLog \${APACHE_LOG_DIR}/7rnx-${h.name}-access.log combined`) |
| … | … | @@ -149,7 +171,7 @@ export function generateConfig (registry) { |
| 149 | 171 | // separately by the TCP health checks below (a dead member is marked down in |
| 150 | 172 | // ~hcinterval*hcfails = ~6s), so the request timeout can be generous without |
| 151 | 173 | // slowing failover. Override per service with `svc.timeout`. |
| 152 | | const memberTimeout = svc.timeout ? Number(svc.timeout) : 300 |
| 174 | const memberTimeout = svc.timeout ? Number(svc.timeout) : DEFAULT_PROXY_TIMEOUT |
| 153 | 175 | |
| 154 | 176 | lines.push(`# === ${svc.name} (balancer) ===`) |
| 155 | 177 | lines.push('') |
| … | … | @@ -229,7 +251,7 @@ export function generateConfig (registry) { |
| 229 | 251 | const backend = `http://${node.ip}:${svc.port}` |
| 230 | 252 | |
| 231 | 253 | lines.push(`<Location ${urlPathClean}>`) |
| 232 | | lines.push(` ProxyPass ${backend}`) |
| 254 | lines.push(` ProxyPass ${backend} timeout=${svc.timeout ? Number(svc.timeout) : DEFAULT_PROXY_TIMEOUT}`) |
| 233 | 255 | lines.push(` ProxyPassReverse ${backend}`) |
| 234 | 256 | lines.push(' ProxyPreserveHost On') |
| 235 | 257 | lines.push(' RequestHeader set X-Forwarded-Proto "https"') |