OverviewFilesCommitsIssuesDocsReleasesPeers

pgh / lib / supervisor.js

205 lines · javascript ref:
/**
 * App supervisor — spawned detached by `pgh app start`. Runs a respawn
 * loop around the real app process, forwards SIGTERM/SIGINT, rotates
 * on crash with bounded backoff.
 *
 * Invocation (from pgh app start):
 *   node lib/supervisor.js
 *
 * Config is read from PGH_SUP_* env vars, set by the parent:
 *   PGH_SUP_NAME         app name
 *   PGH_SUP_ENTRYPOINT   absolute path to the app's entry file
 *   PGH_SUP_RUNTIME      "node" | "bare"
 *   PGH_SUP_ARGS_JSON    JSON array of CLI args
 *   PGH_SUP_CHILD_ENV    JSON object of the child's env (merged by caller)
 *
 * Stdout/stderr are inherited from the parent's detached spawn, which
 * points them at app.log. The supervisor prints lifecycle events
 * prefixed with [supervisor]; the app's own output interleaves.
 *
 * OTA hook: child exit code 75 (EX_TEMPFAIL) respawns immediately
 * regardless of backoff, so the app can request a relaunch against a
 * freshly materialized tree.
 */

import process from '#node:process'
import path from '#node:path'
import { spawn } from '#node:child_process'
import { writeState, writePid, clearPid, readInstallManifest, effectiveEnv } from './app-state.js'
import { resolveChildEnv as resolveEnv, envDelta } from './child-env.js'
import { TreeStore } from './store.js'
import { applyStaged, hasFlock, lockPathFor } from './materializers/atomic.js'

const NAME = process.env.PGH_SUP_NAME
const ENTRYPOINT = process.env.PGH_SUP_ENTRYPOINT
const RUNTIME = process.env.PGH_SUP_RUNTIME || 'node'
// Absolute path to the child's interpreter. The parent (`pgh app start`)
// sets this when it can resolve the child runtime locally — typically to
// the bundled bare on bare-bootstrap installs. Falls back to literal
// RUNTIME (PATH lookup) when unset.
const RUNTIME_PATH = process.env.PGH_SUP_RUNTIME_PATH ||
  (path.basename(process.execPath) === RUNTIME ? process.execPath : RUNTIME)
const ARGS = JSON.parse(process.env.PGH_SUP_ARGS_JSON || '[]')
// The env the PARENT computed at `pgh app start`. Used for the first spawn and as the fallback
// whenever a re-read fails — never as the permanent answer. See resolveChildEnv().
const CHILD_ENV = JSON.parse(process.env.PGH_SUP_CHILD_ENV || '{}')

// Env is RE-RESOLVED every spawn — the logic (and the reason) lives in child-env.js, which is
// importable and therefore testable; this script is not. Same discipline as the re-resolved `cwd`
// below: a long-lived supervisor must not cache anything the install can change underneath it.
const childEnvFor = () => resolveEnv({
  name: NAME,
  readInstallManifest,
  effectiveEnv,
  fallback: CHILD_ENV,
  onWarn: (m) => log(m)
})

// Say it ONCE when it changes. The failure this fixes was invisible from every surface.
let prevEnv = null
function logEnvDelta (env) {
  const changed = envDelta(prevEnv, env)
  if (prevEnv && changed.length) log('child env changed since last spawn: ' + changed.join(', '))
  prevEnv = env
}

const BACKOFF_MIN_MS = 1000
const BACKOFF_MAX_MS = 30000
const FAST_FAIL_S = 5
const OTA_EXIT_CODE = 75
const SHUTDOWN_GRACE_MS = 10000

if (!NAME || !ENTRYPOINT) {
  // eslint-disable-next-line no-console
  console.error('[supervisor] missing PGH_SUP_NAME or PGH_SUP_ENTRYPOINT')
  process.exit(2)
}

// The supervisor owns app.pid: it records its OWN real pid here. `pgh app
// start` spawns us wrapped in `setsid` (so we outlive the CLI's session),
// and setsid forks the real supervisor — so the pid the parent observes
// from spawn() is the short-lived setsid wrapper, which dies the instant it
// forks us. Writing process.pid here makes app.pid authoritative for the
// live process, so `status` (isPidAlive) and `stop` (kill of the process
// group, of which we are the session leader) both target the right pid.
// We write it BEFORE spawning the child, so by the time `start`'s pidfile
// readiness probe observes state.childPid, app.pid is already correct.
writePid(NAME, process.pid)
writeState(NAME, { startedAt: new Date().toISOString(), supervisorPid: process.pid })

let child = null
let shuttingDown = false
let backoff = BACKOFF_MIN_MS

function log (msg) {
  // eslint-disable-next-line no-console
  console.error('[supervisor] ' + msg)
}

function onShutdown (sig) {
  if (shuttingDown) return
  shuttingDown = true
  log('received ' + sig + ' — stopping child')
  if (child && !child.killed) {
    try { child.kill('SIGTERM') } catch {}
    const forceTimer = setTimeout(() => {
      if (child && !child.killed) {
        log('child did not exit in ' + SHUTDOWN_GRACE_MS + 'ms, SIGKILL')
        try { child.kill('SIGKILL') } catch {}
      }
    }, SHUTDOWN_GRACE_MS)
    child.once('exit', () => clearTimeout(forceTimer))
  }
}

process.on('SIGTERM', () => onShutdown('SIGTERM'))
process.on('SIGINT', () => onShutdown('SIGINT'))

async function loop () {
  while (!shuttingDown) {
    const startedAtMs = Date.now()
    log('spawning ' + RUNTIME_PATH + ' ' + ENTRYPOINT + (ARGS.length ? ' ' + ARGS.join(' ') : ''))
    // IN-USE CONTRACT (materializers/atomic.js): no child is running here, so an
    // update that was STAGED while it ran goes live now — this is what makes an
    // exit-75 OTA respawn, or any restart, pick it up. Then the child runs
    // holding a SHARED flock on the app's lock, so an update arriving while it
    // runs stages instead of swapping its tree. `exec` keeps the child's pid,
    // so SIGTERM/SIGKILL above still reach the app itself.
    let cmd = RUNTIME_PATH
    let argv = [ENTRYPOINT, ...ARGS]
    try {
      const store = new TreeStore()
      store.init()
      const a = applyStaged(store, NAME)
      if (a.applied) log('applied the staged update before spawning')
      else if (a.reason === 'in-use') log('a staged update is waiting: something else holds ' + NAME + '\'s lock')
      if (hasFlock()) {
        cmd = 'sh'
        argv = ['-c', 'if : >>"$0" 2>/dev/null; then exec 9>>"$0"; flock -s -w 30 9 || true; fi; exec "$@"',
          lockPathFor(store, NAME), RUNTIME_PATH, ENTRYPOINT, ...ARGS]
      }
    } catch (err) {
      log('staged-update check failed (spawning as before): ' + (err && err.message))
    }
    // Re-resolved every spawn, so an OTA respawn picks up the new tree's manifest env — and ⛔ AFTER
    // applyStaged(): the install manifest lives INSIDE the app dir, so before the swap it is still the
    // previous release's. Resolved first, a staged update (the normal OTA path while an app runs) started
    // the new code with the old defaults.env; favela-gateway 0.3.0 came up without its new role
    // (stortron, 2026-10-14). test/supervisor-staged-env.test.js.
    const childEnv = childEnvFor()
    logEnvDelta(childEnv)
    child = spawn(cmd, argv, {
      stdio: ['ignore', 'inherit', 'inherit'],
      env: childEnv,
      // Launch from the app dir, RE-RESOLVED every spawn. The app's OTA
      // (pgh-runtime) atomically swaps its install dir to a new inode; this
      // supervisor is a separate pgh process that keeps running, so its own
      // cwd becomes the DELETED old inode. Without an explicit cwd the child
      // inherits that dead directory and bare/node abort with ENOENT (exit
      // 134) on every respawn. dirname(ENTRYPOINT) is an absolute path string
      // re-resolved at spawn time, so it always points at the LIVE app dir.
      cwd: path.dirname(ENTRYPOINT)
    })
    writeState(NAME, { childPid: child.pid })

    const exit = await new Promise(resolve => {
      child.once('exit', (code, signal) => resolve({ code, signal }))
      child.once('error', (err) => resolve({ code: null, signal: null, err: err.message }))
    })
    const runS = Math.round((Date.now() - startedAtMs) / 1000)
    log('child exited code=' + exit.code + ' signal=' + exit.signal + ' after ' + runS + 's' + (exit.err ? ' err=' + exit.err : ''))
    writeState(NAME, {
      lastExit: new Date().toISOString(),
      lastExitCode: exit.code,
      lastExitSignal: exit.signal,
      childPid: null
    })

    if (shuttingDown) break

    if (exit.code === OTA_EXIT_CODE) {
      log('OTA respawn requested (exit 75)')
      backoff = BACKOFF_MIN_MS
      continue
    }

    if (runS >= FAST_FAIL_S) backoff = BACKOFF_MIN_MS
    log('respawn in ' + backoff + 'ms')
    await sleep(backoff)
    backoff = Math.min(backoff * 2, BACKOFF_MAX_MS)
  }
  log('shutting down clean')
  clearPid(NAME)
  process.exit(0)
}

function sleep (ms) {
  return new Promise(r => setTimeout(r, ms))
}

loop().catch(err => {
  log('loop error: ' + (err && err.stack || err))
  clearPid(NAME)
  process.exit(1)
})