favela-gateway / lib / watch.js

// The watcher — checks the gateway's health on its own and keeps the alert state.
//
// ⛔ WHY IT RUNS ON A TIMER when the probes otherwise run only on demand: a manual fix nobody hears about
// is the outage (loser-club was invisible to bsky for 2½ months, and the relay had us "offline" with
// nothing reporting it). So: one check 2 minutes after boot (an OTA restart is when the relay drops us),
// then every 15 minutes — the relay guard's own cadence — each with FRESH probes. The gateway's RSS is
// one of the alerts, so if repeated probing leaked (as favela-web's HTTPS did) it would say so.
//
// State lives in a JSON file beside the gateway's socket, so an alert's "open since" survives a restart.

import fs from 'bare-fs'
import path from 'bare-path'
import { evaluateAlerts } from './alerts.js'

const RESOLVED_KEEP = 20

export function createWatcher ({ metrics, stateFile = null, firstMs = 2 * 60 * 1000, intervalMs = 15 * 60 * 1000, now = () => Date.now(), log = () => {} } = {}) {
  let saved = { state: {}, lastCheckAt: null, recentlyResolved: [] }
  if (stateFile) {
    try { saved = { ...saved, ...JSON.parse(fs.readFileSync(stateFile, 'utf8')) } } catch {}
  }
  let open = Object.values(saved.state).filter((e) => e.open).map((e) => ({ id: e.id, severity: e.severity, title: e.title, detail: e.detail, fix: e.fix, since: e.firstSeen, lastSeen: e.lastSeen }))
  let first = null
  let timer = null
  let running = null

  function persist () {
    if (!stateFile) return
    try {
      fs.mkdirSync(path.dirname(stateFile), { recursive: true })
      fs.writeFileSync(stateFile + '.tmp', JSON.stringify(saved))
      fs.renameSync(stateFile + '.tmp', stateFile)
    } catch (err) { log({ msg: 'alerts: could not persist state', error: err.message }) }
  }

  async function check () {
    let m
    try { m = await metrics({ refresh: true }) } catch (err) {
      log({ msg: 'alerts: metrics failed — state unchanged', error: (err && err.message) || String(err) })
      return
    }
    const t = now()
    const r = evaluateAlerts(m, saved.state, t)
    for (const a of r.open) if (!open.some((o) => o.id === a.id)) log({ msg: 'ALERT open', id: a.id, severity: a.severity, title: a.title, fix: a.fix })
    for (const a of r.resolved) log({ msg: 'alert resolved', id: a.id, openFor: Math.round((t - a.since) / 1000) + 's' })
    open = r.open
    saved = { state: r.state, lastCheckAt: t, recentlyResolved: [...r.resolved.reverse(), ...saved.recentlyResolved].slice(0, RESOLVED_KEEP) }
    persist()
  }

  // One check at a time; a request during a check waits for it.
  function checkNow () {
    if (!running) running = check().finally(() => { running = null })
    return running
  }

  return {
    checkNow,
    current: () => ({ open, recentlyResolved: saved.recentlyResolved, lastCheckAt: saved.lastCheckAt }),
    start () {
      first = setTimeout(() => { checkNow() }, firstMs)
      timer = setInterval(() => { checkNow() }, intervalMs)
      if (first.unref) first.unref()
      if (timer.unref) timer.unref()
    },
    stop () { clearTimeout(first); clearInterval(timer) }
  }
}

export default createWatcher