| … | … | @@ -90,7 +90,9 @@ if (roles.includes('lens')) { |
| 90 | 90 | configs.lens = feeds.length ? { feeds } : new Error('lens config: LENS_FEEDS has no id=roomKey pair') |
| 91 | 91 | } |
| 92 | 92 | |
| 93 | const gateway = await startGateway({ roles, log, version, ...configs }) | |
| 93 | // Alert state lives beside the control socket unless told otherwise. | |
| 94 | const stateDir = process.env.GATEWAY_STATE_DIR || path.dirname(sock) | |
| 95 | const gateway = await startGateway({ roles, log, version, watch: true, stateDir, ...configs }) | |
| 94 | 96 | |
| 95 | 97 | // The one control socket. A stale socket file from a killed process blocks bind; we only get here once |
| 96 | 98 | // the supervisor decided no live instance owns this app, so an existing file is orphaned. |
| … | … | @@ -0,0 +1,111 @@ |
| 1 | // Alerts — what a person must act on, derived from `metrics` (hyper-social #80; operator 2026-10-13: | |
| 2 | // manual re-syncs are fine only if the gateway SAYS when one is needed). | |
| 3 | // | |
| 4 | // Every alert has a stable id (so "open since" survives checks and restarts), a severity, the problem in | |
| 5 | // plain words, and the FIX: the command to run or what to check. Pure: evaluateAlerts(metrics, state, now) | |
| 6 | // returns { open, resolved, state }; the watcher persists `state` and the panel and CLI render `open`. | |
| 7 | // | |
| 8 | // Thresholds: conditions that blip (the relay a rev behind while it ingests, the AppView briefly not | |
| 9 | // resolving) open only after TWO checks in a row; peers gone must last an HOUR; everything that is wrong | |
| 10 | // the moment it is seen (relay offline, relay newer than us, PLC elsewhere, a role down) opens at once. | |
| 11 | // | |
| 12 | // ⛔ A check that could not see bsky (probes unreachable → metrics.bsky null) neither opens nor RESOLVES | |
| 13 | // bsky alerts: absence of evidence is not a fix. | |
| 14 | ||
| 15 | const HOUR = 60 * 60 * 1000 | |
| 16 | const RSS_CEILING = 2e9 | |
| 17 | ||
| 18 | function candidates (m) { | |
| 19 | const out = [] | |
| 20 | const add = (id, severity, scope, title, detail, fix, need = {}) => out.push({ id, severity, scope, title, detail, fix, checks: need.checks || 1, ms: need.ms || 0 }) | |
| 21 | ||
| 22 | for (const [name, r] of Object.entries(m.roles || {})) { | |
| 23 | if (!r.up) add('role-down:' + name, 'critical', 'local', `gateway role "${name}" is down`, r.error || 'no error recorded', 'favela-gateway status; pgh app log favela-gateway') | |
| 24 | } | |
| 25 | ||
| 26 | if (m.process && m.process.rss > RSS_CEILING) { | |
| 27 | add('memory', 'warning', 'local', 'gateway memory is high', `RSS ${Math.round(m.process.rss / 1e6)} MB (ceiling ${RSS_CEILING / 1e6} MB) — repeated outbound HTTPS under Bare has leaked before`, 'favela-gateway metrics (watch RSS); pgh app restart favela-gateway if it keeps growing') | |
| 28 | } | |
| 29 | ||
| 30 | if (m.bsky) { | |
| 31 | const r = m.bsky.relay || {} | |
| 32 | if (r.error) add('relay-unreachable', 'warning', 'bsky', 'cannot reach bsky\'s relay', r.error, 'check outbound network from the gateway host', { checks: 2 }) | |
| 33 | else if (r.status && r.status !== 'active') { | |
| 34 | add('relay-offline', 'critical', 'bsky', `bsky's relay has ${m.bsky.hostname} ${r.status}`, `relay seq ${r.seq}; nothing we post reaches bsky until it is active`, 'the gateway asks the relay to crawl by itself (at most every 15 min); if it stays offline: favela-gateway metrics refresh=true, then check that https://' + m.bsky.hostname + ' answers from outside') | |
| 35 | } | |
| 36 | for (const d of m.bsky.dids || []) { | |
| 37 | if (d.relayRev && d.ourRev && d.relayRev > d.ourRev) { | |
| 38 | add('relay-newer:' + d.did, 'critical', 'bsky', `bsky's relay holds a NEWER commit for ${d.did} than we serve`, | |
| 39 | `relay ${d.relayRev}, ours ${d.ourRev} — the relay drops every commit at or below its rev, so this DID's updates do not reach bsky`, | |
| 40 | `first confirm no co-owner (a phone, a desktop) holds a newer head we have not seen; then: favela-gateway emit-sync did=${d.did}`) | |
| 41 | } else if (d.relayRev && d.ourRev && d.relayRev < d.ourRev) { | |
| 42 | add('relay-behind:' + d.did, 'warning', 'bsky', `bsky's relay is behind on ${d.did}`, `relay ${d.relayRev}, ours ${d.ourRev}`, 'usually clears as the relay ingests; if it persists: favela-gateway emit-sync did=' + d.did, { checks: 2 }) | |
| 43 | } | |
| 44 | if (d.plcPds && d.plcPds !== 'https://' + m.bsky.hostname) { | |
| 45 | add('plc-away:' + d.did, 'critical', 'bsky', `${d.did}'s PLC document points elsewhere`, `#atproto_pds = ${d.plcPds}, not https://${m.bsky.hostname}`, 'the DID\'s owner must sign a PLC update back to this host (or stop fronting it here)') | |
| 46 | } | |
| 47 | if (d.appview && d.appview.resolves === false) { | |
| 48 | add('appview-unresolved:' + d.did, 'warning', 'bsky', `bsky's AppView does not resolve ${d.did}`, 'getProfile fails', 'favela-gateway probe-front did=' + d.did + '; check the relay alerts first', { checks: 2 }) | |
| 49 | } | |
| 50 | } | |
| 51 | } | |
| 52 | ||
| 53 | for (const d of (m.membrane && m.membrane.dids) || []) { | |
| 54 | if (d.live === 0) add('peers-offline:' + d.did, 'warning', 'local', `no co-owner of ${d.handle || d.did} is connected`, `${d.fronts} cached front(s), none live — new posts cannot arrive`, 'check that the owner\'s node (favela-host / hpp) is running and online', { ms: HOUR }) | |
| 55 | } | |
| 56 | ||
| 57 | const f = m.fediverse | |
| 58 | if (f) { | |
| 59 | if (f.queue.dead > 0) add('delivery-dead', 'warning', 'local', `${f.queue.dead} fediverse deliveries gave up`, 'after 8 attempts over ~24 h', 'favela-gateway notifications; look at the receiving servers in the admin panel') | |
| 60 | for (const e of f.queue.failing || []) { | |
| 61 | if (e.attempts >= 3) add('delivery-failing:' + e.host, 'warning', 'local', `deliveries to ${e.host} keep failing`, `${e.attempts} attempts — ${e.lastError || '?'}`, 'check whether ' + e.host + ' is up; the queue retries by itself') | |
| 62 | } | |
| 63 | } | |
| 64 | ||
| 65 | for (const feed of (m.lens && m.lens.feeds) || []) { | |
| 66 | if (feed.unfronted && feed.unfronted.length) { | |
| 67 | add('lens-unfronted:' + feed.id, 'warning', 'local', `${feed.id}: ${feed.unfronted.length} member(s) not fronted — their posts are missing from the feed`, feed.unfronted.join(', '), 'front them on the membrane (the member\'s node asks through the swarm door), or check the roster') | |
| 68 | } | |
| 69 | for (const p of feed.perMember || []) { | |
| 70 | if (p.error) add('lens-index:' + p.did, 'warning', 'local', `${feed.id}: cannot index ${p.did}`, p.error, 'favela-gateway probe-front did=' + p.did) | |
| 71 | } | |
| 72 | } | |
| 73 | return out | |
| 74 | } | |
| 75 | ||
| 76 | export function evaluateAlerts (m, prev = {}, now = Date.now()) { | |
| 77 | const state = {} | |
| 78 | const open = [] | |
| 79 | const resolved = [] | |
| 80 | const seen = new Set() | |
| 81 | ||
| 82 | for (const c of candidates(m)) { | |
| 83 | seen.add(c.id) | |
| 84 | const p = prev[c.id] | |
| 85 | const e = { ...c, firstSeen: p ? p.firstSeen : now, count: (p ? p.count : 0) + 1, lastSeen: now } | |
| 86 | e.open = e.count >= c.checks && now - e.firstSeen >= c.ms | |
| 87 | state[c.id] = e | |
| 88 | if (e.open) open.push(view(e)) | |
| 89 | } | |
| 90 | ||
| 91 | for (const [id, p] of Object.entries(prev)) { | |
| 92 | if (seen.has(id)) continue | |
| 93 | // Could not look: keep it exactly as it was (still open if it was). | |
| 94 | if (p.scope === 'bsky' && !m.bsky) { | |
| 95 | state[id] = p | |
| 96 | if (p.open) open.push(view(p)) | |
| 97 | continue | |
| 98 | } | |
| 99 | if (p.open) resolved.push({ ...view(p), resolvedAt: now }) | |
| 100 | } | |
| 101 | ||
| 102 | const rank = { critical: 0, warning: 1 } | |
| 103 | open.sort((a, b) => (rank[a.severity] - rank[b.severity]) || (a.since - b.since)) | |
| 104 | return { open, resolved, state } | |
| 105 | } | |
| 106 | ||
| 107 | function view (e) { | |
| 108 | return { id: e.id, severity: e.severity, title: e.title, detail: e.detail, fix: e.fix, since: e.firstSeen, lastSeen: e.lastSeen } | |
| 109 | } | |
| 110 | ||
| 111 | export default evaluateAlerts |
| … | … | @@ -8,6 +8,7 @@ export const USAGE = `favela-gateway — the favela gateway (bsky membrane + fed |
| 8 | 8 | favela-gateway run the gateway (pgh does this; refuses if one is already running) |
| 9 | 9 | favela-gateway metrics [--json] health: roles, peers, bsky's view of us, fediverse delivery |
| 10 | 10 | favela-gateway metrics refresh=true probe bsky now instead of using the 5-minute cache |
| 11 | favela-gateway alerts [refresh=true] open alerts, each with the command that fixes it | |
| 11 | 12 | favela-gateway status roles and ports |
| 12 | 13 | favela-gateway help the verbs the running gateway serves |
| 13 | 14 | favela-gateway <verb> [key=value …] any verb, e.g. probe-front did=did:plc:…, invite, notifications |
| … | … | @@ -48,10 +49,25 @@ const mb = (n) => (n == null ? '?' : Math.round(n / 1e6) + ' MB') |
| 48 | 49 | const dur = (s) => (s >= 3600 ? Math.floor(s / 3600) + 'h' + String(Math.floor((s % 3600) / 60)).padStart(2, '0') + 'm' : Math.floor(s / 60) + 'm' + (s % 60) + 's') |
| 49 | 50 | |
| 50 | 51 | /** The `metrics` answer as an operator reads it: what is up, what bsky sees, what is wrong. */ |
| 52 | /** Open alerts first: the part an operator must act on, with the command that fixes each. */ | |
| 53 | export function renderAlerts (a, { now = Date.now() } = {}) { | |
| 54 | const L = [] | |
| 55 | if (!a) return 'alerts: the watcher is not running' | |
| 56 | L.push(`alerts (last check ${ago(a.lastCheckAt, now)})`) | |
| 57 | if (!a.open.length) L.push(' none open') | |
| 58 | for (const x of a.open) { | |
| 59 | L.push(` ${x.severity === 'critical' ? 'CRITICAL' : 'warning '} ${x.title} (open ${ago(x.since, now).replace(' ago', '')})`) | |
| 60 | L.push(` ${x.detail}`) | |
| 61 | L.push(` fix: ${x.fix}`) | |
| 62 | } | |
| 63 | return L.join('\n') | |
| 64 | } | |
| 65 | ||
| 51 | 66 | export function renderMetrics (m, { now = Date.now() } = {}) { |
| 52 | 67 | const L = [] |
| 53 | 68 | L.push(`favela-gateway ${m.version || '?'} up ${dur(m.process.uptimeS)} rss ${mb(m.process.rss)} heap ${mb(m.process.heapUsed)}/${mb(m.process.heapTotal)}`) |
| 54 | 69 | L.push('') |
| 70 | if (m.alerts) { L.push(renderAlerts(m.alerts, { now })); L.push('') } | |
| 55 | 71 | L.push('roles') |
| 56 | 72 | for (const [name, r] of Object.entries(m.roles)) L.push(` ${name.padEnd(10)} ${r.up ? 'up' : 'DOWN ' + (r.error || '')}`) |
| 57 | 73 | |
| … | … | @@ -137,6 +153,7 @@ export async function runCli (argv, { sock, out = (s) => console.log(s), err = ( |
| 137 | 153 | } |
| 138 | 154 | const r = await c.call(verb, params) |
| 139 | 155 | if (verb === 'metrics' && !json) out(renderMetrics(r)) |
| 156 | else if (verb === 'alerts' && !json && r && r.open) out(renderAlerts(r)) | |
| 140 | 157 | else out(JSON.stringify(r, null, 2)) |
| 141 | 158 | return r && r.ok === false ? 1 : 0 |
| 142 | 159 | } catch (e) { |
| … | … | @@ -21,6 +21,7 @@ import { startApBridge } from 'ap-bridge/lib/bridge.js' |
| 21 | 21 | import os from 'bare-os' |
| 22 | 22 | import { createProbes } from './probes.js' |
| 23 | 23 | import { createLens } from './lens.js' |
| 24 | import { createWatcher } from './watch.js' | |
| 24 | 25 | |
| 25 | 26 | const ROLES = { |
| 26 | 27 | membrane: async (cfg, opts) => startMembrane(cfg, opts), |
| … | … | @@ -46,7 +47,7 @@ const defaultLog = (o) => console.log(JSON.stringify(o)) |
| 46 | 47 | * @param {Array} [opts.bootstrap] DHT bootstrap for every role's swarm (tests pass []) |
| 47 | 48 | * @param {string} [opts.version] |
| 48 | 49 | */ |
| 49 | export async function startGateway ({ roles, log = defaultLog, bootstrap = null, version = null, fetch = null, ...configs }) { | |
| 50 | export async function startGateway ({ roles, log = defaultLog, bootstrap = null, version = null, fetch = null, stateDir = null, watch = false, ...configs }) { | |
| 50 | 51 | for (const name of roles) if (!ROLES[name]) throw new Error('unknown role: ' + name) |
| 51 | 52 | const startedAt = Date.now() |
| 52 | 53 | |
| … | … | @@ -119,16 +120,36 @@ export async function startGateway ({ roles, log = defaultLog, bootstrap = null, |
| 119 | 120 | const dids = out.membrane.dids.map((d) => ({ did: d.did, rev: d.rev })) |
| 120 | 121 | out.bsky = await probes.get({ dids, emitted: out.membrane.relay.emitted }, { force: !!params.refresh }) |
| 121 | 122 | } |
| 123 | if (watcher) out.alerts = watcher.current() | |
| 122 | 124 | return out |
| 123 | 125 | } |
| 124 | 126 | |
| 127 | // The watcher: checks on its own and keeps alert state on disk (lib/watch.js). Its metrics calls skip | |
| 128 | // `alerts` (it is computing them). Off unless asked, so tests and one-off runs do not schedule checks. | |
| 129 | let watcher = null | |
| 130 | if (watch) { | |
| 131 | watcher = createWatcher({ | |
| 132 | metrics: async (p) => { const m = await metrics(p); delete m.alerts; return m }, | |
| 133 | stateFile: stateDir ? stateDir.replace(/\/$/, '') + '/alerts.json' : null, | |
| 134 | log: (o) => log({ role: 'alerts', ...o }) | |
| 135 | }) | |
| 136 | watcher.start() | |
| 137 | steps.unshift(['alerts', () => watcher.stop()]) | |
| 138 | } | |
| 139 | ||
| 125 | 140 | // ONE verb map. The roles' verbs do not overlap except `status`, which the gateway answers for all |
| 126 | 141 | // of them; a collision would be a bug, so it is refused here rather than silently shadowed. |
| 127 | 142 | const handlers = { status, metrics } |
| 143 | // Open alerts, recently resolved, last check. `refresh: true` checks now (fresh probes) and answers after. | |
| 144 | handlers.alerts = async (params = {}) => { | |
| 145 | if (!watcher) return { ok: false, error: 'the alert watcher is not running' } | |
| 146 | if (params.refresh) await watcher.checkNow() | |
| 147 | return watcher.current() | |
| 148 | } | |
| 128 | 149 | for (const name of roles) { |
| 129 | 150 | if (!running[name]) continue |
| 130 | 151 | for (const [verb, fn] of Object.entries(running[name].handlers)) { |
| 131 | if (verb === 'status' || verb === 'metrics') continue | |
| 152 | if (verb === 'status' || verb === 'metrics' || verb === 'alerts') continue | |
| 132 | 153 | if (handlers[verb]) throw new Error(`verb ${verb} is claimed by two roles`) |
| 133 | 154 | handlers[verb] = fn |
| 134 | 155 | } |
| … | … | @@ -0,0 +1,70 @@ |
| 1 | // The watcher — checks the gateway's health on its own and keeps the alert state. | |
| 2 | // | |
| 3 | // ⛔ WHY IT RUNS ON A TIMER when the probes otherwise run only on demand: a manual fix nobody hears about | |
| 4 | // is the outage (loser-club was invisible to bsky for 2½ months, and the relay had us "offline" with | |
| 5 | // nothing reporting it). So: one check 2 minutes after boot (an OTA restart is when the relay drops us), | |
| 6 | // then every 15 minutes — the relay guard's own cadence — each with FRESH probes. The gateway's RSS is | |
| 7 | // one of the alerts, so if repeated probing leaked (as favela-web's HTTPS did) it would say so. | |
| 8 | // | |
| 9 | // State lives in a JSON file beside the gateway's socket, so an alert's "open since" survives a restart. | |
| 10 | ||
| 11 | import fs from 'bare-fs' | |
| 12 | import path from 'bare-path' | |
| 13 | import { evaluateAlerts } from './alerts.js' | |
| 14 | ||
| 15 | const RESOLVED_KEEP = 20 | |
| 16 | ||
| 17 | export function createWatcher ({ metrics, stateFile = null, firstMs = 2 * 60 * 1000, intervalMs = 15 * 60 * 1000, now = () => Date.now(), log = () => {} } = {}) { | |
| 18 | let saved = { state: {}, lastCheckAt: null, recentlyResolved: [] } | |
| 19 | if (stateFile) { | |
| 20 | try { saved = { ...saved, ...JSON.parse(fs.readFileSync(stateFile, 'utf8')) } } catch {} | |
| 21 | } | |
| 22 | let open = Object.values(saved.state).filter((e) => e.open).map((e) => ({ id: e.id, severity: e.severity, title: e.title, detail: e.detail, fix: e.fix, since: e.firstSeen, lastSeen: e.lastSeen })) | |
| 23 | let first = null | |
| 24 | let timer = null | |
| 25 | let running = null | |
| 26 | ||
| 27 | function persist () { | |
| 28 | if (!stateFile) return | |
| 29 | try { | |
| 30 | fs.mkdirSync(path.dirname(stateFile), { recursive: true }) | |
| 31 | fs.writeFileSync(stateFile + '.tmp', JSON.stringify(saved)) | |
| 32 | fs.renameSync(stateFile + '.tmp', stateFile) | |
| 33 | } catch (err) { log({ msg: 'alerts: could not persist state', error: err.message }) } | |
| 34 | } | |
| 35 | ||
| 36 | async function check () { | |
| 37 | let m | |
| 38 | try { m = await metrics({ refresh: true }) } catch (err) { | |
| 39 | log({ msg: 'alerts: metrics failed — state unchanged', error: (err && err.message) || String(err) }) | |
| 40 | return | |
| 41 | } | |
| 42 | const t = now() | |
| 43 | const r = evaluateAlerts(m, saved.state, t) | |
| 44 | for (const a of r.open) if (!open.some((o) => o.id === a.id)) log({ msg: 'ALERT open', id: a.id, severity: a.severity, title: a.title, fix: a.fix }) | |
| 45 | for (const a of r.resolved) log({ msg: 'alert resolved', id: a.id, openFor: Math.round((t - a.since) / 1000) + 's' }) | |
| 46 | open = r.open | |
| 47 | saved = { state: r.state, lastCheckAt: t, recentlyResolved: [...r.resolved.reverse(), ...saved.recentlyResolved].slice(0, RESOLVED_KEEP) } | |
| 48 | persist() | |
| 49 | } | |
| 50 | ||
| 51 | // One check at a time; a request during a check waits for it. | |
| 52 | function checkNow () { | |
| 53 | if (!running) running = check().finally(() => { running = null }) | |
| 54 | return running | |
| 55 | } | |
| 56 | ||
| 57 | return { | |
| 58 | checkNow, | |
| 59 | current: () => ({ open, recentlyResolved: saved.recentlyResolved, lastCheckAt: saved.lastCheckAt }), | |
| 60 | start () { | |
| 61 | first = setTimeout(() => { checkNow() }, firstMs) | |
| 62 | timer = setInterval(() => { checkNow() }, intervalMs) | |
| 63 | if (first.unref) first.unref() | |
| 64 | if (timer.unref) timer.unref() | |
| 65 | }, | |
| 66 | stop () { clearTimeout(first); clearInterval(timer) } | |
| 67 | } | |
| 68 | } | |
| 69 | ||
| 70 | export default createWatcher |
| … | … | @@ -1,12 +1,12 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "favela-gateway", |
| 3 | "version": "0.3.0", | |
| 3 | "version": "0.4.0", | |
| 4 | 4 | "lockfileVersion": 3, |
| 5 | 5 | "requires": true, |
| 6 | 6 | "packages": { |
| 7 | 7 | "": { |
| 8 | 8 | "name": "favela-gateway", |
| 9 | "version": "0.3.0", | |
| 9 | "version": "0.4.0", | |
| 10 | 10 | "dependencies": { |
| 11 | 11 | "ap-bridge": "file:../ap-bridge", |
| 12 | 12 | "atproto-bridge": "file:../atproto-bridge", |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "favela-gateway", |
| 3 | "version": "0.3.0", | |
| 3 | "version": "0.4.0", | |
| 4 | 4 | "description": "The favela gateway: the bsky membrane (atproto-bridge) and the fediverse bridge (ap-bridge) in one process, keyless towards atproto. Later roles: the bsky lens and a blind seeder (hyper-social #80).", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "lib/gateway.js", |
| … | … | @@ -0,0 +1,120 @@ |
| 1 | // Alerts: what a person must act on, derived from `metrics` (hyper-social #80, operator 2026-10-13). | |
| 2 | // | |
| 3 | // ⛔ WHY. loser-club was invisible to bsky for 2½ months (the relay held a newer July rev) and the relay | |
| 4 | // had us "offline" with nothing reporting either. Fixing those by hand is fine — the operator chose manual | |
| 5 | // re-syncs — only if the gateway SAYS so: each alert names the problem, how long it has been open, and the | |
| 6 | // command that fixes it. Momentary blips need two checks in a row; sustained ones (peers gone) a duration. | |
| 7 | ||
| 8 | import test from 'brittle' | |
| 9 | import { evaluateAlerts } from '../lib/alerts.js' | |
| 10 | ||
| 11 | const HOST = 'pds.7rnx.net' | |
| 12 | const WJ = 'did:plc:wj62ebbusln6gxo5azo6ogzo' | |
| 13 | const LC = 'did:plc:wqnwulql7ndcz6xmtvaovl6u' | |
| 14 | ||
| 15 | function healthy () { | |
| 16 | return { | |
| 17 | process: { rss: 300e6 }, | |
| 18 | roles: { membrane: { up: true }, fediverse: { up: true }, lens: { up: true } }, | |
| 19 | membrane: { dids: [{ did: WJ, handle: 'p2p.7rnx.net', rev: 'b', live: 2, fronts: 2 }], relay: { emitted: 10 }, guard: { heals: 0 } }, | |
| 20 | fediverse: { queue: { pending: 0, dead: 0, failing: [] } }, | |
| 21 | lens: { feeds: [{ id: 'loser-club', unfronted: [], perMember: [] }] }, | |
| 22 | bsky: { hostname: HOST, relay: { status: 'active', seq: 10 }, dids: [{ did: WJ, ourRev: 'b', relayRev: 'b', inSync: true, plcPds: 'https://' + HOST, appview: { resolves: true } }] } | |
| 23 | } | |
| 24 | } | |
| 25 | ||
| 26 | const ids = (r) => r.open.map((a) => a.id).sort() | |
| 27 | ||
| 28 | test('★ healthy: nothing open', (t) => { | |
| 29 | const r = evaluateAlerts(healthy(), {}, 1000) | |
| 30 | t.alike(r.open, []) | |
| 31 | }) | |
| 32 | ||
| 33 | test('⛔ relay newer than us: critical at once, with the emit-sync command and the caution', (t) => { | |
| 34 | const m = healthy() | |
| 35 | m.bsky.dids[0] = { ...m.bsky.dids[0], relayRev: 'z', inSync: false } | |
| 36 | const r = evaluateAlerts(m, {}, 1000) | |
| 37 | const a = r.open.find((x) => x.id === 'relay-newer:' + WJ) | |
| 38 | t.ok(a, 'opened on the first check') | |
| 39 | t.is(a.severity, 'critical') | |
| 40 | t.ok(a.fix.includes('favela-gateway emit-sync did=' + WJ), 'names the command') | |
| 41 | t.ok(/co-owner/.test(a.fix), 'and what to confirm first') | |
| 42 | }) | |
| 43 | ||
| 44 | test('⛔ the relay has us offline: critical at once', (t) => { | |
| 45 | const m = healthy(); m.bsky.relay.status = 'offline' | |
| 46 | t.ok(ids(evaluateAlerts(m, {}, 1)).includes('relay-offline')) | |
| 47 | }) | |
| 48 | ||
| 49 | test('PLC pointing elsewhere: critical at once; a role down: critical at once', (t) => { | |
| 50 | const m = healthy() | |
| 51 | m.bsky.dids[0].plcPds = 'https://bsky.social' | |
| 52 | m.roles.lens = { up: false, error: 'boom' } | |
| 53 | const r = evaluateAlerts(m, {}, 1) | |
| 54 | t.ok(ids(r).includes('plc-away:' + WJ)) | |
| 55 | const down = r.open.find((a) => a.id === 'role-down:lens') | |
| 56 | t.ok(down && /boom/.test(down.detail)) | |
| 57 | }) | |
| 58 | ||
| 59 | test('a blip is not an alert: relay behind / AppView unresolved need two checks in a row', (t) => { | |
| 60 | const m = healthy() | |
| 61 | m.bsky.dids[0] = { ...m.bsky.dids[0], relayRev: 'a', inSync: false, appview: { resolves: false } } | |
| 62 | const one = evaluateAlerts(m, {}, 1000) | |
| 63 | t.alike(ids(one), [], 'first sighting: watched, not open') | |
| 64 | const two = evaluateAlerts(m, one.state, 2000) | |
| 65 | t.alike(ids(two), ['appview-unresolved:' + WJ, 'relay-behind:' + WJ], 'second check in a row: open') | |
| 66 | }) | |
| 67 | ||
| 68 | test('peers offline must last an hour before it is an alert', (t) => { | |
| 69 | const m = healthy(); m.membrane.dids[0].live = 0 | |
| 70 | const a = evaluateAlerts(m, {}, 0) | |
| 71 | const b = evaluateAlerts(m, a.state, 59 * 60e3) | |
| 72 | t.alike(ids(b), [], '59 minutes: not yet') | |
| 73 | const c = evaluateAlerts(m, b.state, 61 * 60e3) | |
| 74 | t.ok(ids(c).includes('peers-offline:' + WJ), '61 minutes: open') | |
| 75 | }) | |
| 76 | ||
| 77 | test('fediverse: dead deliveries and a server failing repeatedly', (t) => { | |
| 78 | const m = healthy() | |
| 79 | m.fediverse.queue = { pending: 2, dead: 1, failing: [{ host: 'prsm.space', attempts: 4, lastError: 'HTTP 502' }] } | |
| 80 | const r = evaluateAlerts(m, {}, 1) | |
| 81 | t.ok(ids(r).includes('delivery-dead')) | |
| 82 | const f = r.open.find((a) => a.id === 'delivery-failing:prsm.space') | |
| 83 | t.ok(f && /502/.test(f.detail)) | |
| 84 | }) | |
| 85 | ||
| 86 | test('lens: a member the membrane does not front, and a member it cannot index', (t) => { | |
| 87 | const m = healthy() | |
| 88 | m.lens.feeds[0].unfronted = ['did:plc:stranger'] | |
| 89 | m.lens.feeds[0].perMember = [{ did: WJ, error: 'no frontable co-owner' }] | |
| 90 | const r = evaluateAlerts(m, {}, 1) | |
| 91 | t.ok(ids(r).includes('lens-unfronted:loser-club')) | |
| 92 | t.ok(ids(r).includes('lens-index:' + WJ)) | |
| 93 | }) | |
| 94 | ||
| 95 | test('memory over the ceiling (a probe leak would show here)', (t) => { | |
| 96 | const m = healthy(); m.process.rss = 3e9 | |
| 97 | t.ok(ids(evaluateAlerts(m, {}, 1)).includes('memory')) | |
| 98 | }) | |
| 99 | ||
| 100 | test('★ open since is kept across checks; a cleared condition resolves and is reported once', (t) => { | |
| 101 | const m = healthy(); m.bsky.relay.status = 'offline' | |
| 102 | const a = evaluateAlerts(m, {}, 1000) | |
| 103 | const b = evaluateAlerts(m, a.state, 5000) | |
| 104 | t.is(b.open[0].since, 1000, 'since = first sighting, not the latest check') | |
| 105 | const ok = healthy() | |
| 106 | const c = evaluateAlerts(ok, b.state, 9000) | |
| 107 | t.alike(c.open, []) | |
| 108 | t.alike(c.resolved.map((x) => x.id), ['relay-offline'], 'reported as resolved') | |
| 109 | t.is(c.resolved[0].since, 1000) | |
| 110 | const d = evaluateAlerts(ok, c.state, 12000) | |
| 111 | t.alike(d.resolved, [], 'and only once') | |
| 112 | }) | |
| 113 | ||
| 114 | test('a check that could not reach bsky does not resolve bsky alerts it could not see', (t) => { | |
| 115 | const m = healthy(); m.bsky.relay.status = 'offline' | |
| 116 | const a = evaluateAlerts(m, {}, 1000) | |
| 117 | const blind = healthy(); blind.bsky = null | |
| 118 | const b = evaluateAlerts(blind, a.state, 2000) | |
| 119 | t.ok(ids(b).includes('relay-offline'), 'still open: nothing showed it cleared') | |
| 120 | }) |
| … | … | @@ -89,3 +89,11 @@ test('metrics --json prints the raw answer', async (t) => { |
| 89 | 89 | await runCli(['metrics', '--json'], { sock: '/x', out: out.write, err: out.write, connect: fakeConnect({ metrics: sampleMetrics }) }) |
| 90 | 90 | t.is(JSON.parse(out.text()).version, '0.2.0') |
| 91 | 91 | }) |
| 92 | ||
| 93 | test('alerts render first, each with its fix', async (t) => { | |
| 94 | const { renderAlerts } = await import('../lib/cli.js') | |
| 95 | const s = renderAlerts({ lastCheckAt: 1e12, open: [{ id: 'relay-newer:x', severity: 'critical', title: 'relay newer', detail: 'relay z, ours a', fix: 'favela-gateway emit-sync did=x', since: 1e12 - 7200e3 }], recentlyResolved: [] }, { now: 1e12 }) | |
| 96 | t.ok(/CRITICAL relay newer/.test(s)) | |
| 97 | t.ok(s.includes('fix: favela-gateway emit-sync did=x')) | |
| 98 | t.ok(/open 2h/.test(s), 'how long it has been open') | |
| 99 | }) |
| … | … | @@ -143,3 +143,15 @@ test('⛔ the lens without a running membrane is down, with the reason — it se |
| 143 | 143 | t.absent(g.roles.lens) |
| 144 | 144 | t.ok(/needs the membrane/.test(g.down.lens), g.down.lens) |
| 145 | 145 | }) |
| 146 | ||
| 147 | test('★ alerts: with the watcher on, `alerts refresh=true` checks now and `metrics` carries the result', async (t) => { | |
| 148 | const offlineRelay = async (url) => ({ ok: true, status: 200, json: async () => (url.includes('getHostStatus') ? { status: 'offline', seq: 5 } : {}) }) | |
| 149 | const g = await startGateway({ roles: ['membrane'], membrane: membraneConfig(), fetch: offlineRelay, watch: true, stateDir: tmp('gwa-st-'), ...quiet }) | |
| 150 | t.teardown(() => stop(g)) | |
| 151 | const a = await g.handlers.alerts({ refresh: true }) | |
| 152 | t.ok(a.open.some((x) => x.id === 'relay-offline'), 'the relay-offline alert is open') | |
| 153 | t.ok(a.lastCheckAt > 0) | |
| 154 | const m = await g.handlers.metrics({}) | |
| 155 | t.ok(m.alerts && m.alerts.open.length, 'metrics carries the alerts') | |
| 156 | t.is(g.steps[0][0], 'alerts', 'the watcher stops first') | |
| 157 | }) |
| … | … | @@ -0,0 +1,61 @@ |
| 1 | // The watcher: checks on its own (agreed 2026-10-13 — a manual fix nobody hears about is the outage), keeps | |
| 2 | // alert state on disk so "open since" survives a restart, and remembers what resolved. | |
| 3 | import test from 'brittle' | |
| 4 | import fs from 'bare-fs' | |
| 5 | import path from 'bare-path' | |
| 6 | import { createWatcher } from '../lib/watch.js' | |
| 7 | ||
| 8 | const tmp = () => fs.mkdtempSync(path.join('/tmp', 'gw-watch-')) | |
| 9 | const offline = () => ({ roles: {}, process: { rss: 1 }, bsky: { hostname: 'h', relay: { status: 'offline', seq: 1 }, dids: [] } }) | |
| 10 | const healthy = () => ({ roles: {}, process: { rss: 1 }, bsky: { hostname: 'h', relay: { status: 'active', seq: 1 }, dids: [] } }) | |
| 11 | ||
| 12 | test('★ a check opens alerts, persists them, and a new watcher (a restart) keeps "since"', async (t) => { | |
| 13 | const dir = tmp() | |
| 14 | let now = 1000 | |
| 15 | const calls = [] | |
| 16 | const w1 = createWatcher({ metrics: async (p) => { calls.push(p); return offline() }, stateFile: path.join(dir, 'alerts.json'), now: () => now, log: () => {} }) | |
| 17 | await w1.checkNow() | |
| 18 | t.alike(calls, [{ refresh: true }], 'checks with FRESH probes, not the cache') | |
| 19 | t.alike(w1.current().open.map((a) => a.id), ['relay-offline']) | |
| 20 | w1.stop() | |
| 21 | now = 900000 | |
| 22 | const w2 = createWatcher({ metrics: async () => offline(), stateFile: path.join(dir, 'alerts.json'), now: () => now, log: () => {} }) | |
| 23 | t.is(w2.current().open[0].since, 1000, 'read back from disk before any new check') | |
| 24 | await w2.checkNow() | |
| 25 | t.is(w2.current().open[0].since, 1000, 'and kept by the next check') | |
| 26 | t.is(w2.current().lastCheckAt, 900000) | |
| 27 | w2.stop() | |
| 28 | }) | |
| 29 | ||
| 30 | test('resolved alerts are remembered (newest first, bounded) for the panel', async (t) => { | |
| 31 | const dir = tmp() | |
| 32 | let m = offline() | |
| 33 | const w = createWatcher({ metrics: async () => m, stateFile: path.join(dir, 'a.json'), log: () => {} }) | |
| 34 | t.teardown(() => w.stop()) | |
| 35 | await w.checkNow() | |
| 36 | m = healthy() | |
| 37 | await w.checkNow() | |
| 38 | t.alike(w.current().open, []) | |
| 39 | t.alike(w.current().recentlyResolved.map((a) => a.id), ['relay-offline']) | |
| 40 | }) | |
| 41 | ||
| 42 | test('a failing metrics call is logged and leaves the state untouched', async (t) => { | |
| 43 | const lines = [] | |
| 44 | const w = createWatcher({ metrics: async () => { throw new Error('boom') }, stateFile: null, log: (o) => lines.push(o) }) | |
| 45 | t.teardown(() => w.stop()) | |
| 46 | await w.checkNow() | |
| 47 | t.ok(lines.some((l) => /boom/.test(l.error || ''))) | |
| 48 | t.alike(w.current().open, []) | |
| 49 | }) | |
| 50 | ||
| 51 | test('start() schedules the first check and the interval; stop() clears both', async (t) => { | |
| 52 | let n = 0 | |
| 53 | const w = createWatcher({ metrics: async () => { n++; return healthy() }, stateFile: null, firstMs: 20, intervalMs: 30, log: () => {} }) | |
| 54 | w.start() | |
| 55 | await new Promise((resolve) => setTimeout(resolve, 120)) | |
| 56 | w.stop() | |
| 57 | const after = n | |
| 58 | t.ok(after >= 2, 'first check plus at least one interval check (' + after + ')') | |
| 59 | await new Promise((resolve) => setTimeout(resolve, 80)) | |
| 60 | t.is(n, after, 'nothing runs after stop') | |
| 61 | }) |