// Alerts — what a person must act on, derived from `metrics` (hyper-social #80; operator 2026-10-13:
// manual re-syncs are fine only if the gateway SAYS when one is needed).
//
// Every alert has a stable id (so "open since" survives checks and restarts), a severity, the problem in
// plain words, and the FIX: the command to run or what to check. Pure: evaluateAlerts(metrics, state, now)
// returns { open, resolved, state }; the watcher persists `state` and the panel and CLI render `open`.
//
// Thresholds: conditions that blip (the relay a rev behind while it ingests, the AppView briefly not
// resolving) open only after TWO checks in a row; peers gone must last an HOUR; everything that is wrong
// the moment it is seen (relay offline, relay newer than us, PLC elsewhere, a role down) opens at once.
//
// ⛔ A check that could not see bsky (probes unreachable → metrics.bsky null) neither opens nor RESOLVES
// bsky alerts: absence of evidence is not a fix.
const HOUR = 60 * 60 * 1000
const RSS_CEILING = 2e9
function candidates (m) {
const out = []
const add = (id, severity, scope, title, detail, fix, need = {}) => out.push({ id, severity, scope, title, detail, fix, checks: need.checks || 1, ms: need.ms || 0 })
for (const [name, r] of Object.entries(m.roles || {})) {
if (!r.up) add('role-down:' + name, 'critical', 'local', `gateway role "${name}" is down`, r.error || 'no error recorded', 'favela-gateway status; pgh app log favela-gateway')
}
if (m.process && m.process.rss > RSS_CEILING) {
add('memory', 'warning', 'local', 'gateway memory is high', `RSS ${Math.round(m.process.rss / 1e6)} MB (ceiling ${RSS_CEILING / 1e6} MB) — repeated outbound HTTPS under Bare has leaked before`, 'favela-gateway metrics (watch RSS); pgh app restart favela-gateway if it keeps growing')
}
if (m.bsky) {
const r = m.bsky.relay || {}
if (r.error) add('relay-unreachable', 'warning', 'bsky', 'cannot reach bsky\'s relay', r.error, 'check outbound network from the gateway host', { checks: 2 })
else if (r.status && r.status !== 'active') {
add('relay-offline', 'critical', 'bsky', `bsky's relay has ${m.bsky.hostname} ${r.status}`, `relay seq ${r.seq}; nothing we post reaches bsky until it is active`, 'the gateway asks the relay to crawl by itself (at most every 15 min); if it stays offline: favela-gateway metrics refresh=true, then check that https://' + m.bsky.hostname + ' answers from outside')
}
for (const d of m.bsky.dids || []) {
if (d.relayRev && d.ourRev && d.relayRev > d.ourRev) {
add('relay-newer:' + d.did, 'critical', 'bsky', `bsky's relay holds a NEWER commit for ${d.did} than we serve`,
`relay ${d.relayRev}, ours ${d.ourRev} — the relay drops every commit at or below its rev, so this DID's updates do not reach bsky`,
`first confirm no co-owner (a phone, a desktop) holds a newer head we have not seen; then: favela-gateway emit-sync did=${d.did}`)
} else if (d.relayRev && d.ourRev && d.relayRev < d.ourRev) {
add('relay-behind:' + d.did, 'warning', 'bsky', `bsky's relay is behind on ${d.did}`, `relay ${d.relayRev}, ours ${d.ourRev}`, 'usually clears as the relay ingests; if it persists: favela-gateway emit-sync did=' + d.did, { checks: 2 })
}
if (d.plcPds && d.plcPds !== 'https://' + m.bsky.hostname) {
add('plc-away:' + d.did, 'critical', 'bsky', `${d.did}'s PLC document points elsewhere`, `#atproto_pds = ${d.plcPds}, not https://${m.bsky.hostname}`, 'the DID\'s owner must sign a PLC update back to this host (or stop fronting it here)')
}
if (d.appview && d.appview.resolves === false) {
add('appview-unresolved:' + d.did, 'warning', 'bsky', `bsky's AppView does not resolve ${d.did}`, 'getProfile fails', 'favela-gateway probe-front did=' + d.did + '; check the relay alerts first', { checks: 2 })
}
}
}
for (const d of (m.membrane && m.membrane.dids) || []) {
if (d.live === 0) add('peers-offline:' + d.did, 'warning', 'local', `no co-owner of ${d.handle || d.did} is connected`, `${d.fronts} cached front(s), none live — new posts cannot arrive`, 'check that the owner\'s node (favela-host / hpp) is running and online', { ms: HOUR })
}
const f = m.fediverse
if (f) {
if (f.queue.dead > 0) add('delivery-dead', 'warning', 'local', `${f.queue.dead} fediverse deliveries gave up`, 'after 8 attempts over ~24 h', 'favela-gateway notifications; look at the receiving servers in the admin panel')
for (const e of f.queue.failing || []) {
if (e.attempts >= 3) add('delivery-failing:' + e.host, 'warning', 'local', `deliveries to ${e.host} keep failing`, `${e.attempts} attempts — ${e.lastError || '?'}`, 'check whether ' + e.host + ' is up; the queue retries by itself')
}
}
for (const feed of (m.lens && m.lens.feeds) || []) {
if (feed.unfronted && feed.unfronted.length) {
add('lens-unfronted:' + feed.id, 'warning', 'local', `${feed.id}: ${feed.unfronted.length} member(s) not fronted — their posts are missing from the feed`, feed.unfronted.join(', '), 'front them on the membrane (the member\'s node asks through the swarm door), or check the roster')
}
for (const p of feed.perMember || []) {
if (p.error) add('lens-index:' + p.did, 'warning', 'local', `${feed.id}: cannot index ${p.did}`, p.error, 'favela-gateway probe-front did=' + p.did)
}
}
return out
}
export function evaluateAlerts (m, prev = {}, now = Date.now()) {
const state = {}
const open = []
const resolved = []
const seen = new Set()
for (const c of candidates(m)) {
seen.add(c.id)
const p = prev[c.id]
const e = { ...c, firstSeen: p ? p.firstSeen : now, count: (p ? p.count : 0) + 1, lastSeen: now }
e.open = e.count >= c.checks && now - e.firstSeen >= c.ms
state[c.id] = e
if (e.open) open.push(view(e))
}
for (const [id, p] of Object.entries(prev)) {
if (seen.has(id)) continue
// Could not look: keep it exactly as it was (still open if it was).
if (p.scope === 'bsky' && !m.bsky) {
state[id] = p
if (p.open) open.push(view(p))
continue
}
if (p.open) resolved.push({ ...view(p), resolvedAt: now })
}
const rank = { critical: 0, warning: 1 }
open.sort((a, b) => (rank[a.severity] - rank[b.severity]) || (a.since - b.since))
return { open, resolved, state }
}
function view (e) {
return { id: e.id, severity: e.severity, title: e.title, detail: e.detail, fix: e.fix, since: e.firstSeen, lastSeen: e.lastSeen }
}
export default evaluateAlerts