OverviewFilesCommitsIssuesDocsReleasesPeers

favela-gateway / test / alerts.test.js

121 lines · javascript ref:
// Alerts: what a person must act on, derived from `metrics` (hyper-social #80, operator 2026-10-13).
//
// ⛔ WHY. loser-club was invisible to bsky for 2½ months (the relay held a newer July rev) and the relay
// had us "offline" with nothing reporting either. Fixing those by hand is fine — the operator chose manual
// re-syncs — only if the gateway SAYS so: each alert names the problem, how long it has been open, and the
// command that fixes it. Momentary blips need two checks in a row; sustained ones (peers gone) a duration.

import test from 'brittle'
import { evaluateAlerts } from '../lib/alerts.js'

const HOST = 'pds.7rnx.net'
const WJ = 'did:plc:wj62ebbusln6gxo5azo6ogzo'
const LC = 'did:plc:wqnwulql7ndcz6xmtvaovl6u'

function healthy () {
  return {
    process: { rss: 300e6 },
    roles: { membrane: { up: true }, fediverse: { up: true }, lens: { up: true } },
    membrane: { dids: [{ did: WJ, handle: 'p2p.7rnx.net', rev: 'b', live: 2, fronts: 2 }], relay: { emitted: 10 }, guard: { heals: 0 } },
    fediverse: { queue: { pending: 0, dead: 0, failing: [] } },
    lens: { feeds: [{ id: 'loser-club', unfronted: [], perMember: [] }] },
    bsky: { hostname: HOST, relay: { status: 'active', seq: 10 }, dids: [{ did: WJ, ourRev: 'b', relayRev: 'b', inSync: true, plcPds: 'https://' + HOST, appview: { resolves: true } }] }
  }
}

const ids = (r) => r.open.map((a) => a.id).sort()

test('★ healthy: nothing open', (t) => {
  const r = evaluateAlerts(healthy(), {}, 1000)
  t.alike(r.open, [])
})

test('⛔ relay newer than us: critical at once, with the emit-sync command and the caution', (t) => {
  const m = healthy()
  m.bsky.dids[0] = { ...m.bsky.dids[0], relayRev: 'z', inSync: false }
  const r = evaluateAlerts(m, {}, 1000)
  const a = r.open.find((x) => x.id === 'relay-newer:' + WJ)
  t.ok(a, 'opened on the first check')
  t.is(a.severity, 'critical')
  t.ok(a.fix.includes('favela-gateway emit-sync did=' + WJ), 'names the command')
  t.ok(/co-owner/.test(a.fix), 'and what to confirm first')
})

test('⛔ the relay has us offline: critical at once', (t) => {
  const m = healthy(); m.bsky.relay.status = 'offline'
  t.ok(ids(evaluateAlerts(m, {}, 1)).includes('relay-offline'))
})

test('PLC pointing elsewhere: critical at once; a role down: critical at once', (t) => {
  const m = healthy()
  m.bsky.dids[0].plcPds = 'https://bsky.social'
  m.roles.lens = { up: false, error: 'boom' }
  const r = evaluateAlerts(m, {}, 1)
  t.ok(ids(r).includes('plc-away:' + WJ))
  const down = r.open.find((a) => a.id === 'role-down:lens')
  t.ok(down && /boom/.test(down.detail))
})

test('a blip is not an alert: relay behind / AppView unresolved need two checks in a row', (t) => {
  const m = healthy()
  m.bsky.dids[0] = { ...m.bsky.dids[0], relayRev: 'a', inSync: false, appview: { resolves: false } }
  const one = evaluateAlerts(m, {}, 1000)
  t.alike(ids(one), [], 'first sighting: watched, not open')
  const two = evaluateAlerts(m, one.state, 2000)
  t.alike(ids(two), ['appview-unresolved:' + WJ, 'relay-behind:' + WJ], 'second check in a row: open')
})

test('peers offline must last an hour before it is an alert', (t) => {
  const m = healthy(); m.membrane.dids[0].live = 0
  const a = evaluateAlerts(m, {}, 0)
  const b = evaluateAlerts(m, a.state, 59 * 60e3)
  t.alike(ids(b), [], '59 minutes: not yet')
  const c = evaluateAlerts(m, b.state, 61 * 60e3)
  t.ok(ids(c).includes('peers-offline:' + WJ), '61 minutes: open')
})

test('fediverse: dead deliveries and a server failing repeatedly', (t) => {
  const m = healthy()
  m.fediverse.queue = { pending: 2, dead: 1, failing: [{ host: 'prsm.space', attempts: 4, lastError: 'HTTP 502' }] }
  const r = evaluateAlerts(m, {}, 1)
  t.ok(ids(r).includes('delivery-dead'))
  const f = r.open.find((a) => a.id === 'delivery-failing:prsm.space')
  t.ok(f && /502/.test(f.detail))
})

test('lens: a member the membrane does not front, and a member it cannot index', (t) => {
  const m = healthy()
  m.lens.feeds[0].unfronted = ['did:plc:stranger']
  m.lens.feeds[0].perMember = [{ did: WJ, error: 'no frontable co-owner' }]
  const r = evaluateAlerts(m, {}, 1)
  t.ok(ids(r).includes('lens-unfronted:loser-club'))
  t.ok(ids(r).includes('lens-index:' + WJ))
})

test('memory over the ceiling (a probe leak would show here)', (t) => {
  const m = healthy(); m.process.rss = 3e9
  t.ok(ids(evaluateAlerts(m, {}, 1)).includes('memory'))
})

test('★ open since is kept across checks; a cleared condition resolves and is reported once', (t) => {
  const m = healthy(); m.bsky.relay.status = 'offline'
  const a = evaluateAlerts(m, {}, 1000)
  const b = evaluateAlerts(m, a.state, 5000)
  t.is(b.open[0].since, 1000, 'since = first sighting, not the latest check')
  const ok = healthy()
  const c = evaluateAlerts(ok, b.state, 9000)
  t.alike(c.open, [])
  t.alike(c.resolved.map((x) => x.id), ['relay-offline'], 'reported as resolved')
  t.is(c.resolved[0].since, 1000)
  const d = evaluateAlerts(ok, c.state, 12000)
  t.alike(d.resolved, [], 'and only once')
})

test('a check that could not reach bsky does not resolve bsky alerts it could not see', (t) => {
  const m = healthy(); m.bsky.relay.status = 'offline'
  const a = evaluateAlerts(m, {}, 1000)
  const blind = healthy(); blind.bsky = null
  const b = evaluateAlerts(blind, a.state, 2000)
  t.ok(ids(b).includes('relay-offline'), 'still open: nothing showed it cleared')
})