| … | … | @@ -22,10 +22,12 @@ import os from 'bare-os' |
| 22 | 22 | import { createProbes } from './probes.js' |
| 23 | 23 | import { createLens } from './lens.js' |
| 24 | 24 | import { createWatcher } from './watch.js' |
| 25 | import { renderStatusPage } from './panel.js' | |
| 25 | 26 | |
| 26 | 27 | const ROLES = { |
| 27 | 28 | membrane: async (cfg, opts) => startMembrane(cfg, opts), |
| 28 | fediverse: async (cfg, opts) => startApBridge(cfg, opts), | |
| 29 | // The fediverse role's admin surface (:3401, authenticated) also serves the gateway's status page. | |
| 30 | fediverse: async (cfg, opts) => startApBridge(cfg, { ...opts, adminPages: opts.adminPages }), | |
| 29 | 31 | // The lens serves THROUGH the membrane (its XRPC extensions, its store, its swarm, its fronts), so it |
| 30 | 32 | // needs the membrane role up and started first. |
| 31 | 33 | lens: async (cfg, opts, running) => { |
| … | … | @@ -50,6 +52,8 @@ const defaultLog = (o) => console.log(JSON.stringify(o)) |
| 50 | 52 | export async function startGateway ({ roles, log = defaultLog, bootstrap = null, version = null, fetch = null, stateDir = null, watch = false, ...configs }) { |
| 51 | 53 | for (const name of roles) if (!ROLES[name]) throw new Error('unknown role: ' + name) |
| 52 | 54 | const startedAt = Date.now() |
| 55 | // Admin pages for the fediverse role's authenticated surface; filled once every role has started. | |
| 56 | const adminPages = new Map() | |
| 53 | 57 | |
| 54 | 58 | const running = {} |
| 55 | 59 | const down = {} |
| … | … | @@ -58,7 +62,7 @@ export async function startGateway ({ roles, log = defaultLog, bootstrap = null, |
| 58 | 62 | try { |
| 59 | 63 | // A config the entry could not load arrives as an Error: that role is down, for that reason. |
| 60 | 64 | if (configs[name] instanceof Error) throw configs[name] |
| 61 | running[name] = await ROLES[name](configs[name], { log: roleLog, bootstrap, version }, running) | |
| 65 | running[name] = await ROLES[name](configs[name], { log: roleLog, bootstrap, version, adminPages }, running) | |
| 62 | 66 | log({ msg: 'role up', role: name }) |
| 63 | 67 | } catch (err) { |
| 64 | 68 | down[name] = (err && err.message) || String(err) |
| … | … | @@ -155,10 +159,13 @@ export async function startGateway ({ roles, log = defaultLog, bootstrap = null, |
| 155 | 159 | } |
| 156 | 160 | } |
| 157 | 161 | |
| 162 | // The status page on the admin surface: both bridges' health, alerts first (lib/panel.js). | |
| 163 | adminPages.set('/admin/status', async () => renderStatusPage(await metrics({}))) | |
| 164 | ||
| 158 | 165 | // The verbs this gateway serves, for `favela-gateway help`. release-ready is the process owner's. |
| 159 | 166 | handlers.verbs = async () => ({ verbs: [...Object.keys(handlers), 'release-ready'].sort() }) |
| 160 | 167 | |
| 161 | return { roles: running, down, handlers, status, metrics, steps } | |
| 168 | return { roles: running, down, handlers, status, metrics, adminPages, steps } | |
| 162 | 169 | } |
| 163 | 170 | |
| 164 | 171 | export default startGateway |
| … | … | @@ -0,0 +1,96 @@ |
| 1 | // The gateway status page — `metrics` as one admin page, alerts first (hyper-social #80; operator | |
| 2 | // 2026-10-13: "metrics / errors for both ap / atproto bridges via webui"). | |
| 3 | // | |
| 4 | // Served on ap-bridge's authenticated admin surface (:3401) as an added page (ap-bridge 0.10.17 | |
| 5 | // adminPages), so it sits behind the same login. Server-rendered HTML, no client JS, in that surface's | |
| 6 | // own chrome. Everything that came from outside — handles, error strings, alert details — is escaped. | |
| 7 | ||
| 8 | const esc = (s) => String(s == null ? '' : s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"') | |
| 9 | ||
| 10 | const ago = (ts, now) => { | |
| 11 | if (!ts) return 'never' | |
| 12 | const s = Math.max(0, Math.round((now - ts) / 1000)) | |
| 13 | if (s < 90) return s + 's ago' | |
| 14 | if (s < 90 * 60) return Math.round(s / 60) + 'm ago' | |
| 15 | if (s < 36 * 3600) return Math.round(s / 3600) + 'h ago' | |
| 16 | return Math.round(s / 86400) + 'd ago' | |
| 17 | } | |
| 18 | const mb = (n) => (n == null ? '?' : Math.round(n / 1e6) + ' MB') | |
| 19 | const dur = (s) => (s >= 86400 ? Math.floor(s / 86400) + 'd ' : '') + Math.floor((s % 86400) / 3600) + 'h ' + Math.floor((s % 3600) / 60) + 'm' | |
| 20 | const ok = (b) => (b ? '<span>ok</span>' : '<span class="warn">NO</span>') | |
| 21 | const table = (head, rows) => '<table><tr>' + head.map((h) => '<th>' + esc(h) + '</th>').join('') + '</tr>' + rows.join('') + '</table>' | |
| 22 | const row = (cells) => '<tr>' + cells.map((c) => '<td>' + c + '</td>').join('') + '</tr>' | |
| 23 | ||
| 24 | function alertsSection (a, now) { | |
| 25 | if (!a) return '<h2>Alerts</h2><p class="warn">The alert watcher is not running — problems are not being watched.</p>' | |
| 26 | const open = a.open.length | |
| 27 | ? table(['', 'problem', 'since', 'fix'], a.open.map((x) => row([ | |
| 28 | x.severity === 'critical' ? '<strong class="warn">CRITICAL</strong>' : 'warning', | |
| 29 | '<strong>' + esc(x.title) + '</strong><br><span class="who">' + esc(x.detail) + '</span>', | |
| 30 | 'open ' + esc(ago(x.since, now).replace(' ago', '')), | |
| 31 | '<code>' + esc(x.fix) + '</code>' | |
| 32 | ]))) | |
| 33 | : '<p>No open alerts.</p>' | |
| 34 | const resolved = (a.recentlyResolved || []).slice(0, 5) | |
| 35 | const res = resolved.length | |
| 36 | ? '<p class="who">Recently resolved: ' + resolved.map((x) => esc(x.title) + ' (resolved ' + esc(ago(x.resolvedAt, now)) + ')').join(' · ') + '</p>' | |
| 37 | : '' | |
| 38 | return '<h2>Alerts</h2><p class="who">Last check ' + esc(ago(a.lastCheckAt, now)) + ' (every 15 min; <code>favela-gateway alerts refresh=true</code> checks now).</p>' + open + res | |
| 39 | } | |
| 40 | ||
| 41 | export function renderStatusPage (m, { now = Date.now() } = {}) { | |
| 42 | const parts = [] | |
| 43 | parts.push('<p class="who">favela-gateway ' + esc(m.version || '?') + ' · up ' + esc(dur(m.process.uptimeS)) + ' · RSS ' + esc(mb(m.process.rss)) + ' · heap ' + esc(mb(m.process.heapUsed)) + '/' + esc(mb(m.process.heapTotal)) + '</p>') | |
| 44 | parts.push(alertsSection(m.alerts, now)) | |
| 45 | ||
| 46 | parts.push('<h2>Roles</h2>' + table(['role', 'state'], Object.entries(m.roles).map(([n, r]) => row([esc(n), r.up ? 'up' : '<strong class="warn">DOWN</strong> ' + esc(r.error || '')])))) | |
| 47 | ||
| 48 | if (m.membrane) { | |
| 49 | const g = m.membrane.guard || {} | |
| 50 | const b = m.bsky | |
| 51 | parts.push('<h2>bsky</h2>' + (b | |
| 52 | ? '<p>Relay sees <code>' + esc(b.hostname) + '</code> as <strong' + (b.relay.status === 'active' ? '' : ' class="warn"') + '>' + esc(b.relay.error ? 'UNREACHABLE ' + b.relay.error : b.relay.status) + '</strong> · relay seq ' + esc(b.relay.seq) + ' · ours ' + esc(m.membrane.relay.emitted) + ' · probed ' + esc(ago(b.at, now)) + '</p>' | |
| 53 | : '<p class="warn">No bsky probe yet.</p>') + | |
| 54 | '<p class="who">Relay guard: ' + (g.configured ? 'last pass ' + esc(ago(g.lastAt, now)) + (g.last && g.last.relayStatus ? ' (saw ' + esc(g.last.relayStatus) + ')' : '') + ' · self-heals ' + esc(g.heals || 0) + (g.lastHealAt ? ' (last ' + esc(ago(g.lastHealAt, now)) + ')' : '') : '<span class="warn">off</span>') + '</p>') | |
| 55 | ||
| 56 | const byDid = new Map(((b && b.dids) || []).map((d) => [d.did, d])) | |
| 57 | parts.push('<h2>Served DIDs</h2>' + table(['account', 'our head', 'bsky relay', 'peers', 'last append', 'PLC', 'AppView'], m.membrane.dids.map((d) => { | |
| 58 | const p = byDid.get(d.did) || {} | |
| 59 | const relay = p.inSync === true ? 'in sync' : p.relayRev ? '<span class="warn">' + esc(p.relayRev) + '</span>' : '?' | |
| 60 | const plc = p.plcPds ? ok(p.plcPds === 'https://' + (b && b.hostname)) : '?' | |
| 61 | const av = p.appview ? (p.appview.resolves ? esc(p.appview.postsCount) + ' posts, newest ' + esc((p.appview.newestIndexedAt || '?').slice(0, 10)) : '<span class="warn">does not resolve</span>') : '?' | |
| 62 | return row([ | |
| 63 | '<strong>' + esc(d.handle || p.handle || '') + '</strong><br><code>' + esc(d.did) + '</code>', | |
| 64 | '<code>' + esc(d.rev || '?') + '</code>', | |
| 65 | relay, | |
| 66 | (d.live ? '' : '<span class="warn">') + esc(d.live) + '/' + esc(d.fronts) + ' live' + (d.live ? '' : '</span>') + (d.forkRejected ? '<br>' + esc(d.forkRejected) + ' rival-line rejections' : ''), | |
| 67 | esc(ago(d.lastAppendAt, now)), | |
| 68 | plc, | |
| 69 | av | |
| 70 | ]) | |
| 71 | }))) | |
| 72 | } | |
| 73 | ||
| 74 | if (m.lens) { | |
| 75 | parts.push('<h2>Lens (bsky custom feeds)</h2>' + table(['feed', 'posts', 'members', 'newest', 'notes'], m.lens.feeds.map((f) => row([ | |
| 76 | '<strong>' + esc(f.id) + '</strong>' + (f.feedUri ? '<br><code>' + esc(f.feedUri) + '</code>' : ''), | |
| 77 | f.communityDid ? esc(f.posts) + ' posts' : '<span class="warn">waiting for the room</span>', | |
| 78 | esc(f.members), | |
| 79 | esc(f.newest ? f.newest.slice(0, 10) : '—'), | |
| 80 | [f.curation !== 'roster' ? '<span class="warn">curation ' + esc(f.curation) + ' not implemented — serving roster</span>' : '', | |
| 81 | f.unfronted.length ? '<span class="warn">not fronted: ' + f.unfronted.map(esc).join(', ') + '</span>' : '', | |
| 82 | ...f.perMember.filter((p) => p.error).map((p) => '<span class="warn">' + esc(p.did) + ': ' + esc(p.error) + '</span>')].filter(Boolean).join('<br>') || '—' | |
| 83 | ])))) | |
| 84 | } | |
| 85 | ||
| 86 | if (m.fediverse) { | |
| 87 | const f = m.fediverse | |
| 88 | parts.push('<h2>Fediverse</h2>' + table(['actor', 'followers', 'delivered', ''], f.users.map((u) => row(['<code>@' + esc(u.username) + '</code>', esc(u.followers), esc(u.delivered) + ' delivered', u.primary ? 'primary' : u.held ? 'held' : '<span class="warn">not held</span>']))) + | |
| 89 | '<p>Queue: ' + esc(f.queue.pending) + ' pending, ' + (f.queue.dead ? '<strong class="warn">' + esc(f.queue.dead) + ' dead</strong>' : '0 dead') + (f.queue.oldestPendingAt ? ', oldest ' + esc(ago(f.queue.oldestPendingAt, now)) : '') + ' · inbound ' + esc(f.inbound.count) + ', last ' + esc(ago(f.inbound.lastReceivedAt, now)) + ' · registered ' + esc(f.registrations) + '</p>' + | |
| 90 | (f.queue.failing.length ? table(['server', 'attempts', 'last error'], f.queue.failing.map((e) => row([esc(e.host), esc(e.attempts), esc(e.lastError || '?')]))) : '')) | |
| 91 | } | |
| 92 | ||
| 93 | return { title: 'Gateway status', body: parts.join('') } | |
| 94 | } | |
| 95 | ||
| 96 | export default renderStatusPage |
| … | … | @@ -155,3 +155,12 @@ test('★ alerts: with the watcher on, `alerts refresh=true` checks now and `met |
| 155 | 155 | t.ok(m.alerts && m.alerts.open.length, 'metrics carries the alerts') |
| 156 | 156 | t.is(g.steps[0][0], 'alerts', 'the watcher stops first') |
| 157 | 157 | }) |
| 158 | ||
| 159 | test('★ the status page is registered on the fediverse role\'s admin surface and renders every role', async (t) => { | |
| 160 | const g = await startGateway({ roles: ['membrane', 'fediverse'], membrane: membraneConfig(), fediverse: apConfig(), fetch: offlineFetch, ...quiet }) | |
| 161 | t.teardown(() => stop(g)) | |
| 162 | t.ok(g.adminPages.has('/admin/status')) | |
| 163 | const p = await g.adminPages.get('/admin/status')() | |
| 164 | t.is(p.title, 'Gateway status') | |
| 165 | for (const s of ['Roles', 'Served DIDs', 'Fediverse', 'membrane', 'fediverse']) t.ok(p.body.includes(s), s) | |
| 166 | }) |
| … | … | @@ -0,0 +1,49 @@ |
| 1 | // The gateway status page on the admin surface: both bridges' health, alerts first, each with its fix. | |
| 2 | import test from 'brittle' | |
| 3 | import { renderStatusPage } from '../lib/panel.js' | |
| 4 | ||
| 5 | const NOW = 1e12 | |
| 6 | const WJ = 'did:plc:wj62ebbusln6gxo5azo6ogzo' | |
| 7 | function sample () { | |
| 8 | return { | |
| 9 | version: '0.4.0', | |
| 10 | process: { uptimeS: 7300, rss: 280e6, heapUsed: 60e6, heapTotal: 110e6 }, | |
| 11 | roles: { membrane: { up: true }, fediverse: { up: true }, lens: { up: false, error: 'lens config: <bad>' } }, | |
| 12 | membrane: { dids: [{ did: WJ, handle: 'p2p.7rnx.net', rev: '3mrhxnkbhv622', fronts: 2, live: 1, lastAppendAt: NOW - 120e3, forkRejected: 0, denied: 0 }], relay: { emitted: 40269, seqBase: 35200 }, guard: { configured: true, lastAt: NOW - 60e3, last: { relayStatus: 'active' }, heals: 1, lastHealAt: NOW - 86400e3 } }, | |
| 13 | fediverse: { users: [{ username: 'p2p', primary: true, held: false, followers: 1, delivered: 172 }], queue: { pending: 1, dead: 0, oldestPendingAt: NOW - 600e3, failing: [{ host: 'prsm.space', attempts: 2, lastError: 'HTTP 502' }] }, inbound: { count: 10, lastReceivedAt: NOW - 3600e3 }, registrations: 2 }, | |
| 14 | lens: { feeds: [{ id: 'loser-club', communityDid: 'did:plc:wqnwul', curation: 'roster', members: 2, unfronted: [], posts: 299, newest: '2026-09-15T14:05:29Z', perMember: [] }] }, | |
| 15 | bsky: { at: NOW - 30e3, hostname: 'pds.7rnx.net', relay: { status: 'active', seq: 40269 }, dids: [{ did: WJ, ourRev: '3mrhxnkbhv622', relayRev: '3mrhxnkbhv622', inSync: true, plcPds: 'https://pds.7rnx.net', handle: 'p2p.7rnx.net', appview: { resolves: true, postsCount: 201, newestIndexedAt: '2026-09-15T14:05:39Z' } }], problems: [] }, | |
| 16 | alerts: { lastCheckAt: NOW - 60e3, open: [{ id: 'relay-newer:x', severity: 'critical', title: 'relay holds a NEWER commit', detail: 'relay <z>, ours a', fix: 'favela-gateway emit-sync did=x', since: NOW - 7200e3 }], recentlyResolved: [{ id: 'relay-offline', title: 'relay offline', since: NOW - 90000e3, resolvedAt: NOW - 86000e3 }] } | |
| 17 | } | |
| 18 | } | |
| 19 | ||
| 20 | test('★ alerts come first, critical marked, each with its fix', (t) => { | |
| 21 | const p = renderStatusPage(sample(), { now: NOW }) | |
| 22 | t.is(p.title, 'Gateway status') | |
| 23 | const a = p.body.indexOf('relay holds a NEWER commit') | |
| 24 | t.ok(a > -1 && a < p.body.indexOf('Served DIDs'), 'before every other section') | |
| 25 | t.ok(p.body.includes('favela-gateway emit-sync did=x'), 'the fix') | |
| 26 | t.ok(/open 2h/.test(p.body), 'how long it has been open') | |
| 27 | t.ok(p.body.includes('relay offline'), 'recently resolved is listed too') | |
| 28 | }) | |
| 29 | ||
| 30 | test('⛔ everything that came from outside is escaped', (t) => { | |
| 31 | const p = renderStatusPage(sample(), { now: NOW }) | |
| 32 | t.absent(p.body.includes('<z>'), 'alert detail escaped') | |
| 33 | t.absent(p.body.includes('<bad>'), 'role error escaped') | |
| 34 | t.ok(p.body.includes('<z>')) | |
| 35 | }) | |
| 36 | ||
| 37 | test('every section is present: roles, bsky, served DIDs, lens, fediverse, process', (t) => { | |
| 38 | const b = renderStatusPage(sample(), { now: NOW }).body | |
| 39 | for (const s of ['Roles', 'bsky', 'Served DIDs', 'Lens', 'Fediverse', 'p2p.7rnx.net', 'in sync', '1/2 live', '299 posts', '172 delivered', 'prsm.space', 'HTTP 502', '280 MB']) t.ok(b.includes(s), s) | |
| 40 | t.ok(/DOWN/.test(b), 'the down role is visible') | |
| 41 | }) | |
| 42 | ||
| 43 | test('a gateway with only the membrane still renders (missing sections are omitted, not broken)', (t) => { | |
| 44 | const m = sample(); m.fediverse = null; m.lens = null; m.alerts = null; m.bsky = null | |
| 45 | const b = renderStatusPage(m, { now: NOW }).body | |
| 46 | t.ok(b.includes('Served DIDs')) | |
| 47 | t.absent(b.includes('Fediverse</h2>')) | |
| 48 | t.ok(/alert watcher is not running|no bsky probe/i.test(b), 'says what it could not show') | |
| 49 | }) |