| … | … | @@ -0,0 +1,135 @@ |
| 1 | #!/usr/bin/env node |
| 2 | /** |
| 3 | * probe-fleet-services.mjs — what each relay is ACTUALLY running. |
| 4 | * |
| 5 | * The fleet's service picture has been assembled from config: from |
| 6 | * `services.json`, from systemd `Environment=` lines, from what someone |
| 7 | * remembers enabling. This project already learned once that config is not |
| 8 | * runtime truth — the "dubai incident", where services were configured and did |
| 9 | * not load, because `services.json` is the runtime authority and a plugin name |
| 10 | * that does not match a registered builtin fails silently. |
| 11 | * |
| 12 | * So this reads the runtime instead: `GET /api/v1/services` on each relay, |
| 13 | * which is the registry's own catalogue of what is loaded right now. Where the |
| 14 | * HTTP API is not reachable from outside, it falls back to running the same |
| 15 | * request over SSH from the box's own loopback. |
| 16 | * |
| 17 | * Read-only. Makes no changes, needs no auth on the read path. |
| 18 | * |
| 19 | * Usage: |
| 20 | * node scripts/probe-fleet-services.mjs |
| 21 | * node scripts/probe-fleet-services.mjs --json > fleet-services.json |
| 22 | * node scripts/probe-fleet-services.mjs --relays fleet/relays.local.json |
| 23 | * node scripts/probe-fleet-services.mjs --no-ssh # HTTP only |
| 24 | */ |
| 25 | |
| 26 | import { readFileSync, existsSync } from 'node:fs' |
| 27 | import { execFile } from 'node:child_process' |
| 28 | import { join, dirname } from 'node:path' |
| 29 | import { fileURLToPath } from 'node:url' |
| 30 | import { promisify } from 'node:util' |
| 31 | |
| 32 | const execFileAsync = promisify(execFile) |
| 33 | const REPO = join(dirname(fileURLToPath(import.meta.url)), '..') |
| 34 | |
| 35 | const args = process.argv.slice(2) |
| 36 | const asJson = args.includes('--json') |
| 37 | const allowSsh = !args.includes('--no-ssh') |
| 38 | const relaysArg = args[args.indexOf('--relays') + 1] |
| 39 | |
| 40 | function inventoryPath () { |
| 41 | if (relaysArg && args.includes('--relays')) return relaysArg |
| 42 | if (process.env.HIVERELAY_FLEET_INVENTORY) return process.env.HIVERELAY_FLEET_INVENTORY |
| 43 | const local = join(REPO, 'fleet', 'relays.local.json') |
| 44 | return existsSync(local) ? local : join(REPO, 'fleet', 'relays.json') |
| 45 | } |
| 46 | |
| 47 | // Same allowlist discipline as the admin operator: inventory values become argv |
| 48 | // elements, so validate before they reach a child process. |
| 49 | function isSafeHost (host) { |
| 50 | return typeof host === 'string' && host.length > 0 && host.length <= 255 && |
| 51 | /^[A-Za-z0-9.:-]+$/.test(host) |
| 52 | } |
| 53 | |
| 54 | function expandHome (p) { |
| 55 | return typeof p === 'string' ? p.replace(/^~(?=\/|$)/, process.env.HOME || '') : p |
| 56 | } |
| 57 | |
| 58 | async function probeHttp (relay) { |
| 59 | const url = `http://${relay.publicIp}:9100/api/v1/services` |
| 60 | const res = await fetch(url, { signal: AbortSignal.timeout(6000) }) |
| 61 | return { body: await res.json(), status: res.status, via: 'http' } |
| 62 | } |
| 63 | |
| 64 | async function probeSsh (relay) { |
| 65 | if (!isSafeHost(relay.publicIp)) throw new Error('unsafe host in inventory') |
| 66 | const key = expandHome(relay.sshKey) |
| 67 | const keyArgs = key && key !== 'default' ? ['-i', key] : [] |
| 68 | const { stdout } = await execFileAsync('ssh', [ |
| 69 | ...keyArgs, |
| 70 | '-o', 'BatchMode=yes', |
| 71 | '-o', 'ConnectTimeout=8', |
| 72 | '-o', 'StrictHostKeyChecking=accept-new', |
| 73 | `root@${relay.publicIp}`, |
| 74 | 'curl -fsS --max-time 6 http://127.0.0.1:9100/api/v1/services' |
| 75 | ], { timeout: 25000 }) |
| 76 | return { body: JSON.parse(stdout), status: 200, via: 'ssh' } |
| 77 | } |
| 78 | |
| 79 | async function probe (relay) { |
| 80 | const base = { name: relay.name, region: relay.region ?? null, ramGB: relay.ramGB ?? null } |
| 81 | if (!isSafeHost(relay.publicIp)) { |
| 82 | return { ...base, reachable: false, error: 'unsafe host in inventory', services: [] } |
| 83 | } |
| 84 | for (const attempt of allowSsh ? [probeHttp, probeSsh] : [probeHttp]) { |
| 85 | try { |
| 86 | const { body, via } = await attempt(relay) |
| 87 | // 503 {"error":"Services not enabled"} is a real answer, not a failure. |
| 88 | if (body && body.error) { |
| 89 | return { ...base, reachable: true, via, servicesEnabled: false, services: [], note: body.error } |
| 90 | } |
| 91 | const services = Array.isArray(body?.services) |
| 92 | ? body.services.map(s => (typeof s === 'string' ? s : s?.name)).filter(Boolean).sort() |
| 93 | : [] |
| 94 | return { ...base, reachable: true, via, servicesEnabled: true, services, total: body?.total ?? services.length } |
| 95 | } catch (err) { |
| 96 | base.error = err?.message?.slice(0, 120) || String(err) |
| 97 | } |
| 98 | } |
| 99 | return { ...base, reachable: false, services: [] } |
| 100 | } |
| 101 | |
| 102 | const inv = inventoryPath() |
| 103 | const relays = JSON.parse(readFileSync(inv, 'utf8')).relays |
| 104 | const results = await Promise.all(relays.map(probe)) |
| 105 | |
| 106 | if (asJson) { |
| 107 | console.log(JSON.stringify({ |
| 108 | probedAt: new Date().toISOString(), |
| 109 | inventory: inv, |
| 110 | source: 'runtime GET /api/v1/services (not config)', |
| 111 | relays: results |
| 112 | }, null, 2)) |
| 113 | } else { |
| 114 | console.log(`\nRuntime service catalogue — ${relays.length} relays from ${inv}`) |
| 115 | console.log('Source: GET /api/v1/services (what is LOADED, not what is configured)\n') |
| 116 | const w = Math.max(...results.map(r => r.name.length), 6) |
| 117 | for (const r of results) { |
| 118 | const status = !r.reachable ? 'unreachable' : r.servicesEnabled === false ? 'services off' : `${r.services.length} loaded` |
| 119 | const detail = !r.reachable ? (r.error || '') : r.services.join(', ') || (r.note || '') |
| 120 | console.log(` ${r.name.padEnd(w)} ${String(r.region ?? '-').padEnd(5)} ${status.padEnd(13)} ${detail}`) |
| 121 | } |
| 122 | |
| 123 | const reachable = results.filter(r => r.reachable) |
| 124 | const union = [...new Set(reachable.flatMap(r => r.services))].sort() |
| 125 | console.log(`\n reachable: ${reachable.length}/${results.length}`) |
| 126 | console.log(` distinct services running across the fleet: ${union.length}${union.length ? ' — ' + union.join(', ') : ''}`) |
| 127 | |
| 128 | // The number that actually matters for diversity: a service on one box is a |
| 129 | // single point of failure regardless of how many relays exist. |
| 130 | const counts = new Map() |
| 131 | for (const r of reachable) for (const s of r.services) counts.set(s, (counts.get(s) || 0) + 1) |
| 132 | const singletons = [...counts].filter(([, n]) => n === 1).map(([s]) => s).sort() |
| 133 | if (singletons.length) console.log(` running on exactly ONE relay (SPOF): ${singletons.join(', ')}`) |
| 134 | console.log() |
| 135 | } |