| … | … | @@ -2,6 +2,7 @@ import test from 'brittle' |
| 2 | 2 | import { readFileSync } from 'node:fs' |
| 3 | 3 | |
| 4 | 4 | const updater = readFileSync('fleet/updater.sh', 'utf8') |
| 5 | const watchdog = readFileSync('fleet/health-watchdog.sh', 'utf8') |
| 5 | 6 | const fleetStatus = readFileSync('fleet/fleet-status.sh', 'utf8') |
| 6 | 7 | const deployVps = readFileSync('scripts/deploy-vps.sh', 'utf8') |
| 7 | 8 | const rolloutCheck = readFileSync('scripts/check-fleet-rollout.mjs', 'utf8') |
| … | … | @@ -20,7 +21,16 @@ test('fleet scripts pass channel names into JSON parsing as data', (t) => { |
| 20 | 21 | test('fleet updater routes dependency-install failures through rollback', (t) => { |
| 21 | 22 | t.ok(updater.includes('rollback_to_previous')) |
| 22 | 23 | t.ok(updater.includes('deps_if_changed "$CUR_SHA" "$TARGET_SHA" || rollback_to_previous')) |
| 23 | | t.ok(updater.includes('if ! git checkout --quiet "$CUR_SHA"; then')) |
| 24 | // --force is required, and this assertion previously locked in its absence. |
| 25 | // npm ci rewrites package-lock.json, so by the time a dependency install has |
| 26 | // failed the tree is dirty and a plain checkout refuses — stranding the box on |
| 27 | // the NEW tree with a half-built node_modules, which is the opposite of a |
| 28 | // rollback. Observed on utah 2026-07-28: "CRITICAL could not checkout previous |
| 29 | // SHA". Safe because the pre-update dirty-tree guard already refused to start |
| 30 | // on a dirty tree, so anything dirty at rollback time was created by that run. |
| 31 | t.ok(updater.includes('if ! git checkout --quiet --force "$CUR_SHA"; then')) |
| 32 | t.absent(updater.includes('if ! git checkout --quiet "$CUR_SHA"; then'), |
| 33 | 'a rollback checkout without --force cannot survive npm ci dirtying the lockfile') |
| 24 | 34 | t.ok(updater.includes('CRITICAL could not checkout previous SHA')) |
| 25 | 35 | t.ok(updater.includes('if ! deps_if_changed "$TARGET_SHA" "$CUR_SHA"; then')) |
| 26 | 36 | t.absent(updater.includes('git checkout --quiet "$CUR_SHA" || log "CRITICAL could not checkout previous SHA"')) |
| … | … | @@ -125,3 +135,57 @@ test('relay CLI keeps high-frequency status output off service logs', (t) => { |
| 125 | 135 | t.ok(cli.includes('stats.served ? stats.served.totalBytesServed')) |
| 126 | 136 | t.absent(cli.includes('if (!args.quiet) {\n statusInterval = setInterval(() =>')) |
| 127 | 137 | }) |
| 138 | |
| 139 | test('fleet updater proves the runtime boots before restarting the live service', (t) => { |
| 140 | // npm ci succeeding does not mean the relay can start. utah-0.5gb installed |
| 141 | // cleanly on 2026-07-28 and then crash-looped: require-addon resolved the |
| 142 | // package root to "/" and looked for /prebuilds/linux-x64/sodium-native.node |
| 143 | // while the binary sat correct inside node_modules. Same tag, same install — |
| 144 | // the difference was Node v22.22.2 vs v22.22.0, and `engines: >=20.0.0` |
| 145 | // admits both, so a version pin would not have caught it either. |
| 146 | t.ok(updater.includes('preflight_runtime'), 'preflight helper exists') |
| 147 | t.ok(updater.includes('require("hyperswarm")'), |
| 148 | 'loads the chain that actually broke: hyperswarm -> hyperdht -> dht-rpc -> udx-native -> require-addon -> sodium-native') |
| 149 | |
| 150 | // Ordering is the whole point: prove the runtime AFTER deps are installed and |
| 151 | // BEFORE the service is restarted, so a bad install rolls back instead of |
| 152 | // leaving a crash-looping relay. |
| 153 | const preflightAt = updater.indexOf('preflight_runtime || rollback_to_previous') |
| 154 | const depsAt = updater.indexOf('deps_if_changed "$CUR_SHA" "$TARGET_SHA"') |
| 155 | // `systemctl restart` appears twice — the first is inside |
| 156 | // rollback_to_previous(), which is defined above the update path. Search from |
| 157 | // the preflight so this compares against the UPDATE path's restart. |
| 158 | const restartAt = updater.indexOf('systemctl restart "$SERVICE"', preflightAt) |
| 159 | t.ok(preflightAt > depsAt, 'preflight runs after the dependency install') |
| 160 | t.ok(restartAt > preflightAt, 'preflight runs before the service restart') |
| 161 | }) |
| 162 | |
| 163 | test('fleet updater refreshes its own agent so update-path fixes can reach a box', (t) => { |
| 164 | // The agent runs from /usr/local/bin and nothing reinstalled it, so a bug in |
| 165 | // the update path could only ever be fixed by hand-visiting every relay. That |
| 166 | // is why rc.6's --force rollback fix reached no box at all. |
| 167 | t.ok(updater.includes('SELF_PATH'), 'knows where it is installed') |
| 168 | t.ok(updater.includes('agent self-updated'), 'reports the refresh') |
| 169 | |
| 170 | // Atomic install: write beside the target, syntax-check, then rename. A crash |
| 171 | // mid-copy must not leave a partial interpreter script at a path systemd runs. |
| 172 | t.ok(updater.includes('bash -n "$SELF_PATH.next"'), 'syntax-checks before adopting') |
| 173 | t.ok(updater.includes('mv -f "$SELF_PATH.next" "$SELF_PATH"'), 'adopts by rename, not in-place write') |
| 174 | |
| 175 | // Must come after the signature gate — never install an agent from an |
| 176 | // unverified tree — and after a deps install that has proven itself. |
| 177 | t.ok(updater.indexOf('verify_tag "$TARGET"') < updater.indexOf('SELF_PATH.next'), |
| 178 | 'self-update happens only after the supply-chain gate has passed') |
| 179 | }) |
| 180 | |
| 181 | test('health watchdog does not kill a relay that is answering honestly', (t) => { |
| 182 | // This watchdog exists for event-loop hangs that systemd cannot see. A |
| 183 | // structured reply proves the loop is turning. `curl -f` discarded the body on |
| 184 | // non-2xx, so a deliberate 503 drain was indistinguishable from a hang — and |
| 185 | // since a restart frees no disk, the box looped SIGKILL every ~4 minutes. |
| 186 | t.absent(watchdog.includes('curl -fsS --max-time "$TIMEOUT" "$URL"'), |
| 187 | '-f discards the body that distinguishes a drain from a hang') |
| 188 | t.ok(watchdog.includes('curl -sS --max-time "$TIMEOUT" "$URL"')) |
| 189 | t.ok(watchdog.includes('"reason"[[:space:]]*:'), 'stands down on any reasoned reply') |
| 190 | t.ok(watchdog.includes('not a hang; not restarting')) |
| 191 | }) |