| … | … | @@ -153,6 +153,21 @@ export function isPidAlive (pid) { |
| 153 | 153 | try { process.kill(pid, 0); return true } catch { return false } |
| 154 | 154 | } |
| 155 | 155 | |
| 156 | // When a process actually STARTED, from the kernel rather than from our own bookkeeping. | |
| 157 | // | |
| 158 | // `state.startedAt` records when pgh spawned a supervisor, which is not the same thing: a process | |
| 159 | // started outside pgh, or one whose state file was rewritten, reports a time that has nothing to do with | |
| 160 | // the code it is executing. The /proc/<pid> directory's ctime is the process start time (verified against | |
| 161 | // both process.uptime() and `ps -o lstart` — agreement within milliseconds), so it is the ground truth | |
| 162 | // for "is this process older than the tree it is supposed to be running". | |
| 163 | // | |
| 164 | // Linux-only by construction; returns null anywhere /proc is absent, and every caller treats null as | |
| 165 | // "unknown" rather than "fine". | |
| 166 | export function processStartedAt (pid) { | |
| 167 | if (!pid) return null | |
| 168 | try { return new Date(fs.statSync('/proc/' + pid).ctimeMs).toISOString() } catch { return null } | |
| 169 | } | |
| 170 | ||
| 156 | 171 | // Discover a running app supervisor by its PGH_SUP_NAME env, independent |
| 157 | 172 | // of the pid file. The supervisor is spawned with PGH_SUP_NAME=<name> in |
| 158 | 173 | // its environment (see `cmdStart`), so when the pid file is missing or |
| … | … | @@ -5,7 +5,7 @@ import { spawn } from '#node:child_process' |
| 5 | 5 | import { execSync } from '../_compat-exec.js' |
| 6 | 6 | import path from '#node:path' |
| 7 | 7 | import { fileURLToPath } from '#node:url' |
| 8 | import { homedir, tmpdir } from '#node:os' | |
| 8 | import { homedir, tmpdir, hostname } from '#node:os' | |
| 9 | 9 | import fs from '#node:fs' |
| 10 | 10 | import b4a from 'b4a' |
| 11 | 11 | |
| … | … | @@ -20,7 +20,7 @@ import { |
| 20 | 20 | appDir, pidFile, logFile, installManifestFile, |
| 21 | 21 | readInstallManifest, writeInstallManifest, readPackageJsonPgh, |
| 22 | 22 | readUserConfig, writeUserConfig, |
| 23 | readPid, clearPid, isPidAlive, findRunningSupervisors, | |
| 23 | readPid, clearPid, isPidAlive, processStartedAt, findRunningSupervisors, | |
| 24 | 24 | readState, |
| 25 | 25 | rotateLogIfNeeded, openLogForAppend, |
| 26 | 26 | effectiveEnv, effectiveArgs, effectiveAutostart, |
| … | … | @@ -55,6 +55,7 @@ export const usage = `pgh app — install and run pgh-managed apps |
| 55 | 55 | Process lifecycle (supervisor-managed): |
| 56 | 56 | pgh app start|stop|restart <name> |
| 57 | 57 | pgh app status [<name>] |
| 58 | pgh app stale [--all] installed-vs-RUNNING check for every app on this host | |
| 58 | 59 | pgh app log <name> [-f] [--lines N] |
| 59 | 60 | pgh app autostart <name> on|off |
| 60 | 61 | pgh app boot | shutdown |
| … | … | @@ -400,6 +401,7 @@ export async function run (client, args) { |
| 400 | 401 | if (sub === 'stop') return await cmdStop(args) |
| 401 | 402 | if (sub === 'restart') return await cmdRestart(args) |
| 402 | 403 | if (sub === 'status') return await cmdStatus(args) |
| 404 | if (sub === 'stale') return cmdStale(args) | |
| 403 | 405 | if (sub === 'log') return await cmdLog(args) |
| 404 | 406 | if (sub === 'autostart') return cmdAutostart(args) |
| 405 | 407 | if (sub === 'boot') return await cmdBoot(args) |
| … | … | @@ -1621,7 +1623,95 @@ function basicStatus (name) { |
| 1621 | 1623 | lastExit: state.lastExit || null, |
| 1622 | 1624 | lastExitCode: state.lastExitCode != null ? state.lastExitCode : null, |
| 1623 | 1625 | autostart: effectiveAutostart(name, manifest), |
| 1624 | log: logFile(name) | |
| 1626 | log: logFile(name), | |
| 1627 | ...staleness(name, { alive, supervisorPid: pid, childPid: state.childPid || null, manifest }) | |
| 1628 | } | |
| 1629 | } | |
| 1630 | ||
| 1631 | // INSTALLED != RUNNING. `version` above is the INSTALL TREE's version — it says nothing about the code | |
| 1632 | // the live process is executing, because `pgh app update` materializes a new tree at the SAME path while | |
| 1633 | // the old process keeps running the old code. Reading a version back off disk therefore cannot detect | |
| 1634 | // this: the path now holds the NEW tree either way. | |
| 1635 | // | |
| 1636 | // What IS decidable: a process that started BEFORE the current tree was installed cannot be running it. | |
| 1637 | // That comparison is the whole check, and it is the one that has repeatedly been done by hand | |
| 1638 | // (`ps -o etimes` against the install tree) — twice in one day on 2026-07-27, finding a box that reported | |
| 1639 | // 0.3.65 while running days-old code. | |
| 1640 | // | |
| 1641 | // Deliberately conservative: `stale` is null (not false) whenever the inputs are unknown, so an app whose | |
| 1642 | // start time or install time cannot be read is never reported as healthy. | |
| 1643 | // How much "process started before the manifest was written" is just the install/respawn race rather | |
| 1644 | // than old code. Measured skew on a healthy app is ~2s; days is the failure being caught. | |
| 1645 | const RESPAWN_SKEW_MS = 60000 | |
| 1646 | ||
| 1647 | function staleness (name, { alive, supervisorPid, childPid, manifest }) { | |
| 1648 | // The CHILD executes the app code; the supervisor merely respawns it. Prefer the child, fall back to | |
| 1649 | // the supervisor, and only then to pgh's own bookkeeping. | |
| 1650 | const pid = alive ? ((childPid && isPidAlive(childPid)) ? childPid : supervisorPid) : null | |
| 1651 | const started = alive ? (processStartedAt(pid) || readState(name).startedAt || null) : null | |
| 1652 | return classifyStale({ running: alive, installedAt: (manifest && manifest.installedAt) || null, startedAt: started, version: manifest && manifest.version }) | |
| 1653 | } | |
| 1654 | ||
| 1655 | // The decision, separated from the I/O so it can be tested exhaustively (same shape as | |
| 1656 | // renderVerboseStatus). `now` is injectable for the same reason. | |
| 1657 | export function classifyStale ({ running, installedAt = null, startedAt = null, version = null, now = Date.now() } = {}) { | |
| 1658 | const started = startedAt | |
| 1659 | if (!running) return { installedAt, processStartedAt: null, uptimeSeconds: null, skewSeconds: null, stale: false, staleReason: 'not running' } | |
| 1660 | const uptimeSeconds = started ? Math.max(0, Math.round((now - Date.parse(started)) / 1000)) : null | |
| 1661 | if (!installedAt) return { installedAt, processStartedAt: started, uptimeSeconds, skewSeconds: null, stale: null, staleReason: 'no installedAt in the install manifest' } | |
| 1662 | if (!started) return { installedAt, processStartedAt: null, uptimeSeconds: null, skewSeconds: null, stale: null, staleReason: 'could not determine process start time' } | |
| 1663 | const manifest = { version } | |
| 1664 | const behindMs = Date.parse(installedAt) - Date.parse(started) | |
| 1665 | // Install and respawn are NOT atomic with respect to each other: on an OTA update the app exits (75), | |
| 1666 | // the supervisor respawns it, and the manifest write races that respawn. Measured on a healthy | |
| 1667 | // tabertron app: processStartedAt 1.7s BEFORE installedAt — which a naive `installedAt > startedAt` | |
| 1668 | // reports as stale. Only a margin well beyond that ordering noise is evidence of old code; the failure | |
| 1669 | // this exists to catch is a process running code that is DAYS old. skewSeconds is always reported, so a | |
| 1670 | // borderline case is visible rather than hidden behind the threshold. | |
| 1671 | const skewSeconds = Math.round(behindMs / 1000) | |
| 1672 | if (behindMs > RESPAWN_SKEW_MS) { | |
| 1673 | return { | |
| 1674 | installedAt, | |
| 1675 | processStartedAt: started, | |
| 1676 | uptimeSeconds, | |
| 1677 | skewSeconds, | |
| 1678 | stale: true, | |
| 1679 | staleReason: `running process started ${skewSeconds}s BEFORE the current tree was installed — it is not running ${manifest.version || 'the installed version'}; restart it` | |
| 1680 | } | |
| 1681 | } | |
| 1682 | return { installedAt, processStartedAt: started, uptimeSeconds, skewSeconds, stale: false, staleReason: null } | |
| 1683 | } | |
| 1684 | ||
| 1685 | // `pgh app stale` — the fleet check, in one invocation per host. | |
| 1686 | // | |
| 1687 | // Answers "is any app on this box running code it is not supposed to be running", which was previously an | |
| 1688 | // ad-hoc sweep comparing `ps -o etimes` against install-tree mtimes, reinvented each time it was needed. | |
| 1689 | // Across hosts this composes the same way as the rest of the stack: `ssh <host> pgh app stale`. | |
| 1690 | // | |
| 1691 | // `stale: null` (unknown) counts as a hit, because "I could not tell" is exactly the answer that must not | |
| 1692 | // be silently rendered as healthy — the whole failure mode here is a green surface over old code. | |
| 1693 | function cmdStale (args) { | |
| 1694 | const all = args.includes('--all') | |
| 1695 | const rows = listAppNames().map(n => basicStatus(n)).map(s => ({ | |
| 1696 | name: s.name, | |
| 1697 | running: s.running, | |
| 1698 | installedVersion: s.version || null, | |
| 1699 | installedAt: s.installedAt || null, | |
| 1700 | processStartedAt: s.processStartedAt || null, | |
| 1701 | uptimeSeconds: s.uptimeSeconds, | |
| 1702 | stale: s.stale, | |
| 1703 | staleReason: s.staleReason | |
| 1704 | })) | |
| 1705 | const flagged = rows.filter(r => r.stale === true || r.stale === null) | |
| 1706 | return { | |
| 1707 | host: (() => { try { return hostname() } catch { return null } })(), | |
| 1708 | at: new Date().toISOString(), | |
| 1709 | apps: all ? rows : flagged, | |
| 1710 | checked: rows.length, | |
| 1711 | stale: rows.filter(r => r.stale === true).map(r => r.name), | |
| 1712 | unknown: rows.filter(r => r.stale === null).map(r => r.name), | |
| 1713 | // A restart is the fix; naming it here saves the next person deriving it under pressure. | |
| 1714 | hint: flagged.length ? 'restart the flagged apps: pgh app restart <name>' : 'every running app started after its current tree was installed' | |
| 1625 | 1715 | } |
| 1626 | 1716 | } |
| 1627 | 1717 |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "pgh", |
| 3 | "version": "0.1.122", | |
| 3 | "version": "0.1.123", | |
| 4 | 4 | "description": "pgh — P2P content-addressed package manager and app launcher", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| … | … | @@ -28,8 +28,8 @@ |
| 28 | 28 | * OWNING package already maps that dep to the SAME version. |
| 29 | 29 | */ |
| 30 | 30 | |
| 31 | import fs from 'fs' | |
| 32 | import path from 'path' | |
| 31 | import fs from '#node:fs' // #node:* resolves to bare-fs under bare, fs under node — test:bare imports this | |
| 32 | import path from '#node:path' | |
| 33 | 33 | |
| 34 | 34 | /** |
| 35 | 35 | * Index a single node_modules dir → Map<name, version> for the packages that |
| … | … | @@ -0,0 +1,68 @@ |
| 1 | // INSTALLED != RUNNING — the staleness classifier behind `pgh app stale`. | |
| 2 | // | |
| 3 | // `pgh app status` reports the INSTALL TREE's version, which says nothing about the code a live process is | |
| 4 | // executing: `pgh app update` materializes a new tree at the SAME path while the old process keeps running | |
| 5 | // the old code, so reading a version back off disk cannot detect it — the path holds the new tree either | |
| 6 | // way. This bit the fleet twice in one day (2026-07-27), once on the very box under investigation, and the | |
| 7 | // check was re-derived by hand each time (`ps -o etimes` against the install tree). | |
| 8 | // | |
| 9 | // What IS decidable: a process that started BEFORE the current tree was installed cannot be running it. | |
| 10 | // The classifier is separated from the I/O (same shape as renderVerboseStatus) so every branch is covered | |
| 11 | // here, including the one that a live run caught and a naive implementation gets wrong. | |
| 12 | import test from 'brittle' | |
| 13 | import { classifyStale } from '../lib/commands/app.js' | |
| 14 | ||
| 15 | const iso = (ms) => new Date(ms).toISOString() | |
| 16 | const T = Date.parse('2026-07-28T12:00:00.000Z') | |
| 17 | ||
| 18 | test('a process started BEFORE the current tree was installed is stale', (t) => { | |
| 19 | const r = classifyStale({ | |
| 20 | running: true, | |
| 21 | installedAt: iso(T), | |
| 22 | startedAt: iso(T - 3 * 24 * 3600 * 1000), // days-old process, freshly installed tree | |
| 23 | version: '0.3.65', | |
| 24 | now: T | |
| 25 | }) | |
| 26 | t.is(r.stale, true, 'flagged stale') | |
| 27 | t.ok(/BEFORE the current tree was installed/.test(r.staleReason), 'the reason states the comparison that was made') | |
| 28 | t.ok(/0\.3\.65/.test(r.staleReason), 'and names the version it is NOT running') | |
| 29 | t.ok(/restart/.test(r.staleReason), 'and says what to do about it') | |
| 30 | t.is(r.skewSeconds, 3 * 24 * 3600, 'the skew is reported in full') | |
| 31 | }) | |
| 32 | ||
| 33 | test('the install/respawn race is NOT stale', (t) => { | |
| 34 | // Measured on a healthy tabertron app: the OTA path exits the app (75), the supervisor respawns it, and | |
| 35 | // the manifest write races the respawn — processStartedAt landed 1.7s BEFORE installedAt. The first | |
| 36 | // version of this check reported that as stale, which would have sent an operator restarting a perfectly | |
| 37 | // current app. This is the regression guard for that. | |
| 38 | const r = classifyStale({ running: true, installedAt: iso(T), startedAt: iso(T - 2000), version: '0.2.4', now: T }) | |
| 39 | t.is(r.stale, false, 'a 2s skew is the install/respawn race, not old code') | |
| 40 | t.is(r.staleReason, null, 'and is not reported as a problem') | |
| 41 | t.is(r.skewSeconds, 2, 'but the skew is still VISIBLE rather than hidden behind the threshold') | |
| 42 | }) | |
| 43 | ||
| 44 | test('a process started after its tree is current', (t) => { | |
| 45 | const r = classifyStale({ running: true, installedAt: iso(T - 3600 * 1000), startedAt: iso(T - 60 * 1000), version: '0.3.83', now: T }) | |
| 46 | t.is(r.stale, false, 'not stale') | |
| 47 | t.is(r.staleReason, null, 'no reason to report') | |
| 48 | t.is(r.uptimeSeconds, 60, 'uptime is reported') | |
| 49 | }) | |
| 50 | ||
| 51 | test('unknowns are never reported as healthy', (t) => { | |
| 52 | // The whole failure mode here is a green surface over old code, so "I could not tell" must not render | |
| 53 | // as "fine". null is a distinct third state, and cmdStale flags it alongside true. | |
| 54 | const noInstall = classifyStale({ running: true, installedAt: null, startedAt: iso(T - 1000), now: T }) | |
| 55 | t.is(noInstall.stale, null, 'no installedAt -> unknown, NOT false') | |
| 56 | t.ok(/installedAt/.test(noInstall.staleReason), 'and says which input was missing') | |
| 57 | ||
| 58 | const noStart = classifyStale({ running: true, installedAt: iso(T), startedAt: null, now: T }) | |
| 59 | t.is(noStart.stale, null, 'no process start time -> unknown, NOT false') | |
| 60 | t.ok(/process start time/.test(noStart.staleReason), 'and says which input was missing') | |
| 61 | }) | |
| 62 | ||
| 63 | test('an app that is not running is not stale', (t) => { | |
| 64 | const r = classifyStale({ running: false, installedAt: iso(T), startedAt: null, now: T }) | |
| 65 | t.is(r.stale, false, 'nothing is running, so nothing is running old code') | |
| 66 | t.is(r.staleReason, 'not running', 'stated plainly rather than left blank') | |
| 67 | t.is(r.uptimeSeconds, null, 'and no uptime is invented') | |
| 68 | }) |
| … | … | @@ -1,12 +1,17 @@ |
| 1 | 1 | import test from 'brittle' |
| 2 | import { mergeManifest } from '../lib/commands/bootstrap.js' | |
| 2 | // ⚠ SKIPPED — `mergeManifest` and the whole `pgh bootstrap publish` overlay-manifest flow were REMOVED | |
| 3 | // on purpose in 3120d1a ("#66 bootstrap: tar local app trees on install; drop `pgh bootstrap` command"): | |
| 4 | // the bootstrap app IS the app, so the curl-installable artifacts are now just a tar of the local app | |
| 5 | // tree, built by reconcileBootstrapArtifacts (covered by test/bootstrap-artifacts.test.js). This test file | |
| 6 | // was left behind importing the deleted module. | |
| 7 | // | |
| 8 | // That import threw at MODULE LOAD, and because `npm test` globs test/*.test.js it aborted the ENTIRE pgh | |
| 9 | // suite before any test ran — so no change to this repo could be verified against it, from 3120d1a until | |
| 10 | // now. The import is gone and the cases are skipped rather than deleted: they still document the merge | |
| 11 | // semantics if that conveyance ever comes back. | |
| 12 | // const { mergeManifest } = await import('../lib/commands/bootstrap.js') | |
| 3 | 13 | |
| 4 | // #66 option B: the pgh repo's bootstrap/manifest.json is the cross-machine | |
| 5 | // conveyance. mergeManifest is the pure merge the `pgh bootstrap publish` | |
| 6 | // command uses to insert/replace each app's overlay tarball entry while always | |
| 7 | // keeping install.sh as a git-blob entry. | |
| 8 | ||
| 9 | test('mergeManifest seeds install.sh blob + adds an overlay entry', (t) => { | |
| 14 | test.skip('mergeManifest seeds install.sh blob + adds an overlay entry', (t) => { | |
| 10 | 15 | const m = mergeManifest(null, { name: 'pgh-bootstrap-bare-linux-x64.tar.gz', kind: 'overlay', contentHash: 'aa', size: 10 }) |
| 11 | 16 | t.is(m.version, 1) |
| 12 | 17 | const names = m.files.map(f => f.name) |
| … | … | @@ -16,7 +21,7 @@ test('mergeManifest seeds install.sh blob + adds an overlay entry', (t) => { |
| 16 | 21 | t.is(o.kind, 'overlay'); t.is(o.contentHash, 'aa'); t.is(o.size, 10) |
| 17 | 22 | }) |
| 18 | 23 | |
| 19 | test('mergeManifest accumulates multiple apps + replaces on re-publish', (t) => { | |
| 24 | test.skip('mergeManifest accumulates multiple apps + replaces on re-publish', (t) => { | |
| 20 | 25 | let m = mergeManifest(null, { name: 'pgh-bootstrap-bare-linux-x64.tar.gz', kind: 'overlay', contentHash: 'aa', size: 10 }) |
| 21 | 26 | m = mergeManifest(m, { name: 'pgh-sidecar-bootstrap-bare-linux-x64.tar.gz', kind: 'overlay', contentHash: 'bb', size: 20 }) |
| 22 | 27 | t.is(m.files.length, 3, 'install.sh + pgh + pgh-sidecar') |
| … | … | @@ -28,7 +33,7 @@ test('mergeManifest accumulates multiple apps + replaces on re-publish', (t) => |
| 28 | 33 | t.is(m.files.find(f => f.name === 'pgh-sidecar-bootstrap-bare-linux-x64.tar.gz').contentHash, 'bb', 'sidecar untouched') |
| 29 | 34 | }) |
| 30 | 35 | |
| 31 | test('mergeManifest preserves an existing install.sh entry + is idempotent without an entry', (t) => { | |
| 36 | test.skip('mergeManifest preserves an existing install.sh entry + is idempotent without an entry', (t) => { | |
| 32 | 37 | const seed = { version: 1, files: [{ name: 'install.sh', kind: 'blob' }] } |
| 33 | 38 | const m = mergeManifest(seed, null) |
| 34 | 39 | t.is(m.files.length, 1) |
| … | … | @@ -48,7 +48,11 @@ import os from '#node:os' |
| 48 | 48 | import { DeployServer } from 'hyper-deploy' |
| 49 | 49 | import { FileIndex } from 'hyper-overlay/file-index' |
| 50 | 50 | import { TransferManager } from 'hyper-overlay/transfer' |
| 51 | import { OverlayProtocolV2 } from 'hyper-overlay/protocol-v2' | |
| 51 | // hyper-overlay 0.3.0 renamed lib/protocol-v2.js -> lib/protocol.js and OverlayProtocolV2 -> | |
| 52 | // OverlayProtocol (v1 was dropped; the current class IS the v2 wire protocol). The old subpath is not | |
| 53 | // in the package's exports map, so this import threw at MODULE LOAD and — because npm test globs | |
| 54 | // test/*.test.js — took the whole pgh suite with it. Aliased so the rest of the file is unchanged. | |
| 55 | import { OverlayProtocol as OverlayProtocolV2 } from 'hyper-overlay/protocol' | |
| 52 | 56 | |
| 53 | 57 | function mkdtemp (label) { |
| 54 | 58 | const dir = path.join(os.tmpdir(), label + '-' + crypto.randomBytes(6).toString('hex')) |
| … | … | @@ -22,7 +22,11 @@ import Corestore from 'corestore' |
| 22 | 22 | |
| 23 | 23 | import { FileIndex } from 'hyper-overlay/file-index' |
| 24 | 24 | import { TransferManager } from 'hyper-overlay/transfer' |
| 25 | import { OverlayProtocolV2 } from 'hyper-overlay/protocol-v2' | |
| 25 | // hyper-overlay 0.3.0 renamed lib/protocol-v2.js -> lib/protocol.js and OverlayProtocolV2 -> | |
| 26 | // OverlayProtocol (v1 was dropped; the current class IS the v2 wire protocol). The old subpath is not | |
| 27 | // in the package's exports map, so this import threw at MODULE LOAD and — because npm test globs | |
| 28 | // test/*.test.js — took the whole pgh suite with it. Aliased so the rest of the file is unchanged. | |
| 29 | import { OverlayProtocol as OverlayProtocolV2 } from 'hyper-overlay/protocol' | |
| 26 | 30 | import { computeTreeHash, KIND_FILE } from 'hyper-overlay/tree' |
| 27 | 31 | |
| 28 | 32 | function mktemp (label) { |