supervisor-staged-after-swap.test.js

// Regression: a supervisor must keep applying staged updates after its FIRST apply.
//
// ⛔ THE INCIDENT (stortron, 2026-10-13). ap-bridge 0.10.14 was installed and staged as
// `ap-bridge.next`, the app answered release-ready and exited 75, and the supervisor logged
//   [supervisor] staged-update check failed (spawning as before): no such file or directory
// then respawned 0.10.13. Every OTA after the first one silently ran the OLD code.
//
// Mechanism: `pgh app start` spawns the supervisor with cwd = the app's install dir. applyStaged()
// swaps that dir (live -> .old, .next -> live, rm .old), so the swap that applies an update DELETES
// the supervisor's cwd. The child spawn already passes an explicit cwd (supervisor-cwd-swap.test.js),
// but applyStaged's own `flock` spawnSync inherits the dead cwd, fails with ENOENT, and the catch
// falls back to spawning the old tree (without the flock wrapper, so later installs swap in place
// instead of staging). Found live on 8 supervisors across stortron and tabertron.
//
// This test runs the real supervisor with cwd = the app dir, applies one staged update (which
// deletes that cwd, as in production), stages another, and requires the next respawn to run it.

import test from 'brittle'
import process from '#node:process'
import fs from '#node:fs'
import { join } from '#node:path'
import { tmpdir } from '#node:os'
import { spawn, spawnSync } from '#node:child_process'
import { fileURLToPath } from '#node:url'

import { TreeStore } from '../lib/store.js'
import { materializeAtomic, lockPathFor, stagedPathFor } from '../lib/materializers/atomic.js'

const SUPERVISOR = fileURLToPath(new URL('../lib/supervisor.js', import.meta.url))
const NODE_MODULES = fileURLToPath(new URL('../node_modules', import.meta.url))
const HAS_FLOCK = spawnSync('flock', ['--version']).status === 0

// An app tree for `version`: it appends its VERSION to the marker, then asks for an OTA respawn.
function appMaterializer (version, marker) {
  return {
    materialize ({ store, appName }) {
      const dir = store.appPath(appName)
      fs.mkdirSync(dir, { recursive: true })
      fs.writeFileSync(join(dir, 'package.json'), JSON.stringify({
        type: 'module',
        imports: {
          '#node:fs': { bare: 'bare-fs', default: 'fs' },
          '#node:process': { bare: 'bare-process', default: 'process' }
        }
      }))
      fs.symlinkSync(NODE_MODULES, join(dir, 'node_modules'))
      fs.writeFileSync(join(dir, 'entry.js'),
        'import fs from "#node:fs"\n' +
        'import process from "#node:process"\n' +
        'fs.appendFileSync(' + JSON.stringify(marker) + ', ' + JSON.stringify(version + '\n') + ')\n' +
        'setTimeout(() => process.exit(75), 300)\n')
      return { referenced: [], rootTarget: dir, depCount: 0 }
    }
  }
}

// Hold the app's shared lock the way a running app does, so an install STAGES.
function holdShared (lockPath) {
  return new Promise((resolve, reject) => {
    fs.writeFileSync(lockPath, '', { flag: 'a' })
    const p = spawn('sh', ['-c', 'exec 9>>"$0"; flock -s 9; echo held; exec sleep 60', lockPath])
    p.stdout.on('data', (d) => { if (String(d).includes('held')) resolve(p) })
    p.on('error', reject)
  })
}

async function stage (store, version, marker) {
  const holder = await holdShared(lockPathFor(store, 'app'))
  try {
    const r = materializeAtomic(store, appMaterializer(version, marker), { installName: 'app' })
    if (!r.staged) throw new Error('setup: version ' + version + ' was not staged')
  } finally {
    holder.kill('SIGKILL')
  }
}

const runs = (marker) => (fs.existsSync(marker) ? fs.readFileSync(marker, 'utf8').split('\n').filter(Boolean) : [])
async function until (fn, ms) {
  const end = Date.now() + ms
  while (Date.now() < end) { if (fn()) return true; await new Promise((r) => setTimeout(r, 100)) }
  return false
}

test('⛔ a staged update applies on every OTA respawn, not only the first', { skip: !HAS_FLOCK }, async (t) => {
  const root = fs.mkdtempSync(join(tmpdir(), 'sup-staged-'))
  const storeRoot = join(root, 'store')
  const store = new TreeStore({ root: storeRoot })
  store.init()
  const marker = join(root, 'runs.txt')

  materializeAtomic(store, appMaterializer('1', marker), { installName: 'app' })
  const live = store.appPath('app')

  const sup = spawn(process.execPath, [SUPERVISOR], {
    cwd: live, // exactly what `pgh app start` does (lib/commands/app.js: cwd: sourcePath)
    stdio: ['ignore', 'ignore', 'pipe'],
    env: {
      ...process.env,
      XDG_RUNTIME_DIR: join(root, 'run'),
      PGH_STORE_ROOT: storeRoot,
      PGH_SUP_NAME: 'app',
      PGH_SUP_ENTRYPOINT: join(live, 'entry.js'),
      PGH_SUP_RUNTIME: 'node',
      PGH_SUP_RUNTIME_PATH: process.execPath,
      PGH_SUP_ARGS_JSON: '[]',
      PGH_SUP_CHILD_ENV: '{}'
    }
  })
  let supLog = ''
  sup.stderr.on('data', (d) => { supLog += d })
  t.teardown(async () => {
    try { sup.kill('SIGTERM') } catch {}
    await new Promise((r) => setTimeout(r, 500))
    try { sup.kill('SIGKILL') } catch {}
    fs.rmSync(root, { recursive: true, force: true })
  })

  t.ok(await until(() => runs(marker).includes('1'), 10000), 'version 1 ran')

  await stage(store, '2', marker)
  t.ok(await until(() => runs(marker).includes('2'), 10000), 'the first staged update applied (this part always worked)')

  await stage(store, '3', marker)
  const third = await until(() => runs(marker).includes('3'), 10000)
  t.ok(third, 'the SECOND staged update applied too — the first swap deleted the supervisor\'s cwd')
  t.absent(/staged-update check failed/.test(supLog), 'and the supervisor never fell back to the old tree')
  if (!third) t.comment('supervisor log: ' + supLog.split('\n').filter((l) => /staged|applied/.test(l)).slice(-3).join(' | '))
  t.absent(fs.existsSync(stagedPathFor(store, 'app')), 'nothing is left waiting in .next')
})
pgh / test / supervisor-staged-after-swap.test.js
