atproto-bridge / scripts / remote-deploy.sh

#!/usr/bin/env bash
# Hardened in-place deploy for a pgh app. Runs ON the target host.
#
# Fixes the 2026-07-03 outage: a detached `stop -> install -> start` stranded the daemon
# when `install` stalled AFTER the stop (network topic hang), and overlapping deploys
# thrashed it. Guarantees, by construction:
#   - SERIALIZED     — flock; a second deploy aborts instead of overlapping.
#   - BOUNDED INSTALL— install can't hang forever (timeout); on failure we don't proceed blind.
#   - NEVER STRANDED — the daemon is ALWAYS (re)started afterwards, even if install fails.
#   - VERIFIED       — polls for running:true AND an HTTP health check before declaring success.
#   - ROLLBACK       — if the new tree won't come up healthy, restore the previous tree.
#
# Usage (on the host):  remote-deploy.sh <app> [port] [--check]
#   --check : report current tree/status/health only; make NO changes.

set -uo pipefail
APP="${1:?usage: remote-deploy.sh <app> [port] [--check]}"
PORT="${2:-3399}"
MODE="${3:-}"
LOCK="/tmp/deploy-$APP.lock"

health() { curl -sf --max-time 6 "http://127.0.0.1:$PORT/xrpc/com.atproto.server.describeServer" >/dev/null; }
current_tree() { pgh app current-tree "$APP" 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1; }
status_line() { pgh app status "$APP" 2>/dev/null | grep -E 'running:|version:'; }

if [ "$MODE" = "--check" ]; then
  echo "[check] $APP tree=$(current_tree)"
  status_line
  if health; then echo "[check] HEALTHY"; else echo "[check] UNHEALTHY"; fi
  exit 0
fi

# Serialize: never let two deploys overlap (the other half of the outage).
exec 9>"$LOCK"
if ! flock -n 9; then echo "[deploy] another deploy of $APP in progress — aborting"; exit 1; fi

PREV="$(current_tree)"
echo "[deploy] $APP prev-tree=${PREV:-none}"

pgh app stop "$APP" || true

if timeout 180 pgh app install "$APP"; then
  echo "[deploy] install ok"
else
  echo "[deploy] install FAILED/timed out"
  if [ -n "$PREV" ]; then echo "[deploy] restoring previous tree $PREV"; pgh app install-tree "$APP" --tree "$PREV" || true; fi
fi

# ALWAYS start — the daemon must never be left stopped, whatever happened above.
pgh app start "$APP" || true

ok=""
for _ in $(seq 1 20); do
  pgh app status "$APP" 2>/dev/null | grep -q "running: true" && { ok=1; break; }
  sleep 3
done
if [ -z "$ok" ]; then echo "[deploy] not running yet — forcing start"; pgh app start "$APP" || true; sleep 5; fi

if health; then
  echo "[deploy] HEALTHY — $(status_line)"
  exit 0
fi

echo "[deploy] UNHEALTHY after start"
if [ -n "$PREV" ]; then
  echo "[deploy] rolling back to $PREV"
  pgh app stop "$APP" || true
  pgh app install-tree "$APP" --tree "$PREV" || true
  pgh app start "$APP" || true
  sleep 5
  if health; then echo "[deploy] ROLLED BACK — HEALTHY"; exit 0; fi
fi
echo "[deploy] STILL UNHEALTHY — MANUAL INTERVENTION NEEDED"
exit 1