#!/usr/bin/env bash
# bench-hyper-tps-cross-machine.sh
#
# Cross-machine cold-install bench for hyper-tps. Runs on the CONSUMER
# (e.g. stortron) pulling from a publisher (e.g. tabertron). Captures
# per-phase timing, bytes transferred, peak client RSS, and stall events.
#
# Usage:
# ./bench-hyper-tps-cross-machine.sh [--runs N] [--publisher <host>]
# [--registry-key <hex>]
# [--output <file>]
# [--restart-sidecar]
#
# Defaults:
# --runs 3
# --publisher tabertron.local
# --registry-key 1dd36bdfa9cb7f7d84adbd07f017e7200f7f90a3a546cd0761fac2a1f8a13e86
# --output ./bench-hyper-tps-$(hostname)-$(date +%FT%H-%M-%S).jsonl
#
# What it does per run:
# 1. mktemp a sandbox PGH_STORE_ROOT (no shared state)
# 2. Capture network rx byte counter
# 3. Spawn `pgh app install pgh://<key>/hyper-tps` with PGH_PROFILE=1,
# tee'ing stdout to a per-run log
# 4. Poll /proc/<pid>/status for peak RSS while it runs
# 5. After completion: parse phase timings from stderr, count stall
# events from stdout (gaps >10s between `synced:` lines)
# 6. Compute bytes transferred via interface delta + sandbox du
# 7. Emit one JSON row to the output file
#
# Cleanup: sandbox removed after each run. Stops on first failure unless
# --continue-on-error is set.
set -euo pipefail
# ── Defaults ─────────────────────────────────────────────────────────
RUNS=3
PUBLISHER="tabertron.local"
REGISTRY_KEY="1dd36bdfa9cb7f7d84adbd07f017e7200f7f90a3a546cd0761fac2a1f8a13e86"
OUTPUT="./bench-hyper-tps-$(hostname)-$(date -u +%FT%H-%M-%SZ).jsonl"
RESTART_SIDECAR=0
CONTINUE_ON_ERROR=0
STALL_THRESHOLD_SEC=10
# ── Parse args ───────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
case "$1" in
--runs) RUNS="$2"; shift 2 ;;
--publisher) PUBLISHER="$2"; shift 2 ;;
--registry-key) REGISTRY_KEY="$2"; shift 2 ;;
--output) OUTPUT="$2"; shift 2 ;;
--restart-sidecar) RESTART_SIDECAR=1; shift ;;
--continue-on-error) CONTINUE_ON_ERROR=1; shift ;;
--stall-sec) STALL_THRESHOLD_SEC="$2"; shift 2 ;;
-h|--help) sed -n '2,30p' "$0" | sed 's/^# \?//'; exit 0 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
done
PGH_URL="pgh://${REGISTRY_KEY}/hyper-tps"
# ── Pre-flight ───────────────────────────────────────────────────────
need () { command -v "$1" >/dev/null || { echo "missing: $1" >&2; exit 2; }; }
need pgh
need jq
# Pick the primary RX interface (the one with default route).
PRIMARY_IFACE=$(ip route 2>/dev/null | awk '/^default/ {print $5; exit}')
if [[ -z "${PRIMARY_IFACE:-}" ]]; then
echo "warn: no primary interface detected — bytes-rx metric will be zero" >&2
fi
rx_bytes () {
if [[ -z "${PRIMARY_IFACE:-}" || ! -r "/sys/class/net/${PRIMARY_IFACE}/statistics/rx_bytes" ]]; then
echo 0
else
cat "/sys/class/net/${PRIMARY_IFACE}/statistics/rx_bytes"
fi
}
# Tabertron-side: confirm sidecar is healthy + record state. Non-fatal.
preflight_publisher () {
echo "== publisher pre-flight ==" >&2
if ssh -o ConnectTimeout=5 "$PUBLISHER" 'pgh app status pgh-sidecar 2>&1 | head -8' 2>&1; then
:
else
echo "warn: could not reach $PUBLISHER over ssh — continuing anyway" >&2
fi
}
preflight_publisher
# Optional: restart local sidecar to clear in-memory state before benching.
if [[ "$RESTART_SIDECAR" -eq 1 ]]; then
echo "== restarting local pgh-sidecar ==" >&2
pgh app stop pgh-sidecar
pgh app start pgh-sidecar
# Wait for UDS to come up
for _ in $(seq 1 60); do
[[ -S "$HOME/.pgh/svc.sock" ]] && break
sleep 2
done
fi
# ── Per-run helpers ──────────────────────────────────────────────────
# Sample RSS of $1 every 500ms until the process exits.
# Writes peak-rss-kb to $2.
sample_rss () {
local pid="$1" out="$2" peak=0
while [[ -d "/proc/$pid" ]]; do
if [[ -r "/proc/$pid/status" ]]; then
local rss
rss=$(awk '/^VmRSS:/ {print $2; exit}' "/proc/$pid/status" 2>/dev/null || echo 0)
[[ "$rss" -gt "$peak" ]] && peak="$rss"
fi
sleep 0.5
done
echo "$peak" > "$out"
}
# Count stall events in a stdout log. A stall = gap >STALL_THRESHOLD_SEC
# between consecutive `synced:` lines (we tag each line with timestamp via
# `ts` if available; otherwise rely on log line ordering only — degraded
# metric in that case).
count_stalls () {
local log="$1"
if command -v ts >/dev/null; then
# `ts -i` prints relative seconds since previous line
grep -E "^[ \t]*synced:" "$log" | ts -i '%.s' | awk -v thresh="$STALL_THRESHOLD_SEC" '
$1 > thresh { count++; if ($1 > max) max = $1 }
END { print count " " (max ? max : 0) }
'
else
# No `ts` — emit 0/0 with a warning to stderr.
echo "0 0"
fi
}
# ── Run loop ─────────────────────────────────────────────────────────
echo "== bench config ==" >&2
echo " consumer: $(hostname)" >&2
echo " publisher: $PUBLISHER" >&2
echo " registry key: ${REGISTRY_KEY:0:12}..." >&2
echo " runs: $RUNS" >&2
echo " output: $OUTPUT" >&2
echo " rx iface: ${PRIMARY_IFACE:-<none>}" >&2
echo " stall threshold: ${STALL_THRESHOLD_SEC}s" >&2
echo "" >&2
mkdir -p "$(dirname "$OUTPUT")" 2>/dev/null || true
for run in $(seq 1 "$RUNS"); do
echo "── run $run/$RUNS ──" >&2
SANDBOX=$(mktemp -d -t pgh-bench-XXXXXX)
mkdir -p "$SANDBOX/store/apps" "$SANDBOX/store/dependencies" \
"$SANDBOX/store/meta" "$SANDBOX/store/tmp" \
"$SANDBOX/store/blobs" "$SANDBOX/bin"
echo "{}" > "$SANDBOX/store/refs.json"
STDOUT_LOG="$SANDBOX/install.stdout.log"
STDERR_LOG="$SANDBOX/install.stderr.log"
RSS_OUT="$SANDBOX/peak-rss-kb.txt"
RX_BEFORE=$(rx_bytes)
START_TS=$(date -u +%s.%N)
# Spawn install, capture stdout + stderr separately.
PGH_STORE_ROOT="$SANDBOX/store" \
PGH_BIN_DIR="$SANDBOX/bin" \
PGH_PROFILE=1 \
pgh app install "$PGH_URL" --name hyper-tps-bench \
> "$STDOUT_LOG" 2> "$STDERR_LOG" &
INSTALL_PID=$!
# Start RSS sampler in background, tied to the install pid.
sample_rss "$INSTALL_PID" "$RSS_OUT" &
SAMPLER_PID=$!
set +e
wait "$INSTALL_PID"
EXIT_CODE=$?
set -e
wait "$SAMPLER_PID" 2>/dev/null || true
END_TS=$(date -u +%s.%N)
RX_AFTER=$(rx_bytes)
WALL_SEC=$(echo "$END_TS $START_TS" | awk '{printf "%.3f", $1 - $2}')
RX_DELTA=$(( RX_AFTER - RX_BEFORE ))
PEAK_RSS_KB=$(cat "$RSS_OUT" 2>/dev/null || echo 0)
# Disk size of fetched data in the sandbox.
DEP_BYTES=$(du -sb "$SANDBOX/store/dependencies" 2>/dev/null | awk '{print $1}' || echo 0)
APP_BYTES=$(du -sb "$SANDBOX/store/apps" 2>/dev/null | awk '{print $1}' || echo 0)
# Parse PGH_PROFILE lines from stderr into a JSON object.
PHASES_JSON=$(awk '
/^\[profile\] \+[0-9]+ms / {
ms = $2; sub(/^\+/, "", ms); sub(/ms$/, "", ms);
label = ""
for (i = 3; i <= NF; i++) label = label (i == 3 ? "" : " ") $i
gsub(/"/, "\\\"", label)
out = out (out == "" ? "" : ",") "\"" label "\":" ms
}
END { print "{" out "}" }
' "$STDERR_LOG")
# Stall events.
read -r STALL_COUNT STALL_MAX_SEC <<< "$(count_stalls "$STDOUT_LOG")"
# Defensive defaults — jq --argjson rejects empty strings.
[[ -z "$WALL_SEC" ]] && WALL_SEC=0
[[ -z "$RX_DELTA" ]] && RX_DELTA=0
[[ -z "$DEP_BYTES" ]] && DEP_BYTES=0
[[ -z "$APP_BYTES" ]] && APP_BYTES=0
[[ -z "$PEAK_RSS_KB" ]] && PEAK_RSS_KB=0
[[ -z "$STALL_COUNT" ]] && STALL_COUNT=0
[[ -z "$STALL_MAX_SEC" ]] && STALL_MAX_SEC=0
[[ -z "$PHASES_JSON" ]] && PHASES_JSON='{}'
ROW=$(jq -n \
--arg consumer "$(hostname)" \
--arg publisher "$PUBLISHER" \
--arg registry "${REGISTRY_KEY:0:12}" \
--arg started "$(date -u -d @$(printf '%.0f' "$START_TS") +%FT%TZ)" \
--argjson run "$run" \
--argjson exit "$EXIT_CODE" \
--argjson wall "$WALL_SEC" \
--argjson rx_bytes "$RX_DELTA" \
--argjson dep_bytes "$DEP_BYTES" \
--argjson app_bytes "$APP_BYTES" \
--argjson peak_rss_kb "$PEAK_RSS_KB" \
--argjson stall_count "$STALL_COUNT" \
--argjson stall_max_sec "$STALL_MAX_SEC" \
--argjson phases "$PHASES_JSON" \
'{consumer: $consumer, publisher: $publisher, registry: $registry,
started: $started, run: $run, exit: $exit, wall_sec: $wall,
rx_bytes: $rx_bytes, dep_bytes: $dep_bytes, app_bytes: $app_bytes,
peak_rss_kb: $peak_rss_kb, stall_count: $stall_count,
stall_max_sec: $stall_max_sec, phases: $phases}')
echo "$ROW" >> "$OUTPUT"
# Human-readable line on stderr for visibility.
printf " exit=%d wall=%ss rx=%dMB dep=%dMB rss_peak=%dMB stalls=%d max_gap=%ss\n" \
"$EXIT_CODE" "$WALL_SEC" \
$(( RX_DELTA / 1024 / 1024 )) \
$(( DEP_BYTES / 1024 / 1024 )) \
$(( PEAK_RSS_KB / 1024 )) \
"$STALL_COUNT" "$STALL_MAX_SEC" >&2
# Cleanup sandbox. On failure (and continue-on-error not set), preserve
# logs by moving them next to OUTPUT before bailing.
if [[ "$EXIT_CODE" -ne 0 ]]; then
DIAG_DIR="${OUTPUT%.jsonl}.run-${run}-failed"
mv "$SANDBOX" "$DIAG_DIR" 2>/dev/null || true
echo " failure diagnostic dir preserved at: $DIAG_DIR" >&2
if [[ "$CONTINUE_ON_ERROR" -eq 0 ]]; then
echo " bailing — pass --continue-on-error to keep going" >&2
exit "$EXIT_CODE"
fi
else
rm -rf "$SANDBOX"
fi
done
# ── Summary ──────────────────────────────────────────────────────────
echo "" >&2
echo "== summary ==" >&2
jq -s '
group_by(.consumer + "→" + .publisher) | .[] | {
pair: (.[0].consumer + "→" + .[0].publisher),
runs: length,
wall_sec: { min: (map(.wall_sec) | min), p50: (map(.wall_sec) | sort | .[length/2|floor]), max: (map(.wall_sec) | max) },
rx_mb_p50: (map(.rx_bytes / 1024 / 1024) | sort | .[length/2|floor]),
peak_rss_mb_p50: (map(.peak_rss_kb / 1024) | sort | .[length/2|floor]),
stall_count_sum: (map(.stall_count) | add),
stall_max_sec: (map(.stall_max_sec) | max),
deps_fetched_p50_ms: (map(.phases."deps-fetched" // 0) | sort | .[length/2|floor]),
root_fetched_p50_ms: (map(.phases."root-fetched" // 0) | sort | .[length/2|floor])
}
' "$OUTPUT" >&2
echo "" >&2
echo "wrote $RUNS row(s) to: $OUTPUT" >&2