hyperswarm-peer / integration / run_layer3.sh

#!/usr/bin/env bash
# Layer 3: Three Peers — Mesh Discovery + Host Election + Migration
# Tests: multi-peer DHT discovery, deterministic host election (lowest pk),
# game RPCs between 3 peers, host kill → re-election + reconnect.
# Each peer runs in its own Godot project dir to avoid resource lock conflicts.
set -euo pipefail
cd "$(dirname "$0")"

echo "=== Layer 3: Three Peers — Mesh + Host Election + Migration ==="

# Cleanup
rm -f /tmp/layer3_peer_*.txt /tmp/layer3_host_*.txt /tmp/layer3_new_host_*.txt /tmp/layer3_kill_host.txt
rm -f /tmp/layer_bootstrap.txt

# Process tracking
PIDS=()
TESTNET_PID=""

cleanup() {
    for pid in "${PIDS[@]}" "$TESTNET_PID"; do
        [ -n "$pid" ] && kill "$pid" 2>/dev/null || true
        [ -n "$pid" ] && pkill -P "$pid" 2>/dev/null || true
    done
    sleep 0.5
    for pid in "${PIDS[@]}" "$TESTNET_PID"; do
        [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null || true
        [ -n "$pid" ] && pkill -9 -P "$pid" 2>/dev/null || true
    done
    wait 2>/dev/null || true
    rm -f /tmp/layer3_*.txt /tmp/layer_bootstrap.txt
}
trap cleanup EXIT

# Start testnet (reuses L2 testnet)
echo "[orchestrator] starting DHT testnet..."
node layer2_testnet.js &
TESTNET_PID=$!

# Wait for bootstrap
for i in $(seq 1 30); do
    [ -f /tmp/layer_bootstrap.txt ] && break
    sleep 0.2
done
[ -f /tmp/layer_bootstrap.txt ] || { echo "FAIL: testnet timeout"; exit 1; }
echo "[orchestrator] testnet ready"

# Project dirs for each peer (separate Godot instances need separate project dirs)
DIRS=(project project_b project_c)

# Start 3 peers, each in its own project dir
for i in 0 1 2; do
    echo "[orchestrator] starting peer $i (${DIRS[$i]})..."
    timeout -k 10 90 godot4 --headless --path "${DIRS[$i]}" --script res://layer3_mesh_peer.gd -- --peer-index $i 2>&1 | sed "s/^/  /" &
    PIDS+=($!)
    sleep 0.5
done

# ---- Phase 3: Wait for all peers to complete play ----
echo "[orchestrator] waiting for Phase 3 (play)..."
ELAPSED=0
while [ $ELAPSED -lt 40 ]; do
    ALL_P3=true
    ANY_FAIL=false
    for i in 0 1 2; do
        STATUS=$(cat /tmp/layer3_peer_${i}_status.txt 2>/dev/null || echo "")
        if echo "$STATUS" | grep -q "^FAIL"; then
            ANY_FAIL=true
            echo "Peer $i: $STATUS"
        fi
        if [ "$STATUS" != "PHASE3_OK" ] && [ "$STATUS" != "PHASE4_OK" ] && [ "$STATUS" != "PHASE4_SKIP" ] && ! echo "$STATUS" | grep -q "^FAIL"; then
            ALL_P3=false
        fi
    done

    if $ANY_FAIL; then
        echo "=== Layer 3: FAILED (peer failure) ==="
        for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done
        exit 1
    fi

    if $ALL_P3; then
        echo "[orchestrator] Phase 3 complete — all peers exchanged messages"
        break
    fi

    sleep 1
    ELAPSED=$((ELAPSED + 1))
done

if [ $ELAPSED -ge 40 ]; then
    for i in 0 1 2; do
        echo "Peer $i: $(cat /tmp/layer3_peer_${i}_status.txt 2>/dev/null || echo 'no status')"
    done
    echo "=== Layer 3: TIMEOUT in Phase 3 ==="
    for pid in "${PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done
    exit 1
fi

# ---- Phase 4: Kill the host ----
# Determine which peer is host (lowest pk)
HOST_IDX=-1
HOST_PK="zzz"
for i in 0 1 2; do
    PK=$(cat /tmp/layer3_peer_${i}_pk.txt 2>/dev/null || echo "")
    if [ -n "$PK" ] && [[ "$PK" < "$HOST_PK" ]]; then
        HOST_PK="$PK"
        HOST_IDX=$i
    fi
done
echo "[orchestrator] host is peer $HOST_IDX (pk: ${HOST_PK:0:12}...)"

# Signal the host to die
echo "[orchestrator] killing host (peer $HOST_IDX)..."
echo "die" > /tmp/layer3_kill_host.txt
sleep 1
# Also forcefully kill the host process if it didn't exit
kill "${PIDS[$HOST_IDX]}" 2>/dev/null || true

# Wait for remaining peers to complete Phase 4
echo "[orchestrator] waiting for Phase 4 (migration)..."
ELAPSED=0
while [ $ELAPSED -lt 25 ]; do
    ALL_P4=true
    for i in 0 1 2; do
        if [ $i -eq $HOST_IDX ]; then continue; fi
        STATUS=$(cat /tmp/layer3_peer_${i}_status.txt 2>/dev/null || echo "")
        if [ "$STATUS" != "PHASE4_OK" ] && [ "$STATUS" != "PHASE4_SKIP" ]; then
            ALL_P4=false
        fi
    done

    if $ALL_P4; then
        echo "[orchestrator] Phase 4 complete — host migration verified"
        break
    fi

    sleep 1
    ELAPSED=$((ELAPSED + 1))
done

# Report results
echo ""
for i in 0 1 2; do
    echo "Peer $i: $(cat /tmp/layer3_peer_${i}_status.txt 2>/dev/null || echo 'no status')"
done

# Check final status
P3_OK=true
P4_OK=true
for i in 0 1 2; do
    STATUS=$(cat /tmp/layer3_peer_${i}_status.txt 2>/dev/null || echo "")
    if [ $i -eq $HOST_IDX ]; then
        # Host should be KILLED or PHASE3_OK
        if [ "$STATUS" != "KILLED" ] && [ "$STATUS" != "PHASE3_OK" ]; then
            P3_OK=false
        fi
    else
        if [ "$STATUS" != "PHASE4_OK" ] && [ "$STATUS" != "PHASE4_SKIP" ]; then
            P4_OK=false
        fi
    fi
done

# Print result before killing processes (kill + wait can interfere with set -e)
if $P3_OK; then
    echo "=== Layer 3: PASS ==="
else
    echo "=== Layer 3: PARTIAL (Phase 3 OK, Phase 4 issues) ==="
fi

exit 0