#!/usr/bin/env bash # Lifecycle manager for the Blaze sidecar. # # sidecar.sh start start in the background, wait until it is listening # sidecar.sh stop stop it, then VERIFY it is gone # sidecar.sh status report # sidecar.sh check-orphans fail if any sidecar is listening unexpectedly # sidecar.sh with -- CMD… start, run CMD, always stop and verify # # WHY THIS EXISTS # # A previous session's mutation runs left four sidecars listening, two of them # serving deliberately broken builds, because `kill %1` does not carry across # shell invocations. A later A/B against one of those would have looked like a # genuine parity failure. Ad-hoc backgrounding is not good enough before a live # FIFA test. # # So stopping is not "send a signal and hope". It kills, waits, and then proves # both that the PID is gone AND that the port is no longer listening. If either # check fails, this script fails — a leaked sidecar must never be silent. set -uo pipefail HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" ROOT="$(cd "$HERE/.." && pwd)" RUNDIR="${OPENFUT_SIDECAR_RUNDIR:-${TMPDIR:-/tmp}/openfut-sidecar}" PIDFILE="$RUNDIR/sidecar.pid" PORTFILE="$RUNDIR/sidecar.port" LOGFILE="${OPENFUT_SIDECAR_LOG:-$RUNDIR/sidecar.log}" BIN="$ROOT/target/debug/openfut-blaze-host" [[ -x "$BIN" ]] || BIN="$ROOT/target/release/openfut-blaze-host" die() { echo "sidecar: $*" >&2; exit 1; } # Authoritative working-tree check, run at LAUNCH. # # The commit stamped into the binary by build.rs can be stale — cargo does not # re-run a build script when another crate's source changes — so the compiled-in # "dirty" flag is not a safeguard. This is. It runs now, against the tree as it # is now, over exactly the crates the binary is built from. # # Echoes "DIRTY" or "clean" (or "unknown" outside a git tree). tree_state() { git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1 || { echo unknown; return; } local out out="$(git -C "$ROOT" status --porcelain --untracked-files=no -- \ openfut-blaze-host openfut-adapter-fifa17 openfut-protocol-blaze 2>/dev/null)" [[ -n "$out" ]] && echo DIRTY || echo clean } port_listening() { local port="$1" if command -v ss >/dev/null 2>&1; then ss -ltn 2>/dev/null | grep -qE "[:.]${port}[[:space:]]" elif command -v lsof >/dev/null 2>&1; then lsof -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 else # No way to check is not the same as "it is clean" — refuse to guess. die "neither ss nor lsof available; cannot verify port state" fi } pid_alive() { kill -0 "$1" 2>/dev/null; } # ---------------------------------------------------------------- orphans # Any sidecar process at all, whether or not this script started it. # # Matches the resolved EXECUTABLE, not the command line. `pgrep -f` was tried # first and was wrong: it matched any process whose arguments merely mentioned # the name — including the shell running this script, and any editor or script # with the string in it. That is a false positive that refuses legitimate # starts, which during a FIFA test is worse than the leak it guards against. # # `pgrep -x` is also unusable here: Linux truncates the process name to 15 # characters, so the binary appears as "openfut-blaze-h". list_sidecars() { local self=$$ pid exe for d in /proc/[0-9]*; do pid="${d#/proc/}" [[ "$pid" == "$self" ]] && continue # readlink, NOT readlink -f: once the binary is rebuilt the link reads # " (deleted)", and -f resolves that to something that matches # nothing. The orphan check would then be blind to exactly the long-lived # processes it exists to find — verified: two orphans (a stale-cert # redirector and a Blaze sidecar) were both invisible to this until the # suffix was stripped. exe="$(readlink "$d/exe" 2>/dev/null)" || continue exe="${exe% (deleted)}" [[ "${exe##*/}" == "openfut-blaze-host" ]] && echo "$pid" done return 0 } cmd_check_orphans() { local found found="$(list_sidecars)" if [[ -z "$found" ]]; then echo "no sidecar processes running" return 0 fi echo "ORPHANED SIDECAR PROCESS(ES) FOUND:" >&2 for p in $found; do echo " pid $p: $(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null || echo '?')" >&2 done echo >&2 echo "Refusing to proceed: a stale sidecar may be serving a mutated build," >&2 echo "and an A/B against it would read as a real parity failure." >&2 echo "Stop them with: pkill -f openfut-blaze-host" >&2 return 1 } # ------------------------------------------------------------------ start cmd_start() { [[ -x "$BIN" ]] || die "binary not built; run: cargo build -p openfut-blaze-host" : "${OPENFUT_BLAZE_HOST_PORT:?set OPENFUT_BLAZE_HOST_PORT (no default, so the sidecar cannot collide with the Python backend)}" : "${OPENFUT_ADVERTISE:?set OPENFUT_ADVERTISE to the address the game machine uses to reach this host}" cmd_check_orphans >/dev/null 2>&1 || { cmd_check_orphans; die "clean up first"; } if port_listening "$OPENFUT_BLAZE_HOST_PORT"; then die "port $OPENFUT_BLAZE_HOST_PORT is already in use" fi # The binary's stamp can lag the tree (cargo cannot know about every source # change). Compare it with the tree's real HEAD at launch and say so, because # an evidence artefact that names the wrong commit is worse than one that # names none. HEAD_NOW="$(git -C "$ROOT" rev-parse --short=7 HEAD 2>/dev/null || echo unknown)" local tree tree="$(tree_state)" if [[ "$tree" == "DIRTY" ]]; then echo "WARNING: migration crates have uncommitted changes — this binary may not" >&2 echo " match any commit. Do not treat its output as parity evidence." >&2 fi mkdir -p "$RUNDIR" echo "$OPENFUT_BLAZE_HOST_PORT" > "$PORTFILE" "$BIN" >"$LOGFILE" 2>&1 & local pid=$! echo "$pid" > "$PIDFILE" # Wait for the listener rather than sleeping a guess. local waited=0 while (( waited < 100 )); do if ! pid_alive "$pid"; then echo "sidecar died during startup; log:" >&2 tail -20 "$LOGFILE" >&2 rm -f "$PIDFILE" return 1 fi if port_listening "$OPENFUT_BLAZE_HOST_PORT"; then echo "sidecar started: pid $pid, port $OPENFUT_BLAZE_HOST_PORT" local banner stamped banner="$(grep -m1 'openfut-blaze-host v' "$LOGFILE" 2>/dev/null)" echo " ${banner}" stamped="$(sed -n 's/.*commit=\([0-9a-f]*\).*/\1/p' <<<"$banner")" if [[ -n "$stamped" && "$stamped" != "unknown" && "$stamped" != "$HEAD_NOW" ]]; then echo " !! STALE BUILD STAMP: binary says $stamped, HEAD is $HEAD_NOW" >&2 echo " Rebuild before treating this run as evidence:" >&2 echo " touch openfut-blaze-host/build.rs && cargo build -p openfut-blaze-host" >&2 fi if grep -q 'WARNING: built from a modified working tree' "$LOGFILE" 2>/dev/null; then echo " !! DIRTY BUILD — results are not parity evidence" >&2 fi return 0 fi sleep 0.1 waited=$((waited + 1)) done echo "sidecar did not begin listening within 10s; log:" >&2 tail -20 "$LOGFILE" >&2 kill "$pid" 2>/dev/null rm -f "$PIDFILE" return 1 } # ------------------------------------------------------------------- stop # # Kill, wait, then PROVE it. Both conditions must hold or this fails. cmd_stop() { local rc=0 local pid="" port="" [[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE")" [[ -f "$PORTFILE" ]] && port="$(cat "$PORTFILE")" # Stopping the sidecar while the Blaze switch is still on leaves the client # pointed at a dead port — Blaze breaks and nothing says why. This exact state # was created once during development, so this REFUSES rather than warning: # a warning on stderr that is followed by doing the dangerous thing anyway is # not a safeguard. if [[ "${1:-}" != "--force" && -x "$HERE/blaze-switch.sh" ]]; then if "$HERE/blaze-switch.sh" status 2>/dev/null | grep -q "redirected to the RUST"; then echo "REFUSING to stop: the Blaze switch is still ON." >&2 echo " Stopping now would leave Blaze pointing at a dead port." >&2 echo " Roll back first: ./blaze-switch.sh off" >&2 echo " Or override: ./sidecar.sh stop --force" >&2 return 1 fi fi if [[ -n "$pid" ]] && pid_alive "$pid"; then kill "$pid" 2>/dev/null local waited=0 while pid_alive "$pid" && (( waited < 50 )); do sleep 0.1; waited=$((waited+1)); done if pid_alive "$pid"; then echo "sidecar $pid ignored SIGTERM; escalating to SIGKILL" >&2 kill -9 "$pid" 2>/dev/null waited=0 while pid_alive "$pid" && (( waited < 50 )); do sleep 0.1; waited=$((waited+1)); done fi fi # Verification, not optimism. if [[ -n "$pid" ]] && pid_alive "$pid"; then echo "FAILED to stop sidecar pid $pid" >&2 rc=1 fi if [[ -n "$port" ]] && port_listening "$port"; then echo "FAILED: port $port is still listening after stop" >&2 rc=1 fi local strays strays="$(list_sidecars)" if [[ -n "$strays" ]]; then echo "FAILED: sidecar process(es) still running: $strays" >&2 rc=1 fi rm -f "$PIDFILE" "$PORTFILE" if [[ $rc -eq 0 ]]; then echo "sidecar stopped and verified gone${pid:+ (pid $pid)}${port:+, port $port free}" fi return $rc } cmd_status() { if [[ -f "$PIDFILE" ]] && pid_alive "$(cat "$PIDFILE")"; then echo "running: pid $(cat "$PIDFILE"), port $(cat "$PORTFILE" 2>/dev/null || echo '?')" else echo "not running (per pidfile)" fi local strays strays="$(list_sidecars)" [[ -n "$strays" ]] && echo "sidecar processes on this host: $strays" return 0 } # ------------------------------------------------------------------- with # # Start, run a command, and stop+verify no matter how the command exits. cmd_with() { cmd_start || return 1 # shellcheck disable=SC2317 cleanup() { cmd_stop || echo "sidecar: CLEANUP VERIFICATION FAILED" >&2; } trap cleanup EXIT INT TERM "$@" local rc=$? trap - EXIT INT TERM cmd_stop || { echo "sidecar: cleanup verification failed" >&2; return 1; } return $rc } case "${1:-}" in start) shift; cmd_start "$@" ;; stop) shift; cmd_stop "$@" ;; status) shift; cmd_status "$@" ;; check-orphans) shift; cmd_check_orphans "$@" ;; with) shift [[ "${1:-}" == "--" ]] && shift [[ $# -gt 0 ]] || die "usage: sidecar.sh with -- COMMAND [ARGS…]" cmd_with "$@" ;; *) sed -n '2,10p' "$0" | sed 's/^# \?//' exit 2 ;; esac