OpenFUT/openfut-blaze-host/sidecar.sh

#!/usr/bin/env bash
# Lifecycle manager for the Blaze sidecar.
#
#   sidecar.sh start          start in the background, wait until it is listening
#   sidecar.sh stop           stop it, then VERIFY it is gone
#   sidecar.sh status         report
#   sidecar.sh check-orphans  fail if any sidecar is listening unexpectedly
#   sidecar.sh with -- CMD…   start, run CMD, always stop and verify
#
# WHY THIS EXISTS
#
# A previous session's mutation runs left four sidecars listening, two of them
# serving deliberately broken builds, because `kill %1` does not carry across
# shell invocations. A later A/B against one of those would have looked like a
# genuine parity failure. Ad-hoc backgrounding is not good enough before a live
# FIFA test.
#
# So stopping is not "send a signal and hope". It kills, waits, and then proves
# both that the PID is gone AND that the port is no longer listening. If either
# check fails, this script fails — a leaked sidecar must never be silent.
set -uo pipefail

HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
ROOT="$(cd "$HERE/.." && pwd)"
RUNDIR="${OPENFUT_SIDECAR_RUNDIR:-${TMPDIR:-/tmp}/openfut-sidecar}"
PIDFILE="$RUNDIR/sidecar.pid"
PORTFILE="$RUNDIR/sidecar.port"
LOGFILE="${OPENFUT_SIDECAR_LOG:-$RUNDIR/sidecar.log}"

BIN="$ROOT/target/debug/openfut-blaze-host"
[[ -x "$BIN" ]] || BIN="$ROOT/target/release/openfut-blaze-host"

die() { echo "sidecar: $*" >&2; exit 1; }

# Authoritative working-tree check, run at LAUNCH.
#
# The commit stamped into the binary by build.rs can be stale — cargo does not
# re-run a build script when another crate's source changes — so the compiled-in
# "dirty" flag is not a safeguard. This is. It runs now, against the tree as it
# is now, over exactly the crates the binary is built from.
#
# Echoes "DIRTY" or "clean" (or "unknown" outside a git tree).
tree_state() {
  git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1 || { echo unknown; return; }
  local out
  out="$(git -C "$ROOT" status --porcelain --untracked-files=no -- \
           openfut-blaze-host openfut-adapter-fifa17 openfut-protocol-blaze 2>/dev/null)"
  [[ -n "$out" ]] && echo DIRTY || echo clean
}

port_listening() {
  local port="$1"
  if command -v ss >/dev/null 2>&1; then
    ss -ltn 2>/dev/null | grep -qE "[:.]${port}[[:space:]]"
  elif command -v lsof >/dev/null 2>&1; then
    lsof -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1
  else
    # No way to check is not the same as "it is clean" — refuse to guess.
    die "neither ss nor lsof available; cannot verify port state"
  fi
}

pid_alive() { kill -0 "$1" 2>/dev/null; }

# ---------------------------------------------------------------- orphans

# Any sidecar process at all, whether or not this script started it.
#
# Matches the resolved EXECUTABLE, not the command line. `pgrep -f` was tried
# first and was wrong: it matched any process whose arguments merely mentioned
# the name — including the shell running this script, and any editor or script
# with the string in it. That is a false positive that refuses legitimate
# starts, which during a FIFA test is worse than the leak it guards against.
#
# `pgrep -x` is also unusable here: Linux truncates the process name to 15
# characters, so the binary appears as "openfut-blaze-h".
list_sidecars() {
  local self=$$ pid exe
  for d in /proc/[0-9]*; do
    pid="${d#/proc/}"
    [[ "$pid" == "$self" ]] && continue
    # readlink, NOT readlink -f: once the binary is rebuilt the link reads
    # "<path> (deleted)", and -f resolves that to something that matches
    # nothing. The orphan check would then be blind to exactly the long-lived
    # processes it exists to find — verified: two orphans (a stale-cert
    # redirector and a Blaze sidecar) were both invisible to this until the
    # suffix was stripped.
    exe="$(readlink "$d/exe" 2>/dev/null)" || continue
    exe="${exe% (deleted)}"
    [[ "${exe##*/}" == "openfut-blaze-host" ]] && echo "$pid"
  done
  return 0
}

cmd_check_orphans() {
  local found
  found="$(list_sidecars)"
  if [[ -z "$found" ]]; then
    echo "no sidecar processes running"
    return 0
  fi
  echo "ORPHANED SIDECAR PROCESS(ES) FOUND:" >&2
  for p in $found; do
    echo "  pid $p: $(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null || echo '?')" >&2
  done
  echo >&2
  echo "Refusing to proceed: a stale sidecar may be serving a mutated build," >&2
  echo "and an A/B against it would read as a real parity failure." >&2
  echo "Stop them with: pkill -f openfut-blaze-host" >&2
  return 1
}

# ------------------------------------------------------------------ start

cmd_start() {
  [[ -x "$BIN" ]] || die "binary not built; run: cargo build -p openfut-blaze-host"
  : "${OPENFUT_BLAZE_HOST_PORT:?set OPENFUT_BLAZE_HOST_PORT (no default, so the sidecar cannot collide with the Python backend)}"
  : "${OPENFUT_ADVERTISE:?set OPENFUT_ADVERTISE to the address the game machine uses to reach this host}"

  cmd_check_orphans >/dev/null 2>&1 || { cmd_check_orphans; die "clean up first"; }

  if port_listening "$OPENFUT_BLAZE_HOST_PORT"; then
    die "port $OPENFUT_BLAZE_HOST_PORT is already in use"
  fi

  # The binary's stamp can lag the tree (cargo cannot know about every source
  # change). Compare it with the tree's real HEAD at launch and say so, because
  # an evidence artefact that names the wrong commit is worse than one that
  # names none.
  HEAD_NOW="$(git -C "$ROOT" rev-parse --short=7 HEAD 2>/dev/null || echo unknown)"

  local tree
  tree="$(tree_state)"
  if [[ "$tree" == "DIRTY" ]]; then
    echo "WARNING: migration crates have uncommitted changes — this binary may not" >&2
    echo "         match any commit. Do not treat its output as parity evidence." >&2
  fi

  mkdir -p "$RUNDIR"
  echo "$OPENFUT_BLAZE_HOST_PORT" > "$PORTFILE"

  "$BIN" >"$LOGFILE" 2>&1 &
  local pid=$!
  echo "$pid" > "$PIDFILE"

  # Wait for the listener rather than sleeping a guess.
  local waited=0
  while (( waited < 100 )); do
    if ! pid_alive "$pid"; then
      echo "sidecar died during startup; log:" >&2
      tail -20 "$LOGFILE" >&2
      rm -f "$PIDFILE"
      return 1
    fi
    if port_listening "$OPENFUT_BLAZE_HOST_PORT"; then
      echo "sidecar started: pid $pid, port $OPENFUT_BLAZE_HOST_PORT"
      local banner stamped
      banner="$(grep -m1 'openfut-blaze-host v' "$LOGFILE" 2>/dev/null)"
      echo "  ${banner}"
      stamped="$(sed -n 's/.*commit=\([0-9a-f]*\).*/\1/p' <<<"$banner")"
      if [[ -n "$stamped" && "$stamped" != "unknown" && "$stamped" != "$HEAD_NOW" ]]; then
        echo "  !! STALE BUILD STAMP: binary says $stamped, HEAD is $HEAD_NOW" >&2
        echo "     Rebuild before treating this run as evidence:" >&2
        echo "       touch openfut-blaze-host/build.rs && cargo build -p openfut-blaze-host" >&2
      fi
      if grep -q 'WARNING: built from a modified working tree' "$LOGFILE" 2>/dev/null; then
        echo "  !! DIRTY BUILD — results are not parity evidence" >&2
      fi
      return 0
    fi
    sleep 0.1
    waited=$((waited + 1))
  done

  echo "sidecar did not begin listening within 10s; log:" >&2
  tail -20 "$LOGFILE" >&2
  kill "$pid" 2>/dev/null
  rm -f "$PIDFILE"
  return 1
}

# ------------------------------------------------------------------- stop
#
# Kill, wait, then PROVE it. Both conditions must hold or this fails.

cmd_stop() {
  local rc=0
  local pid="" port=""
  [[ -f "$PIDFILE"  ]] && pid="$(cat "$PIDFILE")"
  [[ -f "$PORTFILE" ]] && port="$(cat "$PORTFILE")"

  # Stopping the sidecar while the Blaze switch is still on leaves the client
  # pointed at a dead port — Blaze breaks and nothing says why. This exact state
  # was created once during development, so this REFUSES rather than warning:
  # a warning on stderr that is followed by doing the dangerous thing anyway is
  # not a safeguard.
  if [[ "${1:-}" != "--force" && -x "$HERE/blaze-switch.sh" ]]; then
    if "$HERE/blaze-switch.sh" status 2>/dev/null | grep -q "redirected to the RUST"; then
      echo "REFUSING to stop: the Blaze switch is still ON." >&2
      echo "  Stopping now would leave Blaze pointing at a dead port." >&2
      echo "  Roll back first:  ./blaze-switch.sh off" >&2
      echo "  Or override:      ./sidecar.sh stop --force" >&2
      return 1
    fi
  fi

  if [[ -n "$pid" ]] && pid_alive "$pid"; then
    kill "$pid" 2>/dev/null
    local waited=0
    while pid_alive "$pid" && (( waited < 50 )); do sleep 0.1; waited=$((waited+1)); done
    if pid_alive "$pid"; then
      echo "sidecar $pid ignored SIGTERM; escalating to SIGKILL" >&2
      kill -9 "$pid" 2>/dev/null
      waited=0
      while pid_alive "$pid" && (( waited < 50 )); do sleep 0.1; waited=$((waited+1)); done
    fi
  fi

  # Verification, not optimism.
  if [[ -n "$pid" ]] && pid_alive "$pid"; then
    echo "FAILED to stop sidecar pid $pid" >&2
    rc=1
  fi
  if [[ -n "$port" ]] && port_listening "$port"; then
    echo "FAILED: port $port is still listening after stop" >&2
    rc=1
  fi
  local strays
  strays="$(list_sidecars)"
  if [[ -n "$strays" ]]; then
    echo "FAILED: sidecar process(es) still running: $strays" >&2
    rc=1
  fi

  rm -f "$PIDFILE" "$PORTFILE"
  if [[ $rc -eq 0 ]]; then
    echo "sidecar stopped and verified gone${pid:+ (pid $pid)}${port:+, port $port free}"
  fi
  return $rc
}

cmd_status() {
  if [[ -f "$PIDFILE" ]] && pid_alive "$(cat "$PIDFILE")"; then
    echo "running: pid $(cat "$PIDFILE"), port $(cat "$PORTFILE" 2>/dev/null || echo '?')"
  else
    echo "not running (per pidfile)"
  fi
  local strays
  strays="$(list_sidecars)"
  [[ -n "$strays" ]] && echo "sidecar processes on this host: $strays"
  return 0
}

# ------------------------------------------------------------------- with
#
# Start, run a command, and stop+verify no matter how the command exits.

cmd_with() {
  cmd_start || return 1
  # shellcheck disable=SC2317
  cleanup() { cmd_stop || echo "sidecar: CLEANUP VERIFICATION FAILED" >&2; }
  trap cleanup EXIT INT TERM

  "$@"
  local rc=$?

  trap - EXIT INT TERM
  cmd_stop || { echo "sidecar: cleanup verification failed" >&2; return 1; }
  return $rc
}

case "${1:-}" in
  start)         shift; cmd_start "$@" ;;
  stop)          shift; cmd_stop "$@" ;;
  status)        shift; cmd_status "$@" ;;
  check-orphans) shift; cmd_check_orphans "$@" ;;
  with)
    shift
    [[ "${1:-}" == "--" ]] && shift
    [[ $# -gt 0 ]] || die "usage: sidecar.sh with -- COMMAND [ARGS…]"
    cmd_with "$@"
    ;;
  *)
    sed -n '2,10p' "$0" | sed 's/^# \?//'
    exit 2
    ;;
esac