diff --git a/openfut-redirector-host/redirector.sh b/openfut-redirector-host/redirector.sh index 789d5eb..8ccd627 100755 --- a/openfut-redirector-host/redirector.sh +++ b/openfut-redirector-host/redirector.sh @@ -1,178 +1,26 @@ #!/usr/bin/env bash # Lifecycle for the Rust redirector host. # -# redirector.sh start | stop | status | verify +# redirector.sh start | stop | status | verify | verify-running # -# Mirrors sidecar.sh: refuses to start with an orphan present or the port busy, -# and stop PROVES the process is gone and the port free rather than assuming a -# signal worked. +# A thin wrapper: the rules this project paid for — resolve /proc/PID/exe +# rather than matching command lines, tolerate the "(deleted)" suffix after a +# rebuild, prove a stop rather than assume it, refuse an ambiguous binary, and +# check the RUNNING process's commit rather than the one on disk — all live in +# scripts/host-lifecycle.sh so there is one implementation of them. # -# Additionally REFUSES TO START unless the binary's stamped commit equals HEAD -# and the migration crates are clean — evidence from an unidentifiable binary is -# not evidence. -set -uo pipefail +# Environment: +# OPENFUT_REDIRECTOR_HOST_PORT required; no default, so this can never +# collide with the Python redirector +# OPENFUT_REDIRECTOR_CERT/KEY MUST be the same pair every other +# FIFA-facing service presents +# OPENFUT_REDIRECTOR_BIN explicit binary, if debug and release disagree +HL_NAME="redirector" +HL_CRATE="openfut-redirector-host" +HL_BINNAME="openfut-redirector-host" +HL_PORT_VAR="OPENFUT_REDIRECTOR_HOST_PORT" -HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" -ROOT="$(cd "$HERE/.." && pwd)" -RUNDIR="${OPENFUT_REDIRECTOR_RUNDIR:-${TMPDIR:-/tmp}/openfut-redirector}" -PIDFILE="$RUNDIR/redirector.pid" -PORTFILE="$RUNDIR/redirector.port" -# Commit the RUNNING process was started from. `verify` alone inspects the -# on-disk binary, which a rebuild (even `cargo test`, which re-runs build.rs -# when the branch ref moves) can silently advance past the live process. -STAMPFILE="$RUNDIR/redirector.commit" -LOGFILE="${OPENFUT_REDIRECTOR_LOG:-$RUNDIR/redirector.log}" -# Which artifact is under test must never be ambiguous. Preferring debug and -# falling back to release meant `cargo build --release` could produce a fresh -# binary while this kept launching a stale debug one — the guard then compared -# the WRONG artifact against HEAD and refused with a message naming a commit -# nobody was trying to run. Both present and disagreeing is an error, not a -# preference. Set OPENFUT_REDIRECTOR_BIN to choose deliberately. -DEBUG_BIN="$ROOT/target/debug/openfut-redirector-host" -RELEASE_BIN="$ROOT/target/release/openfut-redirector-host" -BIN_ERR="" -if [[ -n "${OPENFUT_REDIRECTOR_BIN:-}" ]]; then - BIN="$OPENFUT_REDIRECTOR_BIN" - [[ -x "$BIN" ]] || { echo "redirector: OPENFUT_REDIRECTOR_BIN is not executable: $BIN" >&2; exit 1; } -elif [[ -x "$DEBUG_BIN" && -x "$RELEASE_BIN" ]]; then - d="$("$DEBUG_BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1)" - r="$("$RELEASE_BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1)" - if [[ "$d" != "$r" ]]; then - # Recorded, NOT fatal here: `stop` and `status` must keep working no matter - # what state the build tree is in. Rolling back can never be blocked by a - # question about which artifact would have been started. - BIN_ERR="REFUSING — two binaries exist and disagree. - debug $DEBUG_BIN commit=$d - release $RELEASE_BIN commit=$r - Rebuild both, delete one, or set OPENFUT_REDIRECTOR_BIN." - fi - BIN="$RELEASE_BIN" -elif [[ -x "$DEBUG_BIN" ]]; then - BIN="$DEBUG_BIN" -else - BIN="$RELEASE_BIN" -fi +# shellcheck source=../scripts/host-lifecycle.sh +source "$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)/scripts/host-lifecycle.sh" -die() { echo "redirector: $*" >&2; exit 1; } -pid_alive() { kill -0 "$1" 2>/dev/null; } -port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; } - -# Match the resolved executable, not the command line: `pgrep -f` matches any -# shell whose arguments merely mention the name. -list_procs() { - local self=$$ pid exe - for d in /proc/[0-9]*; do - pid="${d#/proc/}"; [[ "$pid" == "$self" ]] && continue - # readlink, NOT readlink -f: once the binary is rebuilt the link reads - # " (deleted)", and -f resolves that to something that matches - # nothing. The orphan check would then be blind to exactly the long-lived - # processes it exists to find — verified: two orphans (a stale-cert - # redirector and a Blaze sidecar) were both invisible to this until the - # suffix was stripped. - exe="$(readlink "$d/exe" 2>/dev/null)" || continue - exe="${exe% (deleted)}" - [[ "${exe##*/}" == "openfut-redirector-host" ]] && echo "$pid" - done - return 0 -} - -# Only the commands that actually run a binary care which one it is. -need_bin() { [[ -z "$BIN_ERR" ]] || die "$BIN_ERR"; } - -# The binary prints `commit=` in its banner; ask it rather than guessing. -stamped_commit() { "$BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; } - -cmd_verify() { - need_bin - local c; c="$(stamped_commit)" - [[ -n "$c" ]] || die "could not read the binary's commit stamp" - "$ROOT/scripts/verify-build-identity.sh" "$c" -} - -cmd_start() { - need_bin - [[ -x "$BIN" ]] || die "not built: cargo build -p openfut-redirector-host" - : "${OPENFUT_REDIRECTOR_HOST_PORT:?set OPENFUT_REDIRECTOR_HOST_PORT (no default: runs beside Python)}" - local strays; strays="$(list_procs)" - [[ -z "$strays" ]] || die "orphan redirector process(es): $strays" - port_listening "$OPENFUT_REDIRECTOR_HOST_PORT" && die "port $OPENFUT_REDIRECTOR_HOST_PORT in use" - - cmd_verify || die "build identity check failed — refusing to start" - - mkdir -p "$RUNDIR"; echo "$OPENFUT_REDIRECTOR_HOST_PORT" > "$PORTFILE" - stamped_commit > "$STAMPFILE" - "$BIN" >"$LOGFILE" 2>&1 & - local pid=$!; echo "$pid" > "$PIDFILE" - local w=0 - while (( w < 100 )); do - pid_alive "$pid" || { echo "died during startup:" >&2; tail -20 "$LOGFILE" >&2; rm -f "$PIDFILE"; return 1; } - if port_listening "$OPENFUT_REDIRECTOR_HOST_PORT"; then - echo "redirector started: pid $pid, port $OPENFUT_REDIRECTOR_HOST_PORT" - grep -E 'SELF-TEST|openfut-redirector-host v' "$LOGFILE" | sed 's/^/ /' - return 0 - fi - sleep 0.1; w=$((w+1)) - done - echo "did not listen within 10s:" >&2; tail -20 "$LOGFILE" >&2 - kill "$pid" 2>/dev/null; rm -f "$PIDFILE"; return 1 -} - -cmd_stop() { - local rc=0 pid="" port="" - [[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE")" - [[ -f "$PORTFILE" ]] && port="$(cat "$PORTFILE")" - if [[ -n "$pid" ]] && pid_alive "$pid"; then - kill "$pid" 2>/dev/null - local w=0; while pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done - pid_alive "$pid" && kill -9 "$pid" 2>/dev/null - sleep 0.2 - fi - [[ -n "$pid" ]] && pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; } - [[ -n "$port" ]] && port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; } - local strays; strays="$(list_procs)" - [[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; } - rm -f "$PIDFILE" "$PORTFILE" - [[ $rc -eq 0 ]] && echo "redirector stopped and verified gone${pid:+ (pid $pid)}" - return $rc -} - -cmd_status() { - if [[ -f "$PIDFILE" ]] && pid_alive "$(cat "$PIDFILE")"; then - echo "running: pid $(cat "$PIDFILE"), port $(cat "$PORTFILE" 2>/dev/null || echo '?')" - else - echo "not running" - fi - local strays; strays="$(list_procs)" - [[ -n "$strays" ]] && echo "redirector processes: $strays" - return 0 -} - -cmd_verify_running() { - # The stamp outlives the process it describes. Without this check the command - # happily reports on a corpse — either "OK" or a REFUSAL naming a commit, - # both implying something is running when nothing is. - local pid="" - [[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE" 2>/dev/null)" - if [[ -z "$pid" ]] || ! pid_alive "$pid"; then - echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2 - return 1 - fi - [[ -f "$STAMPFILE" ]] || die "no running-process stamp — was it started by this script?" - local running head - running="$(cat "$STAMPFILE")"; head="$(git -C "$ROOT" rev-parse --short=7 HEAD 2>/dev/null)" - if [[ "$running" != "$head" ]]; then - echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2 - echo " Restart before treating this run as evidence." >&2 - return 1 - fi - echo "running-process identity OK: started from $running == HEAD" -} - -case "${1:-}" in - start) cmd_start ;; - stop) cmd_stop ;; - status) cmd_status ;; - verify) cmd_verify ;; - verify-running) cmd_verify_running ;; - *) sed -n '2,6p' "$0" | sed 's/^# \?//'; exit 2 ;; -esac +hl_dispatch "${1:-}" diff --git a/openfut-roster-host/roster.sh b/openfut-roster-host/roster.sh new file mode 100755 index 0000000..42affeb --- /dev/null +++ b/openfut-roster-host/roster.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Lifecycle for the Rust FUT roster host. +# +# roster.sh start | stop | status | verify | verify-running +# +# A thin wrapper: everything about process ownership, executable identity, +# orphan detection and proven shutdown lives in scripts/host-lifecycle.sh, so +# there is exactly one implementation of the rules the redirector and Blaze +# hosts paid for. This file supplies only what is specific to roster. +# +# Environment: +# OPENFUT_ROSTER_HOST_PORT required; no default, so this can never collide +# with the Python roster server it runs beside +# OPENFUT_ROSTER_CERT/KEY MUST be the same pair every other FIFA-facing +# service presents +# OPENFUT_ROSTER_BIN explicit binary, if debug and release disagree +HL_NAME="roster" +HL_CRATE="openfut-roster-host" +HL_BINNAME="openfut-roster-host" +HL_PORT_VAR="OPENFUT_ROSTER_HOST_PORT" + +# shellcheck source=../scripts/host-lifecycle.sh +source "$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)/scripts/host-lifecycle.sh" + +hl_dispatch "${1:-}" diff --git a/scripts/host-lifecycle.sh b/scripts/host-lifecycle.sh new file mode 100644 index 0000000..925be50 --- /dev/null +++ b/scripts/host-lifecycle.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Common lifecycle for OpenFUT transport hosts. Sourced, never executed. +# +# Every service-specific script supplies four facts and gets start/stop/status/ +# verify/verify-running for free: +# +# HL_NAME service name, e.g. redirector (also the env prefix) +# HL_CRATE cargo package, e.g. openfut-roster-host +# HL_BINNAME executable basename +# HL_PORT_VAR name of the env var holding the listen port +# +# Environment read (prefix derived from HL_NAME, uppercased): +# OPENFUT__BIN explicit binary, overriding debug/release selection +# OPENFUT__RUNDIR pid/port/commit/log directory +# OPENFUT__LOG log file +# +# ── Rules this file exists to enforce ──────────────────────────────────────── +# +# 1. NEVER `pkill -f` / `pgrep -f`. Matching a process's command line matches +# any shell whose arguments merely mention the name — including the shell +# running the command. That has killed this session's own shell twice. Every +# lookup here resolves /proc/PID/exe and compares the executable. +# +# 2. `readlink`, not `readlink -f`. Once a binary is rebuilt the link reads +# " (deleted)" and -f resolves it to nothing, so the orphan check goes +# blind to exactly the long-lived processes it exists to find. Two orphans +# were hidden that way. +# +# 3. stop PROVES the process is gone and the port free, rather than assuming a +# signal worked. +# +# 4. Which artifact is under test is never ambiguous. Two binaries that disagree +# is an error for `start`/`verify` — but NOT for `stop`/`status`, because +# rollback must never be blocked by a question about the build tree. +# +# 5. The RUNNING process's identity is what counts. `verify` inspects the binary +# on disk, which a rebuild can silently advance past the live process. + +set -uo pipefail + +: "${HL_NAME:?host-lifecycle.sh: set HL_NAME before sourcing}" +: "${HL_CRATE:?host-lifecycle.sh: set HL_CRATE before sourcing}" +: "${HL_BINNAME:?host-lifecycle.sh: set HL_BINNAME before sourcing}" +: "${HL_PORT_VAR:?host-lifecycle.sh: set HL_PORT_VAR before sourcing}" + +HL_PREFIX="OPENFUT_$(printf '%s' "$HL_NAME" | tr '[:lower:]-' '[:upper:]_')" +HL_ROOT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)" + +hl_env() { # hl_env SUFFIX [default] + local var="${HL_PREFIX}_$1" + printf '%s' "${!var:-${2:-}}" +} + +HL_RUNDIR="$(hl_env RUNDIR "${TMPDIR:-/tmp}/openfut-${HL_NAME}")" +HL_PIDFILE="$HL_RUNDIR/${HL_NAME}.pid" +HL_PORTFILE="$HL_RUNDIR/${HL_NAME}.port" +HL_STAMPFILE="$HL_RUNDIR/${HL_NAME}.commit" +HL_LOGFILE="$(hl_env LOG "$HL_RUNDIR/${HL_NAME}.log")" + +hl_die() { echo "${HL_NAME}: $*" >&2; exit 1; } +hl_pid_alive() { kill -0 "$1" 2>/dev/null; } +hl_port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; } + +# ── Which binary ───────────────────────────────────────────────────────────── +HL_DEBUG_BIN="$HL_ROOT/target/debug/$HL_BINNAME" +HL_RELEASE_BIN="$HL_ROOT/target/release/$HL_BINNAME" +HL_BIN_ERR="" +_hl_stamp_of() { "$1" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; } + +_hl_explicit="$(hl_env BIN)" +if [[ -n "$_hl_explicit" ]]; then + HL_BIN="$_hl_explicit" + [[ -x "$HL_BIN" ]] || hl_die "${HL_PREFIX}_BIN is not executable: $HL_BIN" +elif [[ -x "$HL_DEBUG_BIN" && -x "$HL_RELEASE_BIN" ]]; then + _d="$(_hl_stamp_of "$HL_DEBUG_BIN")" + _r="$(_hl_stamp_of "$HL_RELEASE_BIN")" + if [[ "$_d" != "$_r" ]]; then + HL_BIN_ERR="REFUSING — two binaries exist and disagree. + debug $HL_DEBUG_BIN commit=$_d + release $HL_RELEASE_BIN commit=$_r + Rebuild both, delete one, or set ${HL_PREFIX}_BIN." + fi + HL_BIN="$HL_RELEASE_BIN" +elif [[ -x "$HL_DEBUG_BIN" ]]; then + HL_BIN="$HL_DEBUG_BIN" +else + HL_BIN="$HL_RELEASE_BIN" +fi + +hl_need_bin() { [[ -z "$HL_BIN_ERR" ]] || hl_die "$HL_BIN_ERR"; } + +# ── Process lookup by resolved executable ──────────────────────────────────── +hl_list_procs() { + local self=$$ pid exe + for d in /proc/[0-9]*; do + pid="${d#/proc/}" + [[ "$pid" == "$self" ]] && continue + exe="$(readlink "$d/exe" 2>/dev/null)" || continue + exe="${exe% (deleted)}" + [[ "${exe##*/}" == "$HL_BINNAME" ]] && echo "$pid" + done + return 0 +} + +# ── Commands ───────────────────────────────────────────────────────────────── +hl_verify() { + hl_need_bin + local c; c="$(_hl_stamp_of "$HL_BIN")" + [[ -n "$c" ]] || hl_die "could not read the binary's commit stamp" + "$HL_ROOT/scripts/verify-build-identity.sh" "$c" +} + +hl_start() { + hl_need_bin + [[ -x "$HL_BIN" ]] || hl_die "not built: cargo build -p $HL_CRATE" + local port="${!HL_PORT_VAR:-}" + [[ -n "$port" ]] || hl_die "set $HL_PORT_VAR (no default: this host runs beside the Python one)" + + local strays; strays="$(hl_list_procs)" + [[ -z "$strays" ]] || hl_die "orphan ${HL_NAME} process(es): $strays" + hl_port_listening "$port" && hl_die "port $port in use" + + hl_verify || hl_die "build identity check failed — refusing to start" + + mkdir -p "$HL_RUNDIR" + echo "$port" > "$HL_PORTFILE" + _hl_stamp_of "$HL_BIN" > "$HL_STAMPFILE" + "$HL_BIN" >"$HL_LOGFILE" 2>&1 & + local pid=$!; echo "$pid" > "$HL_PIDFILE" + + local w=0 + while (( w < 100 )); do + hl_pid_alive "$pid" || { + echo "died during startup:" >&2; tail -20 "$HL_LOGFILE" >&2 + rm -f "$HL_PIDFILE"; return 1 + } + if hl_port_listening "$port"; then + echo "${HL_NAME} started: pid $pid, port $port" + grep -E "SELF-TEST|$HL_BINNAME v" "$HL_LOGFILE" | sed 's/^/ /' + return 0 + fi + sleep 0.1; w=$((w+1)) + done + echo "did not listen within 10s:" >&2; tail -20 "$HL_LOGFILE" >&2 + kill "$pid" 2>/dev/null; rm -f "$HL_PIDFILE"; return 1 +} + +hl_stop() { + local rc=0 pid="" port="" + [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE")" + [[ -f "$HL_PORTFILE" ]] && port="$(cat "$HL_PORTFILE")" + if [[ -n "$pid" ]] && hl_pid_alive "$pid"; then + kill "$pid" 2>/dev/null + local w=0; while hl_pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done + hl_pid_alive "$pid" && kill -9 "$pid" 2>/dev/null + sleep 0.2 + fi + [[ -n "$pid" ]] && hl_pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; } + [[ -n "$port" ]] && hl_port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; } + local strays; strays="$(hl_list_procs)" + [[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; } + rm -f "$HL_PIDFILE" "$HL_PORTFILE" + [[ $rc -eq 0 ]] && echo "${HL_NAME} stopped and verified gone${pid:+ (pid $pid)}" + return $rc +} + +hl_status() { + if [[ -f "$HL_PIDFILE" ]] && hl_pid_alive "$(cat "$HL_PIDFILE")"; then + echo "running: pid $(cat "$HL_PIDFILE"), port $(cat "$HL_PORTFILE" 2>/dev/null || echo '?')" + else + echo "not running" + fi + local strays; strays="$(hl_list_procs)" + [[ -n "$strays" ]] && echo "${HL_NAME} processes: $strays" + return 0 +} + +hl_verify_running() { + # The stamp outlives the process it describes. Without this the command + # reports on a corpse — "OK" or a REFUSAL naming a commit, both implying + # something is running when nothing is. + local pid="" + [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE" 2>/dev/null)" + if [[ -z "$pid" ]] || ! hl_pid_alive "$pid"; then + echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2 + return 1 + fi + [[ -f "$HL_STAMPFILE" ]] || hl_die "no running-process stamp — was it started by this script?" + local running head + running="$(cat "$HL_STAMPFILE")" + head="$(git -C "$HL_ROOT" rev-parse --short=7 HEAD 2>/dev/null)" + if [[ "$running" != "$head" ]]; then + echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2 + echo " Restart before treating this run as evidence." >&2 + return 1 + fi + echo "running-process identity OK: started from $running == HEAD" +} + +hl_dispatch() { + case "${1:-}" in + start) hl_start ;; + stop) hl_stop ;; + status) hl_status ;; + verify) hl_verify ;; + verify-running) hl_verify_running ;; + *) + echo "usage: $(basename "$0") start | stop | status | verify | verify-running" >&2 + exit 2 ;; + esac +}