From 8f3b659c337b426eaad37e2f03d32c624a3760a2 Mon Sep 17 00:00:00 2001 From: funman300 Date: Tue, 11 Aug 2026 17:45:15 +0000 Subject: [PATCH] lifecycle: one host-lifecycle helper; roster.sh; ban pkill -f redirector.sh and the coming roster.sh needed the same five rules, each of which cost something to learn: * resolve /proc/PID/exe; never match a command line. `pkill -f` / `pgrep -f` match any shell whose ARGUMENTS mention the name, including the shell running the command. That has killed this session's own shell twice, and is now banned in migration tooling -- the helper contains no `-f` matching and the header says why. * `readlink`, not `readlink -f`. After a rebuild the link reads " (deleted)" and -f resolves it to nothing, so the orphan check goes blind to exactly the long-lived processes it exists to find. Two orphans hid there, one serving the wrong certificate. * stop PROVES the process is gone and the port free. * an ambiguous binary is an error for start/verify but NOT for stop/status: rollback must never be blocked by a question about the build tree. * verify the RUNNING process's commit, not the artifact on disk, which a rebuild can silently advance past. Copying those into a second script would have been the same mistake as copying the TLS setup. Instead scripts/host-lifecycle.sh owns them and a service supplies four facts: name, crate, executable, port variable. redirector.sh goes from 178 lines to 26 and roster.sh is 24, with no behaviour change -- the refactored redirector.sh still sees the live armed process (pid 830736, port 42227) and still refuses correctly because HEAD has moved past it. Paths are unchanged (rundir, pidfile, portfile, commit stamp, log), so the currently running redirector stays manageable across this refactor. Co-Authored-By: Claude Opus 5 --- openfut-redirector-host/redirector.sh | 190 +++-------------------- openfut-roster-host/roster.sh | 25 +++ scripts/host-lifecycle.sh | 211 ++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 171 deletions(-) create mode 100755 openfut-roster-host/roster.sh create mode 100644 scripts/host-lifecycle.sh diff --git a/openfut-redirector-host/redirector.sh b/openfut-redirector-host/redirector.sh index 789d5eb..8ccd627 100755 --- a/openfut-redirector-host/redirector.sh +++ b/openfut-redirector-host/redirector.sh @@ -1,178 +1,26 @@ #!/usr/bin/env bash # Lifecycle for the Rust redirector host. # -# redirector.sh start | stop | status | verify +# redirector.sh start | stop | status | verify | verify-running # -# Mirrors sidecar.sh: refuses to start with an orphan present or the port busy, -# and stop PROVES the process is gone and the port free rather than assuming a -# signal worked. +# A thin wrapper: the rules this project paid for — resolve /proc/PID/exe +# rather than matching command lines, tolerate the "(deleted)" suffix after a +# rebuild, prove a stop rather than assume it, refuse an ambiguous binary, and +# check the RUNNING process's commit rather than the one on disk — all live in +# scripts/host-lifecycle.sh so there is one implementation of them. # -# Additionally REFUSES TO START unless the binary's stamped commit equals HEAD -# and the migration crates are clean — evidence from an unidentifiable binary is -# not evidence. -set -uo pipefail +# Environment: +# OPENFUT_REDIRECTOR_HOST_PORT required; no default, so this can never +# collide with the Python redirector +# OPENFUT_REDIRECTOR_CERT/KEY MUST be the same pair every other +# FIFA-facing service presents +# OPENFUT_REDIRECTOR_BIN explicit binary, if debug and release disagree +HL_NAME="redirector" +HL_CRATE="openfut-redirector-host" +HL_BINNAME="openfut-redirector-host" +HL_PORT_VAR="OPENFUT_REDIRECTOR_HOST_PORT" -HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)" -ROOT="$(cd "$HERE/.." && pwd)" -RUNDIR="${OPENFUT_REDIRECTOR_RUNDIR:-${TMPDIR:-/tmp}/openfut-redirector}" -PIDFILE="$RUNDIR/redirector.pid" -PORTFILE="$RUNDIR/redirector.port" -# Commit the RUNNING process was started from. `verify` alone inspects the -# on-disk binary, which a rebuild (even `cargo test`, which re-runs build.rs -# when the branch ref moves) can silently advance past the live process. -STAMPFILE="$RUNDIR/redirector.commit" -LOGFILE="${OPENFUT_REDIRECTOR_LOG:-$RUNDIR/redirector.log}" -# Which artifact is under test must never be ambiguous. Preferring debug and -# falling back to release meant `cargo build --release` could produce a fresh -# binary while this kept launching a stale debug one — the guard then compared -# the WRONG artifact against HEAD and refused with a message naming a commit -# nobody was trying to run. Both present and disagreeing is an error, not a -# preference. Set OPENFUT_REDIRECTOR_BIN to choose deliberately. -DEBUG_BIN="$ROOT/target/debug/openfut-redirector-host" -RELEASE_BIN="$ROOT/target/release/openfut-redirector-host" -BIN_ERR="" -if [[ -n "${OPENFUT_REDIRECTOR_BIN:-}" ]]; then - BIN="$OPENFUT_REDIRECTOR_BIN" - [[ -x "$BIN" ]] || { echo "redirector: OPENFUT_REDIRECTOR_BIN is not executable: $BIN" >&2; exit 1; } -elif [[ -x "$DEBUG_BIN" && -x "$RELEASE_BIN" ]]; then - d="$("$DEBUG_BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1)" - r="$("$RELEASE_BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1)" - if [[ "$d" != "$r" ]]; then - # Recorded, NOT fatal here: `stop` and `status` must keep working no matter - # what state the build tree is in. Rolling back can never be blocked by a - # question about which artifact would have been started. - BIN_ERR="REFUSING — two binaries exist and disagree. - debug $DEBUG_BIN commit=$d - release $RELEASE_BIN commit=$r - Rebuild both, delete one, or set OPENFUT_REDIRECTOR_BIN." - fi - BIN="$RELEASE_BIN" -elif [[ -x "$DEBUG_BIN" ]]; then - BIN="$DEBUG_BIN" -else - BIN="$RELEASE_BIN" -fi +# shellcheck source=../scripts/host-lifecycle.sh +source "$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)/scripts/host-lifecycle.sh" -die() { echo "redirector: $*" >&2; exit 1; } -pid_alive() { kill -0 "$1" 2>/dev/null; } -port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; } - -# Match the resolved executable, not the command line: `pgrep -f` matches any -# shell whose arguments merely mention the name. -list_procs() { - local self=$$ pid exe - for d in /proc/[0-9]*; do - pid="${d#/proc/}"; [[ "$pid" == "$self" ]] && continue - # readlink, NOT readlink -f: once the binary is rebuilt the link reads - # " (deleted)", and -f resolves that to something that matches - # nothing. The orphan check would then be blind to exactly the long-lived - # processes it exists to find — verified: two orphans (a stale-cert - # redirector and a Blaze sidecar) were both invisible to this until the - # suffix was stripped. - exe="$(readlink "$d/exe" 2>/dev/null)" || continue - exe="${exe% (deleted)}" - [[ "${exe##*/}" == "openfut-redirector-host" ]] && echo "$pid" - done - return 0 -} - -# Only the commands that actually run a binary care which one it is. -need_bin() { [[ -z "$BIN_ERR" ]] || die "$BIN_ERR"; } - -# The binary prints `commit=` in its banner; ask it rather than guessing. -stamped_commit() { "$BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; } - -cmd_verify() { - need_bin - local c; c="$(stamped_commit)" - [[ -n "$c" ]] || die "could not read the binary's commit stamp" - "$ROOT/scripts/verify-build-identity.sh" "$c" -} - -cmd_start() { - need_bin - [[ -x "$BIN" ]] || die "not built: cargo build -p openfut-redirector-host" - : "${OPENFUT_REDIRECTOR_HOST_PORT:?set OPENFUT_REDIRECTOR_HOST_PORT (no default: runs beside Python)}" - local strays; strays="$(list_procs)" - [[ -z "$strays" ]] || die "orphan redirector process(es): $strays" - port_listening "$OPENFUT_REDIRECTOR_HOST_PORT" && die "port $OPENFUT_REDIRECTOR_HOST_PORT in use" - - cmd_verify || die "build identity check failed — refusing to start" - - mkdir -p "$RUNDIR"; echo "$OPENFUT_REDIRECTOR_HOST_PORT" > "$PORTFILE" - stamped_commit > "$STAMPFILE" - "$BIN" >"$LOGFILE" 2>&1 & - local pid=$!; echo "$pid" > "$PIDFILE" - local w=0 - while (( w < 100 )); do - pid_alive "$pid" || { echo "died during startup:" >&2; tail -20 "$LOGFILE" >&2; rm -f "$PIDFILE"; return 1; } - if port_listening "$OPENFUT_REDIRECTOR_HOST_PORT"; then - echo "redirector started: pid $pid, port $OPENFUT_REDIRECTOR_HOST_PORT" - grep -E 'SELF-TEST|openfut-redirector-host v' "$LOGFILE" | sed 's/^/ /' - return 0 - fi - sleep 0.1; w=$((w+1)) - done - echo "did not listen within 10s:" >&2; tail -20 "$LOGFILE" >&2 - kill "$pid" 2>/dev/null; rm -f "$PIDFILE"; return 1 -} - -cmd_stop() { - local rc=0 pid="" port="" - [[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE")" - [[ -f "$PORTFILE" ]] && port="$(cat "$PORTFILE")" - if [[ -n "$pid" ]] && pid_alive "$pid"; then - kill "$pid" 2>/dev/null - local w=0; while pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done - pid_alive "$pid" && kill -9 "$pid" 2>/dev/null - sleep 0.2 - fi - [[ -n "$pid" ]] && pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; } - [[ -n "$port" ]] && port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; } - local strays; strays="$(list_procs)" - [[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; } - rm -f "$PIDFILE" "$PORTFILE" - [[ $rc -eq 0 ]] && echo "redirector stopped and verified gone${pid:+ (pid $pid)}" - return $rc -} - -cmd_status() { - if [[ -f "$PIDFILE" ]] && pid_alive "$(cat "$PIDFILE")"; then - echo "running: pid $(cat "$PIDFILE"), port $(cat "$PORTFILE" 2>/dev/null || echo '?')" - else - echo "not running" - fi - local strays; strays="$(list_procs)" - [[ -n "$strays" ]] && echo "redirector processes: $strays" - return 0 -} - -cmd_verify_running() { - # The stamp outlives the process it describes. Without this check the command - # happily reports on a corpse — either "OK" or a REFUSAL naming a commit, - # both implying something is running when nothing is. - local pid="" - [[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE" 2>/dev/null)" - if [[ -z "$pid" ]] || ! pid_alive "$pid"; then - echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2 - return 1 - fi - [[ -f "$STAMPFILE" ]] || die "no running-process stamp — was it started by this script?" - local running head - running="$(cat "$STAMPFILE")"; head="$(git -C "$ROOT" rev-parse --short=7 HEAD 2>/dev/null)" - if [[ "$running" != "$head" ]]; then - echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2 - echo " Restart before treating this run as evidence." >&2 - return 1 - fi - echo "running-process identity OK: started from $running == HEAD" -} - -case "${1:-}" in - start) cmd_start ;; - stop) cmd_stop ;; - status) cmd_status ;; - verify) cmd_verify ;; - verify-running) cmd_verify_running ;; - *) sed -n '2,6p' "$0" | sed 's/^# \?//'; exit 2 ;; -esac +hl_dispatch "${1:-}" diff --git a/openfut-roster-host/roster.sh b/openfut-roster-host/roster.sh new file mode 100755 index 0000000..42affeb --- /dev/null +++ b/openfut-roster-host/roster.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Lifecycle for the Rust FUT roster host. +# +# roster.sh start | stop | status | verify | verify-running +# +# A thin wrapper: everything about process ownership, executable identity, +# orphan detection and proven shutdown lives in scripts/host-lifecycle.sh, so +# there is exactly one implementation of the rules the redirector and Blaze +# hosts paid for. This file supplies only what is specific to roster. +# +# Environment: +# OPENFUT_ROSTER_HOST_PORT required; no default, so this can never collide +# with the Python roster server it runs beside +# OPENFUT_ROSTER_CERT/KEY MUST be the same pair every other FIFA-facing +# service presents +# OPENFUT_ROSTER_BIN explicit binary, if debug and release disagree +HL_NAME="roster" +HL_CRATE="openfut-roster-host" +HL_BINNAME="openfut-roster-host" +HL_PORT_VAR="OPENFUT_ROSTER_HOST_PORT" + +# shellcheck source=../scripts/host-lifecycle.sh +source "$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)/scripts/host-lifecycle.sh" + +hl_dispatch "${1:-}" diff --git a/scripts/host-lifecycle.sh b/scripts/host-lifecycle.sh new file mode 100644 index 0000000..925be50 --- /dev/null +++ b/scripts/host-lifecycle.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Common lifecycle for OpenFUT transport hosts. Sourced, never executed. +# +# Every service-specific script supplies four facts and gets start/stop/status/ +# verify/verify-running for free: +# +# HL_NAME service name, e.g. redirector (also the env prefix) +# HL_CRATE cargo package, e.g. openfut-roster-host +# HL_BINNAME executable basename +# HL_PORT_VAR name of the env var holding the listen port +# +# Environment read (prefix derived from HL_NAME, uppercased): +# OPENFUT__BIN explicit binary, overriding debug/release selection +# OPENFUT__RUNDIR pid/port/commit/log directory +# OPENFUT__LOG log file +# +# ── Rules this file exists to enforce ──────────────────────────────────────── +# +# 1. NEVER `pkill -f` / `pgrep -f`. Matching a process's command line matches +# any shell whose arguments merely mention the name — including the shell +# running the command. That has killed this session's own shell twice. Every +# lookup here resolves /proc/PID/exe and compares the executable. +# +# 2. `readlink`, not `readlink -f`. Once a binary is rebuilt the link reads +# " (deleted)" and -f resolves it to nothing, so the orphan check goes +# blind to exactly the long-lived processes it exists to find. Two orphans +# were hidden that way. +# +# 3. stop PROVES the process is gone and the port free, rather than assuming a +# signal worked. +# +# 4. Which artifact is under test is never ambiguous. Two binaries that disagree +# is an error for `start`/`verify` — but NOT for `stop`/`status`, because +# rollback must never be blocked by a question about the build tree. +# +# 5. The RUNNING process's identity is what counts. `verify` inspects the binary +# on disk, which a rebuild can silently advance past the live process. + +set -uo pipefail + +: "${HL_NAME:?host-lifecycle.sh: set HL_NAME before sourcing}" +: "${HL_CRATE:?host-lifecycle.sh: set HL_CRATE before sourcing}" +: "${HL_BINNAME:?host-lifecycle.sh: set HL_BINNAME before sourcing}" +: "${HL_PORT_VAR:?host-lifecycle.sh: set HL_PORT_VAR before sourcing}" + +HL_PREFIX="OPENFUT_$(printf '%s' "$HL_NAME" | tr '[:lower:]-' '[:upper:]_')" +HL_ROOT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)" + +hl_env() { # hl_env SUFFIX [default] + local var="${HL_PREFIX}_$1" + printf '%s' "${!var:-${2:-}}" +} + +HL_RUNDIR="$(hl_env RUNDIR "${TMPDIR:-/tmp}/openfut-${HL_NAME}")" +HL_PIDFILE="$HL_RUNDIR/${HL_NAME}.pid" +HL_PORTFILE="$HL_RUNDIR/${HL_NAME}.port" +HL_STAMPFILE="$HL_RUNDIR/${HL_NAME}.commit" +HL_LOGFILE="$(hl_env LOG "$HL_RUNDIR/${HL_NAME}.log")" + +hl_die() { echo "${HL_NAME}: $*" >&2; exit 1; } +hl_pid_alive() { kill -0 "$1" 2>/dev/null; } +hl_port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; } + +# ── Which binary ───────────────────────────────────────────────────────────── +HL_DEBUG_BIN="$HL_ROOT/target/debug/$HL_BINNAME" +HL_RELEASE_BIN="$HL_ROOT/target/release/$HL_BINNAME" +HL_BIN_ERR="" +_hl_stamp_of() { "$1" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; } + +_hl_explicit="$(hl_env BIN)" +if [[ -n "$_hl_explicit" ]]; then + HL_BIN="$_hl_explicit" + [[ -x "$HL_BIN" ]] || hl_die "${HL_PREFIX}_BIN is not executable: $HL_BIN" +elif [[ -x "$HL_DEBUG_BIN" && -x "$HL_RELEASE_BIN" ]]; then + _d="$(_hl_stamp_of "$HL_DEBUG_BIN")" + _r="$(_hl_stamp_of "$HL_RELEASE_BIN")" + if [[ "$_d" != "$_r" ]]; then + HL_BIN_ERR="REFUSING — two binaries exist and disagree. + debug $HL_DEBUG_BIN commit=$_d + release $HL_RELEASE_BIN commit=$_r + Rebuild both, delete one, or set ${HL_PREFIX}_BIN." + fi + HL_BIN="$HL_RELEASE_BIN" +elif [[ -x "$HL_DEBUG_BIN" ]]; then + HL_BIN="$HL_DEBUG_BIN" +else + HL_BIN="$HL_RELEASE_BIN" +fi + +hl_need_bin() { [[ -z "$HL_BIN_ERR" ]] || hl_die "$HL_BIN_ERR"; } + +# ── Process lookup by resolved executable ──────────────────────────────────── +hl_list_procs() { + local self=$$ pid exe + for d in /proc/[0-9]*; do + pid="${d#/proc/}" + [[ "$pid" == "$self" ]] && continue + exe="$(readlink "$d/exe" 2>/dev/null)" || continue + exe="${exe% (deleted)}" + [[ "${exe##*/}" == "$HL_BINNAME" ]] && echo "$pid" + done + return 0 +} + +# ── Commands ───────────────────────────────────────────────────────────────── +hl_verify() { + hl_need_bin + local c; c="$(_hl_stamp_of "$HL_BIN")" + [[ -n "$c" ]] || hl_die "could not read the binary's commit stamp" + "$HL_ROOT/scripts/verify-build-identity.sh" "$c" +} + +hl_start() { + hl_need_bin + [[ -x "$HL_BIN" ]] || hl_die "not built: cargo build -p $HL_CRATE" + local port="${!HL_PORT_VAR:-}" + [[ -n "$port" ]] || hl_die "set $HL_PORT_VAR (no default: this host runs beside the Python one)" + + local strays; strays="$(hl_list_procs)" + [[ -z "$strays" ]] || hl_die "orphan ${HL_NAME} process(es): $strays" + hl_port_listening "$port" && hl_die "port $port in use" + + hl_verify || hl_die "build identity check failed — refusing to start" + + mkdir -p "$HL_RUNDIR" + echo "$port" > "$HL_PORTFILE" + _hl_stamp_of "$HL_BIN" > "$HL_STAMPFILE" + "$HL_BIN" >"$HL_LOGFILE" 2>&1 & + local pid=$!; echo "$pid" > "$HL_PIDFILE" + + local w=0 + while (( w < 100 )); do + hl_pid_alive "$pid" || { + echo "died during startup:" >&2; tail -20 "$HL_LOGFILE" >&2 + rm -f "$HL_PIDFILE"; return 1 + } + if hl_port_listening "$port"; then + echo "${HL_NAME} started: pid $pid, port $port" + grep -E "SELF-TEST|$HL_BINNAME v" "$HL_LOGFILE" | sed 's/^/ /' + return 0 + fi + sleep 0.1; w=$((w+1)) + done + echo "did not listen within 10s:" >&2; tail -20 "$HL_LOGFILE" >&2 + kill "$pid" 2>/dev/null; rm -f "$HL_PIDFILE"; return 1 +} + +hl_stop() { + local rc=0 pid="" port="" + [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE")" + [[ -f "$HL_PORTFILE" ]] && port="$(cat "$HL_PORTFILE")" + if [[ -n "$pid" ]] && hl_pid_alive "$pid"; then + kill "$pid" 2>/dev/null + local w=0; while hl_pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done + hl_pid_alive "$pid" && kill -9 "$pid" 2>/dev/null + sleep 0.2 + fi + [[ -n "$pid" ]] && hl_pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; } + [[ -n "$port" ]] && hl_port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; } + local strays; strays="$(hl_list_procs)" + [[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; } + rm -f "$HL_PIDFILE" "$HL_PORTFILE" + [[ $rc -eq 0 ]] && echo "${HL_NAME} stopped and verified gone${pid:+ (pid $pid)}" + return $rc +} + +hl_status() { + if [[ -f "$HL_PIDFILE" ]] && hl_pid_alive "$(cat "$HL_PIDFILE")"; then + echo "running: pid $(cat "$HL_PIDFILE"), port $(cat "$HL_PORTFILE" 2>/dev/null || echo '?')" + else + echo "not running" + fi + local strays; strays="$(hl_list_procs)" + [[ -n "$strays" ]] && echo "${HL_NAME} processes: $strays" + return 0 +} + +hl_verify_running() { + # The stamp outlives the process it describes. Without this the command + # reports on a corpse — "OK" or a REFUSAL naming a commit, both implying + # something is running when nothing is. + local pid="" + [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE" 2>/dev/null)" + if [[ -z "$pid" ]] || ! hl_pid_alive "$pid"; then + echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2 + return 1 + fi + [[ -f "$HL_STAMPFILE" ]] || hl_die "no running-process stamp — was it started by this script?" + local running head + running="$(cat "$HL_STAMPFILE")" + head="$(git -C "$HL_ROOT" rev-parse --short=7 HEAD 2>/dev/null)" + if [[ "$running" != "$head" ]]; then + echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2 + echo " Restart before treating this run as evidence." >&2 + return 1 + fi + echo "running-process identity OK: started from $running == HEAD" +} + +hl_dispatch() { + case "${1:-}" in + start) hl_start ;; + stop) hl_stop ;; + status) hl_status ;; + verify) hl_verify ;; + verify-running) hl_verify_running ;; + *) + echo "usage: $(basename "$0") start | stop | status | verify | verify-running" >&2 + exit 2 ;; + esac +}