ca63095786
`list_procs` matched on `readlink -f /proc/PID/exe`. Once the binary is
rebuilt -- which happens constantly here, `cargo test` alone is enough -- the
link reads "<path> (deleted)" and -f resolves it to something that matches
nothing. The scan then finds zero processes, so `start`'s orphan check passes
and a second instance can be launched alongside a stray.
Not theoretical. Two orphans were running undetected tonight:
pid 592731 :42327 a stale-cert redirector left from testing check-tls-parity,
still serving F9:16:1A -- the exact certificate whose
mismatch cost three live gates
pid 542693 :42230 a Blaze sidecar debug build from 03:12
Neither was in a client path, so neither was doing harm, but a stray listener
serving the known-bad certificate is precisely what should never sit around
unnoticed.
Fixed by using plain readlink and stripping the " (deleted)" suffix. Shown both
ways: with the bug `status` reports no processes at all for a live pid; with the
fix it reports 604454. The pidfile path was unaffected, which is why `stop` kept
working and hid this.
137 lines
5.4 KiB
Bash
Executable File
137 lines
5.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Lifecycle for the Rust redirector host.
|
|
#
|
|
# redirector.sh start | stop | status | verify
|
|
#
|
|
# Mirrors sidecar.sh: refuses to start with an orphan present or the port busy,
|
|
# and stop PROVES the process is gone and the port free rather than assuming a
|
|
# signal worked.
|
|
#
|
|
# Additionally REFUSES TO START unless the binary's stamped commit equals HEAD
|
|
# and the migration crates are clean — evidence from an unidentifiable binary is
|
|
# not evidence.
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
|
|
ROOT="$(cd "$HERE/.." && pwd)"
|
|
RUNDIR="${OPENFUT_REDIRECTOR_RUNDIR:-${TMPDIR:-/tmp}/openfut-redirector}"
|
|
PIDFILE="$RUNDIR/redirector.pid"
|
|
PORTFILE="$RUNDIR/redirector.port"
|
|
# Commit the RUNNING process was started from. `verify` alone inspects the
|
|
# on-disk binary, which a rebuild (even `cargo test`, which re-runs build.rs
|
|
# when the branch ref moves) can silently advance past the live process.
|
|
STAMPFILE="$RUNDIR/redirector.commit"
|
|
LOGFILE="${OPENFUT_REDIRECTOR_LOG:-$RUNDIR/redirector.log}"
|
|
BIN="$ROOT/target/debug/openfut-redirector-host"
|
|
[[ -x "$BIN" ]] || BIN="$ROOT/target/release/openfut-redirector-host"
|
|
|
|
die() { echo "redirector: $*" >&2; exit 1; }
|
|
pid_alive() { kill -0 "$1" 2>/dev/null; }
|
|
port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; }
|
|
|
|
# Match the resolved executable, not the command line: `pgrep -f` matches any
|
|
# shell whose arguments merely mention the name.
|
|
list_procs() {
|
|
local self=$$ pid exe
|
|
for d in /proc/[0-9]*; do
|
|
pid="${d#/proc/}"; [[ "$pid" == "$self" ]] && continue
|
|
# readlink, NOT readlink -f: once the binary is rebuilt the link reads
|
|
# "<path> (deleted)", and -f resolves that to something that matches
|
|
# nothing. The orphan check would then be blind to exactly the long-lived
|
|
# processes it exists to find — verified: two orphans (a stale-cert
|
|
# redirector and a Blaze sidecar) were both invisible to this until the
|
|
# suffix was stripped.
|
|
exe="$(readlink "$d/exe" 2>/dev/null)" || continue
|
|
exe="${exe% (deleted)}"
|
|
[[ "${exe##*/}" == "openfut-redirector-host" ]] && echo "$pid"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# The binary prints `commit=<sha>` in its banner; ask it rather than guessing.
|
|
stamped_commit() { "$BIN" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; }
|
|
|
|
cmd_verify() {
|
|
local c; c="$(stamped_commit)"
|
|
[[ -n "$c" ]] || die "could not read the binary's commit stamp"
|
|
"$ROOT/scripts/verify-build-identity.sh" "$c"
|
|
}
|
|
|
|
cmd_start() {
|
|
[[ -x "$BIN" ]] || die "not built: cargo build -p openfut-redirector-host"
|
|
: "${OPENFUT_REDIRECTOR_HOST_PORT:?set OPENFUT_REDIRECTOR_HOST_PORT (no default: runs beside Python)}"
|
|
local strays; strays="$(list_procs)"
|
|
[[ -z "$strays" ]] || die "orphan redirector process(es): $strays"
|
|
port_listening "$OPENFUT_REDIRECTOR_HOST_PORT" && die "port $OPENFUT_REDIRECTOR_HOST_PORT in use"
|
|
|
|
cmd_verify || die "build identity check failed — refusing to start"
|
|
|
|
mkdir -p "$RUNDIR"; echo "$OPENFUT_REDIRECTOR_HOST_PORT" > "$PORTFILE"
|
|
stamped_commit > "$STAMPFILE"
|
|
"$BIN" >"$LOGFILE" 2>&1 &
|
|
local pid=$!; echo "$pid" > "$PIDFILE"
|
|
local w=0
|
|
while (( w < 100 )); do
|
|
pid_alive "$pid" || { echo "died during startup:" >&2; tail -20 "$LOGFILE" >&2; rm -f "$PIDFILE"; return 1; }
|
|
if port_listening "$OPENFUT_REDIRECTOR_HOST_PORT"; then
|
|
echo "redirector started: pid $pid, port $OPENFUT_REDIRECTOR_HOST_PORT"
|
|
grep -E 'SELF-TEST|openfut-redirector-host v' "$LOGFILE" | sed 's/^/ /'
|
|
return 0
|
|
fi
|
|
sleep 0.1; w=$((w+1))
|
|
done
|
|
echo "did not listen within 10s:" >&2; tail -20 "$LOGFILE" >&2
|
|
kill "$pid" 2>/dev/null; rm -f "$PIDFILE"; return 1
|
|
}
|
|
|
|
cmd_stop() {
|
|
local rc=0 pid="" port=""
|
|
[[ -f "$PIDFILE" ]] && pid="$(cat "$PIDFILE")"
|
|
[[ -f "$PORTFILE" ]] && port="$(cat "$PORTFILE")"
|
|
if [[ -n "$pid" ]] && pid_alive "$pid"; then
|
|
kill "$pid" 2>/dev/null
|
|
local w=0; while pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done
|
|
pid_alive "$pid" && kill -9 "$pid" 2>/dev/null
|
|
sleep 0.2
|
|
fi
|
|
[[ -n "$pid" ]] && pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; }
|
|
[[ -n "$port" ]] && port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; }
|
|
local strays; strays="$(list_procs)"
|
|
[[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; }
|
|
rm -f "$PIDFILE" "$PORTFILE"
|
|
[[ $rc -eq 0 ]] && echo "redirector stopped and verified gone${pid:+ (pid $pid)}"
|
|
return $rc
|
|
}
|
|
|
|
cmd_status() {
|
|
if [[ -f "$PIDFILE" ]] && pid_alive "$(cat "$PIDFILE")"; then
|
|
echo "running: pid $(cat "$PIDFILE"), port $(cat "$PORTFILE" 2>/dev/null || echo '?')"
|
|
else
|
|
echo "not running"
|
|
fi
|
|
local strays; strays="$(list_procs)"
|
|
[[ -n "$strays" ]] && echo "redirector processes: $strays"
|
|
return 0
|
|
}
|
|
|
|
cmd_verify_running() {
|
|
[[ -f "$STAMPFILE" ]] || die "no running-process stamp — was it started by this script?"
|
|
local running head
|
|
running="$(cat "$STAMPFILE")"; head="$(git -C "$ROOT" rev-parse --short=7 HEAD 2>/dev/null)"
|
|
if [[ "$running" != "$head" ]]; then
|
|
echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2
|
|
echo " Restart before treating this run as evidence." >&2
|
|
return 1
|
|
fi
|
|
echo "running-process identity OK: started from $running == HEAD"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
start) cmd_start ;;
|
|
stop) cmd_stop ;;
|
|
status) cmd_status ;;
|
|
verify) cmd_verify ;;
|
|
verify-running) cmd_verify_running ;;
|
|
*) sed -n '2,6p' "$0" | sed 's/^# \?//'; exit 2 ;;
|
|
esac
|