lifecycle: one host-lifecycle helper; roster.sh; ban pkill -f
redirector.sh and the coming roster.sh needed the same five rules, each
of which cost something to learn:
* resolve /proc/PID/exe; never match a command line. `pkill -f` /
`pgrep -f` match any shell whose ARGUMENTS mention the name, including
the shell running the command. That has killed this session's own
shell twice, and is now banned in migration tooling -- the helper
contains no `-f` matching and the header says why.
* `readlink`, not `readlink -f`. After a rebuild the link reads
"<path> (deleted)" and -f resolves it to nothing, so the orphan check
goes blind to exactly the long-lived processes it exists to find. Two
orphans hid there, one serving the wrong certificate.
* stop PROVES the process is gone and the port free.
* an ambiguous binary is an error for start/verify but NOT for
stop/status: rollback must never be blocked by a question about the
build tree.
* verify the RUNNING process's commit, not the artifact on disk, which
a rebuild can silently advance past.
Copying those into a second script would have been the same mistake as
copying the TLS setup. Instead scripts/host-lifecycle.sh owns them and a
service supplies four facts: name, crate, executable, port variable.
redirector.sh goes from 178 lines to 26 and roster.sh is 24, with no
behaviour change -- the refactored redirector.sh still sees the live
armed process (pid 830736, port 42227) and still refuses correctly
because HEAD has moved past it.
Paths are unchanged (rundir, pidfile, portfile, commit stamp, log), so
the currently running redirector stays manageable across this refactor.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
funman300
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
# Common lifecycle for OpenFUT transport hosts. Sourced, never executed.
|
||||
#
|
||||
# Every service-specific script supplies four facts and gets start/stop/status/
|
||||
# verify/verify-running for free:
|
||||
#
|
||||
# HL_NAME service name, e.g. redirector (also the env prefix)
|
||||
# HL_CRATE cargo package, e.g. openfut-roster-host
|
||||
# HL_BINNAME executable basename
|
||||
# HL_PORT_VAR name of the env var holding the listen port
|
||||
#
|
||||
# Environment read (prefix derived from HL_NAME, uppercased):
|
||||
# OPENFUT_<NAME>_BIN explicit binary, overriding debug/release selection
|
||||
# OPENFUT_<NAME>_RUNDIR pid/port/commit/log directory
|
||||
# OPENFUT_<NAME>_LOG log file
|
||||
#
|
||||
# ── Rules this file exists to enforce ────────────────────────────────────────
|
||||
#
|
||||
# 1. NEVER `pkill -f` / `pgrep -f`. Matching a process's command line matches
|
||||
# any shell whose arguments merely mention the name — including the shell
|
||||
# running the command. That has killed this session's own shell twice. Every
|
||||
# lookup here resolves /proc/PID/exe and compares the executable.
|
||||
#
|
||||
# 2. `readlink`, not `readlink -f`. Once a binary is rebuilt the link reads
|
||||
# "<path> (deleted)" and -f resolves it to nothing, so the orphan check goes
|
||||
# blind to exactly the long-lived processes it exists to find. Two orphans
|
||||
# were hidden that way.
|
||||
#
|
||||
# 3. stop PROVES the process is gone and the port free, rather than assuming a
|
||||
# signal worked.
|
||||
#
|
||||
# 4. Which artifact is under test is never ambiguous. Two binaries that disagree
|
||||
# is an error for `start`/`verify` — but NOT for `stop`/`status`, because
|
||||
# rollback must never be blocked by a question about the build tree.
|
||||
#
|
||||
# 5. The RUNNING process's identity is what counts. `verify` inspects the binary
|
||||
# on disk, which a rebuild can silently advance past the live process.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
: "${HL_NAME:?host-lifecycle.sh: set HL_NAME before sourcing}"
|
||||
: "${HL_CRATE:?host-lifecycle.sh: set HL_CRATE before sourcing}"
|
||||
: "${HL_BINNAME:?host-lifecycle.sh: set HL_BINNAME before sourcing}"
|
||||
: "${HL_PORT_VAR:?host-lifecycle.sh: set HL_PORT_VAR before sourcing}"
|
||||
|
||||
HL_PREFIX="OPENFUT_$(printf '%s' "$HL_NAME" | tr '[:lower:]-' '[:upper:]_')"
|
||||
HL_ROOT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)"
|
||||
|
||||
hl_env() { # hl_env SUFFIX [default]
|
||||
local var="${HL_PREFIX}_$1"
|
||||
printf '%s' "${!var:-${2:-}}"
|
||||
}
|
||||
|
||||
HL_RUNDIR="$(hl_env RUNDIR "${TMPDIR:-/tmp}/openfut-${HL_NAME}")"
|
||||
HL_PIDFILE="$HL_RUNDIR/${HL_NAME}.pid"
|
||||
HL_PORTFILE="$HL_RUNDIR/${HL_NAME}.port"
|
||||
HL_STAMPFILE="$HL_RUNDIR/${HL_NAME}.commit"
|
||||
HL_LOGFILE="$(hl_env LOG "$HL_RUNDIR/${HL_NAME}.log")"
|
||||
|
||||
hl_die() { echo "${HL_NAME}: $*" >&2; exit 1; }
|
||||
hl_pid_alive() { kill -0 "$1" 2>/dev/null; }
|
||||
hl_port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; }
|
||||
|
||||
# ── Which binary ─────────────────────────────────────────────────────────────
|
||||
HL_DEBUG_BIN="$HL_ROOT/target/debug/$HL_BINNAME"
|
||||
HL_RELEASE_BIN="$HL_ROOT/target/release/$HL_BINNAME"
|
||||
HL_BIN_ERR=""
|
||||
_hl_stamp_of() { "$1" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; }
|
||||
|
||||
_hl_explicit="$(hl_env BIN)"
|
||||
if [[ -n "$_hl_explicit" ]]; then
|
||||
HL_BIN="$_hl_explicit"
|
||||
[[ -x "$HL_BIN" ]] || hl_die "${HL_PREFIX}_BIN is not executable: $HL_BIN"
|
||||
elif [[ -x "$HL_DEBUG_BIN" && -x "$HL_RELEASE_BIN" ]]; then
|
||||
_d="$(_hl_stamp_of "$HL_DEBUG_BIN")"
|
||||
_r="$(_hl_stamp_of "$HL_RELEASE_BIN")"
|
||||
if [[ "$_d" != "$_r" ]]; then
|
||||
HL_BIN_ERR="REFUSING — two binaries exist and disagree.
|
||||
debug $HL_DEBUG_BIN commit=$_d
|
||||
release $HL_RELEASE_BIN commit=$_r
|
||||
Rebuild both, delete one, or set ${HL_PREFIX}_BIN."
|
||||
fi
|
||||
HL_BIN="$HL_RELEASE_BIN"
|
||||
elif [[ -x "$HL_DEBUG_BIN" ]]; then
|
||||
HL_BIN="$HL_DEBUG_BIN"
|
||||
else
|
||||
HL_BIN="$HL_RELEASE_BIN"
|
||||
fi
|
||||
|
||||
hl_need_bin() { [[ -z "$HL_BIN_ERR" ]] || hl_die "$HL_BIN_ERR"; }
|
||||
|
||||
# ── Process lookup by resolved executable ────────────────────────────────────
|
||||
hl_list_procs() {
|
||||
local self=$$ pid exe
|
||||
for d in /proc/[0-9]*; do
|
||||
pid="${d#/proc/}"
|
||||
[[ "$pid" == "$self" ]] && continue
|
||||
exe="$(readlink "$d/exe" 2>/dev/null)" || continue
|
||||
exe="${exe% (deleted)}"
|
||||
[[ "${exe##*/}" == "$HL_BINNAME" ]] && echo "$pid"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Commands ─────────────────────────────────────────────────────────────────
|
||||
hl_verify() {
|
||||
hl_need_bin
|
||||
local c; c="$(_hl_stamp_of "$HL_BIN")"
|
||||
[[ -n "$c" ]] || hl_die "could not read the binary's commit stamp"
|
||||
"$HL_ROOT/scripts/verify-build-identity.sh" "$c"
|
||||
}
|
||||
|
||||
hl_start() {
|
||||
hl_need_bin
|
||||
[[ -x "$HL_BIN" ]] || hl_die "not built: cargo build -p $HL_CRATE"
|
||||
local port="${!HL_PORT_VAR:-}"
|
||||
[[ -n "$port" ]] || hl_die "set $HL_PORT_VAR (no default: this host runs beside the Python one)"
|
||||
|
||||
local strays; strays="$(hl_list_procs)"
|
||||
[[ -z "$strays" ]] || hl_die "orphan ${HL_NAME} process(es): $strays"
|
||||
hl_port_listening "$port" && hl_die "port $port in use"
|
||||
|
||||
hl_verify || hl_die "build identity check failed — refusing to start"
|
||||
|
||||
mkdir -p "$HL_RUNDIR"
|
||||
echo "$port" > "$HL_PORTFILE"
|
||||
_hl_stamp_of "$HL_BIN" > "$HL_STAMPFILE"
|
||||
"$HL_BIN" >"$HL_LOGFILE" 2>&1 &
|
||||
local pid=$!; echo "$pid" > "$HL_PIDFILE"
|
||||
|
||||
local w=0
|
||||
while (( w < 100 )); do
|
||||
hl_pid_alive "$pid" || {
|
||||
echo "died during startup:" >&2; tail -20 "$HL_LOGFILE" >&2
|
||||
rm -f "$HL_PIDFILE"; return 1
|
||||
}
|
||||
if hl_port_listening "$port"; then
|
||||
echo "${HL_NAME} started: pid $pid, port $port"
|
||||
grep -E "SELF-TEST|$HL_BINNAME v" "$HL_LOGFILE" | sed 's/^/ /'
|
||||
return 0
|
||||
fi
|
||||
sleep 0.1; w=$((w+1))
|
||||
done
|
||||
echo "did not listen within 10s:" >&2; tail -20 "$HL_LOGFILE" >&2
|
||||
kill "$pid" 2>/dev/null; rm -f "$HL_PIDFILE"; return 1
|
||||
}
|
||||
|
||||
hl_stop() {
|
||||
local rc=0 pid="" port=""
|
||||
[[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE")"
|
||||
[[ -f "$HL_PORTFILE" ]] && port="$(cat "$HL_PORTFILE")"
|
||||
if [[ -n "$pid" ]] && hl_pid_alive "$pid"; then
|
||||
kill "$pid" 2>/dev/null
|
||||
local w=0; while hl_pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done
|
||||
hl_pid_alive "$pid" && kill -9 "$pid" 2>/dev/null
|
||||
sleep 0.2
|
||||
fi
|
||||
[[ -n "$pid" ]] && hl_pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; }
|
||||
[[ -n "$port" ]] && hl_port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; }
|
||||
local strays; strays="$(hl_list_procs)"
|
||||
[[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; }
|
||||
rm -f "$HL_PIDFILE" "$HL_PORTFILE"
|
||||
[[ $rc -eq 0 ]] && echo "${HL_NAME} stopped and verified gone${pid:+ (pid $pid)}"
|
||||
return $rc
|
||||
}
|
||||
|
||||
hl_status() {
|
||||
if [[ -f "$HL_PIDFILE" ]] && hl_pid_alive "$(cat "$HL_PIDFILE")"; then
|
||||
echo "running: pid $(cat "$HL_PIDFILE"), port $(cat "$HL_PORTFILE" 2>/dev/null || echo '?')"
|
||||
else
|
||||
echo "not running"
|
||||
fi
|
||||
local strays; strays="$(hl_list_procs)"
|
||||
[[ -n "$strays" ]] && echo "${HL_NAME} processes: $strays"
|
||||
return 0
|
||||
}
|
||||
|
||||
hl_verify_running() {
|
||||
# The stamp outlives the process it describes. Without this the command
|
||||
# reports on a corpse — "OK" or a REFUSAL naming a commit, both implying
|
||||
# something is running when nothing is.
|
||||
local pid=""
|
||||
[[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE" 2>/dev/null)"
|
||||
if [[ -z "$pid" ]] || ! hl_pid_alive "$pid"; then
|
||||
echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2
|
||||
return 1
|
||||
fi
|
||||
[[ -f "$HL_STAMPFILE" ]] || hl_die "no running-process stamp — was it started by this script?"
|
||||
local running head
|
||||
running="$(cat "$HL_STAMPFILE")"
|
||||
head="$(git -C "$HL_ROOT" rev-parse --short=7 HEAD 2>/dev/null)"
|
||||
if [[ "$running" != "$head" ]]; then
|
||||
echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2
|
||||
echo " Restart before treating this run as evidence." >&2
|
||||
return 1
|
||||
fi
|
||||
echo "running-process identity OK: started from $running == HEAD"
|
||||
}
|
||||
|
||||
hl_dispatch() {
|
||||
case "${1:-}" in
|
||||
start) hl_start ;;
|
||||
stop) hl_stop ;;
|
||||
status) hl_status ;;
|
||||
verify) hl_verify ;;
|
||||
verify-running) hl_verify_running ;;
|
||||
*)
|
||||
echo "usage: $(basename "$0") start | stop | status | verify | verify-running" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
}
|
||||
Reference in New Issue
Block a user