#!/usr/bin/env bash # Common lifecycle for OpenFUT transport hosts. Sourced, never executed. # # Every service-specific script supplies four facts and gets start/stop/status/ # verify/verify-running for free: # # HL_NAME service name, e.g. redirector (also the env prefix) # HL_CRATE cargo package, e.g. openfut-roster-host # HL_BINNAME executable basename # HL_PORT_VAR name of the env var holding the listen port # # Environment read (prefix derived from HL_NAME, uppercased): # OPENFUT__BIN explicit binary, overriding debug/release selection # OPENFUT__RUNDIR pid/port/commit/log directory # OPENFUT__LOG log file # # ── Rules this file exists to enforce ──────────────────────────────────────── # # 1. NEVER `pkill -f` / `pgrep -f`. Matching a process's command line matches # any shell whose arguments merely mention the name — including the shell # running the command. That has killed this session's own shell twice. Every # lookup here resolves /proc/PID/exe and compares the executable. # # 2. `readlink`, not `readlink -f`. Once a binary is rebuilt the link reads # " (deleted)" and -f resolves it to nothing, so the orphan check goes # blind to exactly the long-lived processes it exists to find. Two orphans # were hidden that way. # # 3. stop PROVES the process is gone and the port free, rather than assuming a # signal worked. # # 4. Which artifact is under test is never ambiguous. Two binaries that disagree # is an error for `start`/`verify` — but NOT for `stop`/`status`, because # rollback must never be blocked by a question about the build tree. # # 5. The RUNNING process's identity is what counts. `verify` inspects the binary # on disk, which a rebuild can silently advance past the live process. set -uo pipefail : "${HL_NAME:?host-lifecycle.sh: set HL_NAME before sourcing}" : "${HL_CRATE:?host-lifecycle.sh: set HL_CRATE before sourcing}" : "${HL_BINNAME:?host-lifecycle.sh: set HL_BINNAME before sourcing}" : "${HL_PORT_VAR:?host-lifecycle.sh: set HL_PORT_VAR before sourcing}" HL_PREFIX="OPENFUT_$(printf '%s' "$HL_NAME" | tr '[:lower:]-' '[:upper:]_')" HL_ROOT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)" hl_env() { # hl_env SUFFIX [default] local var="${HL_PREFIX}_$1" printf '%s' "${!var:-${2:-}}" } HL_RUNDIR="$(hl_env RUNDIR "${TMPDIR:-/tmp}/openfut-${HL_NAME}")" HL_PIDFILE="$HL_RUNDIR/${HL_NAME}.pid" HL_PORTFILE="$HL_RUNDIR/${HL_NAME}.port" HL_STAMPFILE="$HL_RUNDIR/${HL_NAME}.commit" HL_LOGFILE="$(hl_env LOG "$HL_RUNDIR/${HL_NAME}.log")" hl_die() { echo "${HL_NAME}: $*" >&2; exit 1; } hl_pid_alive() { kill -0 "$1" 2>/dev/null; } hl_port_listening() { ss -ltn 2>/dev/null | grep -qE "[:.]${1}[[:space:]]"; } # ── Which binary ───────────────────────────────────────────────────────────── HL_DEBUG_BIN="$HL_ROOT/target/debug/$HL_BINNAME" HL_RELEASE_BIN="$HL_ROOT/target/release/$HL_BINNAME" HL_BIN_ERR="" _hl_stamp_of() { "$1" --identity 2>&1 | sed -nE 's/.*commit=([0-9a-f]+).*/\1/p' | head -1; } _hl_explicit="$(hl_env BIN)" if [[ -n "$_hl_explicit" ]]; then HL_BIN="$_hl_explicit" [[ -x "$HL_BIN" ]] || hl_die "${HL_PREFIX}_BIN is not executable: $HL_BIN" elif [[ -x "$HL_DEBUG_BIN" && -x "$HL_RELEASE_BIN" ]]; then _d="$(_hl_stamp_of "$HL_DEBUG_BIN")" _r="$(_hl_stamp_of "$HL_RELEASE_BIN")" if [[ "$_d" != "$_r" ]]; then HL_BIN_ERR="REFUSING — two binaries exist and disagree. debug $HL_DEBUG_BIN commit=$_d release $HL_RELEASE_BIN commit=$_r Rebuild both, delete one, or set ${HL_PREFIX}_BIN." fi HL_BIN="$HL_RELEASE_BIN" elif [[ -x "$HL_DEBUG_BIN" ]]; then HL_BIN="$HL_DEBUG_BIN" else HL_BIN="$HL_RELEASE_BIN" fi hl_need_bin() { [[ -z "$HL_BIN_ERR" ]] || hl_die "$HL_BIN_ERR"; } # ── Process lookup by resolved executable ──────────────────────────────────── hl_list_procs() { local self=$$ pid exe for d in /proc/[0-9]*; do pid="${d#/proc/}" [[ "$pid" == "$self" ]] && continue exe="$(readlink "$d/exe" 2>/dev/null)" || continue exe="${exe% (deleted)}" [[ "${exe##*/}" == "$HL_BINNAME" ]] && echo "$pid" done return 0 } # ── Commands ───────────────────────────────────────────────────────────────── hl_verify() { hl_need_bin local c; c="$(_hl_stamp_of "$HL_BIN")" [[ -n "$c" ]] || hl_die "could not read the binary's commit stamp" "$HL_ROOT/scripts/verify-build-identity.sh" "$c" } hl_start() { hl_need_bin [[ -x "$HL_BIN" ]] || hl_die "not built: cargo build -p $HL_CRATE" local port="${!HL_PORT_VAR:-}" [[ -n "$port" ]] || hl_die "set $HL_PORT_VAR (no default: this host runs beside the Python one)" local strays; strays="$(hl_list_procs)" [[ -z "$strays" ]] || hl_die "orphan ${HL_NAME} process(es): $strays" hl_port_listening "$port" && hl_die "port $port in use" hl_verify || hl_die "build identity check failed — refusing to start" mkdir -p "$HL_RUNDIR" echo "$port" > "$HL_PORTFILE" _hl_stamp_of "$HL_BIN" > "$HL_STAMPFILE" "$HL_BIN" >"$HL_LOGFILE" 2>&1 & local pid=$!; echo "$pid" > "$HL_PIDFILE" local w=0 while (( w < 100 )); do hl_pid_alive "$pid" || { echo "died during startup:" >&2; tail -20 "$HL_LOGFILE" >&2 rm -f "$HL_PIDFILE"; return 1 } if hl_port_listening "$port"; then echo "${HL_NAME} started: pid $pid, port $port" grep -E "SELF-TEST|$HL_BINNAME v" "$HL_LOGFILE" | sed 's/^/ /' return 0 fi sleep 0.1; w=$((w+1)) done echo "did not listen within 10s:" >&2; tail -20 "$HL_LOGFILE" >&2 kill "$pid" 2>/dev/null; rm -f "$HL_PIDFILE"; return 1 } hl_stop() { local rc=0 pid="" port="" [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE")" [[ -f "$HL_PORTFILE" ]] && port="$(cat "$HL_PORTFILE")" if [[ -n "$pid" ]] && hl_pid_alive "$pid"; then kill "$pid" 2>/dev/null local w=0; while hl_pid_alive "$pid" && (( w < 50 )); do sleep 0.1; w=$((w+1)); done hl_pid_alive "$pid" && kill -9 "$pid" 2>/dev/null sleep 0.2 fi [[ -n "$pid" ]] && hl_pid_alive "$pid" && { echo "FAILED to stop $pid" >&2; rc=1; } [[ -n "$port" ]] && hl_port_listening "$port" && { echo "FAILED: port $port still listening" >&2; rc=1; } local strays; strays="$(hl_list_procs)" [[ -n "$strays" ]] && { echo "FAILED: still running: $strays" >&2; rc=1; } rm -f "$HL_PIDFILE" "$HL_PORTFILE" [[ $rc -eq 0 ]] && echo "${HL_NAME} stopped and verified gone${pid:+ (pid $pid)}" return $rc } hl_status() { if [[ -f "$HL_PIDFILE" ]] && hl_pid_alive "$(cat "$HL_PIDFILE")"; then echo "running: pid $(cat "$HL_PIDFILE"), port $(cat "$HL_PORTFILE" 2>/dev/null || echo '?')" else echo "not running" fi local strays; strays="$(hl_list_procs)" [[ -n "$strays" ]] && echo "${HL_NAME} processes: $strays" return 0 } hl_verify_running() { # The stamp outlives the process it describes. Without this the command # reports on a corpse — "OK" or a REFUSAL naming a commit, both implying # something is running when nothing is. local pid="" [[ -f "$HL_PIDFILE" ]] && pid="$(cat "$HL_PIDFILE" 2>/dev/null)" if [[ -z "$pid" ]] || ! hl_pid_alive "$pid"; then echo "REFUSING: nothing is running — the stamp describes a process that has exited." >&2 return 1 fi [[ -f "$HL_STAMPFILE" ]] || hl_die "no running-process stamp — was it started by this script?" local running head running="$(cat "$HL_STAMPFILE")" head="$(git -C "$HL_ROOT" rev-parse --short=7 HEAD 2>/dev/null)" if [[ "$running" != "$head" ]]; then echo "REFUSING: the RUNNING process was started from $running but HEAD is $head" >&2 echo " Restart before treating this run as evidence." >&2 return 1 fi echo "running-process identity OK: started from $running == HEAD" } hl_dispatch() { case "${1:-}" in start) hl_start ;; stop) hl_stop ;; status) hl_status ;; verify) hl_verify ;; verify-running) hl_verify_running ;; *) echo "usage: $(basename "$0") start | stop | status | verify | verify-running" >&2 exit 2 ;; esac }