funman300
d7c0a5521d
Three times now the same sequence has broken the client path: a build guard correctly refuses to start the Rust replacement, and the `switch on` that follows in the same script arms anyway, because it never checked whether anything was listening. The redirect then lands on a closed socket and the working Python service is bypassed for no benefit. `on` now refuses unless the target port is listening. ALLOW_DEAD_TARGET=1 overrides it for arming ahead of a service that is about to start, but that has to be deliberate. Verified both ways: rc=2 and nothing installed against a dead port, rc=0 and two rules with the override. The watchdog now writes a pidfile. Stopping it by command-line match is unsafe -- any shell whose arguments merely mention the script name matches too, which has now killed the wrong process twice here (once via `pkill -f`, once via a /proc/*/cmdline substring loop).
116 lines
4.3 KiB
Bash
Executable File
116 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Keep an armed switch honest: if the Rust service stops answering, roll back.
|
|
#
|
|
# openfut-watchdog.sh --name <switch> --probe <host:port> --kind <tls|tcp> \
|
|
# [--interval 60] [--failures 3] [--log <path>]
|
|
#
|
|
# WHY
|
|
#
|
|
# `openfut-switch.sh on` prints a warning that the service MUST stay up, because
|
|
# an armed switch pointing at a dead port means the client hits a closed socket
|
|
# with no fallback. A warning is not a safeguard when nobody is at the terminal.
|
|
#
|
|
# This turns the documented rollback into an automatic one. It fails toward the
|
|
# PYTHON oracle, which is the proven-good path, so the worst case of a spurious
|
|
# trip is that a gate needs re-arming — never that the client is left broken.
|
|
#
|
|
# It only ever removes the switch. It never installs one, never restarts the
|
|
# Rust service, and never touches the Python backend. Recovery is deliberately
|
|
# a human decision: an unexplained rollback is a finding to read in the morning,
|
|
# not something to paper over by flapping the switch back on.
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
|
|
SWITCH="$HERE/openfut-switch.sh"
|
|
|
|
NAME=""; PROBE=""; KIND="tls"; INTERVAL=60; MAX_FAIL=3; LOG=""
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--name) NAME="$2"; shift 2 ;;
|
|
--probe) PROBE="$2"; shift 2 ;;
|
|
--kind) KIND="$2"; shift 2 ;;
|
|
--interval) INTERVAL="$2"; shift 2 ;;
|
|
--failures) MAX_FAIL="$2"; shift 2 ;;
|
|
--log) LOG="$2"; shift 2 ;;
|
|
*) echo "watchdog: unknown argument $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
[[ -n "$NAME" && -n "$PROBE" ]] || { echo "watchdog: --name and --probe are required" >&2; exit 2; }
|
|
[[ -x "$SWITCH" ]] || { echo "watchdog: missing $SWITCH" >&2; exit 2; }
|
|
|
|
say() {
|
|
local line; line="[$(date +%H:%M:%S)] watchdog($NAME): $*"
|
|
echo "$line"
|
|
[[ -n "$LOG" ]] && echo "$line" >> "$LOG"
|
|
}
|
|
|
|
# A probe must exercise the SAME path the client uses — through the switch, and
|
|
# through TLS where the client speaks TLS. A bare TCP connect would succeed
|
|
# against a process that has wedged mid-handshake.
|
|
probe_once() {
|
|
local host="${PROBE%:*}" port="${PROBE##*:}"
|
|
if [[ "$KIND" == "tcp" ]]; then
|
|
timeout 8 python3 -c "
|
|
import socket,sys
|
|
try:
|
|
socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6).close()
|
|
except Exception:
|
|
sys.exit(1)" "$host" "$port"
|
|
else
|
|
timeout 12 python3 -c "
|
|
import socket,ssl,sys
|
|
try:
|
|
c=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT); c.check_hostname=False
|
|
c.verify_mode=ssl.CERT_NONE; c.set_ciphers('ALL:@SECLEVEL=0')
|
|
s=c.wrap_socket(socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6),
|
|
server_hostname='watchdog')
|
|
s.sendall(b'POST /redirector/getServerInstance HTTP/1.1\r\nContent-Length: 0\r\n\r\n')
|
|
d=b''
|
|
while True:
|
|
x=s.recv(4096)
|
|
if not x: break
|
|
d+=x
|
|
sys.exit(0 if len(d)>0 else 1)
|
|
except Exception:
|
|
sys.exit(1)" "$host" "$port"
|
|
fi
|
|
}
|
|
|
|
# A pidfile, because stopping this by command-line match is unsafe: any shell
|
|
# whose arguments merely mention the script name matches too. That mistake has
|
|
# killed the wrong process twice in this project.
|
|
PIDFILE="${OPENFUT_WATCHDOG_PIDFILE:-${TMPDIR:-/tmp}/openfut-watchdog-$NAME.pid}"
|
|
if [[ -f "$PIDFILE" ]] && kill -0 "$(cat "$PIDFILE" 2>/dev/null)" 2>/dev/null; then
|
|
echo "watchdog: already running for '$NAME' (pid $(cat "$PIDFILE"))" >&2; exit 1
|
|
fi
|
|
echo $$ > "$PIDFILE"
|
|
trap 'rm -f "$PIDFILE"' EXIT
|
|
|
|
say "started: probing $PROBE every ${INTERVAL}s ($KIND); rolls back after $MAX_FAIL consecutive failures"
|
|
fails=0
|
|
while true; do
|
|
# Nothing to guard once the switch is off — exit rather than spin forever.
|
|
if "$SWITCH" status --name "$NAME" 2>/dev/null | grep -q '^INACTIVE'; then
|
|
say "switch is no longer armed; nothing to guard, exiting"
|
|
exit 0
|
|
fi
|
|
|
|
if probe_once; then
|
|
[[ $fails -gt 0 ]] && say "recovered after $fails failure(s)"
|
|
fails=0
|
|
else
|
|
fails=$((fails + 1))
|
|
say "probe FAILED ($fails/$MAX_FAIL)"
|
|
if [[ $fails -ge $MAX_FAIL ]]; then
|
|
say "rolling back to the Python oracle"
|
|
if "$SWITCH" off --name "$NAME" >/dev/null 2>&1; then
|
|
say "ROLLED BACK — switch removed and verified. Not re-arming; this needs a human."
|
|
else
|
|
say "ROLLBACK FAILED — switch rules may remain. NEEDS IMMEDIATE ATTENTION."
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
sleep "$INTERVAL"
|
|
done
|