tooling: watchdog that rolls an armed switch back when the service stops answering

`openfut-switch.sh on` prints "the service MUST stay up" -- true, and useless when nobody is at the terminal. An armed switch pointing at a dead port means the client hits a closed socket with no fallback. This turns the documented rollback into an automatic one, failing toward the Python oracle. The worst case of a spurious trip is a gate needing re-arming; it can never leave the client broken. It only ever REMOVES a switch. It does not install one, restart the Rust service, or touch Python, and it does not re-arm after tripping -- an unexplained rollback should be a finding to read, not something hidden by flapping the switch back on. The probe goes through the switch and speaks TLS, because a bare TCP connect would succeed against a process wedged mid-handshake. Tested both directions, not just the happy path: quiet for 45s against a healthy service, and against a stopped one it failed 3/3 in 9s, rolled back, and left Python serving -- verified by re-reading all four tables and by which implementation's log grew.
2026-08-11 05:11:41 +00:00
parent cfb0435d96
commit 2ae90b1ea9
1 changed files with 105 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Keep an armed switch honest: if the Rust service stops answering, roll back.
+#
+#   openfut-watchdog.sh --name <switch> --probe <host:port> --kind <tls|tcp> \
+#                       [--interval 60] [--failures 3] [--log <path>]
+#
+# WHY
+#
+# `openfut-switch.sh on` prints a warning that the service MUST stay up, because
+# an armed switch pointing at a dead port means the client hits a closed socket
+# with no fallback. A warning is not a safeguard when nobody is at the terminal.
+#
+# This turns the documented rollback into an automatic one. It fails toward the
+# PYTHON oracle, which is the proven-good path, so the worst case of a spurious
+# trip is that a gate needs re-arming — never that the client is left broken.
+#
+# It only ever removes the switch. It never installs one, never restarts the
+# Rust service, and never touches the Python backend. Recovery is deliberately
+# a human decision: an unexplained rollback is a finding to read in the morning,
+# not something to paper over by flapping the switch back on.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
+SWITCH="$HERE/openfut-switch.sh"
+
+NAME=""; PROBE=""; KIND="tls"; INTERVAL=60; MAX_FAIL=3; LOG=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --name)     NAME="$2"; shift 2 ;;
+    --probe)    PROBE="$2"; shift 2 ;;
+    --kind)     KIND="$2"; shift 2 ;;
+    --interval) INTERVAL="$2"; shift 2 ;;
+    --failures) MAX_FAIL="$2"; shift 2 ;;
+    --log)      LOG="$2"; shift 2 ;;
+    *) echo "watchdog: unknown argument $1" >&2; exit 2 ;;
+  esac
+done
+[[ -n "$NAME" && -n "$PROBE" ]] || { echo "watchdog: --name and --probe are required" >&2; exit 2; }
+[[ -x "$SWITCH" ]] || { echo "watchdog: missing $SWITCH" >&2; exit 2; }
+
+say() {
+  local line; line="[$(date +%H:%M:%S)] watchdog($NAME): $*"
+  echo "$line"
+  [[ -n "$LOG" ]] && echo "$line" >> "$LOG"
+}
+
+# A probe must exercise the SAME path the client uses — through the switch, and
+# through TLS where the client speaks TLS. A bare TCP connect would succeed
+# against a process that has wedged mid-handshake.
+probe_once() {
+  local host="${PROBE%:*}" port="${PROBE##*:}"
+  if [[ "$KIND" == "tcp" ]]; then
+    timeout 8 python3 -c "
+import socket,sys
+try:
+    socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6).close()
+except Exception:
+    sys.exit(1)" "$host" "$port"
+  else
+    timeout 12 python3 -c "
+import socket,ssl,sys
+try:
+    c=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT); c.check_hostname=False
+    c.verify_mode=ssl.CERT_NONE; c.set_ciphers('ALL:@SECLEVEL=0')
+    s=c.wrap_socket(socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6),
+                    server_hostname='watchdog')
+    s.sendall(b'POST /redirector/getServerInstance HTTP/1.1\r\nContent-Length: 0\r\n\r\n')
+    d=b''
+    while True:
+        x=s.recv(4096)
+        if not x: break
+        d+=x
+    sys.exit(0 if len(d)>0 else 1)
+except Exception:
+    sys.exit(1)" "$host" "$port"
+  fi
+}
+
+say "started: probing $PROBE every ${INTERVAL}s ($KIND); rolls back after $MAX_FAIL consecutive failures"
+fails=0
+while true; do
+  # Nothing to guard once the switch is off — exit rather than spin forever.
+  if "$SWITCH" status --name "$NAME" 2>/dev/null | grep -q '^INACTIVE'; then
+    say "switch is no longer armed; nothing to guard, exiting"
+    exit 0
+  fi
+
+  if probe_once; then
+    [[ $fails -gt 0 ]] && say "recovered after $fails failure(s)"
+    fails=0
+  else
+    fails=$((fails + 1))
+    say "probe FAILED ($fails/$MAX_FAIL)"
+    if [[ $fails -ge $MAX_FAIL ]]; then
+      say "rolling back to the Python oracle"
+      if "$SWITCH" off --name "$NAME" >/dev/null 2>&1; then
+        say "ROLLED BACK — switch removed and verified. Not re-arming; this needs a human."
+      else
+        say "ROLLBACK FAILED — switch rules may remain. NEEDS IMMEDIATE ATTENTION."
+      fi
+      exit 1
+    fi
+  fi
+  sleep "$INTERVAL"
+done