#!/usr/bin/env bash
# Keep an armed switch honest: if the Rust service stops answering, roll back.
#
#   openfut-watchdog.sh --name <switch> --probe <host:port> --kind <tls|tcp> \
#                       [--interval 60] [--failures 3] [--log <path>]
#
# WHY
#
# `openfut-switch.sh on` prints a warning that the service MUST stay up, because
# an armed switch pointing at a dead port means the client hits a closed socket
# with no fallback. A warning is not a safeguard when nobody is at the terminal.
#
# This turns the documented rollback into an automatic one. It fails toward the
# PYTHON oracle, which is the proven-good path, so the worst case of a spurious
# trip is that a gate needs re-arming — never that the client is left broken.
#
# It only ever removes the switch. It never installs one, never restarts the
# Rust service, and never touches the Python backend. Recovery is deliberately
# a human decision: an unexplained rollback is a finding to read in the morning,
# not something to paper over by flapping the switch back on.
set -uo pipefail

HERE="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
SWITCH="$HERE/openfut-switch.sh"

NAME=""; PROBE=""; KIND="tls"; INTERVAL=60; MAX_FAIL=3; LOG=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    --name)     NAME="$2"; shift 2 ;;
    --probe)    PROBE="$2"; shift 2 ;;
    --kind)     KIND="$2"; shift 2 ;;
    --interval) INTERVAL="$2"; shift 2 ;;
    --failures) MAX_FAIL="$2"; shift 2 ;;
    --log)      LOG="$2"; shift 2 ;;
    *) echo "watchdog: unknown argument $1" >&2; exit 2 ;;
  esac
done
[[ -n "$NAME" && -n "$PROBE" ]] || { echo "watchdog: --name and --probe are required" >&2; exit 2; }
[[ -x "$SWITCH" ]] || { echo "watchdog: missing $SWITCH" >&2; exit 2; }

say() {
  local line; line="[$(date +%H:%M:%S)] watchdog($NAME): $*"
  echo "$line"
  [[ -n "$LOG" ]] && echo "$line" >> "$LOG"
}

# A probe must exercise the SAME path the client uses — through the switch, and
# through TLS where the client speaks TLS. A bare TCP connect would succeed
# against a process that has wedged mid-handshake.
probe_once() {
  local host="${PROBE%:*}" port="${PROBE##*:}"
  if [[ "$KIND" == "tcp" ]]; then
    timeout 8 python3 -c "
import socket,sys
try:
    socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6).close()
except Exception:
    sys.exit(1)" "$host" "$port"
  else
    timeout 12 python3 -c "
import socket,ssl,sys
try:
    c=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT); c.check_hostname=False
    c.verify_mode=ssl.CERT_NONE; c.set_ciphers('ALL:@SECLEVEL=0')
    s=c.wrap_socket(socket.create_connection((sys.argv[1],int(sys.argv[2])),timeout=6),
                    server_hostname='watchdog')
    s.sendall(b'POST /redirector/getServerInstance HTTP/1.1\r\nContent-Length: 0\r\n\r\n')
    d=b''
    while True:
        x=s.recv(4096)
        if not x: break
        d+=x
    sys.exit(0 if len(d)>0 else 1)
except Exception:
    sys.exit(1)" "$host" "$port"
  fi
}

say "started: probing $PROBE every ${INTERVAL}s ($KIND); rolls back after $MAX_FAIL consecutive failures"
fails=0
while true; do
  # Nothing to guard once the switch is off — exit rather than spin forever.
  if "$SWITCH" status --name "$NAME" 2>/dev/null | grep -q '^INACTIVE'; then
    say "switch is no longer armed; nothing to guard, exiting"
    exit 0
  fi

  if probe_once; then
    [[ $fails -gt 0 ]] && say "recovered after $fails failure(s)"
    fails=0
  else
    fails=$((fails + 1))
    say "probe FAILED ($fails/$MAX_FAIL)"
    if [[ $fails -ge $MAX_FAIL ]]; then
      say "rolling back to the Python oracle"
      if "$SWITCH" off --name "$NAME" >/dev/null 2>&1; then
        say "ROLLED BACK — switch removed and verified. Not re-arming; this needs a human."
      else
        say "ROLLBACK FAILED — switch rules may remain. NEEDS IMMEDIATE ATTENTION."
      fi
      exit 1
    fi
  fi
  sleep "$INTERVAL"
done