funman300
fc1fdcc5ab
Two things surfaced by the production promotion. `status` counted namespace mounts with an unanchored grep, so on production "run/netns/openfut" also matched "openfut-staging" and reported a phantom "2 = leaked stack" against a perfectly healthy host. A status command that invents a fault is the same class of bug as a unit that reports active while serving nobody, so it is fixed with an exact mount-point match. The bind and reconcile logic is untouched; it always umounted an exact path. openfut-rollback-detached.sh makes the documented rollback executable rather than a paragraph in a runbook: it removes supervision, resolves the anchor's CURRENT pid from Docker, and relaunches the incumbent detached pair with the environment replayed from the captured env.json. --dry-run prints the exact commands and touches nothing, which is how it was validated while production was still being served by the processes it would restore.
102 lines
4.4 KiB
Bash
Executable File
102 lines
4.4 KiB
Bash
Executable File
#!/bin/sh
|
|
# Install or remove the OpenFUT systemd supervision set for one environment.
|
|
#
|
|
# Environments are symmetric on purpose: staging and production differ only in
|
|
# unit prefix, anchor container and EnvironmentFile location, so what staging
|
|
# proved is what production gets.
|
|
#
|
|
# install copy units, daemon-reload, enable (does NOT start)
|
|
# start start in dependency order and report the namespace agreement
|
|
# status one-screen health: units, anchor, netns agreement, mounts
|
|
# uninstall disable + stop + remove units (leaves binaries, DB and env alone)
|
|
#
|
|
# The install step deliberately does NOT start anything: on production the
|
|
# changeover has to be sequenced against retiring the existing detached
|
|
# processes, which is an operator decision, not a script's.
|
|
#
|
|
# usage: openfut-supervision-install.sh <install|start|status|uninstall> <staging|production>
|
|
set -eu
|
|
|
|
ACTION="${1:?install|start|status|uninstall}"
|
|
ENVNAME="${2:?staging|production}"
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
|
|
case "$ENVNAME" in
|
|
staging)
|
|
PREFIX="openfut-staging"
|
|
ANCHOR="openfut-staging-anchor"
|
|
NSNAME="openfut-staging"
|
|
UNITS="openfut-staging-netns.service openfut-staging-core.service openfut-staging-host.service openfut-staging-netns-reconcile.service openfut-staging-netns-reconcile.timer"
|
|
ENABLE="openfut-staging-netns.service openfut-staging-core.service openfut-staging-host.service openfut-staging-netns-reconcile.timer"
|
|
;;
|
|
production)
|
|
PREFIX="openfut"
|
|
ANCHOR="openfut-fut-backend"
|
|
NSNAME="openfut"
|
|
UNITS="openfut-netns.service openfut-core.service openfut-host.service openfut-netns-reconcile.service openfut-netns-reconcile.timer"
|
|
ENABLE="openfut-netns.service openfut-core.service openfut-host.service openfut-netns-reconcile.timer"
|
|
;;
|
|
*) echo "unknown environment '$ENVNAME'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
case "$ACTION" in
|
|
install)
|
|
for u in $UNITS; do
|
|
[ -f "$HERE/$u" ] || { echo "missing unit $HERE/$u" >&2; exit 1; }
|
|
install -m 0644 "$HERE/$u" "/etc/systemd/system/$u"
|
|
echo "installed /etc/systemd/system/$u"
|
|
done
|
|
systemctl daemon-reload
|
|
for u in $UNITS; do systemd-analyze verify "/etc/systemd/system/$u" || true; done
|
|
# shellcheck disable=SC2086
|
|
systemctl enable $ENABLE
|
|
echo "enabled (NOT started — start explicitly once the old processes are retired)"
|
|
;;
|
|
|
|
start)
|
|
systemctl start "${PREFIX}-netns.service"
|
|
systemctl start "${PREFIX}-core.service"
|
|
systemctl start "${PREFIX}-host.service"
|
|
systemctl start "${PREFIX}-netns-reconcile.timer"
|
|
sleep 3
|
|
"$0" status "$ENVNAME"
|
|
;;
|
|
|
|
status)
|
|
printf '%-42s %s\n' "unit" "state"
|
|
for u in $UNITS; do printf ' %-40s %s\n' "$u" "$(systemctl is-active "$u" 2>/dev/null || true)"; done
|
|
cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)"
|
|
if [ "$cpid" != "0" ] && [ -e "/proc/$cpid/ns/net" ]; then
|
|
want="$(readlink "/proc/$cpid/ns/net")"
|
|
else
|
|
want="(anchor absent)"
|
|
fi
|
|
echo " anchor $ANCHOR pid=$cpid ns=$want"
|
|
for u in "${PREFIX}-core.service" "${PREFIX}-host.service"; do
|
|
mp="$(systemctl show -p MainPID --value "$u" 2>/dev/null || echo 0)"
|
|
ns="-"; [ "$mp" != "0" ] && [ -e "/proc/$mp/ns/net" ] && ns="$(readlink "/proc/$mp/ns/net")"
|
|
match="MISMATCH"; [ "$ns" = "$want" ] && match="ok"
|
|
printf ' %-40s pid=%-8s ns=%-18s %s\n' "$u" "$mp" "$ns" "$match"
|
|
done
|
|
# Exact mount-point match. A substring grep for "openfut" also counts
|
|
# "openfut-staging" and reports a phantom leaked stack on production —
|
|
# a monitoring lie of exactly the kind this tooling exists to remove.
|
|
mounts="$(awk -v t="/run/netns/$NSNAME" '$2==t {n++} END {print n+0}' /proc/mounts)"
|
|
echo " netns mounts on /run/netns/$NSNAME: $mounts (1 = healthy, >1 = leaked stack)"
|
|
;;
|
|
|
|
uninstall)
|
|
# shellcheck disable=SC2086
|
|
systemctl disable --now $ENABLE 2>/dev/null || true
|
|
systemctl stop "${PREFIX}-netns-reconcile.service" 2>/dev/null || true
|
|
for u in $UNITS; do rm -f "/etc/systemd/system/$u"; echo "removed /etc/systemd/system/$u"; done
|
|
systemctl daemon-reload
|
|
systemctl reset-failed 2>/dev/null || true
|
|
# The bind mount is intentionally left: the namespace belongs to the
|
|
# container, and tearing it down is not part of removing supervision.
|
|
echo "uninstalled. Binaries, EnvironmentFiles, /run/netns and the database are untouched."
|
|
;;
|
|
|
|
*) echo "unknown action '$ACTION'" >&2; exit 2 ;;
|
|
esac
|