From 1e8d46b2583827c2ec882cbeedd0a383e1f1ca6a Mon Sep 17 00:00:00 2001 From: funman300 Date: Sat, 22 Aug 2026 21:14:23 +0000 Subject: [PATCH] ops(systemd): follow the anchor container's netns across recreation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recreating the Docker anchor left the supervised Core and host stranded in the dead namespace while systemd still reported them active — serving nobody, invisible to any monitoring that trusts unit state. Reproduced on staging: netns 4026539938 -> 4026540033, both pids unchanged in the old one, both units "active", traffic ConnectionResetError. There is no systemd-native edge signal to bind to. Containers do appear as units, but the scope name embeds the container ID (docker-.scope), which changes on every recreate, so BindsTo= has no stable target; NetworkNamespacePath= resolves once at start; a .path unit on /run/netns would watch the file this tooling maintains. So: a level-triggered reconcile on a 10s timer, comparing the namespace the services are ACTUALLY in against the anchor's CURRENT one, acting only on a real difference. That cannot miss an event while the watcher restarts or dockerd is down, and needs no debounce — a burst of three recreations produced exactly one rebind. The trigger stays separable: a docker-events unit could invoke the same script. Anchor absent stops the dependants rather than falling back to host networking; docker unavailable logs once and retries on the next tick. Two defects found while testing and fixed here: - mount --bind STACKS when the old mount is busy, silently leaking nsfs entries; the bind helper now drains stale mounts in a loop. - reconcile must stop -> rebind -> start, not rebind -> restart: a running service holds the old namespace open and makes the umount fail busy. Staging also gained a faithful anchor container so the reproduction is structural rather than mocked. Economy state was byte-identical across every lifecycle test. Production units are templates only and remain uninstalled. --- scripts/systemd/README.md | 39 +++-- scripts/systemd/openfut-netns-bind.sh | 17 +- .../systemd/openfut-netns-reconcile.service | 23 +++ scripts/systemd/openfut-netns-reconcile.sh | 147 ++++++++++++++++++ scripts/systemd/openfut-netns-reconcile.timer | 14 ++ scripts/systemd/openfut-staging-core.service | 11 +- scripts/systemd/openfut-staging-host.service | 6 + .../openfut-staging-netns-reconcile.service | 22 +++ .../openfut-staging-netns-reconcile.timer | 16 ++ scripts/systemd/openfut-staging-netns.service | 30 ++++ .../systemd/openfut-supervision-install.sh | 97 ++++++++++++ 11 files changed, 409 insertions(+), 13 deletions(-) create mode 100644 scripts/systemd/openfut-netns-reconcile.service create mode 100755 scripts/systemd/openfut-netns-reconcile.sh create mode 100644 scripts/systemd/openfut-netns-reconcile.timer create mode 100644 scripts/systemd/openfut-staging-netns-reconcile.service create mode 100644 scripts/systemd/openfut-staging-netns-reconcile.timer create mode 100644 scripts/systemd/openfut-staging-netns.service create mode 100755 scripts/systemd/openfut-supervision-install.sh diff --git a/scripts/systemd/README.md b/scripts/systemd/README.md index 3339f1b..5904cf7 100644 --- a/scripts/systemd/README.md +++ b/scripts/systemd/README.md @@ -6,13 +6,18 @@ persistence and no supervisor-visible logs. | file | scope | installed? | |---|---|---| -| `openfut-staging-core.service` | staging Core, port 18081 | **yes** — proving ground | +| `openfut-staging-netns.service` | binds the staging anchor netns | **yes** — proving ground | +| `openfut-staging-core.service` | staging Core, port 18081 | **yes** | | `openfut-staging-host.service` | staging host, port 8299 | **yes** | -| `openfut-netns.service` | publishes the container netns to `/run/netns/openfut` | template only | +| `openfut-staging-netns-reconcile.{service,timer}` | staging netns lifecycle | **yes** | +| `openfut-netns.service` | binds the production anchor netns to `/run/netns/openfut` | template only | | `openfut-core.service` | production Core, port 18080 | template only | | `openfut-host.service` | production host, port 8099 | template only | -| `openfut-netns-bind.sh` | resolves the container netns by NAME, idempotently | helper | +| `openfut-netns-reconcile.{service,timer}` | production netns lifecycle | template only | +| `openfut-netns-bind.sh` | resolves the anchor netns by NAME, idempotently, drains stale mounts | helper | +| `openfut-netns-reconcile.sh` | keeps services in the anchor's CURRENT netns | helper | | `openfut-wait-tcp.sh` | bounded readiness gate | helper | +| `openfut-supervision-install.sh` | install / start / status / uninstall per environment | helper | Production templates are **not installed**. Deploy only via the plan in `OpenFUT-Vault/06 Operations/OpenFUT Service Supervision (staging-proven).md`. @@ -42,20 +47,32 @@ interfaces, silently serving nobody. `openfut-netns-bind.sh` re-resolves by name and refreshes a stale bind mount; `NetworkNamespacePath=` then enters it declaratively. -> **Still open:** a container restart strands *already-running* services in the -> dead namespace. Re-running `openfut-netns.service` plus restarting Core/host -> repairs it, but nothing triggers that automatically yet. See the promotion -> plan's "Residual gap". +**4. Stale namespaces are repaired automatically (this closes the old gap).** +Recreating the anchor strands already-running services in the dead namespace, +and — the dangerous part — *systemd still reports them `active`*. Measured before +the fix: anchor `net:[4026539938] → net:[4026540033]`, Core and host unchanged in +the old one, both units `active`, traffic `ConnectionResetError`. + +`openfut-netns-reconcile.sh` on a 10s timer compares the namespace the services +are **actually in** against the anchor's **current** one and, only on a real +difference, performs one stop → rebind → start cycle. It is level-triggered, so +it cannot miss an event and needs no debounce: a burst of three back-to-back +recreations produced exactly **one** rebind. ## Operating ```bash -systemctl status openfut-staging-core openfut-staging-host -journalctl -u openfut-staging-core -f -sudo systemctl restart openfut-staging-core # host survives and recovers -sudo systemctl stop openfut-staging-host openfut-staging-core +sudo ./openfut-supervision-install.sh status staging # units, anchor, netns agreement, mounts +journalctl -u openfut-staging-netns-reconcile -f # silent unless it acts +sudo systemctl restart openfut-staging-core # host survives and recovers +sudo ./openfut-supervision-install.sh install staging # copy + enable (does not start) +sudo ./openfut-supervision-install.sh uninstall staging # disable, stop, remove units ``` +`status` is the one command worth knowing: it prints each unit's state, the +anchor's current netns, the namespace each service is *actually* in with an +`ok`/`MISMATCH` verdict, and the mount count (`1` healthy, `>1` a leaked stack). + Config lives in `EnvironmentFile`s (`…/systemd/core.env`, `host.env`), generated from the live process environment so supervision changed *how* the processes start and nothing about *what* they do. Binaries are immutable copies, so a diff --git a/scripts/systemd/openfut-netns-bind.sh b/scripts/systemd/openfut-netns-bind.sh index 4b93cb3..19efaa5 100755 --- a/scripts/systemd/openfut-netns-bind.sh +++ b/scripts/systemd/openfut-netns-bind.sh @@ -49,7 +49,22 @@ if mountpoint -q "$TARGET" 2>/dev/null; then exit 0 fi echo "openfut-netns-bind: $TARGET is STALE ($HAVE, want $WANT) — refreshing" - umount "$TARGET" || true + # DRAIN, do not just pop. `mount --bind` STACKS: binding over a busy mount + # silently leaves the old one underneath, and staging grew two nsfs entries + # on the first rebind before this loop existed. Left alone that is one + # leaked mount per container recreation, and the buried namespaces are + # exactly the dead ones we are trying to get rid of. + # + # A umount can legitimately fail while a service still holds the old + # namespace open; the caller's job is to stop dependants FIRST. If it is + # still busy we stack rather than fail — a current top-of-stack mount is + # correct, just untidy — and say so. + while mountpoint -q "$TARGET" 2>/dev/null; do + umount "$TARGET" 2>/dev/null || { + echo "openfut-netns-bind: $TARGET still busy; stacking a current mount over it" >&2 + break + } + done fi [ -e "$TARGET" ] || touch "$TARGET" diff --git a/scripts/systemd/openfut-netns-reconcile.service b/scripts/systemd/openfut-netns-reconcile.service new file mode 100644 index 0000000..8a5fbe4 --- /dev/null +++ b/scripts/systemd/openfut-netns-reconcile.service @@ -0,0 +1,23 @@ +[Unit] +Description=OpenFUT: reconcile services with the anchor container network namespace +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md +After=docker.service +Wants=docker.service +# PRODUCTION TEMPLATE — NOT INSTALLED. +# +# Deliberately NO Requires=docker.service: a docker outage must be a quiet +# retry, not a failed unit. The script decides that itself and exits 0. + +[Service] +Type=oneshot +# Level-triggered. It compares the namespace Core and the host are ACTUALLY in +# against the anchor's CURRENT one, so a burst of container events collapses +# into at most one rebind per tick and no separate debounce is needed. +ExecStart=/home/alex/OpenFUT/scripts/systemd/openfut-netns-reconcile.sh \ + openfut-fut-backend openfut \ + openfut-netns.service \ + openfut-core.service openfut-host.service + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=openfut-netns-reconcile diff --git a/scripts/systemd/openfut-netns-reconcile.sh b/scripts/systemd/openfut-netns-reconcile.sh new file mode 100755 index 0000000..648c0d7 --- /dev/null +++ b/scripts/systemd/openfut-netns-reconcile.sh @@ -0,0 +1,147 @@ +#!/bin/sh +# Keep supervised native services inside the CURRENT network namespace of a +# Docker anchor container, and stop them when that anchor is gone. +# +# ── THE FAILURE THIS EXISTS FOR ──────────────────────────────────────────── +# OpenFUT's Core and host are native binaries that must run inside the anchor +# container's netns. Recreating (or merely restarting) that container gives it a +# NEW netns; the already-running services stay in the old one. Measured on +# staging 2026-08-22: +# +# anchor 37288a0fe816 net:[4026539938] -> 55541a03b66e net:[4026540033] +# core pid 2081773 net:[4026539938] (unchanged, now orphaned) +# host pid 2081791 net:[4026539938] (unchanged, now orphaned) +# systemd: both "active" traffic: ConnectionResetError +# +# Both units report HEALTHY while serving nobody. That is the whole problem: +# the failure is invisible to the supervisor. +# +# ── WHY A RECONCILE AND NOT AN EVENT WATCHER ─────────────────────────────── +# There is no systemd-native edge signal to bind to. Containers do appear as +# `docker-.scope` units (cgroup driver is systemd), but the scope NAME +# embeds the container ID, and the ID changes on recreate — so there is no +# stable unit for BindsTo=. `NetworkNamespacePath` is resolved at unit start +# only, and a .path unit on /run/netns/ would watch the very file this +# script maintains (circular). +# +# So the trigger is a timer, and the check is LEVEL-triggered: it compares the +# namespace the services are ACTUALLY in against the anchor's CURRENT one. That +# is strictly more robust than an edge-triggered watcher, which can miss events +# while it is itself restarting or while dockerd is down, and it needs no +# debounce logic — a burst of container events collapses into at most one +# reconcile per tick, because the only question asked is "does the observed +# state differ from the desired state right now?". +# +# The trigger is deliberately separable from the action. If detection latency +# ever matters, a `docker events` unit can invoke THIS SAME script; nothing here +# would change. +# +# ── INVARIANTS ───────────────────────────────────────────────────────────── +# * A container PID is runtime state and is NEVER persisted. It is resolved from +# Docker on every run. +# * Identity is the netns inode of /proc//ns/net, never the +# container name (same name != same namespace) and never a cached value. +# * systemd stays the service authority: this script only requests start/stop/ +# restart, and unit ordering does the sequencing. +# * Silence when correct. It logs only when it acts or fails, so a 10s timer +# does not fill the journal. +# +# usage: openfut-netns-reconcile.sh ... +set -eu + +CONTAINER="${1:?container name}"; shift +NSNAME="${1:?netns name}"; shift +NSUNIT="${1:?netns unit}"; shift +[ "$#" -ge 1 ] || { echo "reconcile: at least one dependent unit required" >&2; exit 2; } +UNITS="$*" +HERE="$(dirname "$0")" + +log() { echo "openfut-netns-reconcile: $*"; } + +# ---- 1. Is Docker even answering? ----------------------------------------- +# A daemon outage must be a clean, quiet failure that the timer retries, never a +# spin and never a destructive action taken on incomplete information. +if ! docker info >/dev/null 2>&1; then + log "docker daemon unavailable — taking NO action, will retry on the next tick" + exit 0 +fi + +# ---- 2. Resolve the anchor, by name, right now ---------------------------- +CPID="$(docker inspect -f '{{.State.Pid}}' "$CONTAINER" 2>/dev/null || true)" + +if [ -z "$CPID" ] || [ "$CPID" = "0" ] || [ ! -e "/proc/$CPID/ns/net" ]; then + # Anchor gone. Services must NOT keep pretending to be healthy inside a + # namespace whose owner has died — and must never fall back to host + # networking. Stop them; a later tick starts them again once the anchor is + # back, which is what makes recovery automatic. + RUNNING="" + for u in $UNITS; do + [ "$(systemctl is-active "$u" 2>/dev/null)" = "active" ] && RUNNING="$RUNNING $u" + done + if [ -n "$RUNNING" ]; then + log "anchor '$CONTAINER' is ABSENT — stopping$RUNNING (no host-network fallback)" + # Reverse order: dependants before the thing they depend on. + # shellcheck disable=SC2086 + systemctl stop $RUNNING || true + systemctl stop "$NSUNIT" || true + fi + exit 0 +fi + +WANT="$(readlink "/proc/$CPID/ns/net")" + +# ---- 3. Compare against where the services ACTUALLY are ------------------- +# Observed state, not a remembered value: this self-heals no matter how the +# drift happened (container recreate, restart, manual nsenter, anything). +NEED_ACTION=0 +REASON="" +for u in $UNITS; do + state="$(systemctl is-active "$u" 2>/dev/null || true)" + if [ "$state" != "active" ]; then + NEED_ACTION=1; REASON="$REASON $u=$state" + continue + fi + mp="$(systemctl show -p MainPID --value "$u" 2>/dev/null || echo 0)" + if [ -z "$mp" ] || [ "$mp" = "0" ] || [ ! -e "/proc/$mp/ns/net" ]; then + NEED_ACTION=1; REASON="$REASON $u=nopid" + continue + fi + have="$(readlink "/proc/$mp/ns/net")" + if [ "$have" != "$WANT" ]; then + NEED_ACTION=1; REASON="$REASON $u=$have" + fi +done + +if [ "$NEED_ACTION" = "0" ]; then + exit 0 # correct and silent +fi + +# ---- 4. One controlled rebind cycle --------------------------------------- +log "anchor '$CONTAINER' pid=$CPID ns=$WANT; drift:$REASON" +log "namespace changed or services adrift — requesting one rebind cycle" + +# STOP FIRST, then rebind, then start — not restart-around-a-rebind. While a +# service is still running it holds the OLD namespace open, the umount fails +# busy, and `mount --bind` silently STACKS a second nsfs entry over it. Measured: +# the first rebind left two mounts on the path. Stopping the dependants releases +# the old namespace so the drain actually succeeds. +# +# Reverse order on the way down (dependants before their dependency), forward on +# the way up — and the way up is systemd's job: `start` honours the units' own +# After=/Requires=, so Core is listening before the host's readiness gate runs. +REV="" +for u in $UNITS; do REV="$u $REV"; done +# shellcheck disable=SC2086 +systemctl stop $REV || true + +"$HERE/openfut-netns-bind.sh" "$CONTAINER" "$NSNAME" +systemctl restart "$NSUNIT" + +# shellcheck disable=SC2086 +systemctl start $UNITS + +for u in $UNITS; do + mp="$(systemctl show -p MainPID --value "$u" 2>/dev/null || echo 0)" + now="$( [ "$mp" != "0" ] && readlink "/proc/$mp/ns/net" || echo '-')" + log "rebound $u pid=$mp ns=$now" +done diff --git a/scripts/systemd/openfut-netns-reconcile.timer b/scripts/systemd/openfut-netns-reconcile.timer new file mode 100644 index 0000000..db218b5 --- /dev/null +++ b/scripts/systemd/openfut-netns-reconcile.timer @@ -0,0 +1,14 @@ +[Unit] +Description=OpenFUT: periodic anchor-namespace reconcile +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md +# PRODUCTION TEMPLATE — NOT INSTALLED. + +[Timer] +# 10s. Anchor recreation is rare and already an outage; converging inside ~10s +# is ample. Measured cost per tick: one `docker inspect` plus two readlinks. +OnBootSec=15s +OnUnitInactiveSec=10s +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/scripts/systemd/openfut-staging-core.service b/scripts/systemd/openfut-staging-core.service index 0bd6713..8ff7a60 100644 --- a/scripts/systemd/openfut-staging-core.service +++ b/scripts/systemd/openfut-staging-core.service @@ -4,8 +4,12 @@ Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md # Staging Core is the write authority for the staging SQLite DB. Nothing else # may hold it open, which is why there is no second instance and no oneshot # migration unit: Core runs its own migrations at startup, before it binds. -After=network-online.target +After=network-online.target docker.service openfut-staging-netns.service Wants=network-online.target +# Requires, not Wants: without the anchor namespace Core would bind the WRONG +# network entirely. Absent anchor must mean "do not start", never "start on +# the host network". +Requires=openfut-staging-netns.service # StartLimit* MUST live in [Unit]: systemd 252 silently IGNORES them in # [Service] (`systemd-analyze verify` flags it), which would have left the @@ -19,6 +23,11 @@ User=alex Group=alex WorkingDirectory=/home/alex/openfut-sold-staging +# Enter the anchor container namespace declaratively. Equivalent to the +# `nsenter --net=/proc//ns/net` production uses today, but with no pid +# baked in: the path is re-resolved by openfut-staging-netns.service. +NetworkNamespacePath=/run/netns/openfut-staging + # Config is DATA, not baked into the unit, so the same unit file promotes to # production with a different EnvironmentFile. EnvironmentFile=/home/alex/openfut-sold-staging/systemd/core.env diff --git a/scripts/systemd/openfut-staging-host.service b/scripts/systemd/openfut-staging-host.service index 373d000..cdad9a5 100644 --- a/scripts/systemd/openfut-staging-host.service +++ b/scripts/systemd/openfut-staging-host.service @@ -8,6 +8,10 @@ Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md # traffic: the host cannot reach "active" while Core is not listening. Wants=openfut-staging-core.service After=openfut-staging-core.service +# Requires on the NAMESPACE (hard) but only Wants on Core (soft) — the host +# must never bind the host network, yet must survive a Core blip. +Requires=openfut-staging-netns.service +After=openfut-staging-netns.service # `Wants`, deliberately NOT `Requires`/`BindsTo`/`PartOf`. Those propagate a # Core stop into a host stop, and — measured, not assumed — a later Core start @@ -32,6 +36,8 @@ User=alex Group=alex WorkingDirectory=/home/alex/openfut-sold-staging +NetworkNamespacePath=/run/netns/openfut-staging + EnvironmentFile=/home/alex/openfut-sold-staging/systemd/host.env # Readiness gate. Bounded, and FAILS rather than blocking forever: a host that diff --git a/scripts/systemd/openfut-staging-netns-reconcile.service b/scripts/systemd/openfut-staging-netns-reconcile.service new file mode 100644 index 0000000..809e62a --- /dev/null +++ b/scripts/systemd/openfut-staging-netns-reconcile.service @@ -0,0 +1,22 @@ +[Unit] +Description=OpenFUT STAGING: reconcile services with the anchor container network namespace +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md +After=docker.service +Wants=docker.service + +# Deliberately NO Requires=docker.service: a docker outage must be a quiet +# retry, not a failed unit. The script itself decides that and exits 0. + +[Service] +Type=oneshot +# Level-triggered: compares the namespace the services are ACTUALLY in against +# the anchor's CURRENT one, so a burst of container events collapses into at +# most one rebind per tick and no separate debounce is needed. +ExecStart=/home/alex/OpenFUT/scripts/systemd/openfut-netns-reconcile.sh \ + openfut-staging-anchor openfut-staging \ + openfut-staging-netns.service \ + openfut-staging-core.service openfut-staging-host.service + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=openfut-staging-netns-reconcile diff --git a/scripts/systemd/openfut-staging-netns-reconcile.timer b/scripts/systemd/openfut-staging-netns-reconcile.timer new file mode 100644 index 0000000..9f8cf75 --- /dev/null +++ b/scripts/systemd/openfut-staging-netns-reconcile.timer @@ -0,0 +1,16 @@ +[Unit] +Description=OpenFUT STAGING: periodic anchor-namespace reconcile +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md + +[Timer] +# 10s cadence. Container recreation is rare and already an outage; converging +# within ~10s is ample, and a level-triggered check at this rate costs one +# `docker inspect` plus two readlinks. +OnBootSec=15s +OnUnitInactiveSec=10s +AccuracySec=1s +# Not Persistent=: there is nothing to catch up on. The check is stateless and +# the next tick after boot is authoritative. + +[Install] +WantedBy=timers.target diff --git a/scripts/systemd/openfut-staging-netns.service b/scripts/systemd/openfut-staging-netns.service new file mode 100644 index 0000000..0bf1c65 --- /dev/null +++ b/scripts/systemd/openfut-staging-netns.service @@ -0,0 +1,30 @@ +[Unit] +Description=OpenFUT STAGING: publish the anchor container network namespace to /run/netns/openfut-staging +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md +After=docker.service +Requires=docker.service + +# Re-resolved on every start, so this unit is also the REPAIR action after the +# anchor container is recreated: `systemctl restart openfut-staging-netns` plus +# a restart of the dependants re-enters the current namespace. +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +Type=oneshot +RemainAfterExit=yes + +# Resolves the anchor BY NAME, never by a stored pid, and exits non-zero when +# the anchor is absent — which is what makes the dependants' Requires= a real +# admission gate rather than decoration. +ExecStart=/home/alex/OpenFUT/scripts/systemd/openfut-netns-bind.sh openfut-staging-anchor openfut-staging + +# No ExecStop unmount: the namespace belongs to the container's lifetime, and +# tearing the bind mount down under a live Core/host would strand them. + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=openfut-staging-netns + +[Install] +WantedBy=multi-user.target diff --git a/scripts/systemd/openfut-supervision-install.sh b/scripts/systemd/openfut-supervision-install.sh new file mode 100755 index 0000000..c491c8b --- /dev/null +++ b/scripts/systemd/openfut-supervision-install.sh @@ -0,0 +1,97 @@ +#!/bin/sh +# Install or remove the OpenFUT systemd supervision set for one environment. +# +# Environments are symmetric on purpose: staging and production differ only in +# unit prefix, anchor container and EnvironmentFile location, so what staging +# proved is what production gets. +# +# install copy units, daemon-reload, enable (does NOT start) +# start start in dependency order and report the namespace agreement +# status one-screen health: units, anchor, netns agreement, mounts +# uninstall disable + stop + remove units (leaves binaries, DB and env alone) +# +# The install step deliberately does NOT start anything: on production the +# changeover has to be sequenced against retiring the existing detached +# processes, which is an operator decision, not a script's. +# +# usage: openfut-supervision-install.sh +set -eu + +ACTION="${1:?install|start|status|uninstall}" +ENVNAME="${2:?staging|production}" +HERE="$(cd "$(dirname "$0")" && pwd)" + +case "$ENVNAME" in + staging) + PREFIX="openfut-staging" + ANCHOR="openfut-staging-anchor" + NSNAME="openfut-staging" + UNITS="openfut-staging-netns.service openfut-staging-core.service openfut-staging-host.service openfut-staging-netns-reconcile.service openfut-staging-netns-reconcile.timer" + ENABLE="openfut-staging-netns.service openfut-staging-core.service openfut-staging-host.service openfut-staging-netns-reconcile.timer" + ;; + production) + PREFIX="openfut" + ANCHOR="openfut-fut-backend" + NSNAME="openfut" + UNITS="openfut-netns.service openfut-core.service openfut-host.service openfut-netns-reconcile.service openfut-netns-reconcile.timer" + ENABLE="openfut-netns.service openfut-core.service openfut-host.service openfut-netns-reconcile.timer" + ;; + *) echo "unknown environment '$ENVNAME'" >&2; exit 2 ;; +esac + +case "$ACTION" in + install) + for u in $UNITS; do + [ -f "$HERE/$u" ] || { echo "missing unit $HERE/$u" >&2; exit 1; } + install -m 0644 "$HERE/$u" "/etc/systemd/system/$u" + echo "installed /etc/systemd/system/$u" + done + systemctl daemon-reload + for u in $UNITS; do systemd-analyze verify "/etc/systemd/system/$u" || true; done + # shellcheck disable=SC2086 + systemctl enable $ENABLE + echo "enabled (NOT started — start explicitly once the old processes are retired)" + ;; + + start) + systemctl start "${PREFIX}-netns.service" + systemctl start "${PREFIX}-core.service" + systemctl start "${PREFIX}-host.service" + systemctl start "${PREFIX}-netns-reconcile.timer" + sleep 3 + "$0" status "$ENVNAME" + ;; + + status) + printf '%-42s %s\n' "unit" "state" + for u in $UNITS; do printf ' %-40s %s\n' "$u" "$(systemctl is-active "$u" 2>/dev/null || true)"; done + cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)" + if [ "$cpid" != "0" ] && [ -e "/proc/$cpid/ns/net" ]; then + want="$(readlink "/proc/$cpid/ns/net")" + else + want="(anchor absent)" + fi + echo " anchor $ANCHOR pid=$cpid ns=$want" + for u in "${PREFIX}-core.service" "${PREFIX}-host.service"; do + mp="$(systemctl show -p MainPID --value "$u" 2>/dev/null || echo 0)" + ns="-"; [ "$mp" != "0" ] && [ -e "/proc/$mp/ns/net" ] && ns="$(readlink "/proc/$mp/ns/net")" + match="MISMATCH"; [ "$ns" = "$want" ] && match="ok" + printf ' %-40s pid=%-8s ns=%-18s %s\n' "$u" "$mp" "$ns" "$match" + done + echo " netns mounts on /run/netns/$NSNAME: $(grep -c "run/netns/$NSNAME" /proc/mounts || true) (1 = healthy, >1 = leaked stack)" + ;; + + uninstall) + # shellcheck disable=SC2086 + systemctl disable --now $ENABLE 2>/dev/null || true + systemctl stop "${PREFIX}-netns-reconcile.service" 2>/dev/null || true + for u in $UNITS; do rm -f "/etc/systemd/system/$u"; echo "removed /etc/systemd/system/$u"; done + systemctl daemon-reload + systemctl reset-failed 2>/dev/null || true + # The bind mount is intentionally left: the namespace belongs to the + # container, and tearing it down is not part of removing supervision. + echo "uninstalled. Binaries, EnvironmentFiles, /run/netns and the database are untouched." + ;; + + *) echo "unknown action '$ACTION'" >&2; exit 2 ;; +esac