diff --git a/scripts/systemd/openfut-boot-evidence.service b/scripts/systemd/openfut-boot-evidence.service new file mode 100644 index 0000000..63912e7 --- /dev/null +++ b/scripts/systemd/openfut-boot-evidence.service @@ -0,0 +1,20 @@ +[Unit] +Description=OpenFUT: record unattended proof that supervision recovered after boot +Documentation=file:///home/alex/OpenFUT/scripts/systemd/README.md +After=openfut-host.service openfut-netns-reconcile.timer docker.service +Wants=openfut-host.service +# Observation only. Deliberately no Requires= and no ordering that anything else +# waits on: this unit must never be able to affect the boot it is measuring. + +[Service] +Type=oneshot +ExecStart=/home/alex/OpenFUT/scripts/systemd/openfut-boot-evidence.sh +TimeoutStartSec=300 +# Never fail the boot over evidence collection. +SuccessExitStatus=0 1 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=openfut-boot-evidence + +[Install] +WantedBy=multi-user.target diff --git a/scripts/systemd/openfut-boot-evidence.sh b/scripts/systemd/openfut-boot-evidence.sh new file mode 100755 index 0000000..badc8a2 --- /dev/null +++ b/scripts/systemd/openfut-boot-evidence.sh @@ -0,0 +1,132 @@ +#!/bin/sh +# Capture unattended proof that the OpenFUT supervision chain came back by +# itself after a boot. +# +# This exists because the reboot-survival gate is, by definition, a test no +# operator can stand inside: the machine under test is the machine running the +# session. So the machine records its own recovery. +# +# It observes only — it never starts, stops or repairs anything. If the chain is +# broken, the evidence file says so, which is the point. +# +# usage: openfut-boot-evidence.sh [outdir] +set -u + +OUTDIR="${1:-/home/alex/openfut-promotion/boot-evidence}" +ANCHOR=openfut-fut-backend +NSNAME=openfut +DB=/home/alex/openfut-promotion/state/prod-core.db +DEADLINE=180 # seconds to allow for docker + anchor + reconcile to settle + +mkdir -p "$OUTDIR" +BOOTID="$(cat /proc/sys/kernel/random/boot_id)" +OUT="$OUTDIR/boot-$(date -u +%Y%m%dT%H%M%SZ)-${BOOTID%%-*}.json" + +# Wait until converged, or until the deadline. Converged means both services are +# active and share the anchor's CURRENT namespace. Polling rather than a fixed +# sleep so a boot-time reconcile retry is captured as "settled late", not as a +# failure, and so a healthy boot is recorded promptly. +i=0 +while [ "$i" -lt "$DEADLINE" ]; do + cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)" + if [ -n "$cpid" ] && [ "$cpid" != "0" ] && [ -e "/proc/$cpid/ns/net" ]; then + want="$(readlink "/proc/$cpid/ns/net")" + cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)" + hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)" + if [ -n "$cm" ] && [ "$cm" != "0" ] && [ -e "/proc/$cm/ns/net" ] && + [ -n "$hm" ] && [ "$hm" != "0" ] && [ -e "/proc/$hm/ns/net" ] && + [ "$(readlink "/proc/$cm/ns/net")" = "$want" ] && + [ "$(readlink "/proc/$hm/ns/net")" = "$want" ]; then + break + fi + fi + i=$((i + 2)) + sleep 2 +done +SETTLED_AFTER="$i" + +cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)" +cid="$(docker inspect -f '{{.Id}}' "$ANCHOR" 2>/dev/null || echo none)" +ans="$([ "$cpid" != "0" ] && readlink "/proc/$cpid/ns/net" 2>/dev/null || echo none)" +cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)" +hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)" +cns="$([ -n "$cm" ] && [ "$cm" != "0" ] && readlink "/proc/$cm/ns/net" 2>/dev/null || echo none)" +hns="$([ -n "$hm" ] && [ "$hm" != "0" ] && readlink "/proc/$hm/ns/net" 2>/dev/null || echo none)" +mounts="$(awk -v t="/run/netns/$NSNAME" '$2==t {n++} END {print n+0}' /proc/mounts)" + +# Non-mutating reads, from inside the anchor namespace for Core (it is not +# published to the host namespace) and on the published port for the host. +# One argument only: a second positional would be unbound under `set -u` and +# would abort the subshell, silently yielding an empty probe result. +probe() { nsenter --net="/proc/$cpid/ns/net" python3 - "$1" <<'PY' 2>/dev/null || echo "ERR" +import sys, urllib.request, urllib.error +try: + r = urllib.request.urlopen(urllib.request.Request(sys.argv[1], headers={"X-OpenFUT-Game": "fifa17", "X-UT-SID": "boot-evidence"}), timeout=8) + print(r.status) +except urllib.error.HTTPError as e: + print(e.code) +except Exception: + print("ERR") +PY +} + +if [ "$cpid" != "0" ]; then + core_health="$(probe http://127.0.0.1:18080/health)" + core_coll="$(probe 'http://127.0.0.1:18080/collection?limit=1')" + host_acct="$(probe http://127.0.0.1:8099/ut/game/fifa17/user/accountinfo)" + host_club="$(probe 'http://127.0.0.1:8099/ut/game/fifa17/club?count=1&start=0')" +else + core_health=none; core_coll=none; host_acct=none; host_club=none +fi + +econ="$(python3 - "$DB" <<'PY' 2>/dev/null || echo '{}' +import sys, sqlite3, json +c = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True) +q = lambda s: c.execute(s).fetchone()[0] +print(json.dumps({ + "schema": q("SELECT MAX(version) FROM _sqlx_migrations"), + "coins": q("SELECT coins FROM clubs"), + "owned": q("SELECT COUNT(*) FROM owned_cards"), + "by_content_kind": dict(c.execute("SELECT COALESCE(content_kind,'(null)'),COUNT(*) FROM owned_cards GROUP BY 1 ORDER BY 1").fetchall()), + "applications": q("SELECT COUNT(*) FROM consumable_applications"), + "contract_sum": q("SELECT COALESCE(SUM(contract_matches),0) FROM owned_cards"), + "squad_players": q("SELECT COUNT(*) FROM squad_players"), + "market_listings": q("SELECT COUNT(*) FROM market_listings"), + "game_entity_ext": q("SELECT COUNT(*) FROM game_entity_ext"), + "integrity": q("PRAGMA integrity_check"), + "fk": len(c.execute("PRAGMA foreign_key_check").fetchall()), +})) +PY +)" + +owner_rust="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=RUST')" +owner_py="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=PYTHON')" + +cat > "$OUT" </dev/null | grep -c 'rebind cycle')}, + "netns_agreement": $([ "$ans" = "$cns" ] && [ "$ans" = "$hns" ] && [ "$ans" != "none" ] && echo true || echo false), + "nsfs_mounts": $mounts, + "probes": {"core_health": "$core_health", "core_collection": "$core_coll", "host_accountinfo": "$host_acct", "host_club": "$host_club"}, + "authority": {"owner_rust": $owner_rust, "owner_python": $owner_py}, + "economy": $econ +} +EOF + +chmod 0644 "$OUT" +ln -sfn "$OUT" "$OUTDIR/latest.json" +echo "openfut-boot-evidence: wrote $OUT (settled after ${SETTLED_AFTER}s)" + +# Also drop the ordering proof for this boot, so ordering is read from real +# timestamps rather than inferred from unit dependencies. +journalctl -b -u openfut-netns -u openfut-core -u openfut-host -u openfut-netns-reconcile \ + -o short-precise --no-pager > "$OUTDIR/latest-journal.txt" 2>/dev/null +exit 0