#!/bin/sh # Capture unattended proof that the OpenFUT supervision chain came back by # itself after a boot. # # This exists because the reboot-survival gate is, by definition, a test no # operator can stand inside: the machine under test is the machine running the # session. So the machine records its own recovery. # # It observes only — it never starts, stops or repairs anything. If the chain is # broken, the evidence file says so, which is the point. # # usage: openfut-boot-evidence.sh [outdir] set -u OUTDIR="${1:-/home/alex/openfut-promotion/boot-evidence}" ANCHOR=openfut-fut-backend NSNAME=openfut DB=/home/alex/openfut-promotion/state/prod-core.db DEADLINE=180 # seconds to allow for docker + anchor + reconcile to settle mkdir -p "$OUTDIR" BOOTID="$(cat /proc/sys/kernel/random/boot_id)" OUT="$OUTDIR/boot-$(date -u +%Y%m%dT%H%M%SZ)-${BOOTID%%-*}.json" # Wait until converged, or until the deadline. Converged means both services are # active and share the anchor's CURRENT namespace. Polling rather than a fixed # sleep so a boot-time reconcile retry is captured as "settled late", not as a # failure, and so a healthy boot is recorded promptly. i=0 while [ "$i" -lt "$DEADLINE" ]; do cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)" if [ -n "$cpid" ] && [ "$cpid" != "0" ] && [ -e "/proc/$cpid/ns/net" ]; then want="$(readlink "/proc/$cpid/ns/net")" cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)" hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)" if [ -n "$cm" ] && [ "$cm" != "0" ] && [ -e "/proc/$cm/ns/net" ] && [ -n "$hm" ] && [ "$hm" != "0" ] && [ -e "/proc/$hm/ns/net" ] && [ "$(readlink "/proc/$cm/ns/net")" = "$want" ] && [ "$(readlink "/proc/$hm/ns/net")" = "$want" ]; then break fi fi i=$((i + 2)) sleep 2 done SETTLED_AFTER="$i" cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)" cid="$(docker inspect -f '{{.Id}}' "$ANCHOR" 2>/dev/null || echo none)" ans="$([ "$cpid" != "0" ] && readlink "/proc/$cpid/ns/net" 2>/dev/null || echo none)" cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)" hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)" cns="$([ -n "$cm" ] && [ "$cm" != "0" ] && readlink "/proc/$cm/ns/net" 2>/dev/null || echo none)" hns="$([ -n "$hm" ] && [ "$hm" != "0" ] && readlink "/proc/$hm/ns/net" 2>/dev/null || echo none)" mounts="$(awk -v t="/run/netns/$NSNAME" '$2==t {n++} END {print n+0}' /proc/mounts)" # Non-mutating reads, from inside the anchor namespace for Core (it is not # published to the host namespace) and on the published port for the host. # One argument only: a second positional would be unbound under `set -u` and # would abort the subshell, silently yielding an empty probe result. probe() { nsenter --net="/proc/$cpid/ns/net" python3 - "$1" <<'PY' 2>/dev/null || echo "ERR" import sys, urllib.request, urllib.error try: r = urllib.request.urlopen(urllib.request.Request(sys.argv[1], headers={"X-OpenFUT-Game": "fifa17", "X-UT-SID": "boot-evidence"}), timeout=8) print(r.status) except urllib.error.HTTPError as e: print(e.code) except Exception: print("ERR") PY } if [ "$cpid" != "0" ]; then core_health="$(probe http://127.0.0.1:18080/health)" core_coll="$(probe 'http://127.0.0.1:18080/collection?limit=1')" host_acct="$(probe http://127.0.0.1:8099/ut/game/fifa17/user/accountinfo)" host_club="$(probe 'http://127.0.0.1:8099/ut/game/fifa17/club?count=1&start=0')" else core_health=none; core_coll=none; host_acct=none; host_club=none fi econ="$(python3 - "$DB" <<'PY' 2>/dev/null || echo '{}' import sys, sqlite3, json c = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True) q = lambda s: c.execute(s).fetchone()[0] print(json.dumps({ "schema": q("SELECT MAX(version) FROM _sqlx_migrations"), "coins": q("SELECT coins FROM clubs"), "owned": q("SELECT COUNT(*) FROM owned_cards"), "by_content_kind": dict(c.execute("SELECT COALESCE(content_kind,'(null)'),COUNT(*) FROM owned_cards GROUP BY 1 ORDER BY 1").fetchall()), "applications": q("SELECT COUNT(*) FROM consumable_applications"), "contract_sum": q("SELECT COALESCE(SUM(contract_matches),0) FROM owned_cards"), "squad_players": q("SELECT COUNT(*) FROM squad_players"), "market_listings": q("SELECT COUNT(*) FROM market_listings"), "game_entity_ext": q("SELECT COUNT(*) FROM game_entity_ext"), "integrity": q("PRAGMA integrity_check"), "fk": len(c.execute("PRAGMA foreign_key_check").fetchall()), })) PY )" owner_rust="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=RUST')" owner_py="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=PYTHON')" cat > "$OUT" </dev/null | grep -c 'rebind cycle')}, "netns_agreement": $([ "$ans" = "$cns" ] && [ "$ans" = "$hns" ] && [ "$ans" != "none" ] && echo true || echo false), "nsfs_mounts": $mounts, "probes": {"core_health": "$core_health", "core_collection": "$core_coll", "host_accountinfo": "$host_acct", "host_club": "$host_club"}, "authority": {"owner_rust": $owner_rust, "owner_python": $owner_py}, "economy": $econ } EOF chmod 0644 "$OUT" ln -sfn "$OUT" "$OUTDIR/latest.json" echo "openfut-boot-evidence: wrote $OUT (settled after ${SETTLED_AFTER}s)" # Also drop the ordering proof for this boot, so ordering is read from real # timestamps rather than inferred from unit dependencies. journalctl -b -u openfut-netns -u openfut-core -u openfut-host -u openfut-netns-reconcile \ -o short-precise --no-pager > "$OUTDIR/latest-journal.txt" 2>/dev/null exit 0