funman300
4156dc5810
The reboot-survival gate is a test no operator can stand inside: the machine under test is the machine running the session. So the machine records its own recovery. openfut-boot-evidence.service polls until the anchor, Core and host agree on a namespace (or a 180s deadline expires), then writes a JSON file with the boot id, anchor/Core/host pids and netns inodes, unit states, restart counts, whether the reconciler had to act this boot, mount count, four non-mutating probes, route ownership, and the full economy snapshot — plus the journal for the boot so ordering is read from real timestamps rather than inferred from unit dependencies. It observes only; it never starts, stops or repairs anything, and carries no Requires= or ordering that anything else waits on, so it cannot affect the boot it is measuring. If the chain is broken the file says so, which is the point. Polling rather than a fixed sleep means a boot-time reconcile retry is recorded as "settled late" rather than as a failure.
133 lines
6.8 KiB
Bash
Executable File
133 lines
6.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Capture unattended proof that the OpenFUT supervision chain came back by
|
|
# itself after a boot.
|
|
#
|
|
# This exists because the reboot-survival gate is, by definition, a test no
|
|
# operator can stand inside: the machine under test is the machine running the
|
|
# session. So the machine records its own recovery.
|
|
#
|
|
# It observes only — it never starts, stops or repairs anything. If the chain is
|
|
# broken, the evidence file says so, which is the point.
|
|
#
|
|
# usage: openfut-boot-evidence.sh [outdir]
|
|
set -u
|
|
|
|
OUTDIR="${1:-/home/alex/openfut-promotion/boot-evidence}"
|
|
ANCHOR=openfut-fut-backend
|
|
NSNAME=openfut
|
|
DB=/home/alex/openfut-promotion/state/prod-core.db
|
|
DEADLINE=180 # seconds to allow for docker + anchor + reconcile to settle
|
|
|
|
mkdir -p "$OUTDIR"
|
|
BOOTID="$(cat /proc/sys/kernel/random/boot_id)"
|
|
OUT="$OUTDIR/boot-$(date -u +%Y%m%dT%H%M%SZ)-${BOOTID%%-*}.json"
|
|
|
|
# Wait until converged, or until the deadline. Converged means both services are
|
|
# active and share the anchor's CURRENT namespace. Polling rather than a fixed
|
|
# sleep so a boot-time reconcile retry is captured as "settled late", not as a
|
|
# failure, and so a healthy boot is recorded promptly.
|
|
i=0
|
|
while [ "$i" -lt "$DEADLINE" ]; do
|
|
cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)"
|
|
if [ -n "$cpid" ] && [ "$cpid" != "0" ] && [ -e "/proc/$cpid/ns/net" ]; then
|
|
want="$(readlink "/proc/$cpid/ns/net")"
|
|
cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)"
|
|
hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)"
|
|
if [ -n "$cm" ] && [ "$cm" != "0" ] && [ -e "/proc/$cm/ns/net" ] &&
|
|
[ -n "$hm" ] && [ "$hm" != "0" ] && [ -e "/proc/$hm/ns/net" ] &&
|
|
[ "$(readlink "/proc/$cm/ns/net")" = "$want" ] &&
|
|
[ "$(readlink "/proc/$hm/ns/net")" = "$want" ]; then
|
|
break
|
|
fi
|
|
fi
|
|
i=$((i + 2))
|
|
sleep 2
|
|
done
|
|
SETTLED_AFTER="$i"
|
|
|
|
cpid="$(docker inspect -f '{{.State.Pid}}' "$ANCHOR" 2>/dev/null || echo 0)"
|
|
cid="$(docker inspect -f '{{.Id}}' "$ANCHOR" 2>/dev/null || echo none)"
|
|
ans="$([ "$cpid" != "0" ] && readlink "/proc/$cpid/ns/net" 2>/dev/null || echo none)"
|
|
cm="$(systemctl show -p MainPID --value openfut-core.service 2>/dev/null)"
|
|
hm="$(systemctl show -p MainPID --value openfut-host.service 2>/dev/null)"
|
|
cns="$([ -n "$cm" ] && [ "$cm" != "0" ] && readlink "/proc/$cm/ns/net" 2>/dev/null || echo none)"
|
|
hns="$([ -n "$hm" ] && [ "$hm" != "0" ] && readlink "/proc/$hm/ns/net" 2>/dev/null || echo none)"
|
|
mounts="$(awk -v t="/run/netns/$NSNAME" '$2==t {n++} END {print n+0}' /proc/mounts)"
|
|
|
|
# Non-mutating reads, from inside the anchor namespace for Core (it is not
|
|
# published to the host namespace) and on the published port for the host.
|
|
# One argument only: a second positional would be unbound under `set -u` and
|
|
# would abort the subshell, silently yielding an empty probe result.
|
|
probe() { nsenter --net="/proc/$cpid/ns/net" python3 - "$1" <<'PY' 2>/dev/null || echo "ERR"
|
|
import sys, urllib.request, urllib.error
|
|
try:
|
|
r = urllib.request.urlopen(urllib.request.Request(sys.argv[1], headers={"X-OpenFUT-Game": "fifa17", "X-UT-SID": "boot-evidence"}), timeout=8)
|
|
print(r.status)
|
|
except urllib.error.HTTPError as e:
|
|
print(e.code)
|
|
except Exception:
|
|
print("ERR")
|
|
PY
|
|
}
|
|
|
|
if [ "$cpid" != "0" ]; then
|
|
core_health="$(probe http://127.0.0.1:18080/health)"
|
|
core_coll="$(probe 'http://127.0.0.1:18080/collection?limit=1')"
|
|
host_acct="$(probe http://127.0.0.1:8099/ut/game/fifa17/user/accountinfo)"
|
|
host_club="$(probe 'http://127.0.0.1:8099/ut/game/fifa17/club?count=1&start=0')"
|
|
else
|
|
core_health=none; core_coll=none; host_acct=none; host_club=none
|
|
fi
|
|
|
|
econ="$(python3 - "$DB" <<'PY' 2>/dev/null || echo '{}'
|
|
import sys, sqlite3, json
|
|
c = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
|
|
q = lambda s: c.execute(s).fetchone()[0]
|
|
print(json.dumps({
|
|
"schema": q("SELECT MAX(version) FROM _sqlx_migrations"),
|
|
"coins": q("SELECT coins FROM clubs"),
|
|
"owned": q("SELECT COUNT(*) FROM owned_cards"),
|
|
"by_content_kind": dict(c.execute("SELECT COALESCE(content_kind,'(null)'),COUNT(*) FROM owned_cards GROUP BY 1 ORDER BY 1").fetchall()),
|
|
"applications": q("SELECT COUNT(*) FROM consumable_applications"),
|
|
"contract_sum": q("SELECT COALESCE(SUM(contract_matches),0) FROM owned_cards"),
|
|
"squad_players": q("SELECT COUNT(*) FROM squad_players"),
|
|
"market_listings": q("SELECT COUNT(*) FROM market_listings"),
|
|
"game_entity_ext": q("SELECT COUNT(*) FROM game_entity_ext"),
|
|
"integrity": q("PRAGMA integrity_check"),
|
|
"fk": len(c.execute("PRAGMA foreign_key_check").fetchall()),
|
|
}))
|
|
PY
|
|
)"
|
|
|
|
owner_rust="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=RUST')"
|
|
owner_py="$(journalctl -u openfut-host -b --no-pager -o cat 2>/dev/null | grep -c 'owner=PYTHON')"
|
|
|
|
cat > "$OUT" <<EOF
|
|
{
|
|
"boot_id": "$BOOTID",
|
|
"captured_utc": "$(date -u +%FT%TZ)",
|
|
"settled_after_seconds": $SETTLED_AFTER,
|
|
"uptime_at_capture": "$(cut -d' ' -f1 /proc/uptime)",
|
|
"anchor": {"container": "${cid%%[!0-9a-f]*}", "pid": "$cpid", "netns": "$ans"},
|
|
"core": {"pid": "$cm", "netns": "$cns", "active": "$(systemctl is-active openfut-core.service)", "enabled": "$(systemctl is-enabled openfut-core.service)", "nrestarts": "$(systemctl show -p NRestarts --value openfut-core.service)"},
|
|
"host": {"pid": "$hm", "netns": "$hns", "active": "$(systemctl is-active openfut-host.service)", "enabled": "$(systemctl is-enabled openfut-host.service)", "nrestarts": "$(systemctl show -p NRestarts --value openfut-host.service)"},
|
|
"netns_unit": {"active": "$(systemctl is-active openfut-netns.service)", "enabled": "$(systemctl is-enabled openfut-netns.service)", "result": "$(systemctl show -p Result --value openfut-netns.service)"},
|
|
"reconcile": {"timer_active": "$(systemctl is-active openfut-netns-reconcile.timer)", "timer_enabled": "$(systemctl is-enabled openfut-netns-reconcile.timer)", "last_result": "$(systemctl show -p Result --value openfut-netns-reconcile.service)", "acted_this_boot": $(journalctl -u openfut-netns-reconcile -b --no-pager -o cat 2>/dev/null | grep -c 'rebind cycle')},
|
|
"netns_agreement": $([ "$ans" = "$cns" ] && [ "$ans" = "$hns" ] && [ "$ans" != "none" ] && echo true || echo false),
|
|
"nsfs_mounts": $mounts,
|
|
"probes": {"core_health": "$core_health", "core_collection": "$core_coll", "host_accountinfo": "$host_acct", "host_club": "$host_club"},
|
|
"authority": {"owner_rust": $owner_rust, "owner_python": $owner_py},
|
|
"economy": $econ
|
|
}
|
|
EOF
|
|
|
|
chmod 0644 "$OUT"
|
|
ln -sfn "$OUT" "$OUTDIR/latest.json"
|
|
echo "openfut-boot-evidence: wrote $OUT (settled after ${SETTLED_AFTER}s)"
|
|
|
|
# Also drop the ordering proof for this boot, so ordering is read from real
|
|
# timestamps rather than inferred from unit dependencies.
|
|
journalctl -b -u openfut-netns -u openfut-core -u openfut-host -u openfut-netns-reconcile \
|
|
-o short-precise --no-pager > "$OUTDIR/latest-journal.txt" 2>/dev/null
|
|
exit 0
|