From f0c6dcf2389c7b4f3f8074a275ff8740560dc708 Mon Sep 17 00:00:00 2001 From: funman300 Date: Sat, 22 Aug 2026 18:40:12 +0000 Subject: [PATCH] tooling: verified backup, state snapshot, retargetable apply validator Promotion prep for the contract-apply cutover, which unlike the quick-sell promotion moves BOTH binaries and applies a schema migration. fifa17-promotion-backup.py uses SQLite's online backup API, not cp. Production runs WAL with a routinely uncheckpointed WAL (515 KB at capture time); copying the main file alone is not atomic against a live writer and carries no guarantee the WAL holds no newer committed state. Emits a checksummed backup, a metadata record and a RESTORE-*.sh that removes the stale -wal/-shm BEFORE restoring -- omit that and SQLite replays the old journal over the file you just put back, resurrecting the state you were abandoning. fifa17-promotion-snapshot.py is read-only (mode=ro) and counts EVERY table rather than a hand-picked list, so a delta cannot hide in a table nobody thought to name. It also fingerprints the ownership rows, catching a row silently rewritten when counts alone would match. fifa17-contract-apply-validate.py gains --host/--db so one tool serves staging, the migration rehearsal and the production acceptance run. Defaults stay staging: there is deliberately no production default, so a bare invocation cannot touch production. --- scripts/fifa17-contract-apply-validate.py | 13 +- scripts/fifa17-promotion-backup.py | 158 ++++++++++++++++++++++ scripts/fifa17-promotion-snapshot.py | 94 +++++++++++++ 3 files changed, 264 insertions(+), 1 deletion(-) create mode 100755 scripts/fifa17-promotion-backup.py create mode 100755 scripts/fifa17-promotion-snapshot.py diff --git a/scripts/fifa17-contract-apply-validate.py b/scripts/fifa17-contract-apply-validate.py index 1339bce..e06f6d6 100755 --- a/scripts/fifa17-contract-apply-validate.py +++ b/scripts/fifa17-contract-apply-validate.py @@ -11,7 +11,9 @@ Exercises the real route end to end and asserts the full observable contract: * replaying the exhausted resource fails closed rather than granting again; * a manager contract and a non-contract consumable both fail closed. -Read-only against production by construction: every URL is the staging port. +Defaults target STAGING. `--host`/`--db` retarget it at a rehearsal or, under +explicit authorization, at the production acceptance run. There is deliberately +no production default: a bare invocation cannot touch production. """ import argparse import json @@ -20,6 +22,9 @@ import sys import urllib.error import urllib.request +# Defaults are STAGING. Override for a rehearsal or the production acceptance +# run; there is deliberately no production default, so a bare invocation can +# never touch production by accident. HOST = "http://127.0.0.1:8299" CORE_DB = "/home/alex/openfut-sold-staging/staging-core.db" HDRS = {"X-OpenFUT-Game": "fifa17"} @@ -123,7 +128,13 @@ def main(): help="contract resource id to apply (default: first owned player contract)") ap.add_argument("--target", type=int, default=None, help="target player wire id (default: lowest-rated owned player)") + ap.add_argument("--host", default=HOST, help=f"utas-host base URL (default {HOST})") + ap.add_argument("--db", default=CORE_DB, help=f"Core SQLite path (default {CORE_DB})") args = ap.parse_args() + # Rebound before any request so every helper below reads the chosen target. + globals()["HOST"] = args.host + globals()["CORE_DB"] = args.db + print(f"target host : {HOST}\ntarget db : {CORE_DB}\n") print("== BEFORE ==") before = core_snapshot() diff --git a/scripts/fifa17-promotion-backup.py b/scripts/fifa17-promotion-backup.py new file mode 100755 index 0000000..66b816d --- /dev/null +++ b/scripts/fifa17-promotion-backup.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Transactionally-consistent backup of a live OpenFUT Core SQLite DB. + +WHY NOT `cp`. Production runs WAL mode with an uncheckpointed WAL that routinely +holds hundreds of KB of committed pages. Copying only `prod-core.db` captures the +main file WITHOUT those pages and silently loses committed transactions; copying +the three files non-atomically can capture a torn set. This uses SQLite's online +backup API, which walks a read transaction and emits ONE standalone, already- +merged database file — no sidecar needed, no writer paused, safe against a live +production process. + +RESTORE HAZARD, read this before restoring. The destination of a restore MUST +have its `-wal` and `-shm` removed first. SQLite treats an existing `-wal` as +newer-than-the-database journal content and will replay it over the file you +just put back, resurrecting exactly the state you were trying to abandon. The +emitted `RESTORE.sh` does this in the right order. + +Usage: + fifa17-promotion-backup.py [--label NAME] +""" +import argparse +import datetime +import hashlib +import json +import os +import sqlite3 +import stat + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def describe(db_path, read_only=True): + uri = f"file:{db_path}?mode=ro" if read_only else db_path + con = sqlite3.connect(uri, uri=True) + one = lambda s: con.execute(s).fetchone()[0] + info = { + "journal_mode": one("PRAGMA journal_mode"), + "page_size": one("PRAGMA page_size"), + "page_count": one("PRAGMA page_count"), + "migration_max": one("SELECT MAX(version) FROM _sqlx_migrations"), + "migration_count": one("SELECT COUNT(*) FROM _sqlx_migrations"), + "integrity_check": one("PRAGMA integrity_check"), + "foreign_key_check": len(con.execute("PRAGMA foreign_key_check").fetchall()), + "owned_cards": one("SELECT COUNT(*) FROM owned_cards"), + "coins": one("SELECT COALESCE(SUM(coins), 0) FROM clubs"), + } + con.close() + return info + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("source") + ap.add_argument("backup_dir") + ap.add_argument("--label", default="core") + args = ap.parse_args() + + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + os.makedirs(args.backup_dir, exist_ok=True) + dest = os.path.join(args.backup_dir, f"{args.label}-{stamp}.db") + + src_info = describe(args.source) + sidecars = { + os.path.basename(args.source) + suf: + (os.path.getsize(args.source + suf) if os.path.exists(args.source + suf) else None) + for suf in ("", "-wal", "-shm") + } + print(f"source : {args.source}") + print(f" journal : {src_info['journal_mode']} migration_max={src_info['migration_max']}") + print(f" sidecars : {sidecars}") + print(f" integrity : {src_info['integrity_check']} fk_violations={src_info['foreign_key_check']}") + + # Online backup API. Source opened READ-ONLY: production is never written to, + # and in WAL mode this does not block the live writer. + src = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True) + dst = sqlite3.connect(dest) + with dst: + src.backup(dst) + dst.close() + src.close() + os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP) # read-only: a backup is not scratch space + + dst_info = describe(dest) + digest = sha256(dest) + + # The backup is only a backup if it independently verifies. A mismatch here + # means DO NOT PROCEED — it does not mean "retry and hope". + checks = { + "integrity_ok": dst_info["integrity_check"] == "ok", + "no_fk_violations": dst_info["foreign_key_check"] == 0, + "migration_matches": dst_info["migration_max"] == src_info["migration_max"], + "owned_matches": dst_info["owned_cards"] == src_info["owned_cards"], + "coins_match": dst_info["coins"] == src_info["coins"], + } + + meta = { + "taken_at": datetime.datetime.now().isoformat(timespec="seconds"), + "source": os.path.abspath(args.source), + "source_sidecar_sizes": sidecars, + "source_info": src_info, + "backup_path": os.path.abspath(dest), + "backup_sha256": digest, + "backup_size": os.path.getsize(dest), + "backup_info": dst_info, + "verification": checks, + "method": "sqlite3 online backup API (Connection.backup), source opened mode=ro", + } + meta_path = dest + ".json" + open(meta_path, "w").write(json.dumps(meta, indent=2, sort_keys=True) + "\n") + + restore = os.path.join(args.backup_dir, f"RESTORE-{args.label}-{stamp}.sh") + open(restore, "w").write(f"""#!/bin/sh +# Canonical restore for {os.path.abspath(args.source)} +# Generated {meta['taken_at']} from backup {os.path.basename(dest)} +# +# STOP EVERY WRITER FIRST. Restoring under a live Core corrupts both. +set -eu + +TARGET='{os.path.abspath(args.source)}' +BACKUP='{os.path.abspath(dest)}' + +test "$(sha256sum "$BACKUP" | cut -d' ' -f1)" = '{digest}' \\ + || {{ echo 'FATAL: backup checksum mismatch, refusing to restore'; exit 1; }} + +# The stale -wal/-shm MUST go, or SQLite replays them over the restored file. +rm -f "$TARGET-wal" "$TARGET-shm" +cp "$BACKUP" "$TARGET" +chmod u+w "$TARGET" + +sqlite3 "$TARGET" 'PRAGMA integrity_check;' 'PRAGMA foreign_key_check;' \\ + 'SELECT MAX(version) FROM _sqlx_migrations;' +echo 'restore complete -- now start the PREVIOUS Core and host binaries' +""") + os.chmod(restore, 0o755) + + print(f"\nbackup : {dest}") + print(f" sha256 : {digest}") + print(f" size : {meta['backup_size']:,}") + print(f" integrity : {dst_info['integrity_check']} fk_violations={dst_info['foreign_key_check']}") + print(f" migration : {dst_info['migration_max']} owned={dst_info['owned_cards']} coins={dst_info['coins']:,}") + print(f"metadata : {meta_path}") + print(f"restore : {restore}") + print("\nverification:") + for k, v in checks.items(): + print(f" [{'OK ' if v else 'FAIL'}] {k}") + ok = all(checks.values()) + print("\nRESULT:", "BACKUP VERIFIED" if ok else "BACKUP FAILED VERIFICATION -- DO NOT PROCEED") + raise SystemExit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/fifa17-promotion-snapshot.py b/scripts/fifa17-promotion-snapshot.py new file mode 100755 index 0000000..607118e --- /dev/null +++ b/scripts/fifa17-promotion-snapshot.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""READ-ONLY state capture of an OpenFUT Core SQLite DB, for promotion gating. + +Opened `mode=ro` and never written to, so it is safe against live production. +Emits a deterministic JSON document: run it before a promotion and again after, +then `diff` the two. Every table is counted, so a delta cannot hide in a table +nobody thought to list. + +Usage: + fifa17-promotion-snapshot.py [out.json] +""" +import hashlib +import json +import sqlite3 +import sys + + +def snapshot(db_path): + con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + con.row_factory = sqlite3.Row + one = lambda s: con.execute(s).fetchone()[0] + + snap = { + "db_path": db_path, + "journal_mode": one("PRAGMA journal_mode"), + "page_size": one("PRAGMA page_size"), + "page_count": one("PRAGMA page_count"), + "sqlite_version": sqlite3.sqlite_version, + } + + # Schema version + the full applied-migration ledger. A promotion that + # claims "0028 applied" must show it here, with success=1. + snap["migration_max"] = one("SELECT MAX(version) FROM _sqlx_migrations") + snap["migration_count"] = one("SELECT COUNT(*) FROM _sqlx_migrations") + snap["migrations_failed"] = one( + "SELECT COUNT(*) FROM _sqlx_migrations WHERE success <> 1") + + # Every table, counted. Deliberately not a hand-picked list. + tables = [r[0] for r in con.execute( + "SELECT name FROM sqlite_master WHERE type='table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name")] + snap["table_counts"] = {t: one(f'SELECT COUNT(*) FROM "{t}"') for t in tables} + + # The economically load-bearing figures, called out so a diff is readable. + snap["coins"] = dict(con.execute("SELECT id, coins FROM clubs ORDER BY id").fetchall()) + snap["owned_total"] = one("SELECT COUNT(*) FROM owned_cards") + snap["owned_by_kind"] = dict( + con.execute("SELECT content_kind, COUNT(*) FROM owned_cards " + "GROUP BY 1 ORDER BY 1").fetchall()) + snap["squad_players"] = one("SELECT COUNT(*) FROM squad_players") + snap["market_listings"] = one("SELECT COUNT(*) FROM market_listings") + snap["market_listings_sold"] = one("SELECT COUNT(*) FROM market_listings WHERE sold = 1") + snap["market_listings_npc"] = one("SELECT COUNT(*) FROM market_listings WHERE is_npc = 1") + snap["game_entity_ext"] = one("SELECT COUNT(*) FROM game_entity_ext") + snap["consumable_applications"] = one("SELECT COUNT(*) FROM consumable_applications") + snap["packs_total"] = one("SELECT COUNT(*) FROM packs") + snap["packs_unopened"] = one("SELECT COUNT(*) FROM packs WHERE opened = 0") + snap["match_completions"] = one("SELECT COUNT(*) FROM match_completions") + + # Present only after 0028. Absent => pre-0028 DB, which is itself the signal. + cols = [r[1] for r in con.execute("PRAGMA table_info(owned_cards)")] + snap["owned_cards_columns"] = cols + snap["has_0028"] = "contract_matches" in cols + if snap["has_0028"]: + snap["contract_matches_set"] = one( + "SELECT COUNT(*) FROM owned_cards WHERE contract_matches IS NOT NULL") + snap["contract_matches_sum"] = one( + "SELECT COALESCE(SUM(contract_matches), 0) FROM owned_cards") + + # A content fingerprint over ownership: catches a row silently rewritten + # even when every count stays identical. + h = hashlib.sha256() + for r in con.execute( + "SELECT id, club_id, card_id, is_loan, loan_matches_remaining, " + "chemistry_style, position_override, training_bonus, content_kind, " + "quantity FROM owned_cards ORDER BY id"): + h.update(("|".join("" if v is None else str(v) for v in r)).encode()) + snap["owned_cards_fingerprint_pre0028_columns"] = h.hexdigest() + + snap["integrity_check"] = one("PRAGMA integrity_check") + snap["foreign_key_check"] = len(con.execute("PRAGMA foreign_key_check").fetchall()) + con.close() + return snap + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print(__doc__) + raise SystemExit(2) + snap = snapshot(sys.argv[1]) + text = json.dumps(snap, indent=2, sort_keys=True) + print(text) + if len(sys.argv) > 2: + open(sys.argv[2], "w").write(text + "\n")