tooling: verified backup, state snapshot, retargetable apply validator

Promotion prep for the contract-apply cutover, which unlike the quick-sell
promotion moves BOTH binaries and applies a schema migration.

fifa17-promotion-backup.py uses SQLite's online backup API, not cp. Production
runs WAL with a routinely uncheckpointed WAL (515 KB at capture time); copying
the main file alone is not atomic against a live writer and carries no
guarantee the WAL holds no newer committed state. Emits a checksummed backup, a
metadata record and a RESTORE-*.sh that removes the stale -wal/-shm BEFORE
restoring -- omit that and SQLite replays the old journal over the file you just
put back, resurrecting the state you were abandoning.

fifa17-promotion-snapshot.py is read-only (mode=ro) and counts EVERY table
rather than a hand-picked list, so a delta cannot hide in a table nobody thought
to name. It also fingerprints the ownership rows, catching a row silently
rewritten when counts alone would match.

fifa17-contract-apply-validate.py gains --host/--db so one tool serves staging,
the migration rehearsal and the production acceptance run. Defaults stay
staging: there is deliberately no production default, so a bare invocation
cannot touch production.

scripts/fifa17-contract-apply-validate.py

@@ -11,7 +11,9 @@ Exercises the real route end to end and asserts the full observable contract:
   * replaying the exhausted resource fails closed rather than granting again;
   * a manager contract and a non-contract consumable both fail closed.
 
-Read-only against production by construction: every URL is the staging port.
+Defaults target STAGING. `--host`/`--db` retarget it at a rehearsal or, under
+explicit authorization, at the production acceptance run. There is deliberately
+no production default: a bare invocation cannot touch production.
 """
 import argparse
 import json
@@ -20,6 +22,9 @@ import sys
 import urllib.error
 import urllib.request
 
+# Defaults are STAGING. Override for a rehearsal or the production acceptance
+# run; there is deliberately no production default, so a bare invocation can
+# never touch production by accident.
 HOST = "http://127.0.0.1:8299"
 CORE_DB = "/home/alex/openfut-sold-staging/staging-core.db"
 HDRS = {"X-OpenFUT-Game": "fifa17"}
@@ -123,7 +128,13 @@ def main():
                     help="contract resource id to apply (default: first owned player contract)")
     ap.add_argument("--target", type=int, default=None,
                     help="target player wire id (default: lowest-rated owned player)")
+    ap.add_argument("--host", default=HOST, help=f"utas-host base URL (default {HOST})")
+    ap.add_argument("--db", default=CORE_DB, help=f"Core SQLite path (default {CORE_DB})")
     args = ap.parse_args()
+    # Rebound before any request so every helper below reads the chosen target.
+    globals()["HOST"] = args.host
+    globals()["CORE_DB"] = args.db
+    print(f"target host : {HOST}\ntarget db   : {CORE_DB}\n")
 
     print("== BEFORE ==")
     before = core_snapshot()

scripts/fifa17-promotion-backup.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Transactionally-consistent backup of a live OpenFUT Core SQLite DB.
+
+WHY NOT `cp`. Production runs WAL mode with an uncheckpointed WAL that routinely
+holds hundreds of KB of committed pages. Copying only `prod-core.db` captures the
+main file WITHOUT those pages and silently loses committed transactions; copying
+the three files non-atomically can capture a torn set. This uses SQLite's online
+backup API, which walks a read transaction and emits ONE standalone, already-
+merged database file — no sidecar needed, no writer paused, safe against a live
+production process.
+
+RESTORE HAZARD, read this before restoring. The destination of a restore MUST
+have its `-wal` and `-shm` removed first. SQLite treats an existing `-wal` as
+newer-than-the-database journal content and will replay it over the file you
+just put back, resurrecting exactly the state you were trying to abandon. The
+emitted `RESTORE.sh` does this in the right order.
+
+Usage:
+    fifa17-promotion-backup.py <source-db> <backup-dir> [--label NAME]
+"""
+import argparse
+import datetime
+import hashlib
+import json
+import os
+import sqlite3
+import stat
+
+
+def sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def describe(db_path, read_only=True):
+    uri = f"file:{db_path}?mode=ro" if read_only else db_path
+    con = sqlite3.connect(uri, uri=True)
+    one = lambda s: con.execute(s).fetchone()[0]
+    info = {
+        "journal_mode": one("PRAGMA journal_mode"),
+        "page_size": one("PRAGMA page_size"),
+        "page_count": one("PRAGMA page_count"),
+        "migration_max": one("SELECT MAX(version) FROM _sqlx_migrations"),
+        "migration_count": one("SELECT COUNT(*) FROM _sqlx_migrations"),
+        "integrity_check": one("PRAGMA integrity_check"),
+        "foreign_key_check": len(con.execute("PRAGMA foreign_key_check").fetchall()),
+        "owned_cards": one("SELECT COUNT(*) FROM owned_cards"),
+        "coins": one("SELECT COALESCE(SUM(coins), 0) FROM clubs"),
+    }
+    con.close()
+    return info
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("source")
+    ap.add_argument("backup_dir")
+    ap.add_argument("--label", default="core")
+    args = ap.parse_args()
+
+    stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    os.makedirs(args.backup_dir, exist_ok=True)
+    dest = os.path.join(args.backup_dir, f"{args.label}-{stamp}.db")
+
+    src_info = describe(args.source)
+    sidecars = {
+        os.path.basename(args.source) + suf:
+            (os.path.getsize(args.source + suf) if os.path.exists(args.source + suf) else None)
+        for suf in ("", "-wal", "-shm")
+    }
+    print(f"source      : {args.source}")
+    print(f"  journal   : {src_info['journal_mode']}   migration_max={src_info['migration_max']}")
+    print(f"  sidecars  : {sidecars}")
+    print(f"  integrity : {src_info['integrity_check']}  fk_violations={src_info['foreign_key_check']}")
+
+    # Online backup API. Source opened READ-ONLY: production is never written to,
+    # and in WAL mode this does not block the live writer.
+    src = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True)
+    dst = sqlite3.connect(dest)
+    with dst:
+        src.backup(dst)
+    dst.close()
+    src.close()
+    os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP)  # read-only: a backup is not scratch space
+
+    dst_info = describe(dest)
+    digest = sha256(dest)
+
+    # The backup is only a backup if it independently verifies. A mismatch here
+    # means DO NOT PROCEED — it does not mean "retry and hope".
+    checks = {
+        "integrity_ok": dst_info["integrity_check"] == "ok",
+        "no_fk_violations": dst_info["foreign_key_check"] == 0,
+        "migration_matches": dst_info["migration_max"] == src_info["migration_max"],
+        "owned_matches": dst_info["owned_cards"] == src_info["owned_cards"],
+        "coins_match": dst_info["coins"] == src_info["coins"],
+    }
+
+    meta = {
+        "taken_at": datetime.datetime.now().isoformat(timespec="seconds"),
+        "source": os.path.abspath(args.source),
+        "source_sidecar_sizes": sidecars,
+        "source_info": src_info,
+        "backup_path": os.path.abspath(dest),
+        "backup_sha256": digest,
+        "backup_size": os.path.getsize(dest),
+        "backup_info": dst_info,
+        "verification": checks,
+        "method": "sqlite3 online backup API (Connection.backup), source opened mode=ro",
+    }
+    meta_path = dest + ".json"
+    open(meta_path, "w").write(json.dumps(meta, indent=2, sort_keys=True) + "\n")
+
+    restore = os.path.join(args.backup_dir, f"RESTORE-{args.label}-{stamp}.sh")
+    open(restore, "w").write(f"""#!/bin/sh
+# Canonical restore for {os.path.abspath(args.source)}
+# Generated {meta['taken_at']} from backup {os.path.basename(dest)}
+#
+# STOP EVERY WRITER FIRST. Restoring under a live Core corrupts both.
+set -eu
+
+TARGET='{os.path.abspath(args.source)}'
+BACKUP='{os.path.abspath(dest)}'
+
+test "$(sha256sum "$BACKUP" | cut -d' ' -f1)" = '{digest}' \\
+    || {{ echo 'FATAL: backup checksum mismatch, refusing to restore'; exit 1; }}
+
+# The stale -wal/-shm MUST go, or SQLite replays them over the restored file.
+rm -f "$TARGET-wal" "$TARGET-shm"
+cp "$BACKUP" "$TARGET"
+chmod u+w "$TARGET"
+
+sqlite3 "$TARGET" 'PRAGMA integrity_check;' 'PRAGMA foreign_key_check;' \\
+                  'SELECT MAX(version) FROM _sqlx_migrations;'
+echo 'restore complete -- now start the PREVIOUS Core and host binaries'
+""")
+    os.chmod(restore, 0o755)
+
+    print(f"\nbackup      : {dest}")
+    print(f"  sha256    : {digest}")
+    print(f"  size      : {meta['backup_size']:,}")
+    print(f"  integrity : {dst_info['integrity_check']}  fk_violations={dst_info['foreign_key_check']}")
+    print(f"  migration : {dst_info['migration_max']}  owned={dst_info['owned_cards']} coins={dst_info['coins']:,}")
+    print(f"metadata    : {meta_path}")
+    print(f"restore     : {restore}")
+    print("\nverification:")
+    for k, v in checks.items():
+        print(f"   [{'OK ' if v else 'FAIL'}] {k}")
+    ok = all(checks.values())
+    print("\nRESULT:", "BACKUP VERIFIED" if ok else "BACKUP FAILED VERIFICATION -- DO NOT PROCEED")
+    raise SystemExit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/fifa17-promotion-snapshot.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""READ-ONLY state capture of an OpenFUT Core SQLite DB, for promotion gating.
+
+Opened `mode=ro` and never written to, so it is safe against live production.
+Emits a deterministic JSON document: run it before a promotion and again after,
+then `diff` the two. Every table is counted, so a delta cannot hide in a table
+nobody thought to list.
+
+Usage:
+    fifa17-promotion-snapshot.py <db-path> [out.json]
+"""
+import hashlib
+import json
+import sqlite3
+import sys
+
+
+def snapshot(db_path):
+    con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    con.row_factory = sqlite3.Row
+    one = lambda s: con.execute(s).fetchone()[0]
+
+    snap = {
+        "db_path": db_path,
+        "journal_mode": one("PRAGMA journal_mode"),
+        "page_size": one("PRAGMA page_size"),
+        "page_count": one("PRAGMA page_count"),
+        "sqlite_version": sqlite3.sqlite_version,
+    }
+
+    # Schema version + the full applied-migration ledger. A promotion that
+    # claims "0028 applied" must show it here, with success=1.
+    snap["migration_max"] = one("SELECT MAX(version) FROM _sqlx_migrations")
+    snap["migration_count"] = one("SELECT COUNT(*) FROM _sqlx_migrations")
+    snap["migrations_failed"] = one(
+        "SELECT COUNT(*) FROM _sqlx_migrations WHERE success <> 1")
+
+    # Every table, counted. Deliberately not a hand-picked list.
+    tables = [r[0] for r in con.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' "
+        "AND name NOT LIKE 'sqlite_%' ORDER BY name")]
+    snap["table_counts"] = {t: one(f'SELECT COUNT(*) FROM "{t}"') for t in tables}
+
+    # The economically load-bearing figures, called out so a diff is readable.
+    snap["coins"] = dict(con.execute("SELECT id, coins FROM clubs ORDER BY id").fetchall())
+    snap["owned_total"] = one("SELECT COUNT(*) FROM owned_cards")
+    snap["owned_by_kind"] = dict(
+        con.execute("SELECT content_kind, COUNT(*) FROM owned_cards "
+                    "GROUP BY 1 ORDER BY 1").fetchall())
+    snap["squad_players"] = one("SELECT COUNT(*) FROM squad_players")
+    snap["market_listings"] = one("SELECT COUNT(*) FROM market_listings")
+    snap["market_listings_sold"] = one("SELECT COUNT(*) FROM market_listings WHERE sold = 1")
+    snap["market_listings_npc"] = one("SELECT COUNT(*) FROM market_listings WHERE is_npc = 1")
+    snap["game_entity_ext"] = one("SELECT COUNT(*) FROM game_entity_ext")
+    snap["consumable_applications"] = one("SELECT COUNT(*) FROM consumable_applications")
+    snap["packs_total"] = one("SELECT COUNT(*) FROM packs")
+    snap["packs_unopened"] = one("SELECT COUNT(*) FROM packs WHERE opened = 0")
+    snap["match_completions"] = one("SELECT COUNT(*) FROM match_completions")
+
+    # Present only after 0028. Absent => pre-0028 DB, which is itself the signal.
+    cols = [r[1] for r in con.execute("PRAGMA table_info(owned_cards)")]
+    snap["owned_cards_columns"] = cols
+    snap["has_0028"] = "contract_matches" in cols
+    if snap["has_0028"]:
+        snap["contract_matches_set"] = one(
+            "SELECT COUNT(*) FROM owned_cards WHERE contract_matches IS NOT NULL")
+        snap["contract_matches_sum"] = one(
+            "SELECT COALESCE(SUM(contract_matches), 0) FROM owned_cards")
+
+    # A content fingerprint over ownership: catches a row silently rewritten
+    # even when every count stays identical.
+    h = hashlib.sha256()
+    for r in con.execute(
+            "SELECT id, club_id, card_id, is_loan, loan_matches_remaining, "
+            "chemistry_style, position_override, training_bonus, content_kind, "
+            "quantity FROM owned_cards ORDER BY id"):
+        h.update(("|".join("" if v is None else str(v) for v in r)).encode())
+    snap["owned_cards_fingerprint_pre0028_columns"] = h.hexdigest()
+
+    snap["integrity_check"] = one("PRAGMA integrity_check")
+    snap["foreign_key_check"] = len(con.execute("PRAGMA foreign_key_check").fetchall())
+    con.close()
+    return snap
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(__doc__)
+        raise SystemExit(2)
+    snap = snapshot(sys.argv[1])
+    text = json.dumps(snap, indent=2, sort_keys=True)
+    print(text)
+    if len(sys.argv) > 2:
+        open(sys.argv[2], "w").write(text + "\n")