#!/usr/bin/env python3 """Transactionally-consistent backup of a live OpenFUT Core SQLite DB. WHY NOT `cp`. Production runs WAL mode with an uncheckpointed WAL that routinely holds hundreds of KB of committed pages. Copying only `prod-core.db` captures the main file WITHOUT those pages and silently loses committed transactions; copying the three files non-atomically can capture a torn set. This uses SQLite's online backup API, which walks a read transaction and emits ONE standalone, already- merged database file — no sidecar needed, no writer paused, safe against a live production process. RESTORE HAZARD, read this before restoring. The destination of a restore MUST have its `-wal` and `-shm` removed first. SQLite treats an existing `-wal` as newer-than-the-database journal content and will replay it over the file you just put back, resurrecting exactly the state you were trying to abandon. The emitted `RESTORE.sh` does this in the right order. Usage: fifa17-promotion-backup.py [--label NAME] """ import argparse import datetime import hashlib import json import os import sqlite3 import stat def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def describe(db_path, read_only=True): uri = f"file:{db_path}?mode=ro" if read_only else db_path con = sqlite3.connect(uri, uri=True) one = lambda s: con.execute(s).fetchone()[0] info = { "journal_mode": one("PRAGMA journal_mode"), "page_size": one("PRAGMA page_size"), "page_count": one("PRAGMA page_count"), "migration_max": one("SELECT MAX(version) FROM _sqlx_migrations"), "migration_count": one("SELECT COUNT(*) FROM _sqlx_migrations"), "integrity_check": one("PRAGMA integrity_check"), "foreign_key_check": len(con.execute("PRAGMA foreign_key_check").fetchall()), "owned_cards": one("SELECT COUNT(*) FROM owned_cards"), "coins": one("SELECT COALESCE(SUM(coins), 0) FROM clubs"), } con.close() return info def main(): ap = argparse.ArgumentParser() ap.add_argument("source") ap.add_argument("backup_dir") ap.add_argument("--label", default="core") args = ap.parse_args() stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") os.makedirs(args.backup_dir, exist_ok=True) dest = os.path.join(args.backup_dir, f"{args.label}-{stamp}.db") src_info = describe(args.source) sidecars = { os.path.basename(args.source) + suf: (os.path.getsize(args.source + suf) if os.path.exists(args.source + suf) else None) for suf in ("", "-wal", "-shm") } print(f"source : {args.source}") print(f" journal : {src_info['journal_mode']} migration_max={src_info['migration_max']}") print(f" sidecars : {sidecars}") print(f" integrity : {src_info['integrity_check']} fk_violations={src_info['foreign_key_check']}") # Online backup API. Source opened READ-ONLY: production is never written to, # and in WAL mode this does not block the live writer. src = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True) dst = sqlite3.connect(dest) with dst: src.backup(dst) dst.close() src.close() os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP) # read-only: a backup is not scratch space dst_info = describe(dest) digest = sha256(dest) # The backup is only a backup if it independently verifies. A mismatch here # means DO NOT PROCEED — it does not mean "retry and hope". checks = { "integrity_ok": dst_info["integrity_check"] == "ok", "no_fk_violations": dst_info["foreign_key_check"] == 0, "migration_matches": dst_info["migration_max"] == src_info["migration_max"], "owned_matches": dst_info["owned_cards"] == src_info["owned_cards"], "coins_match": dst_info["coins"] == src_info["coins"], } meta = { "taken_at": datetime.datetime.now().isoformat(timespec="seconds"), "source": os.path.abspath(args.source), "source_sidecar_sizes": sidecars, "source_info": src_info, "backup_path": os.path.abspath(dest), "backup_sha256": digest, "backup_size": os.path.getsize(dest), "backup_info": dst_info, "verification": checks, "method": "sqlite3 online backup API (Connection.backup), source opened mode=ro", } meta_path = dest + ".json" open(meta_path, "w").write(json.dumps(meta, indent=2, sort_keys=True) + "\n") restore = os.path.join(args.backup_dir, f"RESTORE-{args.label}-{stamp}.sh") open(restore, "w").write(f"""#!/bin/sh # Canonical restore for {os.path.abspath(args.source)} # Generated {meta['taken_at']} from backup {os.path.basename(dest)} # # STOP EVERY WRITER FIRST. Restoring under a live Core corrupts both. set -eu TARGET='{os.path.abspath(args.source)}' BACKUP='{os.path.abspath(dest)}' test "$(sha256sum "$BACKUP" | cut -d' ' -f1)" = '{digest}' \\ || {{ echo 'FATAL: backup checksum mismatch, refusing to restore'; exit 1; }} # The stale -wal/-shm MUST go, or SQLite replays them over the restored file. rm -f "$TARGET-wal" "$TARGET-shm" cp "$BACKUP" "$TARGET" chmod u+w "$TARGET" sqlite3 "$TARGET" 'PRAGMA integrity_check;' 'PRAGMA foreign_key_check;' \\ 'SELECT MAX(version) FROM _sqlx_migrations;' echo 'restore complete -- now start the PREVIOUS Core and host binaries' """) os.chmod(restore, 0o755) print(f"\nbackup : {dest}") print(f" sha256 : {digest}") print(f" size : {meta['backup_size']:,}") print(f" integrity : {dst_info['integrity_check']} fk_violations={dst_info['foreign_key_check']}") print(f" migration : {dst_info['migration_max']} owned={dst_info['owned_cards']} coins={dst_info['coins']:,}") print(f"metadata : {meta_path}") print(f"restore : {restore}") print("\nverification:") for k, v in checks.items(): print(f" [{'OK ' if v else 'FAIL'}] {k}") ok = all(checks.values()) print("\nRESULT:", "BACKUP VERIFIED" if ok else "BACKUP FAILED VERIFICATION -- DO NOT PROCEED") raise SystemExit(0 if ok else 1) if __name__ == "__main__": main()