OpenFUT/scripts/fifa17-promotion-backup.py

#!/usr/bin/env python3
"""Transactionally-consistent backup of a live OpenFUT Core SQLite DB.

WHY NOT `cp`. Production runs WAL mode with an uncheckpointed WAL that routinely
holds hundreds of KB of committed pages. Copying only `prod-core.db` captures the
main file WITHOUT those pages and silently loses committed transactions; copying
the three files non-atomically can capture a torn set. This uses SQLite's online
backup API, which walks a read transaction and emits ONE standalone, already-
merged database file — no sidecar needed, no writer paused, safe against a live
production process.

RESTORE HAZARD, read this before restoring. The destination of a restore MUST
have its `-wal` and `-shm` removed first. SQLite treats an existing `-wal` as
newer-than-the-database journal content and will replay it over the file you
just put back, resurrecting exactly the state you were trying to abandon. The
emitted `RESTORE.sh` does this in the right order.

Usage:
    fifa17-promotion-backup.py <source-db> <backup-dir> [--label NAME]
"""
import argparse
import datetime
import hashlib
import json
import os
import sqlite3
import stat


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def describe(db_path, read_only=True):
    uri = f"file:{db_path}?mode=ro" if read_only else db_path
    con = sqlite3.connect(uri, uri=True)
    one = lambda s: con.execute(s).fetchone()[0]
    info = {
        "journal_mode": one("PRAGMA journal_mode"),
        "page_size": one("PRAGMA page_size"),
        "page_count": one("PRAGMA page_count"),
        "migration_max": one("SELECT MAX(version) FROM _sqlx_migrations"),
        "migration_count": one("SELECT COUNT(*) FROM _sqlx_migrations"),
        "integrity_check": one("PRAGMA integrity_check"),
        "foreign_key_check": len(con.execute("PRAGMA foreign_key_check").fetchall()),
        "owned_cards": one("SELECT COUNT(*) FROM owned_cards"),
        "coins": one("SELECT COALESCE(SUM(coins), 0) FROM clubs"),
    }
    con.close()
    return info


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("source")
    ap.add_argument("backup_dir")
    ap.add_argument("--label", default="core")
    args = ap.parse_args()

    stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    os.makedirs(args.backup_dir, exist_ok=True)
    dest = os.path.join(args.backup_dir, f"{args.label}-{stamp}.db")

    src_info = describe(args.source)
    sidecars = {
        os.path.basename(args.source) + suf:
            (os.path.getsize(args.source + suf) if os.path.exists(args.source + suf) else None)
        for suf in ("", "-wal", "-shm")
    }
    print(f"source      : {args.source}")
    print(f"  journal   : {src_info['journal_mode']}   migration_max={src_info['migration_max']}")
    print(f"  sidecars  : {sidecars}")
    print(f"  integrity : {src_info['integrity_check']}  fk_violations={src_info['foreign_key_check']}")

    # Online backup API. Source opened READ-ONLY: production is never written to,
    # and in WAL mode this does not block the live writer.
    src = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True)
    dst = sqlite3.connect(dest)
    with dst:
        src.backup(dst)
    dst.close()
    src.close()
    os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP)  # read-only: a backup is not scratch space

    dst_info = describe(dest)
    digest = sha256(dest)

    # The backup is only a backup if it independently verifies. A mismatch here
    # means DO NOT PROCEED — it does not mean "retry and hope".
    checks = {
        "integrity_ok": dst_info["integrity_check"] == "ok",
        "no_fk_violations": dst_info["foreign_key_check"] == 0,
        "migration_matches": dst_info["migration_max"] == src_info["migration_max"],
        "owned_matches": dst_info["owned_cards"] == src_info["owned_cards"],
        "coins_match": dst_info["coins"] == src_info["coins"],
    }

    meta = {
        "taken_at": datetime.datetime.now().isoformat(timespec="seconds"),
        "source": os.path.abspath(args.source),
        "source_sidecar_sizes": sidecars,
        "source_info": src_info,
        "backup_path": os.path.abspath(dest),
        "backup_sha256": digest,
        "backup_size": os.path.getsize(dest),
        "backup_info": dst_info,
        "verification": checks,
        "method": "sqlite3 online backup API (Connection.backup), source opened mode=ro",
    }
    meta_path = dest + ".json"
    open(meta_path, "w").write(json.dumps(meta, indent=2, sort_keys=True) + "\n")

    restore = os.path.join(args.backup_dir, f"RESTORE-{args.label}-{stamp}.sh")
    open(restore, "w").write(f"""#!/bin/sh
# Canonical restore for {os.path.abspath(args.source)}
# Generated {meta['taken_at']} from backup {os.path.basename(dest)}
#
# STOP EVERY WRITER FIRST. Restoring under a live Core corrupts both.
set -eu

TARGET='{os.path.abspath(args.source)}'
BACKUP='{os.path.abspath(dest)}'

test "$(sha256sum "$BACKUP" | cut -d' ' -f1)" = '{digest}' \\
    || {{ echo 'FATAL: backup checksum mismatch, refusing to restore'; exit 1; }}

# The stale -wal/-shm MUST go, or SQLite replays them over the restored file.
rm -f "$TARGET-wal" "$TARGET-shm"
cp "$BACKUP" "$TARGET"
chmod u+w "$TARGET"

sqlite3 "$TARGET" 'PRAGMA integrity_check;' 'PRAGMA foreign_key_check;' \\
                  'SELECT MAX(version) FROM _sqlx_migrations;'
echo 'restore complete -- now start the PREVIOUS Core and host binaries'
""")
    os.chmod(restore, 0o755)

    print(f"\nbackup      : {dest}")
    print(f"  sha256    : {digest}")
    print(f"  size      : {meta['backup_size']:,}")
    print(f"  integrity : {dst_info['integrity_check']}  fk_violations={dst_info['foreign_key_check']}")
    print(f"  migration : {dst_info['migration_max']}  owned={dst_info['owned_cards']} coins={dst_info['coins']:,}")
    print(f"metadata    : {meta_path}")
    print(f"restore     : {restore}")
    print("\nverification:")
    for k, v in checks.items():
        print(f"   [{'OK ' if v else 'FAIL'}] {k}")
    ok = all(checks.values())
    print("\nRESULT:", "BACKUP VERIFIED" if ok else "BACKUP FAILED VERIFICATION -- DO NOT PROCEED")
    raise SystemExit(0 if ok else 1)


if __name__ == "__main__":
    main()