OpenFUT/scripts/sold-staging-down.py

#!/usr/bin/env python3
"""Tear down the isolated FIFA-17 SOLD staging stack brought up by
`scripts/sold-staging-up.py` -- and NOTHING else.

Kill safety is the whole point of this file. `openfut-utas-host` and `openfut-core`
each name TWO live processes on this machine: the staging ones and the PRODUCTION
ones. So there is no pattern matching here at all:

  * every pid comes from the manifest the up script wrote;
  * before any signal, /proc/<pid>/cmdline is read and MUST contain the staging
    directory -- production's cmdline never can, because staging runs binaries
    copied into that directory;
  * the known production pids are refused explicitly, as a second gate;
  * only the process GROUP the up script created (pgid == pid, via
    start_new_session) is signalled, so a responder thread/child cannot be orphaned;
  * afterwards every staging port is proven free and production is proven alive.

    python3 scripts/sold-staging-down.py
    python3 scripts/sold-staging-down.py --purge   # also delete the staging dir
"""

from __future__ import annotations

import argparse
import json
import os
import shutil
import signal
import sys
import time

DEFAULT_STAGING_DIR = "/home/alex/openfut-sold-staging"

FORBIDDEN_PATHS = ("/home/alex/openfut-promotion/state",)
# Production processes that MUST be alive before and after this script runs. These
# two are the ones the batch contract names, and they live in the host pid view.
PROD_PIDS = {3631953: "prod utas-host", 3374264: "prod Core"}
# Reported but not gated: container pids change when the operator restarts the
# container, and a stale entry here would turn a successful teardown into a FATAL.
PROD_PIDS_INFO = {2090886: "prod blaze", 2091170: "prod python oracle",
                  2090888: "prod pow"}
PROD_PORTS = (8099, 8199, 18080, 8443, 42127, 42130, 42131, 4216, 8080, 8081, 8094)


class Fatal(RuntimeError):
    pass


def banner(title: str) -> None:
    print()
    print("=" * 78)
    print(f"== {title}")
    print("=" * 78)


def ok(msg: str) -> None:
    print(f"  [ OK ] {msg}")


def step(msg: str) -> None:
    print(f"  {msg}")


def safe_path(path: str) -> str:
    real = os.path.realpath(path)
    for bad in FORBIDDEN_PATHS:
        if real == bad or real.startswith(bad + os.sep):
            raise Fatal(f"REFUSING to touch production state: {path} -> {real}")
    return path


def pid_alive(pid: int) -> bool:
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    return True


def cmdline_of(pid: int) -> str:
    try:
        with open(f"/proc/{pid}/cmdline", "rb") as fh:
            return fh.read().replace(b"\0", b" ").decode(errors="replace").strip()
    except OSError:
        return ""


def listening_ports() -> set[int]:
    """Ports in state LISTEN, from the kernel socket table. A trial bind() would
    report EADDRINUSE for a stopped server's TIME_WAIT sockets and wrongly claim the
    teardown failed."""
    ports: set[int] = set()
    for path in ("/proc/net/tcp", "/proc/net/tcp6"):
        try:
            with open(path) as fh:
                next(fh, None)  # header
                for line in fh:
                    fields = line.split()
                    if len(fields) < 4 or fields[3] != "0A":  # TCP_LISTEN
                        continue
                    ports.add(int(fields[1].rsplit(":", 1)[1], 16))
        except OSError:
            continue
    return ports


def port_free(port: int) -> bool:
    return port not in listening_ports()


def stop_one(rec: dict, staging_dir: str) -> str:
    """Stop exactly one recorded process. Returns a human-readable outcome."""
    name, pid = rec["name"], int(rec["pid"])

    known_prod = {**PROD_PIDS, **PROD_PIDS_INFO}
    if pid in known_prod:
        raise Fatal(
            f"manifest entry {name} names PRODUCTION pid {pid} ({known_prod[pid]}). "
            "REFUSING to signal anything from this manifest."
        )
    if not pid_alive(pid):
        return f"{name} pid {pid}: already gone"

    live = cmdline_of(pid)
    if staging_dir not in live:
        raise Fatal(
            f"{name} pid {pid} is alive but its cmdline does NOT contain "
            f"{staging_dir!r} -- pid reuse, or the wrong manifest. REFUSING to "
            f"signal it.\n  cmdline: {live!r}"
        )

    try:
        pgid = os.getpgid(pid)
    except OSError:
        pgid = pid
    recorded_pgid = int(rec.get("pgid", pid))
    if pgid != recorded_pgid:
        raise Fatal(
            f"{name} pid {pid} is in process group {pgid} but the manifest recorded "
            f"{recorded_pgid} -- REFUSING to signal a group we did not create."
        )
    if pgid != pid:
        raise Fatal(
            f"{name} pid {pid} is not its own group leader (pgid {pgid}) -- the up "
            "script always starts a new session, so this is not our process."
        )

    os.killpg(pgid, signal.SIGTERM)
    deadline = time.monotonic() + 15.0
    while time.monotonic() < deadline and pid_alive(pid):
        time.sleep(0.1)
    if pid_alive(pid):
        os.killpg(pgid, signal.SIGKILL)
        deadline = time.monotonic() + 10.0
        while time.monotonic() < deadline and pid_alive(pid):
            time.sleep(0.1)
    if pid_alive(pid):
        raise Fatal(f"{name} pid {pid} survived SIGKILL")
    return f"{name} pid {pid} (pgid {pgid}): stopped and verified gone"


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--dir", default=os.environ.get("OPENFUT_SOLD_STAGING_DIR",
                                                   DEFAULT_STAGING_DIR),
                    help=f"staging directory (default: {DEFAULT_STAGING_DIR})")
    ap.add_argument("--purge", action="store_true",
                    help="delete the staging directory after stopping (default: keep "
                         "the databases and logs as evidence)")
    args = ap.parse_args()

    staging_dir = safe_path(os.path.abspath(args.dir))
    manifest_path = safe_path(os.path.join(staging_dir, "manifest.json"))

    try:
        banner("STOPPING THE STAGING STACK (recorded pids only)")
        step(f"staging dir : {staging_dir}")
        if not os.path.exists(manifest_path):
            print(f"  no manifest at {manifest_path} -- nothing was recorded, so "
                  "nothing will be signalled.")
            print("  If a staging process is somehow still running, find it with "
                  "its cmdline (it contains the staging dir) and stop it by pid.")
            return 0
        with open(manifest_path) as fh:
            manifest = json.load(fh)
        if manifest.get("staging_dir") != staging_dir:
            raise Fatal(
                f"manifest staging_dir {manifest.get('staging_dir')!r} != "
                f"{staging_dir!r} -- REFUSING to act on a foreign manifest."
            )
        step(f"variant     : {manifest.get('variant')}")

        for rec in manifest.get("processes", []):
            ok(stop_one(rec, staging_dir))

        banner("PROVE STAGING IS GONE")
        ports = manifest.get("ports", {})
        for name, port in sorted(ports.items(), key=lambda kv: kv[1]):
            if port in PROD_PORTS:
                raise Fatal(f"manifest port {name}={port} is a PRODUCTION port")
            if not port_free(port):
                raise Fatal(f"staging port {name}={port} is STILL listening")
            ok(f"staging port {name} {port} free")

        leftovers = []
        for rec in manifest.get("processes", []):
            pid = int(rec["pid"])
            if pid_alive(pid) and staging_dir in cmdline_of(pid):
                leftovers.append(f"{rec['name']} pid {pid}")
        if leftovers:
            raise Fatal("staging processes still alive: " + ", ".join(leftovers))
        ok("no recorded staging process is alive")

        banner("PROVE PRODUCTION IS STILL UP")
        dead = [f"{what} pid {pid}" for pid, what in PROD_PIDS.items()
                if not pid_alive(pid)]
        for pid, what in PROD_PIDS.items():
            if pid_alive(pid):
                ok(f"{what} pid {pid} alive")
        if dead:
            raise Fatal("production process(es) NOT alive: " + ", ".join(dead))
        for pid, what in PROD_PIDS_INFO.items():
            state = "alive" if pid_alive(pid) else "not found (informational only)"
            step(f"{what} pid {pid} {state}")

        if args.purge:
            shutil.rmtree(safe_path(staging_dir), ignore_errors=True)
            ok(f"purged {staging_dir}")
        else:
            os.replace(manifest_path, safe_path(manifest_path + ".stopped"))
            ok(f"kept {staging_dir} (manifest renamed to manifest.json.stopped so a "
               "fresh `up` is allowed)")

        banner("REMINDER: REVERT THE CLIENT")
        print('  On 10.10.0.105, restore "/mnt/games/FIFA 17/openfut.cfg" to:')
        print()
        print("      host=10.10.0.120")
        print("      https_port=8443")
        print("      blaze_redirector_port=42127")
        print("      blaze_main_port=42130")
        print()
        print("  then RELAUNCH the FIFA 17 client. See docs/SOLD_STAGING_RUNBOOK.md.")
        return 0
    except Fatal as exc:
        print(f"\nFATAL: {exc}", file=sys.stderr)
        return 1


if __name__ == "__main__":
    sys.exit(main())