OpenFUT/fifa17-recon/tools/sweep_collect.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Accumulate FUT_ID_SWEEP results into a real player database.

READ-ONLY against the game (it drives card_identity_probe, which only reads
/proc/PID/mem). It writes exactly one file: data/players.json in this repo.

HOW THE ORACLE WORKS, AND ITS ONE FALSE ANSWER
----------------------------------------------
utas_server's FUT_ID_SWEEP serves a window of candidate playerids as a synthetic
club. The client merges its OWN local players table into every item it parses, so
after one club fetch the resolved identity is sitting in the CardsDb map.

A candidate is REAL when the client gives it a name. The false answer to guard
against is the DB's default row: ids with no entry come back named "Jamal
Blackman", byte-identical every time (first "Jamal", last "Blackman"). Six of our
own pool ids hit this, including one that was in VERIFIED_ASSET_IDS -- which is
why the old "verified" list cannot be trusted and this tool exists.

So DEFAULT_NAME below is a rejection filter, not a curiosity. If a genuine Jamal
Blackman is ever needed, take him from his real id, not from this sweep.

Usage:
    sweep_collect.py                 probe now, merge into data/players.json
    sweep_collect.py --show          summarise the accumulated database
"""
import argparse
import json
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import card_identity_probe as P                                    # noqa: E402
import watch_club_model as W                                       # noqa: E402

DB = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                  "..", "data", "players.json")
DEFAULT_NAME = ("Jamal", "Blackman")     # the DB's empty-row placeholder
SENTINEL_RATING = 7                      # what utas_server serves in a sweep


def load():
    try:
        with open(DB) as f:
            return {int(k): v for k, v in json.load(f).items()}
    except (IOError, ValueError):
        return {}


def save(db):
    os.makedirs(os.path.dirname(DB), exist_ok=True)
    with open(DB, "w") as f:
        json.dump({str(k): v for k, v in sorted(db.items())}, f,
                  indent=1, ensure_ascii=False)


def probe():
    pid = W.find_pid()
    if pid is None:
        print("FIFA17.exe is not running.")
        return None
    base = W.dll_base(pid)
    mem = W.Mem(pid)
    obj = mem.q(base + (W.G_CARDSDB - W.IMG_BASE)) if base else None
    if not obj:
        print("CardsDb is not loaded (no FUT session).")
        return None
    cards = [c for c in (P.read_card(mem, n) for n in P.nodes(mem, obj)) if c]
    print("read %d card record(s), failed reads=%d" % (len(cards), mem.fails))
    return cards


def merge(db, cards):
    added = skipped = placeholder = 0
    for c in cards:
        first, last = c["first"], c["last"]
        if (first, last) == DEFAULT_NAME:
            placeholder += 1
            continue
        if not (first or last or c["known"]):
            skipped += 1
            continue
        pid = c["playerid"]
        row = {
            "first": first, "last": last, "known": c["known"],
            "teamid": c["teamid"], "nation": c["nation"], "league": c["league"],
            "position": c["position"],
        }
        # A sweep card carries our sentinel rating, so the DB never learns a rating
        # from it. Ratings from OWNED cards are ours too -- the merge never
        # overwrites a nonzero rating. Rating therefore stays out of this file
        # rather than being recorded as if the game had supplied it.
        if pid not in db:
            added += 1
        db[pid] = row
    return added, skipped, placeholder


def show(db):
    print("%d player(s) in %s" % (len(db), os.path.normpath(DB)))
    if not db:
        return
    ids = sorted(db)
    print("id range %d..%d" % (ids[0], ids[-1]))
    teams = len({r["teamid"] for r in db.values() if r["teamid"]})
    nats = len({r["nation"] for r in db.values() if r["nation"]})
    lgs = len({r["league"] for r in db.values() if r["league"]})
    print("%d club(s), %d nation(s), %d league(s)" % (teams, nats, lgs))
    for pid in ids[:10]:
        r = db[pid]
        print("  %-7s %-28s team=%-6s nat=%-4s league=%s"
              % (pid, ("%s %s" % (r["first"], r["last"])).strip()[:28],
                 r["teamid"], r["nation"], r["league"]))


def watch(db, interval, seconds):
    """Merge continuously while the game pages through an auto sweep.

    THE MAP DOES NOT ACCUMULATE. Every club fetch wipes it and repopulates from
    that response alone, so a single probe at the end of an auto sweep sees only
    the LAST chunk -- which is how 34,000 candidates were nearly thrown away. A
    pass costs 0.03s and chunks are ~12s apart, so polling catches all of them.
    """
    import time
    t0 = time.time()
    last = -1
    while time.time() - t0 < seconds:
        cards = probe_quiet()
        if cards:
            added, _, _ = merge(db, cards)
            if added:
                save(db)
        if len(db) != last:
            last = len(db)
            print("[%4ds] %d player(s)" % (time.time() - t0, len(db)), flush=True)
        time.sleep(interval)
    save(db)
    return 0


def probe_quiet():
    pid = W.find_pid()
    if pid is None:
        return None
    base = W.dll_base(pid)
    if not base:
        return None
    mem = W.Mem(pid)
    obj = mem.q(base + (W.G_CARDSDB - W.IMG_BASE))
    if not obj:
        return None
    return [c for c in (P.read_card(mem, n) for n in P.nodes(mem, obj)) if c]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--show", action="store_true")
    ap.add_argument("--watch", type=float, metavar="SECS",
                    help="poll continuously for SECS while an auto sweep runs")
    ap.add_argument("--interval", type=float, default=1.0)
    a = ap.parse_args()
    db = load()
    if a.show:
        show(db)
        return 0
    if a.watch:
        return watch(db, a.interval, a.watch)
    cards = probe()
    if cards is None:
        return 1
    added, skipped, placeholder = merge(db, cards)
    save(db)
    print("added %d new, %d placeholder row(s) rejected, %d unnamed skipped"
          % (added, placeholder, skipped))
    show(db)
    return 0


if __name__ == "__main__":
    sys.exit(main())