OpenFUT/fifa17-recon/tools/coach_probe.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Read back what the client resolved for STAFF cards (head coach, GK coach,
physio, fitness coach) and grade every one HIT / MISS / WRONG-BRANCH / NO-MERGE.

READ-ONLY. Uses card_identity_probe.nodes() and the same /proc/PID/mem reader;
there is no write path in this file.

WHY A SEPARATE FILE FROM card_identity_probe
--------------------------------------------
card_identity_probe is a PLAYER tool and is actively wrong for staff:

  * classify() returns "NO-MERGE" for anything with cardtype != 1, so all four
    coach families come back NO-MERGE and the tool reports nothing.
  * F_NAME_KNOWN = 0xdd is the player knownAs string. For physio and fitness
    coach the merge writes RAW STAT BYTES into 0xdd..0xe3, and for managers
    talkrating/negotiation land at 0xe2/0xe3. So `known` is garbage for every
    non-player family and must never be read as a name there.
  * F_ATTRS (0x98..0xac) is only meaningful for head coach and GK coach, and
    even there the merge writes exactly ONE element.

THE MECHANISM (FUN_180141660, cardsdll.dll, 2129 bytes / 214 decompiled lines,
read end to end)
------------------------------------------------------------------------------
The merge switches on record+0x4c, which FUN_1800d8330 derives from the JSON
atom 0x6c cardsubtypeid alone:

    cardsubtypeid  ->  +0x4c  ->  table              key column
    5                  3          headcoachcards     carddbid == *(u32*)(rec+0x18)
    8                  4          fitnesscoachcards  carddbid == *(u32*)(rec+0x18)
    7                  5          physiocards        carddbid == *(u32*)(rec+0x18)
    6                  10         gkcoachcards       carddbid == *(u32*)(rec+0x18)
    4                  2          managercards       carddbid == *(u32*)(rec+0x18)
    0..3               1          players            playerid == rec+0x18 & 0xffffff

record+0x18 is atom 0x287 resourceId. THE COACH BRANCHES DO NOT MASK IT: unlike
FUN_180135890 (players), which does `& 0xffffff` twice, the four staff branches
and the manager branch pass the raw u32 straight into the `==` predicate. So a
version nibble in the high byte of resourceId breaks every staff lookup silently.

MISS FINGERPRINTS -- these are NOT uniform, contrary to earlier notes
--------------------------------------------------------------------
All four write firstname/lastname "DB Error", rating(+0xb4) 0x32 and rare(+0x58)
1, plus a TABLE-UNIQUE fallback assetid at +0x20. Only head coach and GK coach
write 0xf into the attribute array; physio writes 0xf into a BYTE at +0xdd, and
fitness coach writes no 0xf at all -- it writes fieldpos 1 / posbonus 7 /
amount 1 into +0xdd/+0xde/+0xdf.

The fallback assetid is what makes a negative interpretable: it names the branch
that ran. A head-coach id that comes back with assetId 3000259 means the FITNESS
branch ran, which is a different bug from "the id is wrong".

Two facts checked against the on-disk dumps in data/tables/ and used as oracles:
  * no row in any of the four tables has value == 50, so rating 0x32 is an
    unambiguous MISS for all four families;
  * no fitnesscoachcards row has (fieldpos, posbonus, amount) == (1, 7, 1), so
    that byte triple is an unambiguous MISS for fitness coach on its own.

Usage:
    python3 coach_probe.py                  # graded table + tally
    python3 coach_probe.py --json out.json
    python3 coach_probe.py --raw            # + hexdump of the first staff record
"""
import argparse
import json
import os
import struct
import sys

import card_identity_probe as P
import watch_club_model as W

TABLES = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                      "..", "data", "tables")

# cardsubtypeid -> (family, table file, +0x4c value, fallback assetid)
FAMILY = {
    5: ("headcoach",    "headcoachcards.json",     3, 0x1E8514),   # 2000148
    8: ("fitnesscoach", "fitnesscoachcards.json",  4, 0x2DC7C3),   # 3000259
    7: ("physio",       "physiocards.json",        5, 0x3D0992),   # 4000146
    6: ("gkcoach",      "gkcoachcards.json",      10, 0x895542),   # 9000258
    4: ("manager",      "managercards.json",       2, None),       # NO miss-fill
}

F_ATTRS_I32 = (0x98, 0x9C, 0xA0, 0xA4, 0xA8, 0xAC)   # head/GK coach boost slot
F_STAT_BYTES = 0xDD                                   # 0xdd..0xe3, 7 bytes
MISS_RATING = 0x32
MISS_NAME = "DB Error"


def load_tables():
    """{cardsubtypeid: {carddbid: row}} from the read-only on-disk dumps."""
    out = {}
    for sub, (_, fn, _, _) in FAMILY.items():
        path = os.path.join(TABLES, fn)
        try:
            with open(path) as f:
                rows = json.load(f)["rows"]
        except (IOError, OSError, ValueError):
            continue
        out[sub] = {r["carddbid"]: r for r in rows}
    return out


def read_staff(mem, node):
    buf = mem.read(node + P.REC, P.REC_LEN)
    if buf is None or len(buf) < P.REC_LEN:
        return None
    return {
        "node": node,
        "id": P.u32(buf, P.F_ID),
        "resourceId": P.u32(buf, P.F_RESOURCE),
        "assetId": P.u32(buf, P.F_ASSET),
        "cardtype": P.u32(buf, P.F_CARDTYPE),
        "subtype": P.u32(buf, P.F_SUBTYPE),
        "rating": P.u8(buf, P.F_RATING),
        "rare": P.u32(buf, 0x58),
        "tier": P.u32(buf, 0x54),
        "first": P.cstr(buf, P.F_NAME_FIRST, 0x10),
        "last": P.cstr(buf, P.F_NAME_LAST, 0x15),
        # what OUR json set and the coach branches never touch:
        "teamid": P.u32(buf, P.F_TEAM),
        "position": P.u8(buf, P.F_POSITION),
        "nation": P.u16(buf, P.F_NATION),
        "league": P.u32(buf, P.F_LEAGUE),
        # the two stat regions, read as raw numbers, never as a string:
        "attrs": [P.u32(buf, o) for o in F_ATTRS_I32],
        "stat_bytes": list(buf[F_STAT_BYTES:F_STAT_BYTES + 7]),
        "_raw": buf,
    }


def expected_tier(rating):
    """The tail of FUN_180141660, which runs for EVERY family including miss."""
    if rating is None:
        return None
    return 3 if rating >= 0x4B else (2 if rating >= 0x41 else 1)


def grade(c, tables):
    """HIT / MISS / WRONG-BRANCH / NO-MERGE / UNEXPECTED, plus a reason."""
    sub = c["subtype"]
    if sub not in FAMILY:
        return "NOT-STAFF", "cardsubtypeid %s is not a staff family" % sub
    name, _, want_ct, fallback = FAMILY[sub]
    c["family"] = name
    if c["cardtype"] != want_ct:
        return "NO-MERGE", ("cardtype %s, expected %d -- FUN_1800d8330 did not "
                            "map this subtype, so no query ran"
                            % (c["cardtype"], want_ct))

    # A miss anywhere names its own branch through the fallback assetid.
    for osub, (oname, _, _, ofb) in FAMILY.items():
        if ofb is not None and c["assetId"] == ofb and c["rating"] == MISS_RATING:
            if osub == sub:
                row = tables.get(sub, {}).get(c["resourceId"])
                if row is not None:
                    return "UNEXPECTED", ("MISS, but carddbid %d IS in %s -- the "
                                          "key or the field is wrong, not the id"
                                          % (c["resourceId"], name))
                return "MISS", "id %d absent from %s (as designed)" % (
                    c["resourceId"], name)
            return "WRONG-BRANCH", ("fallback assetid %d belongs to %s, but we "
                                    "sent cardsubtypeid %d (%s)"
                                    % (c["assetId"], oname, sub, name))

    if c["rating"] == MISS_RATING or MISS_NAME in (c["first"], c["last"]):
        return "UNEXPECTED", ("miss fingerprint without a known fallback assetid "
                              "(assetId=%s)" % c["assetId"])

    row = tables.get(sub, {}).get(c["resourceId"])
    if row is None:
        if sub == 4:
            return "SILENT", ("managercards writes NO miss-fill; a wrong id is "
                              "indistinguishable from a wrong mechanism")
        return "UNEXPECTED", ("no miss fingerprint, but id %d is absent from %s"
                              % (c["resourceId"], name))

    # HIT: every column we can see must agree with the on-disk row.
    bad = []
    if c["rating"] != row["value"]:
        bad.append("rating %s != value %s" % (c["rating"], row["value"]))
    if c["assetId"] != row["assetid"]:
        bad.append("assetId %s != assetid %s" % (c["assetId"], row["assetid"]))
    if c["rare"] != (1 if row["rare"] == 1 else 0):
        bad.append("rare %s != %s" % (c["rare"], row["rare"]))
    if c["tier"] != expected_tier(c["rating"]):
        bad.append("tier %s != %s" % (c["tier"], expected_tier(c["rating"])))
    if sub in (5, 6):                       # head coach / GK coach
        got = c["attrs"][row["attribute"]]
        if got != row["amount"]:
            bad.append("attrs[%d] %s != amount %s"
                       % (row["attribute"], got, row["amount"]))
    elif sub == 7:                          # physio, via FUN_180136270
        got = c["stat_bytes"][row["attribute"]]
        if got != row["amount"]:
            bad.append("+%#x %s != amount %s"
                       % (0xDD + row["attribute"], got, row["amount"]))
    elif sub == 8:                          # fitness coach: three columns at once
        for i, col in enumerate(("fieldpos", "posbonus", "amount")):
            if c["stat_bytes"][i] != row[col]:
                bad.append("+%#x %s != %s %s"
                           % (0xDD + i, c["stat_bytes"][i], col, row[col]))
    if bad:
        return "HIT-MISMATCH", "; ".join(bad)
    return "HIT", "every visible column agrees with %s row %d" % (
        name, c["resourceId"])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--raw", action="store_true")
    ap.add_argument("--json", metavar="PATH")
    ap.add_argument("--all", action="store_true",
                    help="also list player/unknown cards instead of skipping them")
    a = ap.parse_args()

    tables = load_tables()
    if not tables:
        print("no table dumps under %s -- run tools/db_dump.py first" % TABLES)
        return 1

    pid = W.find_pid()
    if pid is None:
        print("FIFA17.exe is not running.")
        return 1
    base = W.dll_base(pid)
    if base is None:
        print("pid %d is up but %s is not mapped yet." % (pid, W.DLL))
        return 1
    mem = W.Mem(pid)
    obj = mem.q(base + (W.G_CARDSDB - W.IMG_BASE))
    if not obj:
        print("CardsDb singleton is NULL (no FUT session loaded).")
        return 1

    ns = P.nodes(mem, obj)
    print("pid=%d  CardsDb=%#x  size(+0x160e8)=%s  walked=%d"
          % (pid, obj, mem.i32(obj + W.TREE_SIZE), len(ns)))

    cards = []
    for n in ns:
        c = read_staff(mem, n)
        if not c:
            continue
        c["verdict"], c["why"] = grade(c, tables)
        cards.append(c)
    if not a.all:
        cards = [c for c in cards if c["verdict"] != "NOT-STAFF"]
    cards.sort(key=lambda c: (c["subtype"] or 0, c["resourceId"] or 0))

    print()
    print("%-13s %-9s %-8s %-4s %-4s %-4s %-22s %-13s %s"
          % ("family", "resource", "assetId", "rat", "tie", "rar", "name",
             "verdict", "why"))
    for c in cards:
        nm = ("%s %s" % (c["first"], c["last"])).strip()[:22]
        print("%-13s %-9s %-8s %-4s %-4s %-4s %-22s %-13s %s"
              % (c.get("family", "?"), c["resourceId"], c["assetId"],
                 c["rating"], c["tier"], c["rare"], nm, c["verdict"], c["why"]))

    tally = {}
    for c in cards:
        tally[c["verdict"]] = tally.get(c["verdict"], 0) + 1
    print("\n" + "  ".join("%s=%d" % kv for kv in sorted(tally.items())))
    print("\nfields the coach branches NEVER write, i.e. OURS on screen if they "
          "render at all:")
    for c in cards[:8]:
        print("  %-9s teamid=%-6s leagueId=%-6s nation=%-5s position=%s"
              % (c["resourceId"], c["teamid"], c["league"], c["nation"],
                 c["position"]))

    if a.raw and cards:
        b = cards[0]["_raw"]
        print("\nrecord %#x:" % (cards[0]["node"] + P.REC))
        for off in range(0, P.REC_LEN, 16):
            row = b[off:off + 16]
            print("  +%03x  %-47s  %s" % (
                off, " ".join("%02x" % x for x in row),
                "".join(chr(x) if 32 <= x < 127 else "." for x in row)))

    if a.json:
        for c in cards:
            c.pop("_raw", None)
        with open(a.json, "w") as f:
            json.dump(cards, f, indent=1)
        print("\nwrote %s" % a.json)

    print("\nfailed reads=%d" % mem.fails)
    return 0


if __name__ == "__main__":
    sys.exit(main())