OpenFUT/fifa17-recon/tools/extract_player_ids.py

#!/usr/bin/env python3
"""Extract the set of REAL FIFA 17 playerids (with names) from the game's own files.

WHY THIS EXISTS
---------------
FUT cards render generic because the client resolves identity from its LOCAL
`players` table, keyed by `playerid = resourceId & 0xffffff` (see docs/CARD_SYSTEM.md).
An id that is not in that table produces the MISS fingerprint: rating 0x32 (50),
teamid 0x78d (1933), nation 0xe (14), position 2, all attributes 1, name " ".
So the one thing the server needs is a list of playerids that actually exist.

WHERE THE IDS COME FROM
-----------------------
Frostbite bundle indexes. `Data/Win32/contentsb.{toc,sb}`, `contentlaunchsb.{toc,sb}`
and their `Update/Patch/` counterparts store asset paths as plain ASCII, and the
player face assets are named:

    content/character/player/player_<bucket>/<name>_<playerid>_starhead_brt
    content/character/player/player_<bucket>/<name>_<playerid>/hair_<playerid>_0_0_...

<bucket> is floor(playerid/500)*500, which this script uses as a self-check: an id is
only accepted if it falls inside its own directory's bucket. No decryption, no
Frostbite parsing, no cas archives -- the paths are literally in the clear in the
index files. dbdata.dll is NOT involved (its single export `getTableData` is an
anti-tamper attestation routine; see dbdata_probe.c).

COVERAGE, STATED HONESTLY
-------------------------
This yields every player who has a scanned STARHEAD (real face) asset: 1677 ids in
this install. That is NOT the whole `players` table (~18k rows including generic-face
players) -- it is the subset with real faces, which is also the subset whose cards
look best. Getting the full table needs the encrypted dbdata.dll payload or a live
memory read, neither of which this script attempts.

The extracted names are ASSET FILE names (lowercase, ASCII-folded, e.g.
`cristiano_ronaldo`), not the client's display names. You do not need them for the
wire: on a DB hit the client writes the display name, the face, and -- if you send
them as ZERO -- the nation and teamid itself. Only rating, position and the six
attributes are left as the server sent them. So the id alone buys a correct card.

ANCHOR CHECK
------------
playerid 20801 must map to cristiano_ronaldo. The script fails loudly if it does not.

WHAT THIS ALREADY CORRECTED IN fut_cards.VERIFIED_ASSET_IDS
----------------------------------------------------------
16 of the 18 ids there are confirmed by this extract. Two are not:
  * 169193 was labelled "Alonso". This build ships
    `player_45000/xabi_alonso_45197_launch_starhead_brt`, so Xabi Alonso is 45197
    here. 169193 is not him; it may or may not be some other valid row.
  * 200389 was labelled "Oblak". No `oblak` asset and no `200389` string appears
    anywhere in the bundle indexes, so it is unconfirmed.
Absence from this list is NOT proof an id is invalid -- players without a scanned
face have no starhead asset but are still in the `players` table. This list is a
lower bound on the valid id set, not the id set.

Also note playerid 0 (`chris_head`) is a developer placeholder head; drop it before
using the list as a card pool.

USAGE
-----
    ./extract_player_ids.py                       # summary + anchor check
    ./extract_player_ids.py --tsv  out.tsv        # playerid<TAB>asset_name
    ./extract_player_ids.py --json out.json
    FIFA17_DIR=/path/to/FIFA\\ 17 ./extract_player_ids.py
"""
import argparse
import json
import os
import re
import sys

GAME_DIR = os.environ.get("FIFA17_DIR", "/mnt/games/FIFA 17")

# Only the bundle indexes hold plaintext paths; the 30GB of .cas archives do not
# need to be touched.
INDEX_SUFFIXES = (".toc", ".sb")
SCAN_ROOTS = ("Data", "Update")

PLAYER_PATH = re.compile(rb"content/character/player/player_(\d+)/([a-z0-9_\-\.]+)")
NAMED_LEAF = re.compile(r"^([a-z][a-z_\-\.]*?)_(\d+)(?:_launch)?(?:_starhead_brt)?$")
HAIR_LEAF = re.compile(r"^hair_(\d+)_")

BUCKET = 500  # player_9500/ holds playerids 9500..9999

ANCHOR = (20801, "cristiano_ronaldo")


def index_files(game_dir):
    out = []
    for root_name in SCAN_ROOTS:
        base = os.path.join(game_dir, root_name)
        if not os.path.isdir(base):
            continue
        for root, _dirs, files in os.walk(base):
            for f in files:
                if f.endswith(INDEX_SUFFIXES):
                    out.append(os.path.join(root, f))
    return sorted(out)


def extract(game_dir):
    """-> (names: {playerid: asset_name}, faceless: set[playerid], stats: dict)"""
    names, faceless = {}, set()
    leaves = set()
    files = index_files(game_dir)
    for path in files:
        try:
            data = open(path, "rb").read()
        except OSError as exc:
            print("  skip %s: %s" % (path, exc), file=sys.stderr)
            continue
        for m in PLAYER_PATH.finditer(data):
            leaves.add((int(m.group(1)), m.group(2).decode("ascii", "replace")))

    unmatched = 0
    for bucket, leaf in leaves:
        m = NAMED_LEAF.match(leaf)
        if m and bucket <= int(m.group(2)) < bucket + BUCKET:
            names.setdefault(int(m.group(2)), m.group(1))
            continue
        m = HAIR_LEAF.match(leaf)
        if m and bucket <= int(m.group(1)) < bucket + BUCKET:
            faceless.add(int(m.group(1)))
            continue
        unmatched += 1

    stats = {"index_files": len(files), "leaf_paths": len(leaves),
             "unmatched_leaves": unmatched}
    return names, faceless - set(names), stats


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--tsv")
    ap.add_argument("--json")
    ap.add_argument("--dir", default=GAME_DIR)
    args = ap.parse_args()

    if not os.path.isdir(args.dir):
        sys.exit("game dir not found: %s (set FIFA17_DIR)" % args.dir)

    names, extra, stats = extract(args.dir)
    if not names:
        sys.exit("no player asset paths found under %s -- wrong dir?" % args.dir)

    pid, expect = ANCHOR
    got = names.get(pid)
    if got != expect:
        sys.exit("ANCHOR CHECK FAILED: playerid %d -> %r, expected %r. "
                 "The parse is wrong, not the game." % (pid, got, expect))

    print("scanned %d bundle index files, %d player asset leaf paths"
          % (stats["index_files"], stats["leaf_paths"]))
    print("playerids with a real starhead: %d  (id range %d..%d)"
          % (len(names), min(names), max(names)))
    print("hair-only playerids (no named face asset): %d" % len(extra))
    print("unmatched leaf paths: %d" % stats["unmatched_leaves"])
    print("anchor OK: %d -> %s" % (pid, got))

    if args.tsv:
        with open(args.tsv, "w") as fh:
            fh.write("playerid\tasset_name\n")
            for k in sorted(names):
                fh.write("%d\t%s\n" % (k, names[k]))
        print("wrote %s" % args.tsv)
    if args.json:
        with open(args.json, "w") as fh:
            json.dump({str(k): names[k] for k in sorted(names)}, fh, indent=1)
        print("wrote %s" % args.json)


if __name__ == "__main__":
    main()