OpenFUT/fifa17-recon/tools/fut_cards.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""The FUT card pool: the REAL FIFA 17 roster, 17,547 players.

WHAT CHANGED, AND WHY IT MATTERS
--------------------------------
This module used to hold 79 hand-written rows whose asset ids were mostly
invented, on the premise (from an older CARD_SYSTEM.md) that the client's card
map is empty offline so no id could ever render. That premise was wrong.

Card identity does not come from us. The client inserts every item we serve into
its CardsDb map and, just before that, merges in its OWN local players table
keyed on `resourceId & 0xffffff`. An invented id renders as a blank generic card;
a real one renders as a real player. Proven live: a pack showed SILVA and NOWAK
with real names, badges and flags beside three blanks at rating 50.

So the pool is now built from the game's own roster, extracted from a running
FIFA17.exe by tools/dbdata_extract.py into data/roster.json. It was cross-checked
against a completely independent method -- the sweep oracle in
tools/sweep_collect.py, which reads back the identity the CLIENT resolved -- and
573 of 573 overlapping names agreed exactly.

WHICH FIELDS ARE REAL AND WHICH ARE NOT. Be honest about this when reading a card:

    playerid   REAL   data/roster.json
    rating     REAL   same
    name       REAL   resolved by the client from the id; we never send a name
    club       REAL   we send teamid 0 and the client fills its own value
    nation     REAL   we send nation 0, same mechanism
    league     REAL   the client always recomputes leagueid on a DB hit
    position   PARTLY 59 ids are known (data/positions.json); the rest are
                      SYNTHETIC, assigned deterministically per id
    attributes SYNTH  derived from rating and position

The merge fills nation/teamid ONLY when they arrive as zero, and never touches
rating, position or attributes. That asymmetry is the whole design of this file:
send zero for everything the client knows better than us, and send our own value
only where the client has nothing.

THE MISSING COLUMNS, AND THE LEADS FOR THEM
-------------------------------------------
position / nationality / teamId / attributes are NOT in the rating index. Two
live sources exist and BOTH are per-materialised-card caches, not tables:

  * the 0x180-stride resolved card records (attributes + names), and
  * a 32-byte-stride keyed container, entries {playerId, position | hash<<32,
    rating, ?}, found at 0x42e8dbe8 inside the 238 MB heap region.

data/positions.json comes from the second one. Sweeping 5,000 ids did NOT
populate it, so it caches what the game itself materialises rather than what we
ask about. 59 entries survived validation (each entry's rating had to match the
roster's). Anyone extending this: a full-roster position source has not been
found, and the FUT rating index does not contain one.

Position codes are the standard FIFA enum, decoded against our own club cards
read live: GK 0, CB 5, LB 7, CM 14, LM 16, RW 23, ST 25, LW 27.
"""
import json
import os

_HERE = os.path.dirname(os.path.abspath(__file__))
_DATA = os.path.join(_HERE, "..", "data")

# Position code -> the string the client's parser expects in preferredPosition.
POSITION_BY_CODE = {
    0: "GK", 1: "SW", 2: "RWB", 3: "RB", 4: "CB", 5: "CB", 6: "CB",
    7: "LB", 8: "LWB", 9: "CDM", 10: "CDM", 11: "CDM", 12: "RM",
    13: "CM", 14: "CM", 15: "CM", 16: "LM", 17: "CAM", 18: "CAM",
    19: "CAM", 20: "RF", 21: "CF", 22: "LF", 23: "RW", 24: "ST",
    25: "ST", 26: "ST", 27: "LW",
}

# Synthetic-position distribution, shaped like a real squad (one keeper, four at
# the back, four in midfield, three forward) so a random pack looks like a
# football team rather than eleven strikers.
_SYNTH_POSITIONS = (["GK"] +
                    ["RB", "CB", "CB", "LB"] +
                    ["CDM", "CM", "CM", "CAM"] +
                    ["RW", "ST", "LW"])

# Attribute profiles: (pace, shooting, passing, dribbling, defending, physical)
# as multipliers on the rating. The GK profile stands in for the six keeper
# stats the card face shows in that slot instead.
_PROFILE = {
    "GK":  (0.68, 0.70, 0.40, 0.66, 0.20, 0.68),
    "RB":  (1.02, 0.72, 0.90, 0.92, 1.00, 0.95),
    "LB":  (1.02, 0.72, 0.90, 0.92, 1.00, 0.95),
    "RWB": (1.05, 0.75, 0.92, 0.95, 0.96, 0.92),
    "LWB": (1.05, 0.75, 0.92, 0.95, 0.96, 0.92),
    "CB":  (0.82, 0.55, 0.78, 0.75, 1.05, 1.05),
    "SW":  (0.82, 0.55, 0.78, 0.75, 1.05, 1.05),
    "CDM": (0.85, 0.75, 0.98, 0.92, 1.00, 1.00),
    "CM":  (0.90, 0.85, 1.02, 0.98, 0.88, 0.92),
    "RM":  (1.05, 0.88, 0.98, 1.02, 0.72, 0.85),
    "LM":  (1.05, 0.88, 0.98, 1.02, 0.72, 0.85),
    "CAM": (0.95, 0.92, 1.02, 1.04, 0.62, 0.82),
    "RW":  (1.08, 0.92, 0.95, 1.05, 0.60, 0.80),
    "LW":  (1.08, 0.92, 0.95, 1.05, 0.60, 0.80),
    "RF":  (1.02, 0.98, 0.95, 1.04, 0.58, 0.85),
    "LF":  (1.02, 0.98, 0.95, 1.04, 0.58, 0.85),
    "CF":  (1.00, 1.00, 0.95, 1.02, 0.58, 0.88),
    "ST":  (1.00, 1.05, 0.85, 0.98, 0.45, 0.95),
}


def _load(name, default):
    try:
        with open(os.path.join(_DATA, name)) as f:
            return json.load(f)
    except (IOError, ValueError):
        return default


ROSTER = _load("roster.json", [])
KNOWN_POSITIONS = {int(k): v for k, v in _load("positions.json", {}).items()}
# Hand-checked positions, carried over from the curated pool this file replaces.
# They are KNOWLEDGE, not measurement, which is why they rank below the codes the
# game itself supplied. They exist because a synthetic position is unnoticeable on
# an unknown 62-rated defender and glaring on Neuer, and the famous players are
# precisely the ones a pack shows off.
CURATED_POSITIONS = {int(k): v for k, v in _load("positions_curated.json", {}).items()}


def _position(pid):
    """The real position where the game told us one, else curated, else synthetic.

    Deterministic in the id, so a player never changes shape between packs or
    between runs.
    """
    code = KNOWN_POSITIONS.get(pid)
    if code is not None:
        return POSITION_BY_CODE.get(code, "ST")
    if pid in CURATED_POSITIONS:
        return CURATED_POSITIONS[pid]
    return _SYNTH_POSITIONS[pid % len(_SYNTH_POSITIONS)]


def _attrs(rating, pos):
    prof = _PROFILE.get(pos, _PROFILE["CM"])
    out = []
    for i, mult in enumerate(prof):
        # A small, stable per-player wobble so two 82-rated strikers are not
        # byte-identical. Seeded by rating and slot, never by wall-clock, so the
        # pool is reproducible.
        v = int(round(rating * mult)) + ((rating * 7 + i * 13) % 5) - 2
        out.append(max(1, min(99, v)))
    return out


def _build():
    pool = []
    for r in ROSTER:
        pid, rating = r["id"], r["rating"]
        if not pid or rating <= 0:
            # A zero id is not a harmless skip: the registrar writes
            # *(item+0x10) = 0 and the card view-model dereferences item+0x10
            # with no null check, so a zero id reaching the card UI is a crash.
            continue
        pos = _position(pid)
        # nation / league / team are ZERO on purpose -- that is what makes the
        # client fill in the real ones. Do not "improve" this by guessing them.
        pool.append((pid, rating, pos, 0, 0, 0, _attrs(rating, pos)))
    return pool


POOL = _build()

# Kept for the record: the old hand-written "verified" set. 169193 is in it and
# is NOT a real FIFA 17 player -- the client resolves it to the database's empty
# placeholder row, which reads as "Jamal Blackman" on every card. That is exactly
# the failure this rebuild removes, and two independent methods agreed on it. Do
# not restore this set as a source of truth; it is here so older notes stay
# traceable.
VERIFIED_ASSET_IDS = {
    20801, 158023, 176580, 167495, 183907, 155862, 188545, 182521,
    183277, 177003, 192985, 190871, 200389, 197445, 202126, 189332,
    169193, 184941,
}

NAME_BY_ID = {r["id"]: (r["common"] or ("%s %s" % (r["first"], r["last"])).strip())
              for r in ROSTER}


def tier(rating):
    return "gold" if rating >= 75 else "silver" if rating >= 65 else "bronze"


def pool_for(tier_name):
    """Players of one tier. Falls back to the whole pool rather than returning []."""
    sel = [p for p in POOL if tier(p[1]) == tier_name]
    return sel or POOL


def name_of(pid):
    """For LOGS only. The name a player actually sees comes from the client."""
    return NAME_BY_ID.get(pid, "?")


if __name__ == "__main__":
    print("pool: %d players from the real FIFA 17 roster" % len(POOL))
    for name in ("gold", "silver", "bronze"):
        sel = pool_for(name)
        print("  %-7s %5d  ratings %d-%d" % (name, len(sel),
                                             min(p[1] for p in sel),
                                             max(p[1] for p in sel)))
    print("known positions: %d (the rest are synthetic)" % len(KNOWN_POSITIONS))
    print("\ntop 10 by rating:")
    for p in sorted(POOL, key=lambda x: -x[1])[:10]:
        print("  %-7s %-3s %-4s %-26s %s" % (p[0], p[1], p[2], name_of(p[0]), p[6]))