OpenFUT/fifa17-recon/tools/dbschema_probe.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Read FIFA 17's LOADED database schema (and any table's rows) out of a live
FIFA17.exe.  READ-ONLY: /proc/PID/mem is opened 'rb' and only ever seek()/read().
There is no write path in this file.

WHY THIS EXISTS
---------------
`managercards` (and headcoachcards / gkcoachcards / physiocards /
fitnesscoachcards) are queried by CardsDLL through a column called `carddbid`,
and the set of legal carddbid values was unknown -- two blind sweeps (1..5000 and
6000..8000) resolved nothing, and the manager branch writes no miss-fill so a
wrong id is completely silent.  dbdata.dll on disk is packed (see
dbdata_extract.py's docstring: uniform ~7.5 bits/byte entropy, one export that
returns a ~759-byte attestation blob), so the tables cannot be read off disk.

They ARE fully resident in the running game, and unlike the player roster they
are reachable through the DB's own SCHEMA rather than by content-anchoring.

THE STRUCTURES (resolved live, 2026-08-04, pid 11864, sitting in the FUT club UI)
--------------------------------------------------------------------------------
Three cooperating structures, all in the game's rw-p heap:

  * IDENTIFIER INTERN POOL, seen at 0x0770_0000..0x07d0_0000.  Every table name
    and every column name in the whole database, interned once.  Entry:
        +0x00 char*  next-in-bucket
        +0x08 u64    strlen+1
        +0x10 u64    hash
        +0x18 char[] the NUL-terminated identifier, INLINE
    e.g. "managercards" at 0x07be63a0, "carddbid" at 0x078eefc8,
         "talkrating" at 0x078ec188, "headcoachcards" at 0x07be5b80.

  * TABLE DIRECTORY, seen at 0x4254_7ff0.., 573 entries of 40 bytes.  The
    reliable signature is the CONSTANT QWORD 0x07c20760 that every entry carries
    at +0x20 -- anchor on that, not on any fixed address:
        +0x00 char*  table name (into the intern pool)
        +0x08 void*  table descriptor        ("p1")
        +0x10 u32    name hash4 (4 bytes, also repeated inside the descriptor)
        +0x14 u32    column count
        +0x18 void*  column-descriptor bucket ("p2")
        +0x20 void*  0x07c20760              (the anchor)
    Live values read this session:
        managercards       p1=0x42482f08 hash='xIfB' ncol=62
        headcoachcards     p1=0x427429d8 hash='WTdJ' ncol=18
        gkcoachcards       p1=0x42e46648 hash='CZUM' ncol=107
        physiocards        p1=0x42482478 hash='AThf' ncol=9
        fitnesscoachcards  p1=0x42d668e8 hash='MmoU' ncol=11

  * TABLE DESCRIPTOR (at p1).  Self-identifies by repeating the table's own
    hash4; immediately after that hash come
        u32 row_size_bytes, u32 max_bit_index
    (managercards: 0x80 and 0x3ff -- 128 bytes == 1024 bits, consistent), and
    then a column array of 16-byte records:
        u32 type (3 for the scalar columns seen), u32 bit_offset,
        u32 name hash4, u32 bit_width
    First managercards columns read live (bit_offset, width):
        (0x260,7) (0x20,8) (0x267,15) (0x28,8) (0x276,14) (0x284,5)
        (0x30,8) (0x289,7) (0x290,2) (0x38,8) (0x292,18) (0x2a4,...)
    Note the bit offsets run past 416, so a row is NOT the 0x34-byte
    firstname/lastname record seen near 0x42530928 -- that is a different table.

  * COLUMN-NAME entry (0x30 bytes, in the p2 buckets, also anchored by
    0x07c20760 at +0x20):  +0x04 hash4, +0x28 char* name.  This is what turns a
    column's hash4 back into "carddbid".

WHAT THIS TOOL DOES NOT YET DO
------------------------------
It stops at the schema.  It does not locate the ROW STORAGE for a table -- that
pointer was not identified before the game exited.  Candidate leads recorded at
the time: managercards' descriptor holds 0x426cb278 at p1+0x08 and 0x07822918 at
p1+0x30, and the 16 bytes before p1 read `... 81 04 00 00` (0x481 = 1153, a
plausible row count).  Once row storage is found, carddbid for every row is a
mechanical bit-extract with (bit_offset, bit_width) from --table.

USAGE
-----
    ./dbschema_probe.py                        # list the FUT card tables
    ./dbschema_probe.py --table managercards   # full column list for one table
    ./dbschema_probe.py --list                 # every table in the directory

Requires ptrace access to the FIFA process (ptrace_scope=1 + same uid suffices)
and FIFA17.exe actually running.
"""

import argparse
import os
import re
import struct
import sys

SHARED_ANCHOR = 0x07C20760      # the qword every directory/column entry carries at +0x20
POOL_LO, POOL_HI = 0x07700000, 0x07D00000
HEAP_LO, HEAP_HI = 0x42000000, 0x43000000
IDENT = re.compile(r'[A-Za-z][A-Za-z0-9_]*\Z')

FUT_TABLES = ("managercards", "headcoachcards", "gkcoachcards", "physiocards",
              "fitnesscoachcards", "players", "teams", "leagues", "nations")


def find_pid():
    for d in os.listdir('/proc'):
        if not d.isdigit():
            continue
        try:
            if open('/proc/%s/comm' % d).read().strip() == 'FIFA17.exe':
                return int(d)
        except OSError:
            pass
    raise SystemExit("FIFA17.exe is not running (this tool needs the live game)")


class Mem(object):
    """Read-only window onto /proc/PID/mem, with the two regions pre-slurped."""

    def __init__(self, pid):
        self.fh = open('/proc/%d/mem' % pid, 'rb')
        self.pool = self.read(POOL_LO, POOL_HI - POOL_LO)
        self.heap = self.read(HEAP_LO, HEAP_HI - HEAP_LO)
        if not self.pool or not self.heap:
            raise SystemExit("could not read the DB regions -- is the game past the "
                             "main menu?  (the DB is loaded lazily)")

    def read(self, addr, n):
        if addr < 0 or addr > (1 << 47):
            return b''
        try:
            self.fh.seek(addr)
            return self.fh.read(n)
        except (OSError, ValueError):
            return b''

    def cstr(self, addr, maxlen=64):
        for base, buf in ((POOL_LO, self.pool), (HEAP_LO, self.heap)):
            if base <= addr < base + len(buf):
                i = addr - base
                j = buf.find(b'\x00', i, i + maxlen)
                if j <= i:
                    return None
                s = buf[i:j]
                return s.decode('latin1') if re.fullmatch(rb'[ -~]+', s) else None
        return None


def hash4_to_names(mem):
    """Every column-name entry in the DB, keyed by its 4-byte name hash."""
    out = {}
    for base, buf in ((HEAP_LO, mem.heap), (POOL_LO, mem.pool)):
        for off in range(0, len(buf) - 0x30, 8):
            if struct.unpack_from('<Q', buf, off + 0x20)[0] != SHARED_ANCHOR:
                continue
            nm = mem.cstr(struct.unpack_from('<Q', buf, off + 0x28)[0], 48)
            if not nm or not IDENT.match(nm):
                continue
            out.setdefault(struct.unpack_from('<I', buf, off + 4)[0], set()).add(nm)
    return out


def table_directory(mem):
    """name -> (entry_addr, descriptor, hash4, ncol, colbucket)."""
    out = {}
    buf = mem.heap
    for off in range(0, len(buf) - 0x28, 8):
        if struct.unpack_from('<Q', buf, off + 0x20)[0] != SHARED_ANCHOR:
            continue
        nm = mem.cstr(struct.unpack_from('<Q', buf, off)[0], 48)
        if not nm or not IDENT.match(nm):
            continue
        out[nm] = (HEAP_LO + off,
                   struct.unpack_from('<Q', buf, off + 8)[0],
                   struct.unpack_from('<I', buf, off + 0x10)[0],
                   struct.unpack_from('<I', buf, off + 0x14)[0],
                   struct.unpack_from('<Q', buf, off + 0x18)[0])
    return out


def h4str(v):
    return struct.pack('<I', v).decode('latin1')


def describe(mem, tables, h2n, name):
    if name not in tables:
        print("### %-20s NOT IN DIRECTORY" % name)
        return
    ent, p1, h4, ncol, p2 = tables[name]
    print("\n### %s   dir@%#x  descriptor=%#x  hash=%r  columns=%d"
          % (name, ent, p1, h4str(h4), ncol))
    blk = mem.read(p1, 0x80 + ncol * 16 + 0x80)
    if not blk:
        print("    descriptor unreadable")
        return
    pos = blk.find(struct.pack('<I', h4))
    if pos < 0:
        print("    self-hash not found in the descriptor -- layout changed")
        return
    rowsz, maxbit = struct.unpack_from('<II', blk, pos + 4)
    print("    self-hash at +%#x  row_size=%#x bytes  max_bit=%#x" % (pos, rowsz, maxbit))
    # the column array begins at the first 16-byte record whose hash4 is a known
    # column name and whose width is sane
    st = pos + 12
    while st < len(blk) - 16:
        t, boff, ch, w = struct.unpack_from('<IIII', blk, st)
        if t == 3 and ch in h2n and 0 < w <= 64:
            break
        st += 4
    print("    column array at +%#x" % st)
    cols = []
    for i in range(ncol):
        o = st + i * 16
        if o + 16 > len(blk):
            break
        t, boff, ch, w = struct.unpack_from('<IIII', blk, o)
        cols.append((boff, w, t, ch,
                     "|".join(sorted(h2n.get(ch, {"?" + h4str(ch)})))))
    for boff, w, t, ch, nm in sorted(cols):
        print("      bit %5d  width %-3d type %-2d  %s" % (boff, w, t, nm))
    return cols


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--pid', type=int)
    ap.add_argument('--table', action='append')
    ap.add_argument('--list', action='store_true')
    a = ap.parse_args()

    pid = a.pid or find_pid()
    print("FIFA17.exe pid %d" % pid)
    mem = Mem(pid)
    h2n = hash4_to_names(mem)
    tables = table_directory(mem)
    print("column-name hashes: %d      tables in directory: %d" % (len(h2n), len(tables)))
    if a.list:
        for nm in sorted(tables):
            ent, p1, h4, ncol, p2 = tables[nm]
            print("  %-34s desc=%#012x hash=%r cols=%d" % (nm, p1, h4str(h4), ncol))
        return
    for nm in (a.table or FUT_TABLES):
        describe(mem, tables, h2n, nm)


if __name__ == '__main__':
    main()