OpenFUT/fifa17-recon/tools/db_catalog_walk.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""READ-ONLY: enumerate every resident FIFA 17 DB table and dump its rows.

See dbwalk.py header for the on-heap format. This version locates a table's
layout block by its header signature (ncols<<16 | 0xffff) and matches on the
column tag set. Opens /proc/PID/mem 'rb'; only seek()/read().
"""
import glob, struct, sys, json

CONST = 0x07C20760

def find_pid():
    for d in glob.glob("/proc/[0-9]*"):
        try:
            if open(d + "/comm").read().strip() == "FIFA17.exe":
                return int(d.rsplit("/", 1)[-1])
        except Exception:
            pass

pid = find_pid()
f = open("/proc/%d/mem" % pid, "rb")

def rd(va, n):
    try:
        f.seek(va); b = f.read(n)
        return b if b and len(b) == n else None
    except Exception:
        return None

def q(va):
    b = rd(va, 8)
    return struct.unpack("<Q", b)[0] if b else None

def cstr(va, m=64):
    b = rd(va, m)
    if not b:
        return None
    z = b.find(b"\x00")
    if z <= 0:
        return None
    try:
        return b[:z].decode("ascii")
    except UnicodeDecodeError:
        return None

def regions(lo_lim=0, hi_lim=1 << 62):
    out = []
    for line in open("/proc/%d/maps" % pid):
        p = line.split()
        a, b = p[0].split("-")
        if "r" not in p[1] or "w" not in p[1]:
            continue
        lo, hi = int(a, 16), int(b, 16)
        if hi - lo > 512 << 20 or hi < lo_lim or lo > hi_lim:
            continue
        out.append((max(lo, lo_lim), min(hi, hi_lim)))
    return out

# ------- load the DB heap once -------
CHUNKS = []
for lo, hi in regions(0x06000000, 0x48000000):
    off = lo
    while off < hi:
        n = min(16 << 20, hi - off)
        d = rd(off, n)
        if d:
            CHUNKS.append((off, d))
        off += n
print("loaded %d chunks, %.0f MB" % (len(CHUNKS), sum(len(c[1]) for c in CHUNKS) / 1e6))

def scan(pat):
    out = []
    for base, d in CHUNKS:
        i = d.find(pat)
        while i != -1:
            out.append(base + i)
            i = d.find(pat, i + 1)
    return out

# ------- catalog -------
tables = {}
for c in scan(struct.pack("<Q", CONST)):
    if c % 8:
        continue
    nm = cstr(q(c + 8) or 0)
    if not nm:
        continue
    h = rd(c - 0x10, 0x10)
    if not h:
        continue
    tag, cnt, colarr = struct.unpack("<IIQ", h)
    if not (1 <= cnt <= 200) or colarr < 0x1000 or q(colarr + 0x20) != CONST:
        continue
    cols = []
    for i in range(cnt):
        d = rd(colarr + i * 0x30, 0x30)
        if not d:
            break
        ty, ctag, mn, mx, ln = struct.unpack_from("<IIiII", d, 0)
        cols.append(dict(name=cstr(struct.unpack_from("<Q", d, 0x28)[0]),
                         type=ty, tag=ctag, min=mn, max=mx, len=ln))
    tables.setdefault(nm, []).append(dict(desc=c - 0x10, ncols=cnt, cols=cols))
print("tables: %d" % len(tables))

# ------- layout blocks by header signature -------
ncounts = sorted({t["ncols"] for v in tables.values() for t in v})
blocks = []
for n in ncounts:
    sig = struct.pack("<I", (n << 16) | 0xFFFF)
    for a in scan(sig):
        if a % 4:
            continue
        hdr = a - 8                      # hdr = {cap, rowcount, sig, ?}
        ents = []
        ok = True
        for k in range(n):
            e = rd(hdr + 0x10 + k * 16, 16)
            if not e:
                ok = False; break
            off, tag, w, ty = struct.unpack("<4I", e)
            tb = struct.pack("<I", tag)
            if not (all(0x30 <= x < 0x7B for x in tb) and 0 < w <= 512 and off < 16384):
                ok = False; break
            ents.append((off, tag, w, ty))
        if ok:
            blocks.append((hdr, struct.unpack("<I", rd(hdr + 4, 4))[0], ents))
print("layout blocks: %d" % len(blocks))
byset = {}
for hdr, rc, ents in blocks:
    byset.setdefault(frozenset(e[1] for e in ents), []).append((hdr, rc, ents))

def align4(x):
    return (x + 3) & ~3

out = {}
for name in sorted(tables):
    for t in tables[name]:
        tset = frozenset(c["tag"] for c in t["cols"])
        for hdr, rc, ents in byset.get(tset, []):
            lay = {e[1]: (e[0], e[2], e[3]) for e in ents}
            maxbit = max(o + w for o, w, ty in
                         [(lay[c["tag"]][0], 32 if c["type"] == 1 else lay[c["tag"]][1],
                           0) for c in t["cols"]])
            stride = align4((maxbit + 7) // 8)
            rowptr = q(hdr - 0x48)
            out.setdefault(name, []).append(
                dict(hdr=hdr, rows=rc, rowptr=rowptr, stride=stride,
                     cols=[(c["name"], c["type"], lay[c["tag"]][0],
                            lay[c["tag"]][1], c["min"], c["max"]) for c in t["cols"]]))

for name in sorted(out):
    for b in out[name]:
        print("\n== %-24s rows=%-7d stride=%-3d rowptr=%#x hdr=%#x"
              % (name, b["rows"], b["stride"], b["rowptr"] or 0, b["hdr"]))
        for cn, ty, o, w, mn, mx in sorted(b["cols"], key=lambda x: x[2]):
            print("     bit %-5d w=%-4d %-26s type=%d [%d..%d]" % (o, w, cn, ty, mn, mx))
json.dump({k: [{kk: vv for kk, vv in b.items()} for b in v] for k, v in out.items()},
          open("layouts.json", "w"), indent=1)
print("\nwrote layouts.json for %d tables" % len(out))