feat(import): identity import API + FIFA17 real-profile dry-run importer

openfut-identity:
- insert_existing_mapping(game,kind,core_id,external_id): preserve an existing
  external wire id instead of minting; idempotent for an identical mapping,
  rejects conflicting forward/reverse with IdError::Conflict, persists atomically.
- persisted per-scope allocator watermark (set_watermark/watermark_for) so a
  future mint continues past the source high-water even across burned-id gaps;
  next id = max(base_floor, live_max+1, watermark). Backward-compatible on-disk
  format (legacy bare [Row] still loads). +4 tests (10 total).

openfut-import-fifa17 (new): read-only dry-run analysis of a real FIFA17 Python
profile for a faithful Core import. Enforces disjoint item-class balance;
proposes profile-derived CardDefinitions keyed fifa17_<resourceId> (base vs
versioned never collapse) with a resourceId-group consistency gate (hard-fail on
disagreement, never pick a winner) and honest buildability (roster name +
version formula + metadata, never fabricated); plans owned-instance identity
(preserve Python wire ids, preserve nextItemId watermark); checks active-squad
coverage. --apply/--emit-content refuse to write in this phase. 11 tests.

Real profile (33068179/CAGE) dry-run: 1982 items balance (1962 players + 17
consumables + 3 staff); 1681 supported defs (155 base + 1535 versioned), 9
NoName unsupported, 1 hard conflict (resourceId 169193: one of 4 copies has a
divergent nation/team/league); 1949 importable player instances, watermark
100004617 -> first new alloc 100004617; active squad f433 fully supported.
fmt + clippy -D warnings clean.
This commit is contained in:
funman300
2026-08-12 19:23:19 +00:00
parent 63f02c4fb1
commit a51947562c
8 changed files with 1380 additions and 9 deletions
+698
View File
@@ -0,0 +1,698 @@
//! Evidence-driven import of a real FIFA 17 Python profile into OpenFUT Core.
//!
//! This crate is the **read-only dry-run analysis** half of the importer. It
//! proves — before anything is written — that the migration can be done
//! faithfully:
//!
//! * disjoint item-type accounting balances to the source total (a reporting
//! typo must never become a migration assumption);
//! * every observed player card yields an HONEST [`ObservedDefinition`] keyed by
//! `fifa17_<resourceId>` (so a versioned/special card is a *distinct*
//! definition from its base, never collapsed onto it), sourced from the
//! profile's own evidence and never fabricated;
//! * multiple owned copies of one `resourceId` agree on definition-level data —
//! a disagreement is a HARD conflict, never silently reconciled;
//! * every owned instance keeps its existing Python wire id, and the source
//! allocation watermark (`nextItemId`) is preserved so a future Rust-minted id
//! cannot collide with a historically burned one;
//! * the active squad's players all resolve to supported definitions.
//!
//! What it deliberately does NOT do: write the Core DB, the identity store, or
//! any generated content, and it does not resolve nation/league/club numeric
//! ids to names (that is deterministic table resolution performed in the
//! content-emission phase). Program/promo taxonomy (TOTW/TOTS/…) and `rare=SP`
//! semantics stay UNKNOWN — the profile evidences that a versioned card exists
//! and its stats, not its marketing label.
use std::collections::BTreeMap;
use std::path::Path;
use anyhow::{Context, Result};
/// FIFA 17 owned-item wire-id floor (adapter policy
/// `Fifa17WireItemIdPolicy::owned_item_base_floor`). The first ever allocation
/// in an empty store starts here; a real import overrides it with the source
/// watermark.
pub const OWNED_ITEM_BASE_FLOOR: i64 = 100_000_001;
pub mod model;
use model::{Item, Profile};
// ----------------------------------------------------------------- roster
/// Player names keyed by base asset id, loaded from `roster.json`. Names are
/// evidence for CardDefinitions; a card whose base asset id is absent here
/// cannot be honestly named and is reported unsupported (never fabricated).
pub struct Roster {
names: BTreeMap<i64, String>,
}
impl Roster {
pub fn from_json_str(raw: &str) -> Result<Self> {
#[derive(serde::Deserialize)]
struct Row {
id: i64,
#[serde(default)]
first: String,
#[serde(default)]
last: String,
#[serde(default)]
common: String,
}
let rows: Vec<Row> =
serde_json::from_str(raw).context("parsing roster.json (expected an array)")?;
let mut names = BTreeMap::new();
for r in rows {
let name = if !r.common.trim().is_empty() {
r.common.trim().to_string()
} else {
format!("{} {}", r.first.trim(), r.last.trim())
.trim()
.to_string()
};
if !name.is_empty() {
names.insert(r.id, name);
}
}
Ok(Roster { names })
}
pub fn name_for(&self, asset_id: i64) -> Option<&str> {
self.names.get(&asset_id).map(String::as_str)
}
}
pub fn load_roster(path: impl AsRef<Path>) -> Result<Roster> {
let raw = std::fs::read_to_string(path.as_ref())
.with_context(|| format!("reading roster {}", path.as_ref().display()))?;
Roster::from_json_str(&raw)
}
pub fn load_profile(path: impl AsRef<Path>) -> Result<Profile> {
let raw = std::fs::read_to_string(path.as_ref())
.with_context(|| format!("reading profile {}", path.as_ref().display()))?;
Profile::from_json_str(&raw)
}
// ------------------------------------------------------------- classification
/// The disjoint source item classes. Every source item is exactly one of these.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ItemClass {
/// `itemType=player` with an `attributeList` — a real player card.
PlayerCard,
/// `itemType=player` without an `attributeList` (consumable; has amount/pile).
Consumable,
/// `itemType=staff` (manager/coach/etc.).
Staff,
/// Anything else — must be zero for a clean import.
Other,
}
pub fn classify(item: &Item) -> ItemClass {
match item.item_type.as_str() {
"staff" => ItemClass::Staff,
"player" => {
if item.attribute_list.is_some() {
ItemClass::PlayerCard
} else {
ItemClass::Consumable
}
}
_ => ItemClass::Other,
}
}
/// Disjoint counts; `balances()` is the invariant the dry-run enforces.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ItemCounts {
pub total: usize,
pub player_cards: usize,
pub consumables: usize,
pub staff: usize,
pub other: usize,
}
impl ItemCounts {
pub fn balances(&self) -> bool {
self.player_cards + self.consumables + self.staff + self.other == self.total
}
}
pub fn count_items(profile: &Profile) -> ItemCounts {
let mut c = ItemCounts {
total: profile.items.len(),
..Default::default()
};
for it in &profile.items {
match classify(it) {
ItemClass::PlayerCard => c.player_cards += 1,
ItemClass::Consumable => c.consumables += 1,
ItemClass::Staff => c.staff += 1,
ItemClass::Other => c.other += 1,
}
}
c
}
// -------------------------------------------------------------- definitions
/// The definition-level (per-card, not per-instance) fields of a player card.
/// Two owned copies of one `resourceId` MUST have equal `DefFields`; owned
/// instance fields (wire id, contract, fitness, itemState, owners, untradeable)
/// are deliberately excluded from this identity.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DefFields {
pub asset_id: Option<i64>,
pub version: i64,
pub rating: Option<i64>,
pub position: Option<String>,
pub nation: Option<i64>,
pub league: Option<i64>,
pub team: Option<i64>,
/// The six attributes in `index` order (0..=5), if present.
pub attrs: Option<Vec<i64>>,
pub rareflag: Option<i64>,
}
/// Why a `resourceId` cannot yield an honest CardDefinition.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Unsupported {
/// `resourceId != (version << 24) | assetId` — unexpected id structure.
VersionFormula { resource_id: i64, asset_id: i64 },
/// No player name for the base asset id in the roster (would have to fabricate).
NoName { asset_id: i64 },
/// A required definition field (rating/position/nation/league/team/attrs) is missing.
MissingMetadata { fields: Vec<String> },
}
/// An honest, profile-derived player CardDefinition proposal, keyed by
/// `fifa17_<resourceId>`. Carries provenance, not marketing taxonomy.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ObservedDefinition {
pub card_id: String,
pub resource_id: i64,
pub asset_id: i64,
pub version: i64,
pub name: String,
pub rating: i64,
pub position: String,
pub nation_id: i64,
pub league_id: i64,
pub team_id: i64,
/// pace, shooting, passing, dribbling, defending, physical (index 0..=5).
pub attrs: [i64; 6],
pub rareflag: i64,
/// Owned-copy count of this exact resourceId in the source profile.
pub owned_copies: usize,
}
/// `version = resourceId >> 24`; base cards are version 0 (`resourceId == assetId`).
pub fn version_of(resource_id: i64) -> i64 {
resource_id >> 24
}
fn def_fields(item: &Item) -> DefFields {
let attrs = item.attribute_list.as_ref().map(|list| {
let mut v: Vec<(i64, i64)> = list.iter().map(|a| (a.index, a.value)).collect();
v.sort_by_key(|(i, _)| *i);
v.into_iter().map(|(_, val)| val).collect::<Vec<i64>>()
});
DefFields {
asset_id: item.asset_id,
version: version_of(item.resource_id),
rating: item.rating,
position: item.preferred_position.clone(),
nation: item.nation,
league: item.league_id,
team: item.teamid,
attrs,
rareflag: item.rareflag,
}
}
/// A `resourceId` whose owned copies disagree on definition-level data — a hard
/// conflict the importer must never silently reconcile.
#[derive(Debug, Clone)]
pub struct DefConflict {
pub resource_id: i64,
pub wire_ids: Vec<i64>,
pub distinct: Vec<DefFields>,
}
/// Result of proposing definitions for all player cards.
#[derive(Debug, Default)]
pub struct DefinitionPlan {
pub supported: Vec<ObservedDefinition>,
pub unsupported: Vec<(i64, usize, Unsupported)>, // (resource_id, owned_copies, why)
pub conflicts: Vec<DefConflict>,
pub base_defs: usize,
pub versioned_defs: usize,
}
/// Build the definition proposal from the player cards, applying the
/// resourceId-group consistency gate and the honest-buildability test.
pub fn plan_definitions(profile: &Profile, roster: &Roster) -> DefinitionPlan {
// group player cards by resourceId, preserving wire ids for reporting
let mut groups: BTreeMap<i64, Vec<&Item>> = BTreeMap::new();
for it in &profile.items {
if classify(it) == ItemClass::PlayerCard {
groups.entry(it.resource_id).or_default().push(it);
}
}
let mut plan = DefinitionPlan::default();
for (resource_id, items) in groups {
// 1) consistency gate: all copies must share definition-level data.
let first = def_fields(items[0]);
let mut distinct = vec![first.clone()];
for it in &items[1..] {
let f = def_fields(it);
if !distinct.contains(&f) {
distinct.push(f);
}
}
if distinct.len() > 1 {
plan.conflicts.push(DefConflict {
resource_id,
wire_ids: items.iter().map(|i| i.id).collect(),
distinct,
});
continue; // never pick a winner
}
let owned_copies = items.len();
let f = &first;
let version = f.version;
if version == 0 {
plan.base_defs += 1;
} else {
plan.versioned_defs += 1;
}
// 2) honest buildability.
let asset_id = f.asset_id;
// version formula: resourceId == (version << 24) | assetId
if let Some(asset_id) = asset_id {
if resource_id != (version << 24) | asset_id {
plan.unsupported.push((
resource_id,
owned_copies,
Unsupported::VersionFormula {
resource_id,
asset_id,
},
));
continue;
}
}
// required metadata present?
let mut missing = Vec::new();
if f.rating.is_none() {
missing.push("rating".into());
}
if f.position.is_none() {
missing.push("position".into());
}
if f.nation.is_none() {
missing.push("nation".into());
}
if f.league.is_none() {
missing.push("league".into());
}
if f.team.is_none() {
missing.push("team".into());
}
let attrs6: Option<[i64; 6]> = f.attrs.as_ref().and_then(|a| {
if a.len() == 6 {
Some([a[0], a[1], a[2], a[3], a[4], a[5]])
} else {
None
}
});
if attrs6.is_none() {
missing.push("attributeList[6]".into());
}
if asset_id.is_none() {
missing.push("assetId".into());
}
if !missing.is_empty() {
plan.unsupported.push((
resource_id,
owned_copies,
Unsupported::MissingMetadata { fields: missing },
));
continue;
}
let asset_id = asset_id.unwrap();
// name: evidence, never fabricated.
let name = match roster.name_for(asset_id) {
Some(n) => n.to_string(),
None => {
plan.unsupported.push((
resource_id,
owned_copies,
Unsupported::NoName { asset_id },
));
continue;
}
};
plan.supported.push(ObservedDefinition {
card_id: format!("fifa17_{resource_id}"),
resource_id,
asset_id,
version,
name,
rating: f.rating.unwrap(),
position: f.position.clone().unwrap(),
nation_id: f.nation.unwrap(),
league_id: f.league.unwrap(),
team_id: f.team.unwrap(),
attrs: attrs6.unwrap(),
rareflag: f.rareflag.unwrap_or(0),
owned_copies,
});
}
plan
}
// --------------------------------------------------------------- identity
/// The owned-instance identity plan: preserve every supported player item's
/// existing Python wire id, and preserve the source allocation watermark.
#[derive(Debug, Default)]
pub struct IdentityPlan {
/// Wire ids of the player items that WOULD import (supported definitions).
pub import_wire_ids: Vec<i64>,
/// Duplicate wire ids across ALL items (must be empty — a hard conflict).
pub duplicate_wire_ids: Vec<i64>,
pub live_min: Option<i64>,
pub live_max: Option<i64>,
/// Source `nextItemId` — the next id Python would issue (issue-then-increment).
pub source_watermark: i64,
/// The first id a Rust-minted new item would receive after this import:
/// `max(floor, live_max+1, watermark)`. With the watermark this equals the
/// source `nextItemId`, never a burned-gap id.
pub next_allocation: i64,
}
pub fn plan_identity(profile: &Profile, supported_resource_ids: &[i64]) -> IdentityPlan {
use std::collections::BTreeSet;
let supported: BTreeSet<i64> = supported_resource_ids.iter().copied().collect();
// duplicate detection across ALL items
let mut seen = BTreeSet::new();
let mut dups = BTreeSet::new();
for it in &profile.items {
if !seen.insert(it.id) {
dups.insert(it.id);
}
}
let mut import_wire_ids: Vec<i64> = profile
.items
.iter()
.filter(|it| classify(it) == ItemClass::PlayerCard && supported.contains(&it.resource_id))
.map(|it| it.id)
.collect();
import_wire_ids.sort_unstable();
let live_min = profile.items.iter().map(|i| i.id).min();
let live_max = profile.items.iter().map(|i| i.id).max();
let watermark = profile.next_item_id;
let next_allocation = [
OWNED_ITEM_BASE_FLOOR,
live_max.map(|m| m + 1).unwrap_or(OWNED_ITEM_BASE_FLOOR),
watermark,
]
.into_iter()
.max()
.unwrap();
IdentityPlan {
import_wire_ids,
duplicate_wire_ids: dups.into_iter().collect(),
live_min,
live_max,
source_watermark: watermark,
next_allocation,
}
}
// ----------------------------------------------------------------- squad
/// Coverage of the active squad: every starter/sub must resolve to a supported
/// player definition; the manager stays an opaque staff wire ref in the squad
/// extension (not promoted to a Core OwnedCard).
#[derive(Debug, Default)]
pub struct SquadCoverage {
pub present: bool,
pub formation: String,
pub occupied_slots: usize,
pub supported_slots: usize,
/// (slot index, wire id) for slots whose item is missing/unsupported.
pub unsupported_slots: Vec<(i64, i64)>,
pub manager_wire_ids: Vec<i64>,
pub captain_wire_id: Option<i64>,
}
pub fn plan_squad(profile: &Profile, supported_wire_ids: &[i64]) -> SquadCoverage {
use std::collections::BTreeSet;
let supported: BTreeSet<i64> = supported_wire_ids.iter().copied().collect();
let Some(sq) = profile.squads.first() else {
return SquadCoverage::default();
};
let mut cov = SquadCoverage {
present: true,
formation: sq.formation.clone(),
captain_wire_id: sq.captain.filter(|&c| c != 0),
manager_wire_ids: sq
.manager
.iter()
.filter_map(|m| m.get("id").and_then(|v| v.as_i64()))
.filter(|&id| id != 0)
.collect(),
..Default::default()
};
for slot in &sq.players {
let wid = slot.item_data.id;
if wid == 0 {
continue; // empty slot
}
cov.occupied_slots += 1;
if supported.contains(&wid) {
cov.supported_slots += 1;
} else {
cov.unsupported_slots.push((slot.index, wid));
}
}
cov
}
// ----------------------------------------------------------------- report
/// The full dry-run analysis.
#[derive(Debug)]
pub struct Report {
pub game: String,
pub persona_id: i64,
pub persona_name: String,
pub club_name: String,
pub club_abbr: String,
pub coins: i64,
pub counts: ItemCounts,
pub definitions: DefinitionPlan,
pub identity: IdentityPlan,
pub squad: SquadCoverage,
}
impl Report {
/// Unsafe conditions that MUST fail the dry-run (nonzero exit). Deferred
/// items (unsupported non-squad players, consumables, staff) are NOT here —
/// they are reported, not blocking.
pub fn blockers(&self) -> Vec<String> {
let mut b = Vec::new();
if !self.counts.balances() {
b.push(format!(
"item-type accounting does not balance: {}+{}+{}+{} != {}",
self.counts.player_cards,
self.counts.consumables,
self.counts.staff,
self.counts.other,
self.counts.total
));
}
if self.counts.other > 0 {
b.push(format!(
"{} item(s) fall in no known class (itemType neither player nor staff)",
self.counts.other
));
}
if !self.definitions.conflicts.is_empty() {
b.push(format!(
"{} resourceId group(s) disagree on definition-level data (hard conflict)",
self.definitions.conflicts.len()
));
}
if !self.identity.duplicate_wire_ids.is_empty() {
b.push(format!(
"{} duplicate owned wire id(s) — instance identity is not unique",
self.identity.duplicate_wire_ids.len()
));
}
if self.squad.present && !self.squad.unsupported_slots.is_empty() {
b.push(format!(
"{} active-squad slot(s) reference an unsupported/missing item",
self.squad.unsupported_slots.len()
));
}
b
}
pub fn has_blockers(&self) -> bool {
!self.blockers().is_empty()
}
}
/// Run the full read-only analysis. Pure: no I/O, no writes.
pub fn analyze(profile: &Profile, roster: &Roster) -> Report {
let counts = count_items(profile);
let definitions = plan_definitions(profile, roster);
let supported_rids: Vec<i64> = definitions
.supported
.iter()
.map(|d| d.resource_id)
.collect();
let identity = plan_identity(profile, &supported_rids);
let squad = plan_squad(profile, &identity.import_wire_ids);
Report {
game: "fifa17".to_string(),
persona_id: profile.persona_id,
persona_name: profile.persona_name.clone(),
club_name: profile.club_name.clone(),
club_abbr: profile.club_abbr.clone(),
coins: profile.coins,
counts,
definitions,
identity,
squad,
}
}
impl std::fmt::Display for Report {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let d = &self.definitions;
let id = &self.identity;
let sq = &self.squad;
writeln!(
f,
"OpenFUT FIFA17 real-profile import — DRY RUN (no writes)"
)?;
writeln!(
f,
"========================================================"
)?;
writeln!(
f,
"TARGET game={} persona={}/{} club={}/{} coins={}",
self.game,
self.persona_id,
self.persona_name,
self.club_name,
self.club_abbr,
self.coins
)?;
writeln!(f, "\nSOURCE ITEMS (disjoint classes)")?;
writeln!(
f,
" total={} player_cards={} consumables={} staff={} other={} balances={}",
self.counts.total,
self.counts.player_cards,
self.counts.consumables,
self.counts.staff,
self.counts.other,
self.counts.balances()
)?;
writeln!(f, "\nPLAYER DEFINITIONS (card_id = fifa17_<resourceId>)")?;
writeln!(
f,
" supported={} (base={} versioned={}) unsupported={} conflicts={}",
d.supported.len(),
d.base_defs,
d.versioned_defs,
d.unsupported.len(),
d.conflicts.len()
)?;
for (rid, copies, why) in &d.unsupported {
writeln!(
f,
" UNSUPPORTED resourceId={rid} copies={copies}: {why:?}"
)?;
}
for c in &d.conflicts {
writeln!(
f,
" CONFLICT resourceId={} wire_ids={:?} distinct_defs={}",
c.resource_id,
c.wire_ids,
c.distinct.len()
)?;
}
writeln!(f, "\nIDENTITY (preserve existing Python wire ids)")?;
writeln!(
f,
" import_instances={} live_min={:?} live_max={:?} duplicates={}",
id.import_wire_ids.len(),
id.live_min,
id.live_max,
id.duplicate_wire_ids.len()
)?;
writeln!(
f,
" source_watermark(nextItemId)={} => next_rust_allocation={}",
id.source_watermark, id.next_allocation
)?;
writeln!(f, "\nACTIVE SQUAD")?;
if sq.present {
writeln!(
f,
" formation={} occupied={} supported={} unsupported={} captain={:?} manager_refs={:?}",
sq.formation,
sq.occupied_slots,
sq.supported_slots,
sq.unsupported_slots.len(),
sq.captain_wire_id,
sq.manager_wire_ids
)?;
for (idx, wid) in &sq.unsupported_slots {
writeln!(f, " UNSUPPORTED slot index={idx} wire_id={wid}")?;
}
} else {
writeln!(f, " (no squad in profile)")?;
}
writeln!(f, "\nRESULT")?;
writeln!(
f,
" would_import_players={} deferred_consumables={} deferred_staff={} unsupported_players(defs)={}",
id.import_wire_ids.len(),
self.counts.consumables,
self.counts.staff,
d.unsupported.len()
)?;
let blockers = self.blockers();
if blockers.is_empty() {
writeln!(f, " BLOCKERS: none")?;
} else {
writeln!(f, " BLOCKERS ({}):", blockers.len())?;
for b in &blockers {
writeln!(f, " - {b}")?;
}
}
Ok(())
}
}
#[cfg(test)]
mod tests;