redirector-host: reproduce the oracle's connection lifecycle, not just its bytes
Two live gate attempts failed with "An error occurred downloading the FUT
Squad Update" while the redirect response was verified byte-identical to the
Python oracle. Rolling back to the Python redirector fixed it, so the response
bytes were never the whole contract.
Log archaeology found the discriminator: the client polls
/fifa17/fut/rosterupdate.xml ~4x/min in every successful FUT session, and the
only gap in 300 recorded fetches is 03:47-03:58 -- exactly the two
Rust-redirector sessions. The Blaze RPC sequence over those sessions is
identical (msgNum 0-53), so the divergence is entirely outside Blaze.
A differential lifecycle probe against both redirectors found the two
behaviours this host never reproduced:
* the oracle drains the request body per Content-Length; this host stopped
at the header terminator, leaving unread data in the receive queue, which
makes Linux close with RST rather than FIN
* the oracle holds the connection open ~300ms before closing
(time.sleep(0.3)); this host closed at 0ms
Both are now reproduced. The dwell is a named constant, ORACLE_CLOSE_DWELL,
overridable only so the causal experiment -- set it to 0, confirm the failure
returns -- can be run without a rebuild.
The suite could not have caught either: it sent Content-Length: 0, so there
was never a body to drain. It now POSTs a body, and asserts a split-write body
is fully consumed.
Testing the drain via client-visible symptoms does NOT work -- verified by
mutation: with the drain removed the client still reads the buffered response
and sees close_notify before any reset. So the host records a per-connection
ConnOutcome and the test asserts on that. Both mutations (no-dwell, no-drain)
are now each caught by exactly one test.
This does not yet prove causation for the FUT Squad Update failure; it removes
the only two measured divergences. Gate 6 is the test.
This commit is contained in:
@@ -9,7 +9,7 @@
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpStream;
|
||||
|
||||
use openfut_redirector_host::{bind, tls, RedirectorConfig};
|
||||
use openfut_redirector_host::{bind, tls, BodyRead, RedirectorConfig};
|
||||
use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
|
||||
|
||||
/// Start the real host on an ephemeral port.
|
||||
@@ -24,6 +24,14 @@ fn start(advertise: &str) -> String {
|
||||
addr
|
||||
}
|
||||
|
||||
/// A request body, because a retail client POSTs one.
|
||||
///
|
||||
/// The first version of this suite sent `Content-Length: 0`. That is why it
|
||||
/// passed against a host that never drained the body — the blind spot was in
|
||||
/// the measurement, not the assertion.
|
||||
const REQUEST_BODY: &[u8] = b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||
<serverinstancerequest><name>fifa-2017-pc</name></serverinstancerequest>";
|
||||
|
||||
/// A client that behaves like the captured FIFA 17 ClientHello.
|
||||
fn fifa_like_request(addr: &str) -> (String, Vec<u8>) {
|
||||
let mut b = SslConnector::builder(SslMethod::tls()).expect("connector");
|
||||
@@ -54,14 +62,16 @@ fn fifa_like_request(addr: &str) -> (String, Vec<u8>) {
|
||||
.unwrap_or("?")
|
||||
);
|
||||
|
||||
stream
|
||||
.write_all(
|
||||
b"POST /redirector/getServerInstance HTTP/1.1\r\n\
|
||||
Host: winter15.gosredirector.ea.com\r\n\
|
||||
Content-Type: application/xml\r\n\
|
||||
Content-Length: 0\r\n\r\n",
|
||||
)
|
||||
.expect("write request");
|
||||
let mut req = format!(
|
||||
"POST /redirector/getServerInstance HTTP/1.1\r\n\
|
||||
Host: winter15.gosredirector.ea.com\r\n\
|
||||
Content-Type: application/xml\r\n\
|
||||
Content-Length: {}\r\n\r\n",
|
||||
REQUEST_BODY.len()
|
||||
)
|
||||
.into_bytes();
|
||||
req.extend_from_slice(REQUEST_BODY);
|
||||
stream.write_all(&req).expect("write request");
|
||||
stream.flush().ok();
|
||||
|
||||
let mut out = Vec::new();
|
||||
@@ -136,6 +146,127 @@ fn the_advertised_address_reaches_the_wire() {
|
||||
assert!(!text.contains("198.51.100.7"));
|
||||
}
|
||||
|
||||
/// The body must be DRAINED, not left in the receive queue.
|
||||
///
|
||||
/// A server that answers and closes with unread data makes the kernel send RST
|
||||
/// instead of FIN. Sending the body in a second write, after a pause, is what
|
||||
/// makes this observable: a host that stops at the header terminator has
|
||||
/// already responded by the time the body arrives.
|
||||
#[test]
|
||||
fn the_request_body_is_drained_before_closing() {
|
||||
let mut cfg = RedirectorConfig::for_test("198.51.100.7");
|
||||
cfg.listen_port = 0;
|
||||
let server = bind(cfg).expect("host binds");
|
||||
let addr = server.local_addr.to_string();
|
||||
let outcomes = server.outcomes();
|
||||
std::thread::spawn(move || {
|
||||
let _ = server.run();
|
||||
});
|
||||
|
||||
let mut b = SslConnector::builder(SslMethod::tls()).expect("connector");
|
||||
b.set_cipher_list(tls::OBSERVED_CLIENT_SUITES).expect("list");
|
||||
b.set_verify(SslVerifyMode::NONE);
|
||||
let sock = TcpStream::connect(&addr).expect("connect");
|
||||
let ssl = b
|
||||
.build()
|
||||
.configure()
|
||||
.expect("configure")
|
||||
.verify_hostname(false)
|
||||
.into_ssl("winter15.gosredirector.ea.com")
|
||||
.expect("ssl");
|
||||
let mut stream = openssl::ssl::SslStream::new(ssl, sock).expect("stream");
|
||||
stream.connect().expect("handshake");
|
||||
|
||||
stream
|
||||
.write_all(
|
||||
format!(
|
||||
"POST /redirector/getServerInstance HTTP/1.1\r\n\
|
||||
Content-Length: {}\r\n\r\n",
|
||||
REQUEST_BODY.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.expect("write head");
|
||||
stream.flush().ok();
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
|
||||
// If the host had already answered and closed, this write — or the read
|
||||
// that follows it — is where the reset surfaces.
|
||||
stream.write_all(REQUEST_BODY).expect("body must be accepted");
|
||||
stream.flush().ok();
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut chunk = [0u8; 4096];
|
||||
loop {
|
||||
match stream.read(&mut chunk) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => out.extend_from_slice(&chunk[..n]),
|
||||
Err(e) => panic!("connection broke instead of closing cleanly: {e}"),
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
String::from_utf8_lossy(&out).contains("<serverinstanceinfo>"),
|
||||
"expected the full response after a split-write body, got {} bytes",
|
||||
out.len()
|
||||
);
|
||||
|
||||
// The assertion that carries the weight. Checking only that the client got
|
||||
// its response does NOT detect an undrained body — verified by mutation:
|
||||
// with the drain removed, the client still reads the buffered response and
|
||||
// sees close_notify before any reset. Only the host's own record shows
|
||||
// whether it consumed what the client sent.
|
||||
let rec = outcomes.lock().expect("outcomes");
|
||||
let last = rec.last().expect("one connection was recorded");
|
||||
assert_eq!(
|
||||
last.body,
|
||||
BodyRead::Complete(REQUEST_BODY.len()),
|
||||
"the host did not drain the request body: {last:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The oracle holds the connection open after responding; so must this host.
|
||||
///
|
||||
/// Asserted as a floor rather than an exact figure — this measures scheduling,
|
||||
/// so the guarantee is "at least the oracle's dwell", not "exactly 300ms".
|
||||
#[test]
|
||||
fn the_connection_is_held_open_for_the_oracle_dwell() {
|
||||
use openfut_redirector_host::config::ORACLE_CLOSE_DWELL;
|
||||
|
||||
let addr = start("198.51.100.7");
|
||||
let t0 = std::time::Instant::now();
|
||||
let (_, body) = fifa_like_request(&addr);
|
||||
let held = t0.elapsed();
|
||||
|
||||
assert!(!body.is_empty(), "no response");
|
||||
assert!(
|
||||
held >= ORACLE_CLOSE_DWELL,
|
||||
"closed after {held:?}, before the oracle's {ORACLE_CLOSE_DWELL:?} dwell"
|
||||
);
|
||||
}
|
||||
|
||||
/// The dwell is configurable so the causal experiment can be run, and a bad
|
||||
/// value is refused rather than silently ignored.
|
||||
#[test]
|
||||
fn the_dwell_is_overridable_for_the_causal_experiment() {
|
||||
let mut cfg = RedirectorConfig::for_test("198.51.100.7");
|
||||
cfg.listen_port = 0;
|
||||
cfg.close_dwell = std::time::Duration::ZERO;
|
||||
let server = bind(cfg).expect("binds");
|
||||
let addr = server.local_addr.to_string();
|
||||
std::thread::spawn(move || {
|
||||
let _ = server.run();
|
||||
});
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let (_, body) = fifa_like_request(&addr);
|
||||
assert!(!body.is_empty(), "no response");
|
||||
assert!(
|
||||
t0.elapsed() < std::time::Duration::from_millis(250),
|
||||
"a zero dwell should close promptly, took {:?}",
|
||||
t0.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
/// A client that offers only forward-secret suites — i.e. anything modern —
|
||||
/// must fail, confirming this listener really is the legacy island it claims
|
||||
/// to be and has not silently acquired a modern policy.
|
||||
|
||||
Reference in New Issue
Block a user