funman300
0b66662525
24 transactions across 11 connections from a retail session: login, hub, one pack open, two squad saves, a quick-sell, with before/after state manifests. Raw .ofcap stays gitignored at 0600; the sanitized corpus is committed as adapter fixtures. TWO GAPS FOUND BY AUDITING THE OUTPUT, NOT BY TRUSTING THE SANITISER. 1. `POST /ut/auth` carries `macAddress` and `deviceId`. Session tokens were being redacted correctly and these were not. A committed fixture is a published fixture. 2. Then, with those fixed, the audit fired AGAIN on the file about to be committed: `GET .../phishing/trusteddevice?deviceId=...` puts the id in the QUERY STRING. Three input surfaces carry identifiers -- headers, JSON bodies, and query strings -- and the sanitiser knew about two. Both fixed in the tool rather than by editing the file, with a regression test and a mutation for the query path. AND A THIRD ARTEFACT MIX-UP, in the mutation harness itself. It reported the query-redaction mutation as SURVIVED while a hand-run of the same mutation killed it. Cause: the harness pointed at a stale scratchpad copy of the test that pre-dated the query assertion, so it was faithfully testing the mutated tool against a test that could not detect the mutation. That is the same class as the build guard checking the wrong binary and cargo reusing a binary compiled from mutated source -- the third instance today of measuring the wrong artifact. The harness now resolves ROOT from its own location and runs the COMMITTED test; the stale copy is deleted. Harness committed as scripts/mutate-utas-observe.py so this is repeatable rather than a thing that happened once in a scratch directory. 6/6 killed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
76 lines
3.0 KiB
Python
Executable File
76 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Mutation-test the UTAS recorder. A no-op replacement exits 3 rather than
|
|
counting as a survivor -- a sed that silently does nothing is how a mutation run
|
|
lies about coverage."""
|
|
import os, shutil, subprocess, sys
|
|
|
|
import os as _os
|
|
ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
|
|
# The COMMITTED test, not a scratchpad copy. Pointing this at a stale
|
|
# duplicate made a live mutation look survivable: the old copy had no
|
|
# query-string assertion, so disabling query redaction passed.
|
|
TEST = ROOT + "/scripts/test-utas-observe.py"
|
|
|
|
MUTATIONS = [
|
|
("U1 tee: record but do not forward",
|
|
"scripts/openfut-utas-observe.py",
|
|
" try:\n dst.sendall(data)",
|
|
" try:\n pass"),
|
|
("U2 tee: drop the last byte of every chunk",
|
|
"scripts/openfut-utas-observe.py",
|
|
" dst.sendall(data)",
|
|
" dst.sendall(data[:-1])"),
|
|
("U3 sanitiser: stop redacting secrets",
|
|
"scripts/openfut-utas-observe.py",
|
|
" if any(s in k.lower() for s in SECRET_BODY_KEYS) and isinstance(o[k], str):",
|
|
" if False:"),
|
|
("U4 parser: ignore chunked framing",
|
|
"scripts/openfut-utas-observe.py",
|
|
' if te and "chunked" in te.group(1).lower():',
|
|
" if False:"),
|
|
("U6 sanitiser: stop redacting query parameters",
|
|
"scripts/openfut-utas-observe.py",
|
|
" if eq and any(x in k.lower() for x in SECRET_BODY_KEYS):",
|
|
" if False:"),
|
|
("U5 parser: reverse header order",
|
|
"scripts/openfut-utas-observe.py",
|
|
" out.append([k, v.strip()])",
|
|
" out.insert(0, [k, v.strip()])"),
|
|
]
|
|
|
|
LAST = {}
|
|
def run():
|
|
r = subprocess.run([sys.executable, TEST], cwd=ROOT, capture_output=True, text=True)
|
|
LAST["rc"] = r.returncode
|
|
LAST["out"] = r.stdout
|
|
return r.returncode == 0
|
|
|
|
def main():
|
|
if not run():
|
|
print("BASELINE FAILS"); return 2
|
|
survivors = []
|
|
for name, rel, old, new in MUTATIONS:
|
|
path = os.path.join(ROOT, rel); bak = path + ".mutbak"
|
|
shutil.copy2(path, bak)
|
|
try:
|
|
src = open(path).read()
|
|
mutated = src.replace(old, new, 1)
|
|
if mutated == src:
|
|
print("!! MUTATION DID NOT APPLY: %s" % name); return 3
|
|
open(path, "w").write(mutated)
|
|
killed = not run()
|
|
print(("KILLED " if killed else "SURVIVED ") + name + " [rc=%s]" % LAST.get("rc"))
|
|
if not killed:
|
|
for l in LAST.get("out","").splitlines():
|
|
if "query" in l or "FAIL" in l or "all checks" in l:
|
|
print(" " + l)
|
|
if not killed: survivors.append(name)
|
|
finally:
|
|
shutil.move(bak, path); os.utime(path, None)
|
|
print()
|
|
print("all %d mutations killed" % len(MUTATIONS) if not survivors
|
|
else "%d SURVIVED: %s" % (len(survivors), survivors))
|
|
return 1 if survivors else 0
|
|
|
|
sys.exit(main())
|