#!/usr/bin/env python3 """Mutation-test the UTAS recorder. A no-op replacement exits 3 rather than counting as a survivor -- a sed that silently does nothing is how a mutation run lies about coverage.""" import os, shutil, subprocess, sys import os as _os ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))) # The COMMITTED test, not a scratchpad copy. Pointing this at a stale # duplicate made a live mutation look survivable: the old copy had no # query-string assertion, so disabling query redaction passed. TEST = ROOT + "/scripts/test-utas-observe.py" MUTATIONS = [ ("U1 tee: record but do not forward", "scripts/openfut-utas-observe.py", " try:\n dst.sendall(data)", " try:\n pass"), ("U2 tee: drop the last byte of every chunk", "scripts/openfut-utas-observe.py", " dst.sendall(data)", " dst.sendall(data[:-1])"), ("U3 sanitiser: stop redacting secrets", "scripts/openfut-utas-observe.py", " if any(s in k.lower() for s in SECRET_BODY_KEYS) and isinstance(o[k], str):", " if False:"), ("U4 parser: ignore chunked framing", "scripts/openfut-utas-observe.py", ' if te and "chunked" in te.group(1).lower():', " if False:"), ("U6 sanitiser: stop redacting query parameters", "scripts/openfut-utas-observe.py", " if eq and any(x in k.lower() for x in SECRET_BODY_KEYS):", " if False:"), ("U5 parser: reverse header order", "scripts/openfut-utas-observe.py", " out.append([k, v.strip()])", " out.insert(0, [k, v.strip()])"), ] LAST = {} def run(): r = subprocess.run([sys.executable, TEST], cwd=ROOT, capture_output=True, text=True) LAST["rc"] = r.returncode LAST["out"] = r.stdout return r.returncode == 0 def main(): if not run(): print("BASELINE FAILS"); return 2 survivors = [] for name, rel, old, new in MUTATIONS: path = os.path.join(ROOT, rel); bak = path + ".mutbak" shutil.copy2(path, bak) try: src = open(path).read() mutated = src.replace(old, new, 1) if mutated == src: print("!! MUTATION DID NOT APPLY: %s" % name); return 3 open(path, "w").write(mutated) killed = not run() print(("KILLED " if killed else "SURVIVED ") + name + " [rc=%s]" % LAST.get("rc")) if not killed: for l in LAST.get("out","").splitlines(): if "query" in l or "FAIL" in l or "all checks" in l: print(" " + l) if not killed: survivors.append(name) finally: shutil.move(bak, path); os.utime(path, None) print() print("all %d mutations killed" % len(MUTATIONS) if not survivors else "%d SURVIVED: %s" % (len(survivors), survivors)) return 1 if survivors else 0 sys.exit(main())