#!/usr/bin/env python3 """Unprivileged KPI regressions; never invoke the real Docker daemon.""" import contextlib import importlib.util import io import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from unittest import mock ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location("perf_kpi_report", ROOT / "scripts/perf_kpi_report.py") assert SPEC is not None and SPEC.loader is not None report = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(report) sys.modules["perf_kpi_report"] = report def load_module(name): spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py")) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module runtime = load_module("perf_kpi_runtime") host = load_module("perf_kpi") workload = load_module("perf_kpi_workload") # Daemon RSS per phase of the fixture lane; the v0.2.9 shape is this plus the # documented +130..141 MB growth (build/perf/kpi-v0.2.{8,9}). FIXTURE_RSS = {"startup": 70.0, "idle": 72.0, "storm": 88.0, "large": 89.0, "detect": 89.0, "reload": 106.0} class ReportTests(unittest.TestCase): def fixture(self, path, rss=None): path.mkdir(parents=True, exist_ok=True) (path / "phases.log").write_text("0 start\n1 startup\n2 idle\n3 storm\n4 large\n5 detect\n6 reload\n7 end\n") (path / "final_status.json").write_text(json.dumps({"data": { "timings": {"exec_decision": {"p95_ms": 1}}, "monitor": {"events_dropped": 0}, "pipeline": {"events": 1, "scanned": 1}, }})) self.status_samples(path, rss) def status_samples(self, path, rss): """One daemon status sample per workload phase, carrying resources.rss_mb.""" stamps = {"startup": 1.5, "idle": 2.5, "storm": 3.5, "large": 4.5, "detect": 5.5, "reload": 6.5} samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': rss[name]}}})}" for name, when in stamps.items() if rss and name in rss] (path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples)) def peak_samples(self, path, rss, peak): """Two samples per phase: a mid-phase spike, then the phase-end value.""" stamps = {"startup": (1.2, 1.5), "idle": (2.2, 2.5), "storm": (3.2, 3.5), "large": (4.2, 4.5), "detect": (5.2, 5.5), "reload": (6.2, 6.5)} samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': value}}})}" for name, (spike_at, end_at) in stamps.items() if rss and name in rss for when, value in ((spike_at, peak[name]), (end_at, rss[name]))] (path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples)) def complete(self, path, rss, peak=None): """A run that passes every gate; the caller supplies the daemon RSS samples.""" self.fixture(path, rss) if peak: self.peak_samples(path, rss, peak) (path / "phases.log").write_text((path / "phases.log").read_text() + "storm_execs 2\nlarge_execs 2\ndetect_execs 2\n") (path / "docker_stats.csv").write_text("t,cpu_pct,mem\n" + "".join(f"{stamp},0.1,10MiB\n" for stamp in (1.1, 2.1, 2.2, 2.3, 3.1, 4.1, 5.1, 6.1))) final = {"data": {"timings": {"exec_decision": {"count": 1, "p95_ms": 1}}, "monitor": {"events_dropped": 0}, "pipeline": {"scanned": 1}, "prevention": {"active": True}}} (path / "final_status.json").write_text(json.dumps(final)) def compare(self, a, b): out = io.StringIO() with contextlib.redirect_stdout(out): code = report.main(["report", "--compare", str(a), str(b)]) return code, out.getvalue() def run_report(self, a, b, rss_b, rss_a=None): self.complete(a, FIXTURE_RSS if rss_a is None else rss_a) self.complete(b, rss_b) return self.compare(a, b) def run_peaked_report(self, a, b, peak_b, peak_a=None): self.complete(a, FIXTURE_RSS, peak_a) self.complete(b, FIXTURE_RSS, peak_b) return self.compare(a, b) def test_live_guard_parent_installs_crash_safety(self): source = (ROOT / "tests/exec_guard_live.cpp").read_text() prefix = source.split("int main()", 1)[1].split("ExecGuard guard;", 1)[0] self.assertIn("install_crash_safety();", prefix) def test_missing_samples_fail_closed(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.fixture(path) self.assertTrue(report.summarize(path)["failures"]) def test_complete_measurements_can_pass(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.complete(path, FIXTURE_RSS) summary = report.summarize(path) self.assertEqual(summary["failures"], []) self.assertEqual(summary["phases"]["reload"]["daemon_rss_last_mb"], 106.0) self.assertEqual(summary["phases"]["reload"]["daemon_rss_samples"], 1) def test_compare_propagates_failed_gate(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.fixture(path) self.assertEqual(report.main(["report", "--compare", str(path), str(path)]), 5) def test_missing_daemon_rss_samples_fail_closed(self): # Every other gate passes; only the daemon RSS samples are withheld. with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.complete(path, None) failures = report.summarize(path)["failures"] self.assertIn("daemon RSS samples unavailable in reload", failures) self.assertIn("daemon RSS samples unavailable in idle", failures) with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(report.main(["report", str(path)]), 5) def test_daemon_rss_peak_over_threshold_fails(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.complete(path, dict(FIXTURE_RSS, reload=600.0)) self.assertIn("reload daemon RSS peak 600.0 MB > 512.0 MB", report.summarize(path)["failures"]) def test_compare_fails_on_memory_regression(self): # The documented v0.2.8 -> v0.2.9 shape: every phase grew by ~130 MB. with tempfile.TemporaryDirectory() as directory: a, b = Path(directory) / "a", Path(directory) / "b" code, out = self.run_report(a, b, {name: value + 130 for name, value in FIXTURE_RSS.items()}) self.assertEqual(code, 5) blob = " ".join(out.split()) self.assertIn("reload.daemon_rss_last_mb 106.0 236.0 +130.00", blob) self.assertIn("OVER BUDGET: reload daemon RSS 236.0 MB vs baseline 106.0 MB " "(+130.0 MB > allowed +32.0 MB)", blob) def test_compare_within_repeat_spread_passes(self): with tempfile.TemporaryDirectory() as directory: a, b = Path(directory) / "a", Path(directory) / "b" code, out = self.run_report(a, b, {name: value + 20 for name, value in FIXTURE_RSS.items()}) self.assertEqual(code, 0) self.assertNotIn("OVER BUDGET", out) def test_compare_fails_on_peak_only_regression(self): # Phase-end RSS is identical in every phase; only the peak grew, so a # gate that budgets daemon_rss_last_mb alone lets this run pass. with tempfile.TemporaryDirectory() as directory: a, b = Path(directory) / "a", Path(directory) / "b" code, out = self.run_peaked_report(a, b, {name: value + 200 for name, value in FIXTURE_RSS.items()}) self.assertEqual(code, 5) blob = " ".join(out.split()) self.assertIn("reload.daemon_rss_last_mb 106.0 106.0", blob) self.assertIn("reload.daemon_rss_max_mb 106.0 306.0", blob) self.assertIn("OVER BUDGET: reload daemon RSS peak 306.0 MB vs baseline 106.0 MB " "(+200.0 MB > allowed +64.0 MB)", blob) self.assertNotIn("OVER BUDGET: reload daemon RSS 106.0", blob) def test_compare_within_peak_spread_passes(self): # The worst same-version peak spread measured on the lane (+59.6 MB, # idle, build/perf/kpi-v0.2.8-r1 -> -r2) is spread, not a regression. with tempfile.TemporaryDirectory() as directory: a, b = Path(directory) / "a", Path(directory) / "b" code, out = self.run_peaked_report(a, b, {name: value + 59 for name, value in FIXTURE_RSS.items()}) self.assertEqual(code, 0) self.assertNotIn("OVER BUDGET", out) def test_compare_missing_peak_samples_fail_closed(self): # A run without daemon RSS samples must fail the gate instead of # comparing as "no peak growth". with tempfile.TemporaryDirectory() as directory: a, b = Path(directory) / "a", Path(directory) / "b" self.complete(a, FIXTURE_RSS) self.complete(b, None) code, out = self.compare(a, b) self.assertEqual(code, 5) blob = " ".join(out.split()) self.assertIn("B gate failures: daemon RSS samples unavailable", blob) self.assertIn("reload RSS peak unavailable in A or B", blob) def test_rss_budget_scales_with_baseline(self): # max(32 MB, 15% of A): 40 MB of growth fails on a small baseline and # passes on a large one. with tempfile.TemporaryDirectory() as directory: a, b, c, d = (Path(directory) / name for name in "abcd") code, _ = self.run_report(a, b, {name: value + 40 for name, value in FIXTURE_RSS.items()}) self.assertEqual(code, 5) code, out = self.run_report(c, d, dict.fromkeys(FIXTURE_RSS, 340.0), rss_a=dict.fromkeys(FIXTURE_RSS, 300.0)) self.assertEqual(code, 0) self.assertNotIn("OVER BUDGET", out) def test_compare_reports_gate_failures(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) self.fixture(path) code, out = self.compare(path, path) self.assertEqual(code, 5) blob = " ".join(out.split()) self.assertIn("A gate failures:", blob) self.assertIn("daemon RSS samples unavailable in reload", blob) class RuntimeTests(unittest.TestCase): def test_boottime_timer_ticks(self): ticks = [] runtime.until(runtime.boot() + 0.03, tick=lambda: ticks.append(1), interval=0.005) self.assertTrue(ticks) def test_invalid_timer_does_not_leak(self): before = len(list(Path("/proc/self/fd").iterdir())) for value in (float("nan"), float("inf"), -1, 0): with self.assertRaises(ValueError): runtime.Timer(value) self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir()))) def test_short_child_and_timeout(self): result = runtime.capture([sys.executable, "-c", "raise SystemExit(7)"], 1) self.assertEqual(result.returncode, 7) with self.assertRaises(subprocess.TimeoutExpired): runtime.capture([sys.executable, "-c", "import select;select.select([],[],[],30)"], 0.03) def test_pidfd_failure_closes_timer(self): process = subprocess.Popen([sys.executable, "-c", "import select;select.select([],[],[],30)"], start_new_session=True) try: before = len(list(Path("/proc/self/fd").iterdir())) with mock.patch.object(runtime.os, "pidfd_open", side_effect=OSError("injected")), self.assertRaises(OSError): runtime.until(runtime.boot() + 1, process) self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir()))) finally: runtime.stop(process) def test_clock_source_and_expired_wait(self): with mock.patch.object(runtime.time, "clock_gettime", return_value=10) as clock: self.assertIsNone(runtime.until(9)) clock.assert_called_with(runtime.time.CLOCK_BOOTTIME) class WorkloadTests(unittest.TestCase): def test_daemon_start_failure_stops_before_workload(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) binary = path / "daemon" binary.write_text("#!/usr/bin/env python3\nraise SystemExit(9)\n") binary.chmod(0o700) with mock.patch.object(workload, "OUT", path), mock.patch.object(workload, "BIN", str(binary)), self.assertRaises(RuntimeError): workload.main() self.assertEqual((path / "phases.log").read_text().split()[1], "start") def test_malformed_status_is_not_readiness(self): with self.assertRaises(RuntimeError): workload.status_payload(b"not-json") class HarnessTests(unittest.TestCase): def fake_docker(self, directory): path = directory / "docker" path.write_text('#!/usr/bin/env python3\nimport os,sys,select\n' 'with open(os.environ["FAKE_CALLS"],"a") as f:f.write(sys.argv[1]+"\\n")\n' 'if sys.argv[1]=="run":\n' ' if os.environ.get("FAKE_HANG"):select.select([],[],[],30)\n' ' sys.exit(125)\n' 'if sys.argv[1]=="inspect":\n' ' if os.environ.get("FAKE_FOREIGN"):print("foreign");sys.exit(0)\n' ' print("Error: No such object",file=sys.stderr);sys.exit(1)\n' 'sys.exit(0)\n') path.chmod(0o700) return path def run_fake(self, hang=False): with tempfile.TemporaryDirectory() as directory: path = Path(directory) docker = self.fake_docker(path) environment = dict(os.environ, KPI_DOCKER=str(docker), KPI_SKIP_BUILD="1", KPI_OUT=str(path / "out"), KPI_RULES=str(path / "missing"), KPI_TIMEOUT_S="2", FAKE_CALLS=str(path / "calls")) if hang: environment["FAKE_HANG"] = "1" with mock.patch.dict(os.environ, environment, clear=True): code = host.main(["perf", "fixture"]) state = json.loads((path / "out/lifecycle.json").read_text()) self.assertEqual(code, 124 if hang else 125) self.assertEqual(state["cleanup"], "already_absent") self.assertFalse((path / "out/kpi.json").read_text() == "{}") def test_docker_start_failure_not_masked(self): self.run_fake() def test_docker_hang_is_bounded(self): self.run_fake(hang=True) def test_foreign_container_never_removed(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) docker = self.fake_docker(path) with mock.patch.dict(os.environ, {"FAKE_FOREIGN": "1", "FAKE_CALLS": str(path / "calls")}), self.assertRaises(RuntimeError): host.remove_owned(str(docker), "fixture", "ours") self.assertNotIn("rm", (path / "calls").read_text().splitlines()) if __name__ == "__main__": unittest.main(verbosity=2)