strawberry/tests/test_perf_kpi.py
trvon a5f2dbedfc
KPI: bound collection and reject incomplete measurements
Use suspend-aware BOOTTIME waits and owned-container cleanup without changing the workload. Reject missing coverage and preserve producer/report failures. Include runtime/helper regressions, constrained build-job support and live exec-guard crash safety. Document the failed live/idle gates without relaxing thresholds or claiming production acceptance.
2026-11-03 06:57:00 -07:00

169 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Unprivileged KPI regressions; never invoke the real Docker daemon."""
import importlib.util
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("perf_kpi_report", ROOT / "scripts/perf_kpi_report.py")
assert SPEC is not None and SPEC.loader is not None
report = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(report)
sys.modules["perf_kpi_report"] = report
def load_module(name):
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py"))
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
runtime = load_module("perf_kpi_runtime")
host = load_module("perf_kpi")
workload = load_module("perf_kpi_workload")
class ReportTests(unittest.TestCase):
def fixture(self, path):
(path / "phases.log").write_text("0 start\n1 startup\n2 idle\n3 storm\n4 large\n5 detect\n6 reload\n7 end\n")
(path / "final_status.json").write_text(json.dumps({"data": {
"timings": {"exec_decision": {"p95_ms": 1}},
"monitor": {"events_dropped": 0}, "pipeline": {"events": 1, "scanned": 1},
}}))
def test_live_guard_parent_installs_crash_safety(self):
source = (ROOT / "tests/exec_guard_live.cpp").read_text()
prefix = source.split("int main()", 1)[1].split("ExecGuard guard;", 1)[0]
self.assertIn("install_crash_safety();", prefix)
def test_missing_samples_fail_closed(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
self.assertTrue(report.summarize(path)["failures"])
def test_complete_measurements_can_pass(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
(path / "phases.log").write_text((path / "phases.log").read_text() + "storm_execs 2\nlarge_execs 2\ndetect_execs 2\n")
(path / "docker_stats.csv").write_text("t,cpu_pct,mem\n" + "".join(f"{stamp},0.1,10MiB\n" for stamp in (1.1, 2.1, 2.2, 2.3, 3.1, 4.1, 5.1, 6.1)))
final = {"data": {"timings": {"exec_decision": {"count": 1, "p95_ms": 1}},
"monitor": {"events_dropped": 0}, "pipeline": {"scanned": 1}, "prevention": {"active": True}}}
(path / "final_status.json").write_text(json.dumps(final))
self.assertEqual(report.summarize(path)["failures"], [])
def test_compare_propagates_failed_gate(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
self.assertEqual(report.main(["report", "--compare", str(path), str(path)]), 5)
class RuntimeTests(unittest.TestCase):
def test_boottime_timer_ticks(self):
ticks = []
runtime.until(runtime.boot() + 0.03, tick=lambda: ticks.append(1), interval=0.005)
self.assertTrue(ticks)
def test_invalid_timer_does_not_leak(self):
before = len(list(Path("/proc/self/fd").iterdir()))
for value in (float("nan"), float("inf"), -1, 0):
with self.assertRaises(ValueError):
runtime.Timer(value)
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
def test_short_child_and_timeout(self):
result = runtime.capture([sys.executable, "-c", "raise SystemExit(7)"], 1)
self.assertEqual(result.returncode, 7)
with self.assertRaises(subprocess.TimeoutExpired):
runtime.capture([sys.executable, "-c", "import select;select.select([],[],[],30)"], 0.03)
def test_pidfd_failure_closes_timer(self):
process = subprocess.Popen([sys.executable, "-c", "import select;select.select([],[],[],30)"], start_new_session=True)
try:
before = len(list(Path("/proc/self/fd").iterdir()))
with mock.patch.object(runtime.os, "pidfd_open", side_effect=OSError("injected")), self.assertRaises(OSError):
runtime.until(runtime.boot() + 1, process)
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
finally:
runtime.stop(process)
def test_clock_source_and_expired_wait(self):
with mock.patch.object(runtime.time, "clock_gettime", return_value=10) as clock:
self.assertIsNone(runtime.until(9))
clock.assert_called_with(runtime.time.CLOCK_BOOTTIME)
class WorkloadTests(unittest.TestCase):
def test_daemon_start_failure_stops_before_workload(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
binary = path / "daemon"
binary.write_text("#!/usr/bin/env python3\nraise SystemExit(9)\n")
binary.chmod(0o700)
with mock.patch.object(workload, "OUT", path), mock.patch.object(workload, "BIN", str(binary)), self.assertRaises(RuntimeError):
workload.main()
self.assertEqual((path / "phases.log").read_text().split()[1], "start")
def test_malformed_status_is_not_readiness(self):
with self.assertRaises(RuntimeError):
workload.status_payload(b"not-json")
class HarnessTests(unittest.TestCase):
def fake_docker(self, directory):
path = directory / "docker"
path.write_text('#!/usr/bin/env python3\nimport os,sys,select\n'
'with open(os.environ["FAKE_CALLS"],"a") as f:f.write(sys.argv[1]+"\\n")\n'
'if sys.argv[1]=="run":\n'
' if os.environ.get("FAKE_HANG"):select.select([],[],[],30)\n'
' sys.exit(125)\n'
'if sys.argv[1]=="inspect":\n'
' if os.environ.get("FAKE_FOREIGN"):print("foreign");sys.exit(0)\n'
' print("Error: No such object",file=sys.stderr);sys.exit(1)\n'
'sys.exit(0)\n')
path.chmod(0o700)
return path
def run_fake(self, hang=False):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
docker = self.fake_docker(path)
environment = dict(os.environ, KPI_DOCKER=str(docker), KPI_SKIP_BUILD="1", KPI_OUT=str(path / "out"),
KPI_RULES=str(path / "missing"), KPI_TIMEOUT_S="2", FAKE_CALLS=str(path / "calls"))
if hang:
environment["FAKE_HANG"] = "1"
with mock.patch.dict(os.environ, environment, clear=True):
code = host.main(["perf", "fixture"])
state = json.loads((path / "out/lifecycle.json").read_text())
self.assertEqual(code, 124 if hang else 125)
self.assertEqual(state["cleanup"], "already_absent")
self.assertFalse((path / "out/kpi.json").read_text() == "{}")
def test_docker_start_failure_not_masked(self):
self.run_fake()
def test_docker_hang_is_bounded(self):
self.run_fake(hang=True)
def test_foreign_container_never_removed(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
docker = self.fake_docker(path)
with mock.patch.dict(os.environ, {"FAKE_FOREIGN": "1", "FAKE_CALLS": str(path / "calls")}), self.assertRaises(RuntimeError):
host.remove_owned(str(docker), "fixture", "ours")
self.assertNotIn("rm", (path / "calls").read_text().splitlines())
if __name__ == "__main__":
unittest.main(verbosity=2)