Use suspend-aware BOOTTIME waits and owned-container cleanup without changing the workload. Reject missing coverage and preserve producer/report failures. Include runtime/helper regressions, constrained build-job support and live exec-guard crash safety. Document the failed live/idle gates without relaxing thresholds or claiming production acceptance.
169 lines
7.6 KiB
Python
169 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Unprivileged KPI regressions; never invoke the real Docker daemon."""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SPEC = importlib.util.spec_from_file_location("perf_kpi_report", ROOT / "scripts/perf_kpi_report.py")
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
report = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(report)
|
|
sys.modules["perf_kpi_report"] = report
|
|
|
|
|
|
def load_module(name):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py"))
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
runtime = load_module("perf_kpi_runtime")
|
|
host = load_module("perf_kpi")
|
|
workload = load_module("perf_kpi_workload")
|
|
|
|
|
|
class ReportTests(unittest.TestCase):
|
|
def fixture(self, path):
|
|
(path / "phases.log").write_text("0 start\n1 startup\n2 idle\n3 storm\n4 large\n5 detect\n6 reload\n7 end\n")
|
|
(path / "final_status.json").write_text(json.dumps({"data": {
|
|
"timings": {"exec_decision": {"p95_ms": 1}},
|
|
"monitor": {"events_dropped": 0}, "pipeline": {"events": 1, "scanned": 1},
|
|
}}))
|
|
|
|
def test_live_guard_parent_installs_crash_safety(self):
|
|
source = (ROOT / "tests/exec_guard_live.cpp").read_text()
|
|
prefix = source.split("int main()", 1)[1].split("ExecGuard guard;", 1)[0]
|
|
self.assertIn("install_crash_safety();", prefix)
|
|
|
|
def test_missing_samples_fail_closed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
self.assertTrue(report.summarize(path)["failures"])
|
|
|
|
def test_complete_measurements_can_pass(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
(path / "phases.log").write_text((path / "phases.log").read_text() + "storm_execs 2\nlarge_execs 2\ndetect_execs 2\n")
|
|
(path / "docker_stats.csv").write_text("t,cpu_pct,mem\n" + "".join(f"{stamp},0.1,10MiB\n" for stamp in (1.1, 2.1, 2.2, 2.3, 3.1, 4.1, 5.1, 6.1)))
|
|
final = {"data": {"timings": {"exec_decision": {"count": 1, "p95_ms": 1}},
|
|
"monitor": {"events_dropped": 0}, "pipeline": {"scanned": 1}, "prevention": {"active": True}}}
|
|
(path / "final_status.json").write_text(json.dumps(final))
|
|
self.assertEqual(report.summarize(path)["failures"], [])
|
|
|
|
def test_compare_propagates_failed_gate(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
self.assertEqual(report.main(["report", "--compare", str(path), str(path)]), 5)
|
|
|
|
|
|
class RuntimeTests(unittest.TestCase):
|
|
def test_boottime_timer_ticks(self):
|
|
ticks = []
|
|
runtime.until(runtime.boot() + 0.03, tick=lambda: ticks.append(1), interval=0.005)
|
|
self.assertTrue(ticks)
|
|
|
|
def test_invalid_timer_does_not_leak(self):
|
|
before = len(list(Path("/proc/self/fd").iterdir()))
|
|
for value in (float("nan"), float("inf"), -1, 0):
|
|
with self.assertRaises(ValueError):
|
|
runtime.Timer(value)
|
|
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
|
|
|
|
def test_short_child_and_timeout(self):
|
|
result = runtime.capture([sys.executable, "-c", "raise SystemExit(7)"], 1)
|
|
self.assertEqual(result.returncode, 7)
|
|
with self.assertRaises(subprocess.TimeoutExpired):
|
|
runtime.capture([sys.executable, "-c", "import select;select.select([],[],[],30)"], 0.03)
|
|
|
|
def test_pidfd_failure_closes_timer(self):
|
|
process = subprocess.Popen([sys.executable, "-c", "import select;select.select([],[],[],30)"], start_new_session=True)
|
|
try:
|
|
before = len(list(Path("/proc/self/fd").iterdir()))
|
|
with mock.patch.object(runtime.os, "pidfd_open", side_effect=OSError("injected")), self.assertRaises(OSError):
|
|
runtime.until(runtime.boot() + 1, process)
|
|
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
|
|
finally:
|
|
runtime.stop(process)
|
|
|
|
def test_clock_source_and_expired_wait(self):
|
|
with mock.patch.object(runtime.time, "clock_gettime", return_value=10) as clock:
|
|
self.assertIsNone(runtime.until(9))
|
|
clock.assert_called_with(runtime.time.CLOCK_BOOTTIME)
|
|
|
|
|
|
class WorkloadTests(unittest.TestCase):
|
|
def test_daemon_start_failure_stops_before_workload(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
binary = path / "daemon"
|
|
binary.write_text("#!/usr/bin/env python3\nraise SystemExit(9)\n")
|
|
binary.chmod(0o700)
|
|
with mock.patch.object(workload, "OUT", path), mock.patch.object(workload, "BIN", str(binary)), self.assertRaises(RuntimeError):
|
|
workload.main()
|
|
self.assertEqual((path / "phases.log").read_text().split()[1], "start")
|
|
|
|
def test_malformed_status_is_not_readiness(self):
|
|
with self.assertRaises(RuntimeError):
|
|
workload.status_payload(b"not-json")
|
|
|
|
|
|
class HarnessTests(unittest.TestCase):
|
|
def fake_docker(self, directory):
|
|
path = directory / "docker"
|
|
path.write_text('#!/usr/bin/env python3\nimport os,sys,select\n'
|
|
'with open(os.environ["FAKE_CALLS"],"a") as f:f.write(sys.argv[1]+"\\n")\n'
|
|
'if sys.argv[1]=="run":\n'
|
|
' if os.environ.get("FAKE_HANG"):select.select([],[],[],30)\n'
|
|
' sys.exit(125)\n'
|
|
'if sys.argv[1]=="inspect":\n'
|
|
' if os.environ.get("FAKE_FOREIGN"):print("foreign");sys.exit(0)\n'
|
|
' print("Error: No such object",file=sys.stderr);sys.exit(1)\n'
|
|
'sys.exit(0)\n')
|
|
path.chmod(0o700)
|
|
return path
|
|
|
|
def run_fake(self, hang=False):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
docker = self.fake_docker(path)
|
|
environment = dict(os.environ, KPI_DOCKER=str(docker), KPI_SKIP_BUILD="1", KPI_OUT=str(path / "out"),
|
|
KPI_RULES=str(path / "missing"), KPI_TIMEOUT_S="2", FAKE_CALLS=str(path / "calls"))
|
|
if hang:
|
|
environment["FAKE_HANG"] = "1"
|
|
with mock.patch.dict(os.environ, environment, clear=True):
|
|
code = host.main(["perf", "fixture"])
|
|
state = json.loads((path / "out/lifecycle.json").read_text())
|
|
self.assertEqual(code, 124 if hang else 125)
|
|
self.assertEqual(state["cleanup"], "already_absent")
|
|
self.assertFalse((path / "out/kpi.json").read_text() == "{}")
|
|
|
|
def test_docker_start_failure_not_masked(self):
|
|
self.run_fake()
|
|
|
|
def test_docker_hang_is_bounded(self):
|
|
self.run_fake(hang=True)
|
|
|
|
def test_foreign_container_never_removed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
docker = self.fake_docker(path)
|
|
with mock.patch.dict(os.environ, {"FAKE_FOREIGN": "1", "FAKE_CALLS": str(path / "calls")}), self.assertRaises(RuntimeError):
|
|
host.remove_owned(str(docker), "fixture", "ours")
|
|
self.assertNotIn("rm", (path / "calls").read_text().splitlines())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|