compare() budgeted daemon_rss_last_mb only, so a regression that spiked and then settled passed: the peak was printed but never checked, leaving just the absolute 512 MB ceiling. Both metrics are now budgeted per phase through one helper — max(32 MB, 15% of the baseline) for the phase-end value and max(64 MB, 15%) for the peak. The 64 MB floor is measured, not chosen: with a shared 32 MB floor three same-version repeat pairs (8-r1->8-r2 idle peak +59.6 MB, 9-r1->9-r2 storm +36.3 MB, 9-r1->9-r3 +36.0 MB) would be called regressions, because a single transient sample spreads about twice as wide as the phase-end value. 64 MB clears all nine same-version pairs while every v0.2.8 -> v0.2.9 pair still trips it; re-checked four same-version pairs here with zero over-budget phases. The peak check adds real signal: on the stored runs it catches v0.2.9's idle peak at +137.4 MB where the last-sample check saw +102.4 MB. Missing samples still fail closed and never read as "no growth".
311 lines
15 KiB
Python
311 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Unprivileged KPI regressions; never invoke the real Docker daemon."""
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SPEC = importlib.util.spec_from_file_location("perf_kpi_report", ROOT / "scripts/perf_kpi_report.py")
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
report = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(report)
|
|
sys.modules["perf_kpi_report"] = report
|
|
|
|
|
|
def load_module(name):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py"))
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
runtime = load_module("perf_kpi_runtime")
|
|
host = load_module("perf_kpi")
|
|
workload = load_module("perf_kpi_workload")
|
|
|
|
|
|
# Daemon RSS per phase of the fixture lane; the v0.2.9 shape is this plus the
|
|
# documented +130..141 MB growth (build/perf/kpi-v0.2.{8,9}).
|
|
FIXTURE_RSS = {"startup": 70.0, "idle": 72.0, "storm": 88.0, "large": 89.0, "detect": 89.0, "reload": 106.0}
|
|
|
|
|
|
class ReportTests(unittest.TestCase):
|
|
def fixture(self, path, rss=None):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
(path / "phases.log").write_text("0 start\n1 startup\n2 idle\n3 storm\n4 large\n5 detect\n6 reload\n7 end\n")
|
|
(path / "final_status.json").write_text(json.dumps({"data": {
|
|
"timings": {"exec_decision": {"p95_ms": 1}},
|
|
"monitor": {"events_dropped": 0}, "pipeline": {"events": 1, "scanned": 1},
|
|
}}))
|
|
self.status_samples(path, rss)
|
|
|
|
def status_samples(self, path, rss):
|
|
"""One daemon status sample per workload phase, carrying resources.rss_mb."""
|
|
stamps = {"startup": 1.5, "idle": 2.5, "storm": 3.5, "large": 4.5, "detect": 5.5, "reload": 6.5}
|
|
samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': rss[name]}}})}"
|
|
for name, when in stamps.items() if rss and name in rss]
|
|
(path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples))
|
|
|
|
def peak_samples(self, path, rss, peak):
|
|
"""Two samples per phase: a mid-phase spike, then the phase-end value."""
|
|
stamps = {"startup": (1.2, 1.5), "idle": (2.2, 2.5), "storm": (3.2, 3.5),
|
|
"large": (4.2, 4.5), "detect": (5.2, 5.5), "reload": (6.2, 6.5)}
|
|
samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': value}}})}"
|
|
for name, (spike_at, end_at) in stamps.items() if rss and name in rss
|
|
for when, value in ((spike_at, peak[name]), (end_at, rss[name]))]
|
|
(path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples))
|
|
|
|
def complete(self, path, rss, peak=None):
|
|
"""A run that passes every gate; the caller supplies the daemon RSS samples."""
|
|
self.fixture(path, rss)
|
|
if peak:
|
|
self.peak_samples(path, rss, peak)
|
|
(path / "phases.log").write_text((path / "phases.log").read_text() + "storm_execs 2\nlarge_execs 2\ndetect_execs 2\n")
|
|
(path / "docker_stats.csv").write_text("t,cpu_pct,mem\n" + "".join(f"{stamp},0.1,10MiB\n" for stamp in (1.1, 2.1, 2.2, 2.3, 3.1, 4.1, 5.1, 6.1)))
|
|
final = {"data": {"timings": {"exec_decision": {"count": 1, "p95_ms": 1}},
|
|
"monitor": {"events_dropped": 0}, "pipeline": {"scanned": 1}, "prevention": {"active": True}}}
|
|
(path / "final_status.json").write_text(json.dumps(final))
|
|
|
|
def compare(self, a, b):
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out):
|
|
code = report.main(["report", "--compare", str(a), str(b)])
|
|
return code, out.getvalue()
|
|
|
|
def run_report(self, a, b, rss_b, rss_a=None):
|
|
self.complete(a, FIXTURE_RSS if rss_a is None else rss_a)
|
|
self.complete(b, rss_b)
|
|
return self.compare(a, b)
|
|
|
|
def run_peaked_report(self, a, b, peak_b, peak_a=None):
|
|
self.complete(a, FIXTURE_RSS, peak_a)
|
|
self.complete(b, FIXTURE_RSS, peak_b)
|
|
return self.compare(a, b)
|
|
|
|
def test_live_guard_parent_installs_crash_safety(self):
|
|
source = (ROOT / "tests/exec_guard_live.cpp").read_text()
|
|
prefix = source.split("int main()", 1)[1].split("ExecGuard guard;", 1)[0]
|
|
self.assertIn("install_crash_safety();", prefix)
|
|
|
|
def test_missing_samples_fail_closed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
self.assertTrue(report.summarize(path)["failures"])
|
|
|
|
def test_complete_measurements_can_pass(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.complete(path, FIXTURE_RSS)
|
|
summary = report.summarize(path)
|
|
self.assertEqual(summary["failures"], [])
|
|
self.assertEqual(summary["phases"]["reload"]["daemon_rss_last_mb"], 106.0)
|
|
self.assertEqual(summary["phases"]["reload"]["daemon_rss_samples"], 1)
|
|
|
|
def test_compare_propagates_failed_gate(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
self.assertEqual(report.main(["report", "--compare", str(path), str(path)]), 5)
|
|
|
|
def test_missing_daemon_rss_samples_fail_closed(self):
|
|
# Every other gate passes; only the daemon RSS samples are withheld.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.complete(path, None)
|
|
failures = report.summarize(path)["failures"]
|
|
self.assertIn("daemon RSS samples unavailable in reload", failures)
|
|
self.assertIn("daemon RSS samples unavailable in idle", failures)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(report.main(["report", str(path)]), 5)
|
|
|
|
def test_daemon_rss_peak_over_threshold_fails(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.complete(path, dict(FIXTURE_RSS, reload=600.0))
|
|
self.assertIn("reload daemon RSS peak 600.0 MB > 512.0 MB", report.summarize(path)["failures"])
|
|
|
|
def test_compare_fails_on_memory_regression(self):
|
|
# The documented v0.2.8 -> v0.2.9 shape: every phase grew by ~130 MB.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b = Path(directory) / "a", Path(directory) / "b"
|
|
code, out = self.run_report(a, b, {name: value + 130 for name, value in FIXTURE_RSS.items()})
|
|
self.assertEqual(code, 5)
|
|
blob = " ".join(out.split())
|
|
self.assertIn("reload.daemon_rss_last_mb 106.0 236.0 +130.00", blob)
|
|
self.assertIn("OVER BUDGET: reload daemon RSS 236.0 MB vs baseline 106.0 MB "
|
|
"(+130.0 MB > allowed +32.0 MB)", blob)
|
|
|
|
def test_compare_within_repeat_spread_passes(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b = Path(directory) / "a", Path(directory) / "b"
|
|
code, out = self.run_report(a, b, {name: value + 20 for name, value in FIXTURE_RSS.items()})
|
|
self.assertEqual(code, 0)
|
|
self.assertNotIn("OVER BUDGET", out)
|
|
|
|
def test_compare_fails_on_peak_only_regression(self):
|
|
# Phase-end RSS is identical in every phase; only the peak grew, so a
|
|
# gate that budgets daemon_rss_last_mb alone lets this run pass.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b = Path(directory) / "a", Path(directory) / "b"
|
|
code, out = self.run_peaked_report(a, b, {name: value + 200 for name, value in FIXTURE_RSS.items()})
|
|
self.assertEqual(code, 5)
|
|
blob = " ".join(out.split())
|
|
self.assertIn("reload.daemon_rss_last_mb 106.0 106.0", blob)
|
|
self.assertIn("reload.daemon_rss_max_mb 106.0 306.0", blob)
|
|
self.assertIn("OVER BUDGET: reload daemon RSS peak 306.0 MB vs baseline 106.0 MB "
|
|
"(+200.0 MB > allowed +64.0 MB)", blob)
|
|
self.assertNotIn("OVER BUDGET: reload daemon RSS 106.0", blob)
|
|
|
|
def test_compare_within_peak_spread_passes(self):
|
|
# The worst same-version peak spread measured on the lane (+59.6 MB,
|
|
# idle, build/perf/kpi-v0.2.8-r1 -> -r2) is spread, not a regression.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b = Path(directory) / "a", Path(directory) / "b"
|
|
code, out = self.run_peaked_report(a, b, {name: value + 59 for name, value in FIXTURE_RSS.items()})
|
|
self.assertEqual(code, 0)
|
|
self.assertNotIn("OVER BUDGET", out)
|
|
|
|
def test_compare_missing_peak_samples_fail_closed(self):
|
|
# A run without daemon RSS samples must fail the gate instead of
|
|
# comparing as "no peak growth".
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b = Path(directory) / "a", Path(directory) / "b"
|
|
self.complete(a, FIXTURE_RSS)
|
|
self.complete(b, None)
|
|
code, out = self.compare(a, b)
|
|
self.assertEqual(code, 5)
|
|
blob = " ".join(out.split())
|
|
self.assertIn("B gate failures: daemon RSS samples unavailable", blob)
|
|
self.assertIn("reload RSS peak unavailable in A or B", blob)
|
|
|
|
def test_rss_budget_scales_with_baseline(self):
|
|
# max(32 MB, 15% of A): 40 MB of growth fails on a small baseline and
|
|
# passes on a large one.
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
a, b, c, d = (Path(directory) / name for name in "abcd")
|
|
code, _ = self.run_report(a, b, {name: value + 40 for name, value in FIXTURE_RSS.items()})
|
|
self.assertEqual(code, 5)
|
|
code, out = self.run_report(c, d, dict.fromkeys(FIXTURE_RSS, 340.0), rss_a=dict.fromkeys(FIXTURE_RSS, 300.0))
|
|
self.assertEqual(code, 0)
|
|
self.assertNotIn("OVER BUDGET", out)
|
|
|
|
def test_compare_reports_gate_failures(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
self.fixture(path)
|
|
code, out = self.compare(path, path)
|
|
self.assertEqual(code, 5)
|
|
blob = " ".join(out.split())
|
|
self.assertIn("A gate failures:", blob)
|
|
self.assertIn("daemon RSS samples unavailable in reload", blob)
|
|
|
|
|
|
class RuntimeTests(unittest.TestCase):
|
|
def test_boottime_timer_ticks(self):
|
|
ticks = []
|
|
runtime.until(runtime.boot() + 0.03, tick=lambda: ticks.append(1), interval=0.005)
|
|
self.assertTrue(ticks)
|
|
|
|
def test_invalid_timer_does_not_leak(self):
|
|
before = len(list(Path("/proc/self/fd").iterdir()))
|
|
for value in (float("nan"), float("inf"), -1, 0):
|
|
with self.assertRaises(ValueError):
|
|
runtime.Timer(value)
|
|
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
|
|
|
|
def test_short_child_and_timeout(self):
|
|
result = runtime.capture([sys.executable, "-c", "raise SystemExit(7)"], 1)
|
|
self.assertEqual(result.returncode, 7)
|
|
with self.assertRaises(subprocess.TimeoutExpired):
|
|
runtime.capture([sys.executable, "-c", "import select;select.select([],[],[],30)"], 0.03)
|
|
|
|
def test_pidfd_failure_closes_timer(self):
|
|
process = subprocess.Popen([sys.executable, "-c", "import select;select.select([],[],[],30)"], start_new_session=True)
|
|
try:
|
|
before = len(list(Path("/proc/self/fd").iterdir()))
|
|
with mock.patch.object(runtime.os, "pidfd_open", side_effect=OSError("injected")), self.assertRaises(OSError):
|
|
runtime.until(runtime.boot() + 1, process)
|
|
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
|
|
finally:
|
|
runtime.stop(process)
|
|
|
|
def test_clock_source_and_expired_wait(self):
|
|
with mock.patch.object(runtime.time, "clock_gettime", return_value=10) as clock:
|
|
self.assertIsNone(runtime.until(9))
|
|
clock.assert_called_with(runtime.time.CLOCK_BOOTTIME)
|
|
|
|
|
|
class WorkloadTests(unittest.TestCase):
|
|
def test_daemon_start_failure_stops_before_workload(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
binary = path / "daemon"
|
|
binary.write_text("#!/usr/bin/env python3\nraise SystemExit(9)\n")
|
|
binary.chmod(0o700)
|
|
with mock.patch.object(workload, "OUT", path), mock.patch.object(workload, "BIN", str(binary)), self.assertRaises(RuntimeError):
|
|
workload.main()
|
|
self.assertEqual((path / "phases.log").read_text().split()[1], "start")
|
|
|
|
def test_malformed_status_is_not_readiness(self):
|
|
with self.assertRaises(RuntimeError):
|
|
workload.status_payload(b"not-json")
|
|
|
|
|
|
class HarnessTests(unittest.TestCase):
|
|
def fake_docker(self, directory):
|
|
path = directory / "docker"
|
|
path.write_text('#!/usr/bin/env python3\nimport os,sys,select\n'
|
|
'with open(os.environ["FAKE_CALLS"],"a") as f:f.write(sys.argv[1]+"\\n")\n'
|
|
'if sys.argv[1]=="run":\n'
|
|
' if os.environ.get("FAKE_HANG"):select.select([],[],[],30)\n'
|
|
' sys.exit(125)\n'
|
|
'if sys.argv[1]=="inspect":\n'
|
|
' if os.environ.get("FAKE_FOREIGN"):print("foreign");sys.exit(0)\n'
|
|
' print("Error: No such object",file=sys.stderr);sys.exit(1)\n'
|
|
'sys.exit(0)\n')
|
|
path.chmod(0o700)
|
|
return path
|
|
|
|
def run_fake(self, hang=False):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
docker = self.fake_docker(path)
|
|
environment = dict(os.environ, KPI_DOCKER=str(docker), KPI_SKIP_BUILD="1", KPI_OUT=str(path / "out"),
|
|
KPI_RULES=str(path / "missing"), KPI_TIMEOUT_S="2", FAKE_CALLS=str(path / "calls"))
|
|
if hang:
|
|
environment["FAKE_HANG"] = "1"
|
|
with mock.patch.dict(os.environ, environment, clear=True):
|
|
code = host.main(["perf", "fixture"])
|
|
state = json.loads((path / "out/lifecycle.json").read_text())
|
|
self.assertEqual(code, 124 if hang else 125)
|
|
self.assertEqual(state["cleanup"], "already_absent")
|
|
self.assertFalse((path / "out/kpi.json").read_text() == "{}")
|
|
|
|
def test_docker_start_failure_not_masked(self):
|
|
self.run_fake()
|
|
|
|
def test_docker_hang_is_bounded(self):
|
|
self.run_fake(hang=True)
|
|
|
|
def test_foreign_container_never_removed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory)
|
|
docker = self.fake_docker(path)
|
|
with mock.patch.dict(os.environ, {"FAKE_FOREIGN": "1", "FAKE_CALLS": str(path / "calls")}), self.assertRaises(RuntimeError):
|
|
host.remove_owned(str(docker), "fixture", "ours")
|
|
self.assertNotIn("rm", (path / "calls").read_text().splitlines())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|