strawberry/tests/test_perf_kpi.py
trvon c60b44fb4d
kpi: budget the per-phase RSS peak, not only the last sample
compare() budgeted daemon_rss_last_mb only, so a regression that spiked and
then settled passed: the peak was printed but never checked, leaving just the
absolute 512 MB ceiling. Both metrics are now budgeted per phase through one
helper — max(32 MB, 15% of the baseline) for the phase-end value and
max(64 MB, 15%) for the peak.

The 64 MB floor is measured, not chosen: with a shared 32 MB floor three
same-version repeat pairs (8-r1->8-r2 idle peak +59.6 MB, 9-r1->9-r2 storm
+36.3 MB, 9-r1->9-r3 +36.0 MB) would be called regressions, because a single
transient sample spreads about twice as wide as the phase-end value. 64 MB
clears all nine same-version pairs while every v0.2.8 -> v0.2.9 pair still
trips it; re-checked four same-version pairs here with zero over-budget phases.

The peak check adds real signal: on the stored runs it catches v0.2.9's idle
peak at +137.4 MB where the last-sample check saw +102.4 MB. Missing samples
still fail closed and never read as "no growth".
2026-11-19 16:19:41 -07:00

311 lines
15 KiB
Python

#!/usr/bin/env python3
"""Unprivileged KPI regressions; never invoke the real Docker daemon."""
import contextlib
import importlib.util
import io
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("perf_kpi_report", ROOT / "scripts/perf_kpi_report.py")
assert SPEC is not None and SPEC.loader is not None
report = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(report)
sys.modules["perf_kpi_report"] = report
def load_module(name):
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py"))
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
runtime = load_module("perf_kpi_runtime")
host = load_module("perf_kpi")
workload = load_module("perf_kpi_workload")
# Daemon RSS per phase of the fixture lane; the v0.2.9 shape is this plus the
# documented +130..141 MB growth (build/perf/kpi-v0.2.{8,9}).
FIXTURE_RSS = {"startup": 70.0, "idle": 72.0, "storm": 88.0, "large": 89.0, "detect": 89.0, "reload": 106.0}
class ReportTests(unittest.TestCase):
def fixture(self, path, rss=None):
path.mkdir(parents=True, exist_ok=True)
(path / "phases.log").write_text("0 start\n1 startup\n2 idle\n3 storm\n4 large\n5 detect\n6 reload\n7 end\n")
(path / "final_status.json").write_text(json.dumps({"data": {
"timings": {"exec_decision": {"p95_ms": 1}},
"monitor": {"events_dropped": 0}, "pipeline": {"events": 1, "scanned": 1},
}}))
self.status_samples(path, rss)
def status_samples(self, path, rss):
"""One daemon status sample per workload phase, carrying resources.rss_mb."""
stamps = {"startup": 1.5, "idle": 2.5, "storm": 3.5, "large": 4.5, "detect": 5.5, "reload": 6.5}
samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': rss[name]}}})}"
for name, when in stamps.items() if rss and name in rss]
(path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples))
def peak_samples(self, path, rss, peak):
"""Two samples per phase: a mid-phase spike, then the phase-end value."""
stamps = {"startup": (1.2, 1.5), "idle": (2.2, 2.5), "storm": (3.2, 3.5),
"large": (4.2, 4.5), "detect": (5.2, 5.5), "reload": (6.2, 6.5)}
samples = [f"{when} {json.dumps({'data': {'resources': {'own_cpu_pct': 0.1, 'rss_mb': value}}})}"
for name, (spike_at, end_at) in stamps.items() if rss and name in rss
for when, value in ((spike_at, peak[name]), (end_at, rss[name]))]
(path / "status.jsonl").write_text("".join(sample + "\n" for sample in samples))
def complete(self, path, rss, peak=None):
"""A run that passes every gate; the caller supplies the daemon RSS samples."""
self.fixture(path, rss)
if peak:
self.peak_samples(path, rss, peak)
(path / "phases.log").write_text((path / "phases.log").read_text() + "storm_execs 2\nlarge_execs 2\ndetect_execs 2\n")
(path / "docker_stats.csv").write_text("t,cpu_pct,mem\n" + "".join(f"{stamp},0.1,10MiB\n" for stamp in (1.1, 2.1, 2.2, 2.3, 3.1, 4.1, 5.1, 6.1)))
final = {"data": {"timings": {"exec_decision": {"count": 1, "p95_ms": 1}},
"monitor": {"events_dropped": 0}, "pipeline": {"scanned": 1}, "prevention": {"active": True}}}
(path / "final_status.json").write_text(json.dumps(final))
def compare(self, a, b):
out = io.StringIO()
with contextlib.redirect_stdout(out):
code = report.main(["report", "--compare", str(a), str(b)])
return code, out.getvalue()
def run_report(self, a, b, rss_b, rss_a=None):
self.complete(a, FIXTURE_RSS if rss_a is None else rss_a)
self.complete(b, rss_b)
return self.compare(a, b)
def run_peaked_report(self, a, b, peak_b, peak_a=None):
self.complete(a, FIXTURE_RSS, peak_a)
self.complete(b, FIXTURE_RSS, peak_b)
return self.compare(a, b)
def test_live_guard_parent_installs_crash_safety(self):
source = (ROOT / "tests/exec_guard_live.cpp").read_text()
prefix = source.split("int main()", 1)[1].split("ExecGuard guard;", 1)[0]
self.assertIn("install_crash_safety();", prefix)
def test_missing_samples_fail_closed(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
self.assertTrue(report.summarize(path)["failures"])
def test_complete_measurements_can_pass(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.complete(path, FIXTURE_RSS)
summary = report.summarize(path)
self.assertEqual(summary["failures"], [])
self.assertEqual(summary["phases"]["reload"]["daemon_rss_last_mb"], 106.0)
self.assertEqual(summary["phases"]["reload"]["daemon_rss_samples"], 1)
def test_compare_propagates_failed_gate(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
self.assertEqual(report.main(["report", "--compare", str(path), str(path)]), 5)
def test_missing_daemon_rss_samples_fail_closed(self):
# Every other gate passes; only the daemon RSS samples are withheld.
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.complete(path, None)
failures = report.summarize(path)["failures"]
self.assertIn("daemon RSS samples unavailable in reload", failures)
self.assertIn("daemon RSS samples unavailable in idle", failures)
with contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(report.main(["report", str(path)]), 5)
def test_daemon_rss_peak_over_threshold_fails(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.complete(path, dict(FIXTURE_RSS, reload=600.0))
self.assertIn("reload daemon RSS peak 600.0 MB > 512.0 MB", report.summarize(path)["failures"])
def test_compare_fails_on_memory_regression(self):
# The documented v0.2.8 -> v0.2.9 shape: every phase grew by ~130 MB.
with tempfile.TemporaryDirectory() as directory:
a, b = Path(directory) / "a", Path(directory) / "b"
code, out = self.run_report(a, b, {name: value + 130 for name, value in FIXTURE_RSS.items()})
self.assertEqual(code, 5)
blob = " ".join(out.split())
self.assertIn("reload.daemon_rss_last_mb 106.0 236.0 +130.00", blob)
self.assertIn("OVER BUDGET: reload daemon RSS 236.0 MB vs baseline 106.0 MB "
"(+130.0 MB > allowed +32.0 MB)", blob)
def test_compare_within_repeat_spread_passes(self):
with tempfile.TemporaryDirectory() as directory:
a, b = Path(directory) / "a", Path(directory) / "b"
code, out = self.run_report(a, b, {name: value + 20 for name, value in FIXTURE_RSS.items()})
self.assertEqual(code, 0)
self.assertNotIn("OVER BUDGET", out)
def test_compare_fails_on_peak_only_regression(self):
# Phase-end RSS is identical in every phase; only the peak grew, so a
# gate that budgets daemon_rss_last_mb alone lets this run pass.
with tempfile.TemporaryDirectory() as directory:
a, b = Path(directory) / "a", Path(directory) / "b"
code, out = self.run_peaked_report(a, b, {name: value + 200 for name, value in FIXTURE_RSS.items()})
self.assertEqual(code, 5)
blob = " ".join(out.split())
self.assertIn("reload.daemon_rss_last_mb 106.0 106.0", blob)
self.assertIn("reload.daemon_rss_max_mb 106.0 306.0", blob)
self.assertIn("OVER BUDGET: reload daemon RSS peak 306.0 MB vs baseline 106.0 MB "
"(+200.0 MB > allowed +64.0 MB)", blob)
self.assertNotIn("OVER BUDGET: reload daemon RSS 106.0", blob)
def test_compare_within_peak_spread_passes(self):
# The worst same-version peak spread measured on the lane (+59.6 MB,
# idle, build/perf/kpi-v0.2.8-r1 -> -r2) is spread, not a regression.
with tempfile.TemporaryDirectory() as directory:
a, b = Path(directory) / "a", Path(directory) / "b"
code, out = self.run_peaked_report(a, b, {name: value + 59 for name, value in FIXTURE_RSS.items()})
self.assertEqual(code, 0)
self.assertNotIn("OVER BUDGET", out)
def test_compare_missing_peak_samples_fail_closed(self):
# A run without daemon RSS samples must fail the gate instead of
# comparing as "no peak growth".
with tempfile.TemporaryDirectory() as directory:
a, b = Path(directory) / "a", Path(directory) / "b"
self.complete(a, FIXTURE_RSS)
self.complete(b, None)
code, out = self.compare(a, b)
self.assertEqual(code, 5)
blob = " ".join(out.split())
self.assertIn("B gate failures: daemon RSS samples unavailable", blob)
self.assertIn("reload RSS peak unavailable in A or B", blob)
def test_rss_budget_scales_with_baseline(self):
# max(32 MB, 15% of A): 40 MB of growth fails on a small baseline and
# passes on a large one.
with tempfile.TemporaryDirectory() as directory:
a, b, c, d = (Path(directory) / name for name in "abcd")
code, _ = self.run_report(a, b, {name: value + 40 for name, value in FIXTURE_RSS.items()})
self.assertEqual(code, 5)
code, out = self.run_report(c, d, dict.fromkeys(FIXTURE_RSS, 340.0), rss_a=dict.fromkeys(FIXTURE_RSS, 300.0))
self.assertEqual(code, 0)
self.assertNotIn("OVER BUDGET", out)
def test_compare_reports_gate_failures(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
self.fixture(path)
code, out = self.compare(path, path)
self.assertEqual(code, 5)
blob = " ".join(out.split())
self.assertIn("A gate failures:", blob)
self.assertIn("daemon RSS samples unavailable in reload", blob)
class RuntimeTests(unittest.TestCase):
def test_boottime_timer_ticks(self):
ticks = []
runtime.until(runtime.boot() + 0.03, tick=lambda: ticks.append(1), interval=0.005)
self.assertTrue(ticks)
def test_invalid_timer_does_not_leak(self):
before = len(list(Path("/proc/self/fd").iterdir()))
for value in (float("nan"), float("inf"), -1, 0):
with self.assertRaises(ValueError):
runtime.Timer(value)
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
def test_short_child_and_timeout(self):
result = runtime.capture([sys.executable, "-c", "raise SystemExit(7)"], 1)
self.assertEqual(result.returncode, 7)
with self.assertRaises(subprocess.TimeoutExpired):
runtime.capture([sys.executable, "-c", "import select;select.select([],[],[],30)"], 0.03)
def test_pidfd_failure_closes_timer(self):
process = subprocess.Popen([sys.executable, "-c", "import select;select.select([],[],[],30)"], start_new_session=True)
try:
before = len(list(Path("/proc/self/fd").iterdir()))
with mock.patch.object(runtime.os, "pidfd_open", side_effect=OSError("injected")), self.assertRaises(OSError):
runtime.until(runtime.boot() + 1, process)
self.assertEqual(before, len(list(Path("/proc/self/fd").iterdir())))
finally:
runtime.stop(process)
def test_clock_source_and_expired_wait(self):
with mock.patch.object(runtime.time, "clock_gettime", return_value=10) as clock:
self.assertIsNone(runtime.until(9))
clock.assert_called_with(runtime.time.CLOCK_BOOTTIME)
class WorkloadTests(unittest.TestCase):
def test_daemon_start_failure_stops_before_workload(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
binary = path / "daemon"
binary.write_text("#!/usr/bin/env python3\nraise SystemExit(9)\n")
binary.chmod(0o700)
with mock.patch.object(workload, "OUT", path), mock.patch.object(workload, "BIN", str(binary)), self.assertRaises(RuntimeError):
workload.main()
self.assertEqual((path / "phases.log").read_text().split()[1], "start")
def test_malformed_status_is_not_readiness(self):
with self.assertRaises(RuntimeError):
workload.status_payload(b"not-json")
class HarnessTests(unittest.TestCase):
def fake_docker(self, directory):
path = directory / "docker"
path.write_text('#!/usr/bin/env python3\nimport os,sys,select\n'
'with open(os.environ["FAKE_CALLS"],"a") as f:f.write(sys.argv[1]+"\\n")\n'
'if sys.argv[1]=="run":\n'
' if os.environ.get("FAKE_HANG"):select.select([],[],[],30)\n'
' sys.exit(125)\n'
'if sys.argv[1]=="inspect":\n'
' if os.environ.get("FAKE_FOREIGN"):print("foreign");sys.exit(0)\n'
' print("Error: No such object",file=sys.stderr);sys.exit(1)\n'
'sys.exit(0)\n')
path.chmod(0o700)
return path
def run_fake(self, hang=False):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
docker = self.fake_docker(path)
environment = dict(os.environ, KPI_DOCKER=str(docker), KPI_SKIP_BUILD="1", KPI_OUT=str(path / "out"),
KPI_RULES=str(path / "missing"), KPI_TIMEOUT_S="2", FAKE_CALLS=str(path / "calls"))
if hang:
environment["FAKE_HANG"] = "1"
with mock.patch.dict(os.environ, environment, clear=True):
code = host.main(["perf", "fixture"])
state = json.loads((path / "out/lifecycle.json").read_text())
self.assertEqual(code, 124 if hang else 125)
self.assertEqual(state["cleanup"], "already_absent")
self.assertFalse((path / "out/kpi.json").read_text() == "{}")
def test_docker_start_failure_not_masked(self):
self.run_fake()
def test_docker_hang_is_bounded(self):
self.run_fake(hang=True)
def test_foreign_container_never_removed(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
docker = self.fake_docker(path)
with mock.patch.dict(os.environ, {"FAKE_FOREIGN": "1", "FAKE_CALLS": str(path / "calls")}), self.assertRaises(RuntimeError):
host.remove_owned(str(docker), "fixture", "ours")
self.assertNotIn("rm", (path / "calls").read_text().splitlines())
if __name__ == "__main__":
unittest.main(verbosity=2)