strawberry/scripts/perf_kpi_workload.py
trvon a5f2dbedfc
KPI: bound collection and reject incomplete measurements
Use suspend-aware BOOTTIME waits and owned-container cleanup without changing the workload. Reject missing coverage and preserve producer/report failures. Include runtime/helper regressions, constrained build-job support and live exec-guard crash safety. Document the failed live/idle gates without relaxing thresholds or claiming production acceptance.
2026-11-03 06:57:00 -07:00

162 lines
7 KiB
Python

"""Container-only fixed KPI workload; no host stores/services are mounted.
Original phase lengths, two CPU hogs/eight exec streams, four padded96MiB
executables and forty distinct marker binaries. BOOTTIME timer/pidfd events
bound readiness, sampling, phases and child cleanup, without shell sleep loops.
"""
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
from contextlib import ExitStack
from pathlib import Path
from perf_kpi_runtime import boot, capture, seconds, stop, until
BIN = "/src/builddir/src/strawberry"
CONF = "/kpi/strawberry.conf"
OUT = Path("/kpi")
def stream(argv):
deadline = float(argv[2])
files = argv[3:]
count = 0
while boot() < deadline and count < 1_000_000:
for path in files:
subprocess.run([path, "true"] if path == "/usr/bin/env" else [path], check=True)
count += 1
print(count)
return 0
def status_payload(data):
try:
payload = json.loads(data)
except (UnicodeError, json.JSONDecodeError) as error:
raise RuntimeError("malformed fixture daemon status") from error
if not isinstance(payload, dict):
raise RuntimeError("fixture status must be an object")
return payload
class Ready(Exception):
"""A successful readiness event ends the bounded startup wait."""
def main():
if len(sys.argv) > 1 and sys.argv[1] == "--stream":
return stream(sys.argv)
durations = {key: seconds(os.environ.get(key + "_S", default)) for key, default in
(("STARTUP", "30"), ("IDLE", "60"), ("STORM", "30"), ("LARGE", "20"), ("DETECT", "20"), ("RELOAD", "20"))}
children = []
daemon = None
with (OUT / "daemon.log").open("wb") as log:
def phase(name):
with (OUT / "phases.log").open("a") as phases:
phases.write(f"{time.time():.9f} {name}\n")
print("==", name, flush=True)
def sample():
result = capture([BIN, "--status", "--json", CONF], 2)
if result.returncode:
raise RuntimeError("daemon status request failed")
payload = status_payload(result.stdout)
if not isinstance(payload.get("data"), dict) or not payload["data"]:
raise ValueError("incomplete daemon status")
with (OUT / "status.jsonl").open("a") as statuses:
statuses.write(f"{time.time():.9f} {json.dumps(payload)}\n")
return result.stdout
try:
daemon = subprocess.Popen([BIN, CONF], stdout=log, stderr=log, start_new_session=True)
phase("start")
def readiness():
result = capture([BIN, "--status", "--json", CONF], 2)
if result.returncode == 0:
payload = status_payload(result.stdout)
if isinstance(payload.get("data"), dict) and payload["data"]:
raise Ready()
try:
code = until(boot() + 60, daemon, readiness, 0.5, deadline_is_error=False)
raise RuntimeError(f"daemon not ready (exit={code})")
except Ready:
sample()
with ExitStack() as counter_stack, tempfile.TemporaryDirectory(prefix="strawberry-kpi-") as directory:
temp = Path(directory)
for name in ("startup", "idle", "storm", "large", "detect", "reload"):
phase(name)
deadline = boot() + durations[name.upper()]
workers = []
counters = []
if name == "storm":
for _ in range(2):
child = subprocess.Popen(["sha256sum", "/dev/zero"], stdout=subprocess.DEVNULL, start_new_session=True)
workers.append(child)
children.append(child)
files = ["/bin/true", "/usr/bin/env"]
copies = 8
elif name in ("large", "detect"):
files = []
for index in range(4 if name == "large" else 40):
path = temp / f"{name}-{index}"
shutil.copyfile("/bin/true", path)
with path.open("ab") as binary:
if name == "large":
for _ in range(96):
binary.write(os.urandom(1024 * 1024))
else:
binary.write(f"STRAWBERRY-KPI-DETECT-MARKER {index}".encode())
path.chmod(0o700)
files.append(str(path))
copies = 1
else:
files = []
copies = 0
if name == "reload":
result = capture([BIN, "--reload-rules", CONF], 5)
if result.returncode:
raise RuntimeError("fixture rule reload failed")
for _ in range(copies):
counter = counter_stack.enter_context(tempfile.TemporaryFile())
counters.append(counter)
child = subprocess.Popen([sys.executable, __file__, "--stream", str(deadline), *files], stdout=counter, start_new_session=True)
workers.append(child)
children.append(child)
result = until(deadline, daemon, sample, 2, deadline_is_error=False)
if result is not None:
raise RuntimeError(f"daemon exited during {name}: {result}")
for index, child in enumerate(workers):
if name == "storm" and index < 2:
stop(child) # the two intentionally infinite CPU hogs
else:
code = until(boot() + 2, child)
if code:
raise RuntimeError(f"exec stream failed: {code}")
if counters:
total = 0
for counter in counters:
counter.seek(0)
total += int(counter.read(64).strip())
counter.close()
with (OUT / "phases.log").open("a") as phases:
phases.write(f"{name}_execs {total}\n")
phase("end")
(OUT / "final_status.json").write_bytes(sample())
daemon.terminate()
result = until(boot() + 10, daemon)
if result:
raise RuntimeError(f"daemon shutdown failed: {result}")
print("done", flush=True)
return 0
finally:
for child in children:
if child.poll() is None:
stop(child)
if daemon is not None and daemon.poll() is None:
stop(daemon)
if __name__ == "__main__":
sys.exit(main())