strawberry/scripts/perf_kpi_report.py
trvon 2d5e05e8db
Add resource governor, scan profiles and KPI harness
PSI/cgroup-aware governor with efficient/balanced/aggressive profiles,
background pacing and thread priorities; timing histograms and optional
Tracy zones; a capped Docker KPI harness with report/compare tooling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 18:17:16 -06:00

188 lines
8.1 KiB
Python
Executable file

#!/usr/bin/env python3
"""Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh).
perf_kpi_report.py <run_dir> # writes <run_dir>/kpi.json, prints a table
perf_kpi_report.py --compare <dirA> <dirB> # side-by-side deltas
Exit code 5 when a threshold fails (idle CPU, exec decision p95, dropped
events), mirroring the yams constrained-lane convention.
"""
import json
import sys
from pathlib import Path
THRESHOLDS = {
"idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens
"exec_decision_p95_ms": 250.0, # launch latency added by the exec guard
"events_dropped": 0,
}
def parse_mem_mb(text):
text = text.strip()
units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576}
for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])):
if text.endswith(unit):
try:
return float(text[: -len(unit)]) * factor
except ValueError:
return None
return None
def load_run(run):
run = Path(run)
phases = []
walls = {}
for line in (run / "phases.log").read_text().splitlines():
stamp, _, name = line.partition(" ")
if stamp.endswith("_wall_s") or stamp.endswith("_execs"):
walls[stamp] = float(name)
else:
phases.append((float(stamp), name))
stats = []
stats_file = run / "docker_stats.csv"
if stats_file.exists():
for line in stats_file.read_text().splitlines()[1:]:
parts = line.split(",")
if len(parts) >= 3:
try:
stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2])))
except ValueError:
pass
statuses = []
status_file = run / "status.jsonl"
if status_file.exists():
for line in status_file.read_text().splitlines():
stamp, _, payload = line.partition(" ")
try:
statuses.append((float(stamp), json.loads(payload).get("data", {})))
except (ValueError, json.JSONDecodeError):
pass
final = {}
final_file = run / "final_status.json"
if final_file.exists():
try:
final = json.loads(final_file.read_text()).get("data", {})
except json.JSONDecodeError:
final = {}
return phases, walls, stats, statuses, final
def at(statuses, when, path):
"""Status value at (or just before) a time."""
best = None
for stamp, data in statuses:
if stamp <= when:
best = data
value = best or {}
for key in path:
value = value.get(key, {}) if isinstance(value, dict) else {}
return value if not isinstance(value, dict) else None
def summarize(run):
phases, walls, stats, statuses, final = load_run(run)
bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf"))
for i, (start, name) in enumerate(phases) if name not in ("start", "end")]
report = {"run": str(run), "phases": {}, "walls_s": walls}
for name, start, end in bounds:
cpu = [c for t, c, _ in stats if start <= t < end]
mem = [m for t, _, m in stats if start <= t < end and m is not None]
own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end]
own = [v for v in own if isinstance(v, (int, float)) and v >= 0]
entry = {
# Daemon's own CPU (governor sample, % of the CPUs it may use);
# container CPU also includes the harness's status polling.
"daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None,
"cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None,
"cpu_max_pct": round(max(cpu), 2) if cpu else None,
"mem_max_mb": round(max(mem), 1) if mem else None,
"samples": len(cpu),
}
before = at(statuses, start, ["prevention", "timeouts"])
after = at(statuses, end, ["prevention", "timeouts"])
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
entry["exec_timeouts"] = after - before
report["phases"][name] = entry
timings = final.get("timings", {})
report["timings"] = timings
report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")}
report["events_dropped"] = final.get("monitor", {}).get("events_dropped")
report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")}
report["resources"] = final.get("resources", {})
failures = []
idle_phase = report["phases"].get("idle", {})
idle = idle_phase.get("daemon_cpu_mean_pct")
label = "daemon"
if idle is None: # older builds without resources.own_cpu_pct
idle, label = idle_phase.get("cpu_mean_pct"), "container"
if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]:
failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%")
p95 = timings.get("exec_decision", {}).get("p95_ms")
if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]:
failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms")
dropped = report["events_dropped"]
if dropped not in (None, 0):
failures.append(f"{dropped} events dropped")
if not final:
failures.append("no final status (daemon did not run to the end?)")
report["failures"] = failures
return report
def print_report(report):
print(f"run: {report['run']}")
print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'mem max MB':>11} {'exec timeouts':>14}")
for name, entry in report["phases"].items():
print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} "
f"{str(entry['mem_max_mb']):>11} {str(entry.get('exec_timeouts', '-')):>14}")
for key, value in report["walls_s"].items():
print(f"{key}: {value:g}")
print("timings (ms):")
for name, entry in report["timings"].items():
if isinstance(entry, dict) and entry.get("count"):
unit = "mb" if name == "image_size" else "ms"
print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} "
f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}")
print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}")
print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS")
def compare(a, b):
ra, rb = summarize(a), summarize(b)
print(f"A: {a}\nB: {b}")
print(f"{'metric':32} {'A':>12} {'B':>12}")
for phase in sorted(set(ra["phases"]) | set(rb["phases"])):
for key in ("daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"):
va = ra["phases"].get(phase, {}).get(key)
vb = rb["phases"].get(phase, {}).get(key)
print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12}")
for name in sorted(set(ra["timings"]) | set(rb["timings"])):
ea, eb = ra["timings"].get(name), rb["timings"].get(name)
if isinstance(ea, dict) or isinstance(eb, dict):
unit = "mb" if name == "image_size" else "ms"
for q in ("p50", "p95", "max"):
va = (ea or {}).get(f"{q}_{unit}")
vb = (eb or {}).get(f"{q}_{unit}")
print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}")
for key in ("storm_execs", "large_execs"):
print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}")
def main(argv):
if len(argv) == 4 and argv[1] == "--compare":
compare(argv[2], argv[3])
return 0
if len(argv) != 2:
print(__doc__)
return 2
report = summarize(argv[1])
(Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2))
print_report(report)
return 5 if report["failures"] else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))