PSI/cgroup-aware governor with efficient/balanced/aggressive profiles, background pacing and thread priorities; timing histograms and optional Tracy zones; a capped Docker KPI harness with report/compare tooling. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
188 lines
8.1 KiB
Python
Executable file
188 lines
8.1 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh).
|
|
|
|
perf_kpi_report.py <run_dir> # writes <run_dir>/kpi.json, prints a table
|
|
perf_kpi_report.py --compare <dirA> <dirB> # side-by-side deltas
|
|
|
|
Exit code 5 when a threshold fails (idle CPU, exec decision p95, dropped
|
|
events), mirroring the yams constrained-lane convention.
|
|
"""
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
THRESHOLDS = {
|
|
"idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens
|
|
"exec_decision_p95_ms": 250.0, # launch latency added by the exec guard
|
|
"events_dropped": 0,
|
|
}
|
|
|
|
|
|
def parse_mem_mb(text):
|
|
text = text.strip()
|
|
units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576}
|
|
for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])):
|
|
if text.endswith(unit):
|
|
try:
|
|
return float(text[: -len(unit)]) * factor
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def load_run(run):
|
|
run = Path(run)
|
|
phases = []
|
|
walls = {}
|
|
for line in (run / "phases.log").read_text().splitlines():
|
|
stamp, _, name = line.partition(" ")
|
|
if stamp.endswith("_wall_s") or stamp.endswith("_execs"):
|
|
walls[stamp] = float(name)
|
|
else:
|
|
phases.append((float(stamp), name))
|
|
stats = []
|
|
stats_file = run / "docker_stats.csv"
|
|
if stats_file.exists():
|
|
for line in stats_file.read_text().splitlines()[1:]:
|
|
parts = line.split(",")
|
|
if len(parts) >= 3:
|
|
try:
|
|
stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2])))
|
|
except ValueError:
|
|
pass
|
|
statuses = []
|
|
status_file = run / "status.jsonl"
|
|
if status_file.exists():
|
|
for line in status_file.read_text().splitlines():
|
|
stamp, _, payload = line.partition(" ")
|
|
try:
|
|
statuses.append((float(stamp), json.loads(payload).get("data", {})))
|
|
except (ValueError, json.JSONDecodeError):
|
|
pass
|
|
final = {}
|
|
final_file = run / "final_status.json"
|
|
if final_file.exists():
|
|
try:
|
|
final = json.loads(final_file.read_text()).get("data", {})
|
|
except json.JSONDecodeError:
|
|
final = {}
|
|
return phases, walls, stats, statuses, final
|
|
|
|
|
|
def at(statuses, when, path):
|
|
"""Status value at (or just before) a time."""
|
|
best = None
|
|
for stamp, data in statuses:
|
|
if stamp <= when:
|
|
best = data
|
|
value = best or {}
|
|
for key in path:
|
|
value = value.get(key, {}) if isinstance(value, dict) else {}
|
|
return value if not isinstance(value, dict) else None
|
|
|
|
|
|
def summarize(run):
|
|
phases, walls, stats, statuses, final = load_run(run)
|
|
bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf"))
|
|
for i, (start, name) in enumerate(phases) if name not in ("start", "end")]
|
|
report = {"run": str(run), "phases": {}, "walls_s": walls}
|
|
for name, start, end in bounds:
|
|
cpu = [c for t, c, _ in stats if start <= t < end]
|
|
mem = [m for t, _, m in stats if start <= t < end and m is not None]
|
|
own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end]
|
|
own = [v for v in own if isinstance(v, (int, float)) and v >= 0]
|
|
entry = {
|
|
# Daemon's own CPU (governor sample, % of the CPUs it may use);
|
|
# container CPU also includes the harness's status polling.
|
|
"daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None,
|
|
"cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None,
|
|
"cpu_max_pct": round(max(cpu), 2) if cpu else None,
|
|
"mem_max_mb": round(max(mem), 1) if mem else None,
|
|
"samples": len(cpu),
|
|
}
|
|
before = at(statuses, start, ["prevention", "timeouts"])
|
|
after = at(statuses, end, ["prevention", "timeouts"])
|
|
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
|
|
entry["exec_timeouts"] = after - before
|
|
report["phases"][name] = entry
|
|
timings = final.get("timings", {})
|
|
report["timings"] = timings
|
|
report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")}
|
|
report["events_dropped"] = final.get("monitor", {}).get("events_dropped")
|
|
report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")}
|
|
report["resources"] = final.get("resources", {})
|
|
|
|
failures = []
|
|
idle_phase = report["phases"].get("idle", {})
|
|
idle = idle_phase.get("daemon_cpu_mean_pct")
|
|
label = "daemon"
|
|
if idle is None: # older builds without resources.own_cpu_pct
|
|
idle, label = idle_phase.get("cpu_mean_pct"), "container"
|
|
if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]:
|
|
failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%")
|
|
p95 = timings.get("exec_decision", {}).get("p95_ms")
|
|
if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]:
|
|
failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms")
|
|
dropped = report["events_dropped"]
|
|
if dropped not in (None, 0):
|
|
failures.append(f"{dropped} events dropped")
|
|
if not final:
|
|
failures.append("no final status (daemon did not run to the end?)")
|
|
report["failures"] = failures
|
|
return report
|
|
|
|
|
|
def print_report(report):
|
|
print(f"run: {report['run']}")
|
|
print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'mem max MB':>11} {'exec timeouts':>14}")
|
|
for name, entry in report["phases"].items():
|
|
print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} "
|
|
f"{str(entry['mem_max_mb']):>11} {str(entry.get('exec_timeouts', '-')):>14}")
|
|
for key, value in report["walls_s"].items():
|
|
print(f"{key}: {value:g}")
|
|
print("timings (ms):")
|
|
for name, entry in report["timings"].items():
|
|
if isinstance(entry, dict) and entry.get("count"):
|
|
unit = "mb" if name == "image_size" else "ms"
|
|
print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} "
|
|
f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}")
|
|
print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}")
|
|
print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS")
|
|
|
|
|
|
def compare(a, b):
|
|
ra, rb = summarize(a), summarize(b)
|
|
print(f"A: {a}\nB: {b}")
|
|
print(f"{'metric':32} {'A':>12} {'B':>12}")
|
|
for phase in sorted(set(ra["phases"]) | set(rb["phases"])):
|
|
for key in ("daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"):
|
|
va = ra["phases"].get(phase, {}).get(key)
|
|
vb = rb["phases"].get(phase, {}).get(key)
|
|
print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12}")
|
|
for name in sorted(set(ra["timings"]) | set(rb["timings"])):
|
|
ea, eb = ra["timings"].get(name), rb["timings"].get(name)
|
|
if isinstance(ea, dict) or isinstance(eb, dict):
|
|
unit = "mb" if name == "image_size" else "ms"
|
|
for q in ("p50", "p95", "max"):
|
|
va = (ea or {}).get(f"{q}_{unit}")
|
|
vb = (eb or {}).get(f"{q}_{unit}")
|
|
print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}")
|
|
for key in ("storm_execs", "large_execs"):
|
|
print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}")
|
|
|
|
|
|
def main(argv):
|
|
if len(argv) == 4 and argv[1] == "--compare":
|
|
compare(argv[2], argv[3])
|
|
return 0
|
|
if len(argv) != 2:
|
|
print(__doc__)
|
|
return 2
|
|
report = summarize(argv[1])
|
|
(Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2))
|
|
print_report(report)
|
|
return 5 if report["failures"] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|