#!/usr/bin/env python3 """Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh). perf_kpi_report.py # writes /kpi.json, prints a table perf_kpi_report.py --compare # side-by-side deltas Exit code 5 when a threshold fails (idle CPU, exec decision p95, dropped events), mirroring the yams constrained-lane convention. """ import json import sys from pathlib import Path THRESHOLDS = { "idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens "exec_decision_p95_ms": 250.0, # launch latency added by the exec guard "events_dropped": 0, } def parse_mem_mb(text): text = text.strip() units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576} for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])): if text.endswith(unit): try: return float(text[: -len(unit)]) * factor except ValueError: return None return None def load_run(run): run = Path(run) phases = [] walls = {} for line in (run / "phases.log").read_text().splitlines(): stamp, _, name = line.partition(" ") if stamp.endswith("_wall_s") or stamp.endswith("_execs"): walls[stamp] = float(name) else: phases.append((float(stamp), name)) stats = [] stats_file = run / "docker_stats.csv" if stats_file.exists(): for line in stats_file.read_text().splitlines()[1:]: parts = line.split(",") if len(parts) >= 3: try: stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2]))) except ValueError: pass statuses = [] status_file = run / "status.jsonl" if status_file.exists(): for line in status_file.read_text().splitlines(): stamp, _, payload = line.partition(" ") try: statuses.append((float(stamp), json.loads(payload).get("data", {}))) except (ValueError, json.JSONDecodeError): pass final = {} final_file = run / "final_status.json" if final_file.exists(): try: final = json.loads(final_file.read_text()).get("data", {}) except json.JSONDecodeError: final = {} return phases, walls, stats, statuses, final def at(statuses, when, path): """Status value at (or just before) a time.""" best = None for stamp, data in statuses: if stamp <= when: best = data value = best or {} for key in path: value = value.get(key, {}) if isinstance(value, dict) else {} return value if not isinstance(value, dict) else None def summarize(run): phases, walls, stats, statuses, final = load_run(run) bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf")) for i, (start, name) in enumerate(phases) if name not in ("start", "end")] report = {"run": str(run), "phases": {}, "walls_s": walls} for name, start, end in bounds: cpu = [c for t, c, _ in stats if start <= t < end] mem = [m for t, _, m in stats if start <= t < end and m is not None] own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end] own = [v for v in own if isinstance(v, (int, float)) and v >= 0] entry = { # Daemon's own CPU (governor sample, % of the CPUs it may use); # container CPU also includes the harness's status polling. "daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None, "cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None, "cpu_max_pct": round(max(cpu), 2) if cpu else None, "mem_max_mb": round(max(mem), 1) if mem else None, "samples": len(cpu), } before = at(statuses, start, ["prevention", "timeouts"]) after = at(statuses, end, ["prevention", "timeouts"]) if isinstance(before, (int, float)) and isinstance(after, (int, float)): entry["exec_timeouts"] = after - before report["phases"][name] = entry timings = final.get("timings", {}) report["timings"] = timings report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")} report["events_dropped"] = final.get("monitor", {}).get("events_dropped") report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")} report["resources"] = final.get("resources", {}) failures = [] idle_phase = report["phases"].get("idle", {}) idle = idle_phase.get("daemon_cpu_mean_pct") label = "daemon" if idle is None: # older builds without resources.own_cpu_pct idle, label = idle_phase.get("cpu_mean_pct"), "container" if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]: failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%") p95 = timings.get("exec_decision", {}).get("p95_ms") if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]: failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms") dropped = report["events_dropped"] if dropped not in (None, 0): failures.append(f"{dropped} events dropped") if not final: failures.append("no final status (daemon did not run to the end?)") report["failures"] = failures return report def print_report(report): print(f"run: {report['run']}") print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'mem max MB':>11} {'exec timeouts':>14}") for name, entry in report["phases"].items(): print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} " f"{str(entry['mem_max_mb']):>11} {str(entry.get('exec_timeouts', '-')):>14}") for key, value in report["walls_s"].items(): print(f"{key}: {value:g}") print("timings (ms):") for name, entry in report["timings"].items(): if isinstance(entry, dict) and entry.get("count"): unit = "mb" if name == "image_size" else "ms" print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} " f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}") print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}") print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS") def compare(a, b): ra, rb = summarize(a), summarize(b) print(f"A: {a}\nB: {b}") print(f"{'metric':32} {'A':>12} {'B':>12}") for phase in sorted(set(ra["phases"]) | set(rb["phases"])): for key in ("daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"): va = ra["phases"].get(phase, {}).get(key) vb = rb["phases"].get(phase, {}).get(key) print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12}") for name in sorted(set(ra["timings"]) | set(rb["timings"])): ea, eb = ra["timings"].get(name), rb["timings"].get(name) if isinstance(ea, dict) or isinstance(eb, dict): unit = "mb" if name == "image_size" else "ms" for q in ("p50", "p95", "max"): va = (ea or {}).get(f"{q}_{unit}") vb = (eb or {}).get(f"{q}_{unit}") print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}") for key in ("storm_execs", "large_execs"): print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}") def main(argv): if len(argv) == 4 and argv[1] == "--compare": compare(argv[2], argv[3]) return 0 if len(argv) != 2: print(__doc__) return 2 report = summarize(argv[1]) (Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2)) print_report(report) return 5 if report["failures"] else 0 if __name__ == "__main__": sys.exit(main(sys.argv))