#!/usr/bin/env python3 """Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh). perf_kpi_report.py # writes /kpi.json, prints a table perf_kpi_report.py --compare # side-by-side deltas + daemon RSS budget Daemon memory is gated per phase from the status samples the workload already collects (`resources.rss_mb`): `daemon_rss_max_mb` (peak), `daemon_rss_last_mb` (last sample of the phase) and `daemon_rss_samples`. The container `mem_max_mb` stays a secondary, informational metric. Exit code 5 covers, mirroring the yams constrained-lane convention: * the workload gates (phases, counters, prevention, lifecycle, ...), listed as `failures` by summarize(); * the daemon gates: idle CPU, exec decision p95, dropped events, a phase with no daemon RSS samples (fail closed, never skipped silently), and a phase daemon RSS peak over THRESHOLDS["daemon_rss_max_mb"]; * with --compare, either run failing any of the above, or a phase whose daemon RSS grew over the baseline-relative budget: the larger of a floor and THRESHOLDS["daemon_rss_delta_pct"] percent of that phase in the baseline run. Both the phase-end value (daemon_rss_last_mb, THRESHOLDS["daemon_rss_delta_mb"]) and the phase peak (daemon_rss_max_mb, THRESHOLDS["daemon_rss_peak_delta_mb"], a wider floor because a single transient sample is noisier) are budgeted, so a peak-only regression cannot pass on the last-sample check alone. """ import json import math import sys from pathlib import Path THRESHOLDS = { "idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens "exec_decision_p95_ms": 250.0, # launch latency added by the exec guard "events_dropped": 0, # Daemon RSS peak per phase: 2x the largest lane peak on record (256.6 MB, # build/perf/kpi-v0.2.9-r2, reload phase; v0.2.8 peaks at 105.7 MB). "daemon_rss_max_mb": 512.0, # --compare allowance for daemon RSS growth over the baseline, per phase: # the larger of a floor and a share of the baseline. The floor sits above the # worst same-version repeat spread measured on the lane (+29.1 MB, idle phase, # build/perf/kpi-v0.2.8-r1 -> -r2) and well below the smallest delta the # documented v0.2.8 -> v0.2.9 regression shows (+58.3 MB idle, +132..141 MB # final RSS). "daemon_rss_delta_mb": 32.0, "daemon_rss_delta_pct": 15.0, # --compare allowance for the per-phase daemon RSS *peak* (daemon_rss_max_mb), # same shape as above: max(floor, pct% of A peak), wider floor. One transient # sample is noisier than the phase-end value: the measured same-version peak # spread on the lane is +59.6 MB (idle, build/perf/kpi-v0.2.8-r1 -> -r2) and # +36.3 MB (storm, build/perf/kpi-v0.2.9-r1 -> -r2), while the documented # v0.2.8 -> v0.2.9 peak regression is +76..+151 MB per phase. "daemon_rss_peak_delta_mb": 64.0, } def parse_mem_mb(text): text = text.strip() units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576} for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])): if text.endswith(unit): try: return float(text[: -len(unit)]) * factor except ValueError: return None return None def load_run(run): run = Path(run) phases = [] walls = {} for line in (run / "phases.log").read_text().splitlines(): stamp, _, name = line.partition(" ") if stamp.endswith("_wall_s") or stamp.endswith("_execs"): walls[stamp] = float(name) else: phases.append((float(stamp), name)) stats = [] stats_file = run / "docker_stats.csv" if stats_file.exists(): for line in stats_file.read_text().splitlines()[1:]: parts = line.split(",") if len(parts) >= 3: try: stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2]))) except ValueError as error: raise ValueError("malformed container statistics") from error statuses = [] status_file = run / "status.jsonl" if status_file.exists(): for line in status_file.read_text().splitlines(): stamp, _, payload = line.partition(" ") try: statuses.append((float(stamp), json.loads(payload).get("data", {}))) except (ValueError, json.JSONDecodeError) as error: raise ValueError("malformed daemon status sample") from error final = {} final_file = run / "final_status.json" if final_file.exists(): try: final = json.loads(final_file.read_text()).get("data", {}) except json.JSONDecodeError as error: raise ValueError("malformed final daemon status") from error return phases, walls, stats, statuses, final def at(statuses, when, path): """Status value at (or just before) a time.""" best = None for stamp, data in statuses: if stamp <= when: best = data value = best or {} for key in path: value = value.get(key, {}) if isinstance(value, dict) else {} return value if not isinstance(value, dict) else None def summarize(run): phases, walls, stats, statuses, final = load_run(run) bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf")) for i, (start, name) in enumerate(phases) if name not in ("start", "end")] report = {"run": str(run), "phases": {}, "walls_s": walls} for name, start, end in bounds: cpu = [c for t, c, _ in stats if start <= t < end] mem = [m for t, _, m in stats if start <= t < end and m is not None] own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end] own = [v for v in own if isinstance(v, (int, float)) and v >= 0] # (time, daemon RSS) pairs; max() then yields the phase's last sample # whatever order the producer wrote the lines in. rss = [(t, d.get("resources", {}).get("rss_mb")) for t, d in statuses if start <= t < end] rss = [(t, v) for t, v in rss if isinstance(v, (int, float)) and v >= 0] entry = { # Daemon's own CPU (governor sample, % of the CPUs it may use); # container CPU also includes the harness's status polling. "daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None, "cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None, "cpu_max_pct": round(max(cpu), 2) if cpu else None, # Daemon RSS from the governor's own sample: the memory gate. "daemon_rss_max_mb": round(max(v for _, v in rss), 1) if rss else None, "daemon_rss_last_mb": round(max(rss)[1], 1) if rss else None, "daemon_rss_samples": len(rss), # Container RSS, secondary context only (it also holds the tools). "mem_max_mb": round(max(mem), 1) if mem else None, "samples": len(cpu), } before = at(statuses, start, ["prevention", "timeouts"]) after = at(statuses, end, ["prevention", "timeouts"]) if isinstance(before, (int, float)) and isinstance(after, (int, float)): entry["exec_timeouts"] = after - before report["phases"][name] = entry timings = final.get("timings", {}) report["timings"] = timings report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")} report["events_dropped"] = final.get("monitor", {}).get("events_dropped") report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")} report["resources"] = final.get("resources", {}) failures = [] required = {"startup", "idle", "storm", "large", "detect", "reload"} if set(report["phases"]) != required: failures.append("missing or unexpected workload phases") if any(right[0] <= left[0] for left, right in zip(phases, phases[1:], strict=False)): failures.append("phase timestamps are not strictly increasing") for name in sorted(required): entry = report["phases"].get(name, {}) if entry.get("samples", 0) < (3 if name == "idle" else 1): failures.append(f"insufficient container samples in {name}") # Fail closed: a phase without daemon RSS samples is not "no regression". if entry.get("daemon_rss_samples", 0) < 1: failures.append(f"daemon RSS samples unavailable in {name}") for name, entry in sorted(report["phases"].items()): peak = entry.get("daemon_rss_max_mb") if peak is not None and peak > THRESHOLDS["daemon_rss_max_mb"]: failures.append(f"{name} daemon RSS peak {peak} MB > {THRESHOLDS['daemon_rss_max_mb']} MB") if any(not math.isfinite(cpu) or cpu < 0 or memory is None or not math.isfinite(memory) for _, cpu, memory in stats): failures.append("invalid CPU/memory measurements") lifecycle = Path(run) / "lifecycle.json" if lifecycle.exists(): try: state = json.loads(lifecycle.read_text()) except (OSError, json.JSONDecodeError) as error: raise ValueError("unavailable or malformed KPI lifecycle") from error if state.get("run_exit") != 0 or state.get("cleanup_error") or not state.get("observed_budget_held"): failures.append("workload, cleanup or elapsed-budget failure") if not isinstance(report["pipeline"].get("scanned"), (int, float)) or report["pipeline"]["scanned"] <= 0: failures.append("no scanned-image evidence: scanner coverage unavailable") for counter in ("storm_execs", "large_execs", "detect_execs"): if walls.get(counter, 0) <= 0: failures.append(f"workload counter unavailable: {counter}") if report["prevention"].get("active") is not True or timings.get("exec_decision", {}).get("count", 0) <= 0: failures.append("exec prevention telemetry inactive or unobserved") idle_phase = report["phases"].get("idle", {}) idle = idle_phase.get("daemon_cpu_mean_pct") label = "daemon" if idle is None: # older builds without resources.own_cpu_pct idle, label = idle_phase.get("cpu_mean_pct"), "container" if idle is None: failures.append("idle CPU unavailable") if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]: failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%") p95 = timings.get("exec_decision", {}).get("p95_ms") if p95 is None: failures.append("exec decision latency unavailable") if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]: failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms") dropped = report["events_dropped"] if dropped is None: failures.append("event-drop counter unavailable") if dropped not in (None, 0): failures.append(f"{dropped} events dropped") if not final: failures.append("no final status (daemon did not run to the end?)") report["failures"] = failures return report def print_report(report): print(f"run: {report['run']}") print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'rss max MB':>11} {'rss last MB':>12} " f"{'rss n':>6} {'cont mem MB':>12} {'exec timeouts':>14}") for name, entry in report["phases"].items(): print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} " f"{str(entry['daemon_rss_max_mb']):>11} {str(entry['daemon_rss_last_mb']):>12} " f"{str(entry['daemon_rss_samples']):>6} {str(entry['mem_max_mb']):>12} " f"{str(entry.get('exec_timeouts', '-')):>14}") for key, value in report["walls_s"].items(): print(f"{key}: {value:g}") print("timings (ms):") for name, entry in report["timings"].items(): if isinstance(entry, dict) and entry.get("count"): unit = "mb" if name == "image_size" else "ms" print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} " f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}") print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}") print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS") def rss_growth_budget_mb(baseline_mb, floor_mb=None): """Allowed daemon RSS growth of one phase over its baseline value.""" floor = THRESHOLDS["daemon_rss_delta_mb"] if floor_mb is None else floor_mb return max(floor, baseline_mb * THRESHOLDS["daemon_rss_delta_pct"] / 100) def rss_budget(ra, rb, key, label, column, floor, over): """Print one daemon RSS budget table (phase-end value or phase peak).""" print(f"daemon RSS ({column}) growth over baseline (budget: max({floor:g} MB, " f"{THRESHOLDS['daemon_rss_delta_pct']:g}% of A)):") print(f"{'phase':12} {'A ' + column + ' MB':>10} {'B ' + column + ' MB':>10} " f"{'delta MB':>10} {'allowed MB':>11}") for phase in sorted(set(ra["phases"]) & set(rb["phases"])): va = ra["phases"][phase].get(key) vb = rb["phases"][phase].get(key) if not isinstance(va, (int, float)) or not isinstance(vb, (int, float)): # Fail closed: summarize() already fails this phase's gate, and a # missing sample is never treated as "no growth". print(f"{phase:12} RSS {column} unavailable in A or B") continue allowed = rss_growth_budget_mb(va, floor) print(f"{phase:12} {va:>10.1f} {vb:>10.1f} {vb - va:>+10.1f} {allowed:>+11.1f}") if vb - va > allowed: over.append(f"{phase} {label} {vb:.1f} MB vs baseline {va:.1f} MB " f"(+{vb - va:.1f} MB > allowed +{allowed:.1f} MB)") def delta(a, b): return f"{b - a:+.2f}" if isinstance(a, (int, float)) and isinstance(b, (int, float)) else "-" def compare(a, b): ra, rb = summarize(a), summarize(b) print(f"A: {a}\nB: {b}") print("A gate failures:", "; ".join(ra["failures"]) or "none") print("B gate failures:", "; ".join(rb["failures"]) or "none") print(f"{'metric':32} {'A':>12} {'B':>12} {'delta':>10}") for phase in sorted(set(ra["phases"]) | set(rb["phases"])): # Daemon RSS first: it is the metric this run exists to protect. for key in ("daemon_rss_last_mb", "daemon_rss_max_mb", "daemon_rss_samples", "daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"): va = ra["phases"].get(phase, {}).get(key) vb = rb["phases"].get(phase, {}).get(key) print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12} {delta(va, vb):>10}") for name in sorted(set(ra["timings"]) | set(rb["timings"])): ea, eb = ra["timings"].get(name), rb["timings"].get(name) if isinstance(ea, dict) or isinstance(eb, dict): unit = "mb" if name == "image_size" else "ms" for q in ("p50", "p95", "max"): va = (ea or {}).get(f"{q}_{unit}") vb = (eb or {}).get(f"{q}_{unit}") print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}") for key in ("storm_execs", "large_execs", "detect_execs"): print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}") over = [] # Budget both the phase-end RSS and the phase peak: a peak-only regression # (a spike released before the phase ends) is still a regression. for key, label, column, floor in ( ("daemon_rss_last_mb", "daemon RSS", "last", THRESHOLDS["daemon_rss_delta_mb"]), ("daemon_rss_max_mb", "daemon RSS peak", "peak", THRESHOLDS["daemon_rss_peak_delta_mb"])): rss_budget(ra, rb, key, label, column, floor, over) for item in over: print("OVER BUDGET:", item) return 5 if ra["failures"] or rb["failures"] or over else 0 def main(argv): if len(argv) == 4 and argv[1] == "--compare": return compare(argv[2], argv[3]) if len(argv) != 2: print(__doc__) return 2 report = summarize(argv[1]) (Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2)) print_report(report) return 5 if report["failures"] else 0 if __name__ == "__main__": sys.exit(main(sys.argv))