strawberry/scripts/perf_kpi_report.py
trvon c60b44fb4d
kpi: budget the per-phase RSS peak, not only the last sample
compare() budgeted daemon_rss_last_mb only, so a regression that spiked and
then settled passed: the peak was printed but never checked, leaving just the
absolute 512 MB ceiling. Both metrics are now budgeted per phase through one
helper — max(32 MB, 15% of the baseline) for the phase-end value and
max(64 MB, 15%) for the peak.

The 64 MB floor is measured, not chosen: with a shared 32 MB floor three
same-version repeat pairs (8-r1->8-r2 idle peak +59.6 MB, 9-r1->9-r2 storm
+36.3 MB, 9-r1->9-r3 +36.0 MB) would be called regressions, because a single
transient sample spreads about twice as wide as the phase-end value. 64 MB
clears all nine same-version pairs while every v0.2.8 -> v0.2.9 pair still
trips it; re-checked four same-version pairs here with zero over-budget phases.

The peak check adds real signal: on the stored runs it catches v0.2.9's idle
peak at +137.4 MB where the last-sample check saw +102.4 MB. Missing samples
still fail closed and never read as "no growth".
2026-11-19 16:19:41 -07:00

317 lines
16 KiB
Python
Executable file

#!/usr/bin/env python3
"""Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh).
perf_kpi_report.py <run_dir> # writes <run_dir>/kpi.json, prints a table
perf_kpi_report.py --compare <dirA> <dirB> # side-by-side deltas + daemon RSS budget
Daemon memory is gated per phase from the status samples the workload already
collects (`resources.rss_mb`): `daemon_rss_max_mb` (peak), `daemon_rss_last_mb`
(last sample of the phase) and `daemon_rss_samples`. The container `mem_max_mb`
stays a secondary, informational metric.
Exit code 5 covers, mirroring the yams constrained-lane convention:
* the workload gates (phases, counters, prevention, lifecycle, ...), listed as
`failures` by summarize();
* the daemon gates: idle CPU, exec decision p95, dropped events, a phase with
no daemon RSS samples (fail closed, never skipped silently), and a phase
daemon RSS peak over THRESHOLDS["daemon_rss_max_mb"];
* with --compare, either run failing any of the above, or a phase whose daemon
RSS grew over the baseline-relative budget: the larger of a floor and
THRESHOLDS["daemon_rss_delta_pct"] percent of that phase in the baseline
run. Both the phase-end value (daemon_rss_last_mb,
THRESHOLDS["daemon_rss_delta_mb"]) and the phase peak
(daemon_rss_max_mb, THRESHOLDS["daemon_rss_peak_delta_mb"], a wider floor
because a single transient sample is noisier) are budgeted, so a
peak-only regression cannot pass on the last-sample check alone.
"""
import json
import math
import sys
from pathlib import Path
THRESHOLDS = {
"idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens
"exec_decision_p95_ms": 250.0, # launch latency added by the exec guard
"events_dropped": 0,
# Daemon RSS peak per phase: 2x the largest lane peak on record (256.6 MB,
# build/perf/kpi-v0.2.9-r2, reload phase; v0.2.8 peaks at 105.7 MB).
"daemon_rss_max_mb": 512.0,
# --compare allowance for daemon RSS growth over the baseline, per phase:
# the larger of a floor and a share of the baseline. The floor sits above the
# worst same-version repeat spread measured on the lane (+29.1 MB, idle phase,
# build/perf/kpi-v0.2.8-r1 -> -r2) and well below the smallest delta the
# documented v0.2.8 -> v0.2.9 regression shows (+58.3 MB idle, +132..141 MB
# final RSS).
"daemon_rss_delta_mb": 32.0,
"daemon_rss_delta_pct": 15.0,
# --compare allowance for the per-phase daemon RSS *peak* (daemon_rss_max_mb),
# same shape as above: max(floor, pct% of A peak), wider floor. One transient
# sample is noisier than the phase-end value: the measured same-version peak
# spread on the lane is +59.6 MB (idle, build/perf/kpi-v0.2.8-r1 -> -r2) and
# +36.3 MB (storm, build/perf/kpi-v0.2.9-r1 -> -r2), while the documented
# v0.2.8 -> v0.2.9 peak regression is +76..+151 MB per phase.
"daemon_rss_peak_delta_mb": 64.0,
}
def parse_mem_mb(text):
text = text.strip()
units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576}
for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])):
if text.endswith(unit):
try:
return float(text[: -len(unit)]) * factor
except ValueError:
return None
return None
def load_run(run):
run = Path(run)
phases = []
walls = {}
for line in (run / "phases.log").read_text().splitlines():
stamp, _, name = line.partition(" ")
if stamp.endswith("_wall_s") or stamp.endswith("_execs"):
walls[stamp] = float(name)
else:
phases.append((float(stamp), name))
stats = []
stats_file = run / "docker_stats.csv"
if stats_file.exists():
for line in stats_file.read_text().splitlines()[1:]:
parts = line.split(",")
if len(parts) >= 3:
try:
stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2])))
except ValueError as error:
raise ValueError("malformed container statistics") from error
statuses = []
status_file = run / "status.jsonl"
if status_file.exists():
for line in status_file.read_text().splitlines():
stamp, _, payload = line.partition(" ")
try:
statuses.append((float(stamp), json.loads(payload).get("data", {})))
except (ValueError, json.JSONDecodeError) as error:
raise ValueError("malformed daemon status sample") from error
final = {}
final_file = run / "final_status.json"
if final_file.exists():
try:
final = json.loads(final_file.read_text()).get("data", {})
except json.JSONDecodeError as error:
raise ValueError("malformed final daemon status") from error
return phases, walls, stats, statuses, final
def at(statuses, when, path):
"""Status value at (or just before) a time."""
best = None
for stamp, data in statuses:
if stamp <= when:
best = data
value = best or {}
for key in path:
value = value.get(key, {}) if isinstance(value, dict) else {}
return value if not isinstance(value, dict) else None
def summarize(run):
phases, walls, stats, statuses, final = load_run(run)
bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf"))
for i, (start, name) in enumerate(phases) if name not in ("start", "end")]
report = {"run": str(run), "phases": {}, "walls_s": walls}
for name, start, end in bounds:
cpu = [c for t, c, _ in stats if start <= t < end]
mem = [m for t, _, m in stats if start <= t < end and m is not None]
own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end]
own = [v for v in own if isinstance(v, (int, float)) and v >= 0]
# (time, daemon RSS) pairs; max() then yields the phase's last sample
# whatever order the producer wrote the lines in.
rss = [(t, d.get("resources", {}).get("rss_mb")) for t, d in statuses if start <= t < end]
rss = [(t, v) for t, v in rss if isinstance(v, (int, float)) and v >= 0]
entry = {
# Daemon's own CPU (governor sample, % of the CPUs it may use);
# container CPU also includes the harness's status polling.
"daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None,
"cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None,
"cpu_max_pct": round(max(cpu), 2) if cpu else None,
# Daemon RSS from the governor's own sample: the memory gate.
"daemon_rss_max_mb": round(max(v for _, v in rss), 1) if rss else None,
"daemon_rss_last_mb": round(max(rss)[1], 1) if rss else None,
"daemon_rss_samples": len(rss),
# Container RSS, secondary context only (it also holds the tools).
"mem_max_mb": round(max(mem), 1) if mem else None,
"samples": len(cpu),
}
before = at(statuses, start, ["prevention", "timeouts"])
after = at(statuses, end, ["prevention", "timeouts"])
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
entry["exec_timeouts"] = after - before
report["phases"][name] = entry
timings = final.get("timings", {})
report["timings"] = timings
report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")}
report["events_dropped"] = final.get("monitor", {}).get("events_dropped")
report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")}
report["resources"] = final.get("resources", {})
failures = []
required = {"startup", "idle", "storm", "large", "detect", "reload"}
if set(report["phases"]) != required:
failures.append("missing or unexpected workload phases")
if any(right[0] <= left[0] for left, right in zip(phases, phases[1:], strict=False)):
failures.append("phase timestamps are not strictly increasing")
for name in sorted(required):
entry = report["phases"].get(name, {})
if entry.get("samples", 0) < (3 if name == "idle" else 1):
failures.append(f"insufficient container samples in {name}")
# Fail closed: a phase without daemon RSS samples is not "no regression".
if entry.get("daemon_rss_samples", 0) < 1:
failures.append(f"daemon RSS samples unavailable in {name}")
for name, entry in sorted(report["phases"].items()):
peak = entry.get("daemon_rss_max_mb")
if peak is not None and peak > THRESHOLDS["daemon_rss_max_mb"]:
failures.append(f"{name} daemon RSS peak {peak} MB > {THRESHOLDS['daemon_rss_max_mb']} MB")
if any(not math.isfinite(cpu) or cpu < 0 or memory is None or not math.isfinite(memory) for _, cpu, memory in stats):
failures.append("invalid CPU/memory measurements")
lifecycle = Path(run) / "lifecycle.json"
if lifecycle.exists():
try:
state = json.loads(lifecycle.read_text())
except (OSError, json.JSONDecodeError) as error:
raise ValueError("unavailable or malformed KPI lifecycle") from error
if state.get("run_exit") != 0 or state.get("cleanup_error") or not state.get("observed_budget_held"):
failures.append("workload, cleanup or elapsed-budget failure")
if not isinstance(report["pipeline"].get("scanned"), (int, float)) or report["pipeline"]["scanned"] <= 0:
failures.append("no scanned-image evidence: scanner coverage unavailable")
for counter in ("storm_execs", "large_execs", "detect_execs"):
if walls.get(counter, 0) <= 0:
failures.append(f"workload counter unavailable: {counter}")
if report["prevention"].get("active") is not True or timings.get("exec_decision", {}).get("count", 0) <= 0:
failures.append("exec prevention telemetry inactive or unobserved")
idle_phase = report["phases"].get("idle", {})
idle = idle_phase.get("daemon_cpu_mean_pct")
label = "daemon"
if idle is None: # older builds without resources.own_cpu_pct
idle, label = idle_phase.get("cpu_mean_pct"), "container"
if idle is None:
failures.append("idle CPU unavailable")
if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]:
failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%")
p95 = timings.get("exec_decision", {}).get("p95_ms")
if p95 is None:
failures.append("exec decision latency unavailable")
if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]:
failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms")
dropped = report["events_dropped"]
if dropped is None:
failures.append("event-drop counter unavailable")
if dropped not in (None, 0):
failures.append(f"{dropped} events dropped")
if not final:
failures.append("no final status (daemon did not run to the end?)")
report["failures"] = failures
return report
def print_report(report):
print(f"run: {report['run']}")
print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'rss max MB':>11} {'rss last MB':>12} "
f"{'rss n':>6} {'cont mem MB':>12} {'exec timeouts':>14}")
for name, entry in report["phases"].items():
print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} "
f"{str(entry['daemon_rss_max_mb']):>11} {str(entry['daemon_rss_last_mb']):>12} "
f"{str(entry['daemon_rss_samples']):>6} {str(entry['mem_max_mb']):>12} "
f"{str(entry.get('exec_timeouts', '-')):>14}")
for key, value in report["walls_s"].items():
print(f"{key}: {value:g}")
print("timings (ms):")
for name, entry in report["timings"].items():
if isinstance(entry, dict) and entry.get("count"):
unit = "mb" if name == "image_size" else "ms"
print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} "
f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}")
print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}")
print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS")
def rss_growth_budget_mb(baseline_mb, floor_mb=None):
"""Allowed daemon RSS growth of one phase over its baseline value."""
floor = THRESHOLDS["daemon_rss_delta_mb"] if floor_mb is None else floor_mb
return max(floor, baseline_mb * THRESHOLDS["daemon_rss_delta_pct"] / 100)
def rss_budget(ra, rb, key, label, column, floor, over):
"""Print one daemon RSS budget table (phase-end value or phase peak)."""
print(f"daemon RSS ({column}) growth over baseline (budget: max({floor:g} MB, "
f"{THRESHOLDS['daemon_rss_delta_pct']:g}% of A)):")
print(f"{'phase':12} {'A ' + column + ' MB':>10} {'B ' + column + ' MB':>10} "
f"{'delta MB':>10} {'allowed MB':>11}")
for phase in sorted(set(ra["phases"]) & set(rb["phases"])):
va = ra["phases"][phase].get(key)
vb = rb["phases"][phase].get(key)
if not isinstance(va, (int, float)) or not isinstance(vb, (int, float)):
# Fail closed: summarize() already fails this phase's gate, and a
# missing sample is never treated as "no growth".
print(f"{phase:12} RSS {column} unavailable in A or B")
continue
allowed = rss_growth_budget_mb(va, floor)
print(f"{phase:12} {va:>10.1f} {vb:>10.1f} {vb - va:>+10.1f} {allowed:>+11.1f}")
if vb - va > allowed:
over.append(f"{phase} {label} {vb:.1f} MB vs baseline {va:.1f} MB "
f"(+{vb - va:.1f} MB > allowed +{allowed:.1f} MB)")
def delta(a, b):
return f"{b - a:+.2f}" if isinstance(a, (int, float)) and isinstance(b, (int, float)) else "-"
def compare(a, b):
ra, rb = summarize(a), summarize(b)
print(f"A: {a}\nB: {b}")
print("A gate failures:", "; ".join(ra["failures"]) or "none")
print("B gate failures:", "; ".join(rb["failures"]) or "none")
print(f"{'metric':32} {'A':>12} {'B':>12} {'delta':>10}")
for phase in sorted(set(ra["phases"]) | set(rb["phases"])):
# Daemon RSS first: it is the metric this run exists to protect.
for key in ("daemon_rss_last_mb", "daemon_rss_max_mb", "daemon_rss_samples",
"daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"):
va = ra["phases"].get(phase, {}).get(key)
vb = rb["phases"].get(phase, {}).get(key)
print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12} {delta(va, vb):>10}")
for name in sorted(set(ra["timings"]) | set(rb["timings"])):
ea, eb = ra["timings"].get(name), rb["timings"].get(name)
if isinstance(ea, dict) or isinstance(eb, dict):
unit = "mb" if name == "image_size" else "ms"
for q in ("p50", "p95", "max"):
va = (ea or {}).get(f"{q}_{unit}")
vb = (eb or {}).get(f"{q}_{unit}")
print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}")
for key in ("storm_execs", "large_execs", "detect_execs"):
print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}")
over = []
# Budget both the phase-end RSS and the phase peak: a peak-only regression
# (a spike released before the phase ends) is still a regression.
for key, label, column, floor in (
("daemon_rss_last_mb", "daemon RSS", "last", THRESHOLDS["daemon_rss_delta_mb"]),
("daemon_rss_max_mb", "daemon RSS peak", "peak", THRESHOLDS["daemon_rss_peak_delta_mb"])):
rss_budget(ra, rb, key, label, column, floor, over)
for item in over:
print("OVER BUDGET:", item)
return 5 if ra["failures"] or rb["failures"] or over else 0
def main(argv):
if len(argv) == 4 and argv[1] == "--compare":
return compare(argv[2], argv[3])
if len(argv) != 2:
print(__doc__)
return 2
report = summarize(argv[1])
(Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2))
print_report(report)
return 5 if report["failures"] else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))