compare() budgeted daemon_rss_last_mb only, so a regression that spiked and then settled passed: the peak was printed but never checked, leaving just the absolute 512 MB ceiling. Both metrics are now budgeted per phase through one helper — max(32 MB, 15% of the baseline) for the phase-end value and max(64 MB, 15%) for the peak. The 64 MB floor is measured, not chosen: with a shared 32 MB floor three same-version repeat pairs (8-r1->8-r2 idle peak +59.6 MB, 9-r1->9-r2 storm +36.3 MB, 9-r1->9-r3 +36.0 MB) would be called regressions, because a single transient sample spreads about twice as wide as the phase-end value. 64 MB clears all nine same-version pairs while every v0.2.8 -> v0.2.9 pair still trips it; re-checked four same-version pairs here with zero over-budget phases. The peak check adds real signal: on the stored runs it catches v0.2.9's idle peak at +137.4 MB where the last-sample check saw +102.4 MB. Missing samples still fail closed and never read as "no growth".
317 lines
16 KiB
Python
Executable file
317 lines
16 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Summarize a Strawberry constrained-lane KPI run (scripts/perf_kpi.sh).
|
|
|
|
perf_kpi_report.py <run_dir> # writes <run_dir>/kpi.json, prints a table
|
|
perf_kpi_report.py --compare <dirA> <dirB> # side-by-side deltas + daemon RSS budget
|
|
|
|
Daemon memory is gated per phase from the status samples the workload already
|
|
collects (`resources.rss_mb`): `daemon_rss_max_mb` (peak), `daemon_rss_last_mb`
|
|
(last sample of the phase) and `daemon_rss_samples`. The container `mem_max_mb`
|
|
stays a secondary, informational metric.
|
|
|
|
Exit code 5 covers, mirroring the yams constrained-lane convention:
|
|
* the workload gates (phases, counters, prevention, lifecycle, ...), listed as
|
|
`failures` by summarize();
|
|
* the daemon gates: idle CPU, exec decision p95, dropped events, a phase with
|
|
no daemon RSS samples (fail closed, never skipped silently), and a phase
|
|
daemon RSS peak over THRESHOLDS["daemon_rss_max_mb"];
|
|
* with --compare, either run failing any of the above, or a phase whose daemon
|
|
RSS grew over the baseline-relative budget: the larger of a floor and
|
|
THRESHOLDS["daemon_rss_delta_pct"] percent of that phase in the baseline
|
|
run. Both the phase-end value (daemon_rss_last_mb,
|
|
THRESHOLDS["daemon_rss_delta_mb"]) and the phase peak
|
|
(daemon_rss_max_mb, THRESHOLDS["daemon_rss_peak_delta_mb"], a wider floor
|
|
because a single transient sample is noisier) are budgeted, so a
|
|
peak-only regression cannot pass on the last-sample check alone.
|
|
"""
|
|
import json
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
THRESHOLDS = {
|
|
"idle_cpu_mean_pct": 2.0, # % of one CPU while nothing happens
|
|
"exec_decision_p95_ms": 250.0, # launch latency added by the exec guard
|
|
"events_dropped": 0,
|
|
# Daemon RSS peak per phase: 2x the largest lane peak on record (256.6 MB,
|
|
# build/perf/kpi-v0.2.9-r2, reload phase; v0.2.8 peaks at 105.7 MB).
|
|
"daemon_rss_max_mb": 512.0,
|
|
# --compare allowance for daemon RSS growth over the baseline, per phase:
|
|
# the larger of a floor and a share of the baseline. The floor sits above the
|
|
# worst same-version repeat spread measured on the lane (+29.1 MB, idle phase,
|
|
# build/perf/kpi-v0.2.8-r1 -> -r2) and well below the smallest delta the
|
|
# documented v0.2.8 -> v0.2.9 regression shows (+58.3 MB idle, +132..141 MB
|
|
# final RSS).
|
|
"daemon_rss_delta_mb": 32.0,
|
|
"daemon_rss_delta_pct": 15.0,
|
|
# --compare allowance for the per-phase daemon RSS *peak* (daemon_rss_max_mb),
|
|
# same shape as above: max(floor, pct% of A peak), wider floor. One transient
|
|
# sample is noisier than the phase-end value: the measured same-version peak
|
|
# spread on the lane is +59.6 MB (idle, build/perf/kpi-v0.2.8-r1 -> -r2) and
|
|
# +36.3 MB (storm, build/perf/kpi-v0.2.9-r1 -> -r2), while the documented
|
|
# v0.2.8 -> v0.2.9 peak regression is +76..+151 MB per phase.
|
|
"daemon_rss_peak_delta_mb": 64.0,
|
|
}
|
|
|
|
|
|
def parse_mem_mb(text):
|
|
text = text.strip()
|
|
units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 1048576}
|
|
for unit, factor in sorted(units.items(), key=lambda kv: -len(kv[0])):
|
|
if text.endswith(unit):
|
|
try:
|
|
return float(text[: -len(unit)]) * factor
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def load_run(run):
|
|
run = Path(run)
|
|
phases = []
|
|
walls = {}
|
|
for line in (run / "phases.log").read_text().splitlines():
|
|
stamp, _, name = line.partition(" ")
|
|
if stamp.endswith("_wall_s") or stamp.endswith("_execs"):
|
|
walls[stamp] = float(name)
|
|
else:
|
|
phases.append((float(stamp), name))
|
|
stats = []
|
|
stats_file = run / "docker_stats.csv"
|
|
if stats_file.exists():
|
|
for line in stats_file.read_text().splitlines()[1:]:
|
|
parts = line.split(",")
|
|
if len(parts) >= 3:
|
|
try:
|
|
stats.append((float(parts[0]), float(parts[1]), parse_mem_mb(parts[2])))
|
|
except ValueError as error:
|
|
raise ValueError("malformed container statistics") from error
|
|
statuses = []
|
|
status_file = run / "status.jsonl"
|
|
if status_file.exists():
|
|
for line in status_file.read_text().splitlines():
|
|
stamp, _, payload = line.partition(" ")
|
|
try:
|
|
statuses.append((float(stamp), json.loads(payload).get("data", {})))
|
|
except (ValueError, json.JSONDecodeError) as error:
|
|
raise ValueError("malformed daemon status sample") from error
|
|
final = {}
|
|
final_file = run / "final_status.json"
|
|
if final_file.exists():
|
|
try:
|
|
final = json.loads(final_file.read_text()).get("data", {})
|
|
except json.JSONDecodeError as error:
|
|
raise ValueError("malformed final daemon status") from error
|
|
return phases, walls, stats, statuses, final
|
|
|
|
|
|
def at(statuses, when, path):
|
|
"""Status value at (or just before) a time."""
|
|
best = None
|
|
for stamp, data in statuses:
|
|
if stamp <= when:
|
|
best = data
|
|
value = best or {}
|
|
for key in path:
|
|
value = value.get(key, {}) if isinstance(value, dict) else {}
|
|
return value if not isinstance(value, dict) else None
|
|
|
|
|
|
def summarize(run):
|
|
phases, walls, stats, statuses, final = load_run(run)
|
|
bounds = [(name, start, phases[i + 1][0] if i + 1 < len(phases) else float("inf"))
|
|
for i, (start, name) in enumerate(phases) if name not in ("start", "end")]
|
|
report = {"run": str(run), "phases": {}, "walls_s": walls}
|
|
for name, start, end in bounds:
|
|
cpu = [c for t, c, _ in stats if start <= t < end]
|
|
mem = [m for t, _, m in stats if start <= t < end and m is not None]
|
|
own = [d.get("resources", {}).get("own_cpu_pct") for t, d in statuses if start <= t < end]
|
|
own = [v for v in own if isinstance(v, (int, float)) and v >= 0]
|
|
# (time, daemon RSS) pairs; max() then yields the phase's last sample
|
|
# whatever order the producer wrote the lines in.
|
|
rss = [(t, d.get("resources", {}).get("rss_mb")) for t, d in statuses if start <= t < end]
|
|
rss = [(t, v) for t, v in rss if isinstance(v, (int, float)) and v >= 0]
|
|
entry = {
|
|
# Daemon's own CPU (governor sample, % of the CPUs it may use);
|
|
# container CPU also includes the harness's status polling.
|
|
"daemon_cpu_mean_pct": round(sum(own) / len(own), 2) if own else None,
|
|
"cpu_mean_pct": round(sum(cpu) / len(cpu), 2) if cpu else None,
|
|
"cpu_max_pct": round(max(cpu), 2) if cpu else None,
|
|
# Daemon RSS from the governor's own sample: the memory gate.
|
|
"daemon_rss_max_mb": round(max(v for _, v in rss), 1) if rss else None,
|
|
"daemon_rss_last_mb": round(max(rss)[1], 1) if rss else None,
|
|
"daemon_rss_samples": len(rss),
|
|
# Container RSS, secondary context only (it also holds the tools).
|
|
"mem_max_mb": round(max(mem), 1) if mem else None,
|
|
"samples": len(cpu),
|
|
}
|
|
before = at(statuses, start, ["prevention", "timeouts"])
|
|
after = at(statuses, end, ["prevention", "timeouts"])
|
|
if isinstance(before, (int, float)) and isinstance(after, (int, float)):
|
|
entry["exec_timeouts"] = after - before
|
|
report["phases"][name] = entry
|
|
timings = final.get("timings", {})
|
|
report["timings"] = timings
|
|
report["prevention"] = {k: final.get("prevention", {}).get(k) for k in ("active", "checked", "timeouts", "cache_hits")}
|
|
report["events_dropped"] = final.get("monitor", {}).get("events_dropped")
|
|
report["pipeline"] = {k: final.get("pipeline", {}).get(k) for k in ("events", "scanned", "cache_hits", "images_too_large")}
|
|
report["resources"] = final.get("resources", {})
|
|
|
|
failures = []
|
|
required = {"startup", "idle", "storm", "large", "detect", "reload"}
|
|
if set(report["phases"]) != required:
|
|
failures.append("missing or unexpected workload phases")
|
|
if any(right[0] <= left[0] for left, right in zip(phases, phases[1:], strict=False)):
|
|
failures.append("phase timestamps are not strictly increasing")
|
|
for name in sorted(required):
|
|
entry = report["phases"].get(name, {})
|
|
if entry.get("samples", 0) < (3 if name == "idle" else 1):
|
|
failures.append(f"insufficient container samples in {name}")
|
|
# Fail closed: a phase without daemon RSS samples is not "no regression".
|
|
if entry.get("daemon_rss_samples", 0) < 1:
|
|
failures.append(f"daemon RSS samples unavailable in {name}")
|
|
for name, entry in sorted(report["phases"].items()):
|
|
peak = entry.get("daemon_rss_max_mb")
|
|
if peak is not None and peak > THRESHOLDS["daemon_rss_max_mb"]:
|
|
failures.append(f"{name} daemon RSS peak {peak} MB > {THRESHOLDS['daemon_rss_max_mb']} MB")
|
|
if any(not math.isfinite(cpu) or cpu < 0 or memory is None or not math.isfinite(memory) for _, cpu, memory in stats):
|
|
failures.append("invalid CPU/memory measurements")
|
|
lifecycle = Path(run) / "lifecycle.json"
|
|
if lifecycle.exists():
|
|
try:
|
|
state = json.loads(lifecycle.read_text())
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise ValueError("unavailable or malformed KPI lifecycle") from error
|
|
if state.get("run_exit") != 0 or state.get("cleanup_error") or not state.get("observed_budget_held"):
|
|
failures.append("workload, cleanup or elapsed-budget failure")
|
|
if not isinstance(report["pipeline"].get("scanned"), (int, float)) or report["pipeline"]["scanned"] <= 0:
|
|
failures.append("no scanned-image evidence: scanner coverage unavailable")
|
|
for counter in ("storm_execs", "large_execs", "detect_execs"):
|
|
if walls.get(counter, 0) <= 0:
|
|
failures.append(f"workload counter unavailable: {counter}")
|
|
if report["prevention"].get("active") is not True or timings.get("exec_decision", {}).get("count", 0) <= 0:
|
|
failures.append("exec prevention telemetry inactive or unobserved")
|
|
idle_phase = report["phases"].get("idle", {})
|
|
idle = idle_phase.get("daemon_cpu_mean_pct")
|
|
label = "daemon"
|
|
if idle is None: # older builds without resources.own_cpu_pct
|
|
idle, label = idle_phase.get("cpu_mean_pct"), "container"
|
|
if idle is None:
|
|
failures.append("idle CPU unavailable")
|
|
if idle is not None and idle > THRESHOLDS["idle_cpu_mean_pct"]:
|
|
failures.append(f"idle {label} CPU mean {idle}% > {THRESHOLDS['idle_cpu_mean_pct']}%")
|
|
p95 = timings.get("exec_decision", {}).get("p95_ms")
|
|
if p95 is None:
|
|
failures.append("exec decision latency unavailable")
|
|
if p95 is not None and p95 > THRESHOLDS["exec_decision_p95_ms"]:
|
|
failures.append(f"exec decision p95 {p95} ms > {THRESHOLDS['exec_decision_p95_ms']} ms")
|
|
dropped = report["events_dropped"]
|
|
if dropped is None:
|
|
failures.append("event-drop counter unavailable")
|
|
if dropped not in (None, 0):
|
|
failures.append(f"{dropped} events dropped")
|
|
if not final:
|
|
failures.append("no final status (daemon did not run to the end?)")
|
|
report["failures"] = failures
|
|
return report
|
|
|
|
|
|
def print_report(report):
|
|
print(f"run: {report['run']}")
|
|
print(f"{'phase':10} {'daemon cpu%':>12} {'cont cpu%':>10} {'cpu max%':>9} {'rss max MB':>11} {'rss last MB':>12} "
|
|
f"{'rss n':>6} {'cont mem MB':>12} {'exec timeouts':>14}")
|
|
for name, entry in report["phases"].items():
|
|
print(f"{name:10} {str(entry.get('daemon_cpu_mean_pct')):>12} {str(entry['cpu_mean_pct']):>10} {str(entry['cpu_max_pct']):>9} "
|
|
f"{str(entry['daemon_rss_max_mb']):>11} {str(entry['daemon_rss_last_mb']):>12} "
|
|
f"{str(entry['daemon_rss_samples']):>6} {str(entry['mem_max_mb']):>12} "
|
|
f"{str(entry.get('exec_timeouts', '-')):>14}")
|
|
for key, value in report["walls_s"].items():
|
|
print(f"{key}: {value:g}")
|
|
print("timings (ms):")
|
|
for name, entry in report["timings"].items():
|
|
if isinstance(entry, dict) and entry.get("count"):
|
|
unit = "mb" if name == "image_size" else "ms"
|
|
print(f" {name:14} n={entry['count']:<6} p50={entry.get('p50_' + unit)} p95={entry.get('p95_' + unit)} "
|
|
f"p99={entry.get('p99_' + unit)} max={entry.get('max_' + unit)}")
|
|
print(f"prevention: {report['prevention']} events_dropped: {report['events_dropped']} pipeline: {report['pipeline']}")
|
|
print("RESULT:", "FAIL — " + "; ".join(report["failures"]) if report["failures"] else "PASS")
|
|
|
|
|
|
def rss_growth_budget_mb(baseline_mb, floor_mb=None):
|
|
"""Allowed daemon RSS growth of one phase over its baseline value."""
|
|
floor = THRESHOLDS["daemon_rss_delta_mb"] if floor_mb is None else floor_mb
|
|
return max(floor, baseline_mb * THRESHOLDS["daemon_rss_delta_pct"] / 100)
|
|
|
|
|
|
def rss_budget(ra, rb, key, label, column, floor, over):
|
|
"""Print one daemon RSS budget table (phase-end value or phase peak)."""
|
|
print(f"daemon RSS ({column}) growth over baseline (budget: max({floor:g} MB, "
|
|
f"{THRESHOLDS['daemon_rss_delta_pct']:g}% of A)):")
|
|
print(f"{'phase':12} {'A ' + column + ' MB':>10} {'B ' + column + ' MB':>10} "
|
|
f"{'delta MB':>10} {'allowed MB':>11}")
|
|
for phase in sorted(set(ra["phases"]) & set(rb["phases"])):
|
|
va = ra["phases"][phase].get(key)
|
|
vb = rb["phases"][phase].get(key)
|
|
if not isinstance(va, (int, float)) or not isinstance(vb, (int, float)):
|
|
# Fail closed: summarize() already fails this phase's gate, and a
|
|
# missing sample is never treated as "no growth".
|
|
print(f"{phase:12} RSS {column} unavailable in A or B")
|
|
continue
|
|
allowed = rss_growth_budget_mb(va, floor)
|
|
print(f"{phase:12} {va:>10.1f} {vb:>10.1f} {vb - va:>+10.1f} {allowed:>+11.1f}")
|
|
if vb - va > allowed:
|
|
over.append(f"{phase} {label} {vb:.1f} MB vs baseline {va:.1f} MB "
|
|
f"(+{vb - va:.1f} MB > allowed +{allowed:.1f} MB)")
|
|
|
|
def delta(a, b):
|
|
return f"{b - a:+.2f}" if isinstance(a, (int, float)) and isinstance(b, (int, float)) else "-"
|
|
|
|
def compare(a, b):
|
|
ra, rb = summarize(a), summarize(b)
|
|
print(f"A: {a}\nB: {b}")
|
|
print("A gate failures:", "; ".join(ra["failures"]) or "none")
|
|
print("B gate failures:", "; ".join(rb["failures"]) or "none")
|
|
print(f"{'metric':32} {'A':>12} {'B':>12} {'delta':>10}")
|
|
for phase in sorted(set(ra["phases"]) | set(rb["phases"])):
|
|
# Daemon RSS first: it is the metric this run exists to protect.
|
|
for key in ("daemon_rss_last_mb", "daemon_rss_max_mb", "daemon_rss_samples",
|
|
"daemon_cpu_mean_pct", "cpu_mean_pct", "cpu_max_pct", "mem_max_mb", "exec_timeouts"):
|
|
va = ra["phases"].get(phase, {}).get(key)
|
|
vb = rb["phases"].get(phase, {}).get(key)
|
|
print(f"{phase + '.' + key:32} {str(va):>12} {str(vb):>12} {delta(va, vb):>10}")
|
|
for name in sorted(set(ra["timings"]) | set(rb["timings"])):
|
|
ea, eb = ra["timings"].get(name), rb["timings"].get(name)
|
|
if isinstance(ea, dict) or isinstance(eb, dict):
|
|
unit = "mb" if name == "image_size" else "ms"
|
|
for q in ("p50", "p95", "max"):
|
|
va = (ea or {}).get(f"{q}_{unit}")
|
|
vb = (eb or {}).get(f"{q}_{unit}")
|
|
print(f"{name + '.' + q:32} {str(va):>12} {str(vb):>12}")
|
|
for key in ("storm_execs", "large_execs", "detect_execs"):
|
|
print(f"{key:32} {str(ra['walls_s'].get(key)):>12} {str(rb['walls_s'].get(key)):>12}")
|
|
over = []
|
|
# Budget both the phase-end RSS and the phase peak: a peak-only regression
|
|
# (a spike released before the phase ends) is still a regression.
|
|
for key, label, column, floor in (
|
|
("daemon_rss_last_mb", "daemon RSS", "last", THRESHOLDS["daemon_rss_delta_mb"]),
|
|
("daemon_rss_max_mb", "daemon RSS peak", "peak", THRESHOLDS["daemon_rss_peak_delta_mb"])):
|
|
rss_budget(ra, rb, key, label, column, floor, over)
|
|
for item in over:
|
|
print("OVER BUDGET:", item)
|
|
return 5 if ra["failures"] or rb["failures"] or over else 0
|
|
|
|
|
|
def main(argv):
|
|
if len(argv) == 4 and argv[1] == "--compare":
|
|
return compare(argv[2], argv[3])
|
|
if len(argv) != 2:
|
|
print(__doc__)
|
|
return 2
|
|
report = summarize(argv[1])
|
|
(Path(argv[1]) / "kpi.json").write_text(json.dumps(report, indent=2))
|
|
print_report(report)
|
|
return 5 if report["failures"] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|