strawberry/scripts/perf_kpi.py
trvon a5f2dbedfc
KPI: bound collection and reject incomplete measurements
Use suspend-aware BOOTTIME waits and owned-container cleanup without changing the workload. Reject missing coverage and preserve producer/report failures. Include runtime/helper regressions, constrained build-job support and live exec-guard crash safety. Document the failed live/idle gates without relaxing thresholds or claiming production acceptance.
2026-11-03 06:57:00 -07:00

173 lines
8.6 KiB
Python

"""Constrained Linux KPI supervisor; see perf_kpi.sh. Trusted local inputs only."""
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import time
import uuid
from pathlib import Path
import perf_kpi_report
from perf_kpi_runtime import boot, capture, clocks, seconds, stop, until
ROOT = Path(__file__).resolve().parents[1]
def remove_owned(docker, name, token, timeout: float = 5):
check = capture([docker, "inspect", "--format", '{{index .Config.Labels "strawberry.audit.run"}}', name], timeout)
if check.returncode:
if b"no such" in check.stderr.lower():
return "already_absent"
raise RuntimeError("container ownership inspection failed")
if check.stdout.decode().strip() != token:
raise RuntimeError("refusing cleanup of foreign container")
removed = capture([docker, "rm", "-f", name], timeout)
if removed.returncode:
raise RuntimeError("owned container removal failed")
return "removed_owned"
def prepare_rules(out, source):
rules = out / "rules"
rules.mkdir()
shutil.copyfile(ROOT / "config/example_rules.yar", rules / "example_rules.yar")
(rules / "kpi_detect.yar").write_text('''rule KPI_Detect_Marker {
meta:
severity = "medium"
description = "KPI workload marker (perf_kpi.sh)"
strings:
$m = "STRAWBERRY-KPI-DETECT-MARKER"
condition:
$m
}
''')
files = sorted(source.glob("*.yar")) if source.is_dir() else []
if len(files) > 4096:
raise ValueError("rule file limit exceeded")
for path in files:
if path.name in ("example_rules.yar", "kpi_detect.yar"):
raise ValueError("external rule basename collides with KPI rule")
if path.stat().st_size > 128 * 1024 * 1024:
raise ValueError("rule input exceeds limit")
shutil.copyfile(path, rules / path.name) # ordinary user access, no Docker bypass
return {"external_feed_present": bool(files), "source": str(source), "files": [
{"name": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
for path in sorted(rules.iterdir())]}
def main(argv):
label = argv[1] if len(argv) == 2 else "run"
if len(argv) > 2 or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,48}", label):
raise ValueError("label must be a single safe basename")
profile = os.environ.get("KPI_PROFILE", "balanced")
if profile not in ("efficient", "balanced", "aggressive"):
raise ValueError("invalid KPI_PROFILE")
budget = seconds(os.environ.get("KPI_TIMEOUT_S", "300"))
started = boot()
cleanup_reserve = min(15, budget / 4)
deadline = started + budget - cleanup_reserve
token = uuid.uuid4().hex
name = "strawberry-kpi-" + token
out = Path(os.environ.get("KPI_OUT", str(ROOT / "build/perf" / (time.strftime("%Y%m%d-%H%M%S-") + label + "-" + token[:8])))).resolve()
out.mkdir(parents=True, mode=0o700, exist_ok=False)
state = {"label": label, "clocks_start": clocks(), "timeout_s": budget, "container": name,
"token": token, "run_exit": None, "report_exit": None, "cleanup": None, "sampler_errors": []}
docker = os.environ.get("KPI_DOCKER", "docker")
image = os.environ.get("IMAGE_TAG", "strawberry-linux-test")
(out / "state").mkdir()
def save():
(out / "lifecycle.json").write_text(json.dumps(state, indent=2) + "\n")
save()
try:
provenance = prepare_rules(out, Path(os.environ.get("KPI_RULES", "/var/lib/strawberry/feeds/yara-forge-core/current")))
state["rules"] = provenance
if not provenance["external_feed_present"]:
print("WARNING: starter rules only; not representative", file=sys.stderr)
if os.environ.get("KPI_SKIP_BUILD") != "1":
build = capture([docker, "build", "--cpu-quota=100000", "--cpu-period=100000", "--memory=4g", "--memory-swap=4g",
"--build-arg", "BUILD_JOBS=1", "--target", "test", "-t", image, str(ROOT)], 900)
state["build_elapsed_boottime_s"] = boot() - started
state["clocks_invocation"] = state["clocks_start"]
state["clocks_start"] = clocks()
started = boot()
deadline = started + budget - cleanup_reserve
(out / "docker_build.log").write_bytes(build.stdout + build.stderr)
state["build_exit"] = build.returncode
if build.returncode:
raise RuntimeError("project image build failed")
(out / "strawberry.conf").write_text(
"db_path=/kpi/state/database.db\npolicy_path=/kpi/state/policy.json\nfeeds_dir=/kpi/state/feeds\n"
"yara_rules_path=/kpi/rules\nrequire_rules=true\nenable_bpf_monitoring=true\n"
"scan_existing_processes_on_start=true\ncontrol_socket_path=/kpi/state/control.sock\n"
"control_group=\nopensnitch_integration=false\n" + f"scan_profile={profile}\n" + os.environ.get("KPI_EXTRA_CONFIG", "") + "\n")
for script in ("perf_kpi_workload.py", "perf_kpi_runtime.py"):
shutil.copyfile(ROOT / "scripts" / script, out / script)
state["image"] = image
state["cpus"] = os.environ.get("KPI_CPUS", "2")
state["memory"] = os.environ.get("KPI_MEMORY", "2g")
(out / "meta.txt").write_text(f"label={label} cpus={state['cpus']} memory={state['memory']} profile={profile}\n")
command = [docker, "run", "--rm", "--name", name, "--label", "strawberry.audit.run=" + token,
"--privileged", "--network=none", "--cpus=" + state["cpus"], "--memory=" + state["memory"],
"--memory-swap=" + state["memory"], "--ulimit", "core=0", "-v", "/sys/kernel/tracing:/sys/kernel/tracing",
"-v", "/sys/kernel/debug:/sys/kernel/debug", "-v", str(out) + ":/kpi"]
if os.environ.get("KPI_CPUSET"):
command += ["--cpuset-cpus", os.environ["KPI_CPUSET"]]
for key in ("STARTUP_S", "IDLE_S", "STORM_S", "LARGE_S", "DETECT_S", "RELOAD_S"):
command += ["-e", key]
command += [image, "nice", "-n", "19", "python3", "/kpi/perf_kpi_workload.py"]
with (out / "docker_stats.csv").open("w") as stats, (out / "workload.log").open("wb") as log:
stats.write("t,cpu_pct,mem\n")
stats.flush()
def sample():
try:
result = capture([docker, "stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", name], min(3, max(0.01, deadline - boot())))
if result.returncode:
state["sampler_errors"].append("docker stats nonzero")
return
text = result.stdout.decode("utf-8", "replace").strip()
cpu, memory = text.split(",", 1)
stats.write(f"{time.time():.9f},{cpu.rstrip('%')},{memory.split('/')[0].strip()}\n")
stats.flush()
except (ValueError, OSError, subprocess.TimeoutExpired) as error:
state["sampler_errors"].append(type(error).__name__)
process = subprocess.Popen(command, stdout=log, stderr=log, start_new_session=True)
try:
state["run_exit"] = until(deadline, process, sample)
except BaseException:
stop(process)
raise
except (Exception, KeyboardInterrupt) as error:
state["error"] = str(error)
state["run_exit"] = state["run_exit"] or 124
finally:
try:
state["cleanup"] = remove_owned(docker, name, token, min(5, max(0.01, started + budget - boot())))
except (Exception, KeyboardInterrupt) as error:
state["cleanup_error"] = str(error)
state["clocks_end"] = clocks()
state["elapsed_boottime_s"] = boot() - started
state["observed_budget_held"] = state["elapsed_boottime_s"] <= budget
save()
try:
summary = perf_kpi_report.summarize(out)
state["report_exit"] = 5 if summary["failures"] else 0
except (OSError, ValueError, KeyError, TypeError) as error:
summary = {"failures": ["measurement unavailable: " + str(error)]}
state["report_exit"] = 5
state["clocks_end"] = clocks()
state["elapsed_boottime_s"] = boot() - started
state["observed_budget_held"] = state["elapsed_boottime_s"] <= budget
if not state["observed_budget_held"]:
summary["failures"].append("elapsed budget exceeded including report")
state["report_exit"] = 5
(out / "kpi.json").write_text(json.dumps(summary, indent=2) + "\n")
save()
print(out)
return int(state["run_exit"] or (1 if state.get("cleanup_error") or not state["observed_budget_held"] else 0) or state["report_exit"])
if __name__ == "__main__":
sys.exit(main(sys.argv))