Use suspend-aware BOOTTIME waits and owned-container cleanup without changing the workload. Reject missing coverage and preserve producer/report failures. Include runtime/helper regressions, constrained build-job support and live exec-guard crash safety. Document the failed live/idle gates without relaxing thresholds or claiming production acceptance.
173 lines
8.6 KiB
Python
173 lines
8.6 KiB
Python
"""Constrained Linux KPI supervisor; see perf_kpi.sh. Trusted local inputs only."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import perf_kpi_report
|
|
from perf_kpi_runtime import boot, capture, clocks, seconds, stop, until
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def remove_owned(docker, name, token, timeout: float = 5):
|
|
check = capture([docker, "inspect", "--format", '{{index .Config.Labels "strawberry.audit.run"}}', name], timeout)
|
|
if check.returncode:
|
|
if b"no such" in check.stderr.lower():
|
|
return "already_absent"
|
|
raise RuntimeError("container ownership inspection failed")
|
|
if check.stdout.decode().strip() != token:
|
|
raise RuntimeError("refusing cleanup of foreign container")
|
|
removed = capture([docker, "rm", "-f", name], timeout)
|
|
if removed.returncode:
|
|
raise RuntimeError("owned container removal failed")
|
|
return "removed_owned"
|
|
|
|
|
|
def prepare_rules(out, source):
|
|
rules = out / "rules"
|
|
rules.mkdir()
|
|
shutil.copyfile(ROOT / "config/example_rules.yar", rules / "example_rules.yar")
|
|
(rules / "kpi_detect.yar").write_text('''rule KPI_Detect_Marker {
|
|
meta:
|
|
severity = "medium"
|
|
description = "KPI workload marker (perf_kpi.sh)"
|
|
strings:
|
|
$m = "STRAWBERRY-KPI-DETECT-MARKER"
|
|
condition:
|
|
$m
|
|
}
|
|
''')
|
|
files = sorted(source.glob("*.yar")) if source.is_dir() else []
|
|
if len(files) > 4096:
|
|
raise ValueError("rule file limit exceeded")
|
|
for path in files:
|
|
if path.name in ("example_rules.yar", "kpi_detect.yar"):
|
|
raise ValueError("external rule basename collides with KPI rule")
|
|
if path.stat().st_size > 128 * 1024 * 1024:
|
|
raise ValueError("rule input exceeds limit")
|
|
shutil.copyfile(path, rules / path.name) # ordinary user access, no Docker bypass
|
|
return {"external_feed_present": bool(files), "source": str(source), "files": [
|
|
{"name": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
for path in sorted(rules.iterdir())]}
|
|
|
|
|
|
def main(argv):
|
|
label = argv[1] if len(argv) == 2 else "run"
|
|
if len(argv) > 2 or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,48}", label):
|
|
raise ValueError("label must be a single safe basename")
|
|
profile = os.environ.get("KPI_PROFILE", "balanced")
|
|
if profile not in ("efficient", "balanced", "aggressive"):
|
|
raise ValueError("invalid KPI_PROFILE")
|
|
budget = seconds(os.environ.get("KPI_TIMEOUT_S", "300"))
|
|
started = boot()
|
|
cleanup_reserve = min(15, budget / 4)
|
|
deadline = started + budget - cleanup_reserve
|
|
token = uuid.uuid4().hex
|
|
name = "strawberry-kpi-" + token
|
|
out = Path(os.environ.get("KPI_OUT", str(ROOT / "build/perf" / (time.strftime("%Y%m%d-%H%M%S-") + label + "-" + token[:8])))).resolve()
|
|
out.mkdir(parents=True, mode=0o700, exist_ok=False)
|
|
state = {"label": label, "clocks_start": clocks(), "timeout_s": budget, "container": name,
|
|
"token": token, "run_exit": None, "report_exit": None, "cleanup": None, "sampler_errors": []}
|
|
docker = os.environ.get("KPI_DOCKER", "docker")
|
|
image = os.environ.get("IMAGE_TAG", "strawberry-linux-test")
|
|
(out / "state").mkdir()
|
|
def save():
|
|
(out / "lifecycle.json").write_text(json.dumps(state, indent=2) + "\n")
|
|
save()
|
|
try:
|
|
provenance = prepare_rules(out, Path(os.environ.get("KPI_RULES", "/var/lib/strawberry/feeds/yara-forge-core/current")))
|
|
state["rules"] = provenance
|
|
if not provenance["external_feed_present"]:
|
|
print("WARNING: starter rules only; not representative", file=sys.stderr)
|
|
if os.environ.get("KPI_SKIP_BUILD") != "1":
|
|
build = capture([docker, "build", "--cpu-quota=100000", "--cpu-period=100000", "--memory=4g", "--memory-swap=4g",
|
|
"--build-arg", "BUILD_JOBS=1", "--target", "test", "-t", image, str(ROOT)], 900)
|
|
state["build_elapsed_boottime_s"] = boot() - started
|
|
state["clocks_invocation"] = state["clocks_start"]
|
|
state["clocks_start"] = clocks()
|
|
started = boot()
|
|
deadline = started + budget - cleanup_reserve
|
|
(out / "docker_build.log").write_bytes(build.stdout + build.stderr)
|
|
state["build_exit"] = build.returncode
|
|
if build.returncode:
|
|
raise RuntimeError("project image build failed")
|
|
(out / "strawberry.conf").write_text(
|
|
"db_path=/kpi/state/database.db\npolicy_path=/kpi/state/policy.json\nfeeds_dir=/kpi/state/feeds\n"
|
|
"yara_rules_path=/kpi/rules\nrequire_rules=true\nenable_bpf_monitoring=true\n"
|
|
"scan_existing_processes_on_start=true\ncontrol_socket_path=/kpi/state/control.sock\n"
|
|
"control_group=\nopensnitch_integration=false\n" + f"scan_profile={profile}\n" + os.environ.get("KPI_EXTRA_CONFIG", "") + "\n")
|
|
for script in ("perf_kpi_workload.py", "perf_kpi_runtime.py"):
|
|
shutil.copyfile(ROOT / "scripts" / script, out / script)
|
|
state["image"] = image
|
|
state["cpus"] = os.environ.get("KPI_CPUS", "2")
|
|
state["memory"] = os.environ.get("KPI_MEMORY", "2g")
|
|
(out / "meta.txt").write_text(f"label={label} cpus={state['cpus']} memory={state['memory']} profile={profile}\n")
|
|
command = [docker, "run", "--rm", "--name", name, "--label", "strawberry.audit.run=" + token,
|
|
"--privileged", "--network=none", "--cpus=" + state["cpus"], "--memory=" + state["memory"],
|
|
"--memory-swap=" + state["memory"], "--ulimit", "core=0", "-v", "/sys/kernel/tracing:/sys/kernel/tracing",
|
|
"-v", "/sys/kernel/debug:/sys/kernel/debug", "-v", str(out) + ":/kpi"]
|
|
if os.environ.get("KPI_CPUSET"):
|
|
command += ["--cpuset-cpus", os.environ["KPI_CPUSET"]]
|
|
for key in ("STARTUP_S", "IDLE_S", "STORM_S", "LARGE_S", "DETECT_S", "RELOAD_S"):
|
|
command += ["-e", key]
|
|
command += [image, "nice", "-n", "19", "python3", "/kpi/perf_kpi_workload.py"]
|
|
with (out / "docker_stats.csv").open("w") as stats, (out / "workload.log").open("wb") as log:
|
|
stats.write("t,cpu_pct,mem\n")
|
|
stats.flush()
|
|
def sample():
|
|
try:
|
|
result = capture([docker, "stats", "--no-stream", "--format", "{{.CPUPerc}},{{.MemUsage}}", name], min(3, max(0.01, deadline - boot())))
|
|
if result.returncode:
|
|
state["sampler_errors"].append("docker stats nonzero")
|
|
return
|
|
text = result.stdout.decode("utf-8", "replace").strip()
|
|
cpu, memory = text.split(",", 1)
|
|
stats.write(f"{time.time():.9f},{cpu.rstrip('%')},{memory.split('/')[0].strip()}\n")
|
|
stats.flush()
|
|
except (ValueError, OSError, subprocess.TimeoutExpired) as error:
|
|
state["sampler_errors"].append(type(error).__name__)
|
|
process = subprocess.Popen(command, stdout=log, stderr=log, start_new_session=True)
|
|
try:
|
|
state["run_exit"] = until(deadline, process, sample)
|
|
except BaseException:
|
|
stop(process)
|
|
raise
|
|
except (Exception, KeyboardInterrupt) as error:
|
|
state["error"] = str(error)
|
|
state["run_exit"] = state["run_exit"] or 124
|
|
finally:
|
|
try:
|
|
state["cleanup"] = remove_owned(docker, name, token, min(5, max(0.01, started + budget - boot())))
|
|
except (Exception, KeyboardInterrupt) as error:
|
|
state["cleanup_error"] = str(error)
|
|
state["clocks_end"] = clocks()
|
|
state["elapsed_boottime_s"] = boot() - started
|
|
state["observed_budget_held"] = state["elapsed_boottime_s"] <= budget
|
|
save()
|
|
try:
|
|
summary = perf_kpi_report.summarize(out)
|
|
state["report_exit"] = 5 if summary["failures"] else 0
|
|
except (OSError, ValueError, KeyError, TypeError) as error:
|
|
summary = {"failures": ["measurement unavailable: " + str(error)]}
|
|
state["report_exit"] = 5
|
|
state["clocks_end"] = clocks()
|
|
state["elapsed_boottime_s"] = boot() - started
|
|
state["observed_budget_held"] = state["elapsed_boottime_s"] <= budget
|
|
if not state["observed_budget_held"]:
|
|
summary["failures"].append("elapsed budget exceeded including report")
|
|
state["report_exit"] = 5
|
|
(out / "kpi.json").write_text(json.dumps(summary, indent=2) + "\n")
|
|
save()
|
|
print(out)
|
|
return int(state["run_exit"] or (1 if state.get("cleanup_error") or not state["observed_budget_held"] else 0) or state["report_exit"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|