279 lines
10 KiB
Python
279 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""Artifact-first benchmark for Brick's stable-ID structural diff and gates."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime as dt
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import platform
|
||
import shutil
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
import runner
|
||
|
||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||
PLAN = ROOT / "tests" / "benchmarks" / "plans" / "structural_diff.json"
|
||
COUNT_FIELDS = {
|
||
"added_nodes",
|
||
"removed_nodes",
|
||
"changed_nodes",
|
||
"added_edges",
|
||
"removed_edges",
|
||
"changed_edges",
|
||
}
|
||
|
||
|
||
def load_plan() -> dict:
|
||
plan = json.loads(PLAN.read_text(encoding="utf-8"))
|
||
if plan.get("schema_version") != 1 or plan.get("name") != "structural_diff":
|
||
raise ValueError("invalid structural_diff plan identity")
|
||
if int(plan.get("repeats", 0)) < 3:
|
||
raise ValueError("structural_diff benchmark requires at least three repeats")
|
||
for case in plan.get("cases", []):
|
||
if set(case.get("expect", {})) != COUNT_FIELDS:
|
||
raise ValueError(f"invalid expected diff counts for {case.get('name')!r}")
|
||
for side in ["baseline", "current"]:
|
||
if not (ROOT / case[side]).is_dir():
|
||
raise ValueError(f"missing {side} fixture for {case['name']}")
|
||
if not plan.get("cases"):
|
||
raise ValueError("structural_diff plan has no cases")
|
||
return plan
|
||
|
||
|
||
def run_command(command: list[str], stdout_path: pathlib.Path, stderr_path: pathlib.Path) -> None:
|
||
completed = subprocess.run(
|
||
command,
|
||
cwd=ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
stdout_path.write_text(completed.stdout, encoding="utf-8")
|
||
stderr_path.write_text(completed.stderr, encoding="utf-8")
|
||
if completed.returncode:
|
||
raise RuntimeError(completed.stderr or completed.stdout)
|
||
|
||
|
||
def run_measured(
|
||
command: list[str], stdout_path: pathlib.Path, stderr_path: pathlib.Path
|
||
) -> tuple[float, int]:
|
||
if not hasattr(os, "wait4"):
|
||
raise RuntimeError("RSS measurement requires os.wait4")
|
||
started = time.perf_counter()
|
||
with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr:
|
||
process = subprocess.Popen(command, cwd=ROOT, stdout=stdout, stderr=stderr)
|
||
_, status, usage = os.wait4(process.pid, 0)
|
||
process.returncode = os.waitstatus_to_exitcode(status)
|
||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||
if process.returncode:
|
||
raise RuntimeError(stderr_path.read_text(encoding="utf-8"))
|
||
max_rss_kib = int(usage.ru_maxrss)
|
||
if sys.platform == "darwin":
|
||
max_rss_kib //= 1024
|
||
return round(elapsed_ms, 3), max_rss_kib
|
||
|
||
|
||
def replace_repository(source: pathlib.Path, repository: pathlib.Path) -> None:
|
||
if repository.exists():
|
||
shutil.rmtree(repository)
|
||
shutil.copytree(source, repository)
|
||
|
||
|
||
def build_graph(
|
||
binary: pathlib.Path,
|
||
repository: pathlib.Path,
|
||
graph: pathlib.Path,
|
||
html: pathlib.Path,
|
||
log_prefix: pathlib.Path,
|
||
) -> None:
|
||
run_command(
|
||
[
|
||
str(binary),
|
||
"build",
|
||
str(repository),
|
||
"--json",
|
||
str(graph),
|
||
"--html",
|
||
str(html),
|
||
],
|
||
log_prefix.with_suffix(".stdout"),
|
||
log_prefix.with_suffix(".stderr"),
|
||
)
|
||
|
||
|
||
def run(out_dir: pathlib.Path | None = None) -> pathlib.Path:
|
||
plan = load_plan()
|
||
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||
artifact = (out_dir or ROOT / "build" / "benchmarks" / "structural_diff" / stamp).resolve()
|
||
artifact.mkdir(parents=True, exist_ok=False)
|
||
binary = ROOT / "target" / "debug" / "brick"
|
||
prepare = ["cargo", "build", "--quiet"]
|
||
run_command(prepare, artifact / "prepare.stdout", artifact / "prepare.stderr")
|
||
manifest = {
|
||
"schema_version": 1,
|
||
"started_at": stamp,
|
||
"plan": "structural_diff",
|
||
"root": str(ROOT),
|
||
"provenance": runner.git_provenance(ROOT),
|
||
"host": {
|
||
"platform": platform.platform(),
|
||
"python": platform.python_version(),
|
||
},
|
||
"prepare_command": prepare,
|
||
}
|
||
(artifact / "run_manifest.json").write_text(
|
||
json.dumps(manifest, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
(artifact / "plan.resolved.json").write_text(
|
||
json.dumps(plan, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
|
||
samples: list[dict] = []
|
||
signatures: dict[str, str] = {}
|
||
for repeat in range(plan["repeats"]):
|
||
for case in plan["cases"]:
|
||
name = case["name"]
|
||
repository = artifact / "repositories" / name
|
||
baseline_graph = artifact / f"{name}-baseline-{repeat}.json"
|
||
current_graph = artifact / f"{name}-current-{repeat}.json"
|
||
replace_repository(ROOT / case["baseline"], repository)
|
||
build_graph(
|
||
binary,
|
||
repository,
|
||
baseline_graph,
|
||
artifact / f"{name}-baseline-{repeat}.html",
|
||
artifact / f"{name}-baseline-build-{repeat}",
|
||
)
|
||
if case.get("allow_root_relocation", False):
|
||
repository = artifact / "relocated-repositories" / name
|
||
replace_repository(ROOT / case["current"], repository)
|
||
build_graph(
|
||
binary,
|
||
repository,
|
||
current_graph,
|
||
artifact / f"{name}-current-{repeat}.html",
|
||
artifact / f"{name}-current-build-{repeat}",
|
||
)
|
||
diff_path = artifact / f"{name}-diff-{repeat}.json"
|
||
command = [
|
||
str(binary),
|
||
"check",
|
||
str(current_graph),
|
||
"--min-score",
|
||
"0",
|
||
"--baseline",
|
||
str(baseline_graph),
|
||
"--diff-json",
|
||
str(diff_path),
|
||
]
|
||
if case.get("allow_root_relocation", False):
|
||
strict = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
|
||
(artifact / f"{name}-strict-{repeat}.stderr").write_text(strict.stderr, encoding="utf-8")
|
||
if strict.returncode == 0 or "scan roots differ" not in strict.stderr:
|
||
raise AssertionError(f"{name} must reject different roots without explicit policy")
|
||
command.append("--allow-root-relocation")
|
||
for field, value in sorted(case["expect"].items()):
|
||
command.extend([f"--max-{field.replace('_', '-')}", str(value)])
|
||
elapsed_ms, max_rss_kib = run_measured(
|
||
command,
|
||
artifact / f"{name}-check-{repeat}.stdout",
|
||
artifact / f"{name}-check-{repeat}.stderr",
|
||
)
|
||
value = json.loads(diff_path.read_text(encoding="utf-8"))
|
||
policy = "explicit-root-equivalence" if case.get("allow_root_relocation", False) else "strict"
|
||
baseline_root = json.loads(baseline_graph.read_text(encoding="utf-8"))["root"]
|
||
current_root = json.loads(current_graph.read_text(encoding="utf-8"))["root"]
|
||
if (value.get("diff_schema_version") != 2
|
||
or value.get("comparison_policy") != policy
|
||
or value.get("baseline_root") != baseline_root
|
||
or value.get("current_root") != current_root):
|
||
raise AssertionError(f"{name} invalid diff comparison metadata")
|
||
counts = value.get("counts")
|
||
if counts != case["expect"]:
|
||
raise AssertionError(
|
||
f"{name} diff counts differ: expected {case['expect']}, got {counts}"
|
||
)
|
||
signature = json.dumps(value, sort_keys=True, separators=(",", ":"))
|
||
if name in signatures and signatures[name] != signature:
|
||
raise AssertionError(f"unstable structural diff for {name} repeat {repeat}")
|
||
signatures[name] = signature
|
||
samples.append(
|
||
{
|
||
"repeat": repeat,
|
||
"case": name,
|
||
"command": command,
|
||
"elapsed_ms": elapsed_ms,
|
||
"max_rss_kib": max_rss_kib,
|
||
"counts": counts,
|
||
}
|
||
)
|
||
|
||
cases = {}
|
||
for case in plan["cases"]:
|
||
case_samples = [sample for sample in samples if sample["case"] == case["name"]]
|
||
elapsed = [sample["elapsed_ms"] for sample in case_samples]
|
||
rss = [sample["max_rss_kib"] for sample in case_samples]
|
||
cases[case["name"]] = {
|
||
"identity_stable": True,
|
||
"elapsed_ms_median": round(statistics.median(elapsed), 3),
|
||
"elapsed_ms_min": min(elapsed),
|
||
"elapsed_ms_max": max(elapsed),
|
||
"max_rss_kib_median": statistics.median(rss),
|
||
"max_rss_kib_min": min(rss),
|
||
"max_rss_kib_max": max(rss),
|
||
"counts": case["expect"],
|
||
}
|
||
summary = {
|
||
"schema_version": 1,
|
||
"plan": "structural_diff",
|
||
"repeats": plan["repeats"],
|
||
"cases": cases,
|
||
"samples": samples,
|
||
}
|
||
(artifact / "summary.json").write_text(
|
||
json.dumps(summary, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
report = ["# Brick benchmark: structural_diff", ""]
|
||
for name, result in cases.items():
|
||
report.extend(
|
||
[
|
||
f"## {name}",
|
||
"",
|
||
"- Stable diff: yes",
|
||
f"- Counts: {result['counts']}",
|
||
f"- Median: {result['elapsed_ms_median']} ms",
|
||
f"- Range: {result['elapsed_ms_min']}–{result['elapsed_ms_max']} ms",
|
||
f"- Max RSS median: {result['max_rss_kib_median']} KiB",
|
||
"",
|
||
]
|
||
)
|
||
(artifact / "REPORT.md").write_text("\n".join(report), encoding="utf-8")
|
||
return artifact
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("command", choices=["self-test", "run"])
|
||
parser.add_argument("--out-dir", type=pathlib.Path)
|
||
args = parser.parse_args()
|
||
plan = load_plan()
|
||
if args.command == "self-test":
|
||
print(f"structural diff benchmark self-test passed ({len(plan['cases'])} cases)")
|
||
return 0
|
||
artifact = run(args.out_dir)
|
||
print(f"ARTIFACT={artifact}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
raise SystemExit(main())
|
||
except (OSError, ValueError, RuntimeError, AssertionError) as error:
|
||
print(f"structural_diff benchmark: {error}", file=sys.stderr)
|
||
raise SystemExit(1) from error
|