#!/usr/bin/env python3 """Artifact-first benchmark for Brick's stable-ID structural diff and gates.""" from __future__ import annotations import argparse import datetime as dt import json import os import pathlib import platform import shutil import statistics import subprocess import sys import time import runner ROOT = pathlib.Path(__file__).resolve().parents[2] PLAN = ROOT / "tests" / "benchmarks" / "plans" / "structural_diff.json" COUNT_FIELDS = { "added_nodes", "removed_nodes", "changed_nodes", "added_edges", "removed_edges", "changed_edges", } def load_plan() -> dict: plan = json.loads(PLAN.read_text(encoding="utf-8")) if plan.get("schema_version") != 1 or plan.get("name") != "structural_diff": raise ValueError("invalid structural_diff plan identity") if int(plan.get("repeats", 0)) < 3: raise ValueError("structural_diff benchmark requires at least three repeats") for case in plan.get("cases", []): if set(case.get("expect", {})) != COUNT_FIELDS: raise ValueError(f"invalid expected diff counts for {case.get('name')!r}") for side in ["baseline", "current"]: if not (ROOT / case[side]).is_dir(): raise ValueError(f"missing {side} fixture for {case['name']}") if not plan.get("cases"): raise ValueError("structural_diff plan has no cases") return plan def run_command(command: list[str], stdout_path: pathlib.Path, stderr_path: pathlib.Path) -> None: completed = subprocess.run( command, cwd=ROOT, text=True, capture_output=True, check=False, ) stdout_path.write_text(completed.stdout, encoding="utf-8") stderr_path.write_text(completed.stderr, encoding="utf-8") if completed.returncode: raise RuntimeError(completed.stderr or completed.stdout) def run_measured( command: list[str], stdout_path: pathlib.Path, stderr_path: pathlib.Path ) -> tuple[float, int]: if not hasattr(os, "wait4"): raise RuntimeError("RSS measurement requires os.wait4") started = time.perf_counter() with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: process = subprocess.Popen(command, cwd=ROOT, stdout=stdout, stderr=stderr) _, status, usage = os.wait4(process.pid, 0) process.returncode = os.waitstatus_to_exitcode(status) elapsed_ms = (time.perf_counter() - started) * 1000 if process.returncode: raise RuntimeError(stderr_path.read_text(encoding="utf-8")) max_rss_kib = int(usage.ru_maxrss) if sys.platform == "darwin": max_rss_kib //= 1024 return round(elapsed_ms, 3), max_rss_kib def replace_repository(source: pathlib.Path, repository: pathlib.Path) -> None: if repository.exists(): shutil.rmtree(repository) shutil.copytree(source, repository) def build_graph( binary: pathlib.Path, repository: pathlib.Path, graph: pathlib.Path, html: pathlib.Path, log_prefix: pathlib.Path, ) -> None: run_command( [ str(binary), "build", str(repository), "--json", str(graph), "--html", str(html), ], log_prefix.with_suffix(".stdout"), log_prefix.with_suffix(".stderr"), ) def run(out_dir: pathlib.Path | None = None) -> pathlib.Path: plan = load_plan() stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ") artifact = (out_dir or ROOT / "build" / "benchmarks" / "structural_diff" / stamp).resolve() artifact.mkdir(parents=True, exist_ok=False) binary = ROOT / "target" / "debug" / "brick" prepare = ["cargo", "build", "--quiet"] run_command(prepare, artifact / "prepare.stdout", artifact / "prepare.stderr") manifest = { "schema_version": 1, "started_at": stamp, "plan": "structural_diff", "root": str(ROOT), "provenance": runner.git_provenance(ROOT), "host": { "platform": platform.platform(), "python": platform.python_version(), }, "prepare_command": prepare, } (artifact / "run_manifest.json").write_text( json.dumps(manifest, indent=2) + "\n", encoding="utf-8" ) (artifact / "plan.resolved.json").write_text( json.dumps(plan, indent=2) + "\n", encoding="utf-8" ) samples: list[dict] = [] signatures: dict[str, str] = {} for repeat in range(plan["repeats"]): for case in plan["cases"]: name = case["name"] repository = artifact / "repositories" / name baseline_graph = artifact / f"{name}-baseline-{repeat}.json" current_graph = artifact / f"{name}-current-{repeat}.json" replace_repository(ROOT / case["baseline"], repository) build_graph( binary, repository, baseline_graph, artifact / f"{name}-baseline-{repeat}.html", artifact / f"{name}-baseline-build-{repeat}", ) if case.get("allow_root_relocation", False): repository = artifact / "relocated-repositories" / name replace_repository(ROOT / case["current"], repository) build_graph( binary, repository, current_graph, artifact / f"{name}-current-{repeat}.html", artifact / f"{name}-current-build-{repeat}", ) diff_path = artifact / f"{name}-diff-{repeat}.json" command = [ str(binary), "check", str(current_graph), "--min-score", "0", "--baseline", str(baseline_graph), "--diff-json", str(diff_path), ] if case.get("allow_root_relocation", False): strict = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False) (artifact / f"{name}-strict-{repeat}.stderr").write_text(strict.stderr, encoding="utf-8") if strict.returncode == 0 or "scan roots differ" not in strict.stderr: raise AssertionError(f"{name} must reject different roots without explicit policy") command.append("--allow-root-relocation") for field, value in sorted(case["expect"].items()): command.extend([f"--max-{field.replace('_', '-')}", str(value)]) elapsed_ms, max_rss_kib = run_measured( command, artifact / f"{name}-check-{repeat}.stdout", artifact / f"{name}-check-{repeat}.stderr", ) value = json.loads(diff_path.read_text(encoding="utf-8")) policy = "explicit-root-equivalence" if case.get("allow_root_relocation", False) else "strict" baseline_root = json.loads(baseline_graph.read_text(encoding="utf-8"))["root"] current_root = json.loads(current_graph.read_text(encoding="utf-8"))["root"] if (value.get("diff_schema_version") != 2 or value.get("comparison_policy") != policy or value.get("baseline_root") != baseline_root or value.get("current_root") != current_root): raise AssertionError(f"{name} invalid diff comparison metadata") counts = value.get("counts") if counts != case["expect"]: raise AssertionError( f"{name} diff counts differ: expected {case['expect']}, got {counts}" ) signature = json.dumps(value, sort_keys=True, separators=(",", ":")) if name in signatures and signatures[name] != signature: raise AssertionError(f"unstable structural diff for {name} repeat {repeat}") signatures[name] = signature samples.append( { "repeat": repeat, "case": name, "command": command, "elapsed_ms": elapsed_ms, "max_rss_kib": max_rss_kib, "counts": counts, } ) cases = {} for case in plan["cases"]: case_samples = [sample for sample in samples if sample["case"] == case["name"]] elapsed = [sample["elapsed_ms"] for sample in case_samples] rss = [sample["max_rss_kib"] for sample in case_samples] cases[case["name"]] = { "identity_stable": True, "elapsed_ms_median": round(statistics.median(elapsed), 3), "elapsed_ms_min": min(elapsed), "elapsed_ms_max": max(elapsed), "max_rss_kib_median": statistics.median(rss), "max_rss_kib_min": min(rss), "max_rss_kib_max": max(rss), "counts": case["expect"], } summary = { "schema_version": 1, "plan": "structural_diff", "repeats": plan["repeats"], "cases": cases, "samples": samples, } (artifact / "summary.json").write_text( json.dumps(summary, indent=2) + "\n", encoding="utf-8" ) report = ["# Brick benchmark: structural_diff", ""] for name, result in cases.items(): report.extend( [ f"## {name}", "", "- Stable diff: yes", f"- Counts: {result['counts']}", f"- Median: {result['elapsed_ms_median']} ms", f"- Range: {result['elapsed_ms_min']}–{result['elapsed_ms_max']} ms", f"- Max RSS median: {result['max_rss_kib_median']} KiB", "", ] ) (artifact / "REPORT.md").write_text("\n".join(report), encoding="utf-8") return artifact def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("command", choices=["self-test", "run"]) parser.add_argument("--out-dir", type=pathlib.Path) args = parser.parse_args() plan = load_plan() if args.command == "self-test": print(f"structural diff benchmark self-test passed ({len(plan['cases'])} cases)") return 0 artifact = run(args.out_dir) print(f"ARTIFACT={artifact}") return 0 if __name__ == "__main__": try: raise SystemExit(main()) except (OSError, ValueError, RuntimeError, AssertionError) as error: print(f"structural_diff benchmark: {error}", file=sys.stderr) raise SystemExit(1) from error