fold/bench/perf_audit_run.py

#!/usr/bin/env python3
"""Multi-run perf measurement harness for fold (see docs/internal/perf-audit.md).

Single-run wall-clock diffs at fold's curl-baseline scale carry ~15%
run-to-run variance — too noisy to attribute sub-15% optimization
wins. This script:

  1. Runs `fold scan --profile ...` N times against a fixed target.
  2. Captures the --profile JSON each run.
  3. Reports median + IQR + min/max for total_wall_ms and the new
     per-detector phase timings (walker_ms, encode_ms, hook_ms,
     state_merge_ms).
  4. Optionally diffs against a saved baseline JSON (recorded by an
     earlier invocation with --record).

Convention: snapshots land under docs/internal/perf-audit/runs/<label>.json
so per-commit baselines are diff-able from git.

Usage:
  bench/perf_audit_run.py --label baseline --runs 5
  bench/perf_audit_run.py --label after-fix --runs 5 \\
      --diff docs/internal/perf-audit/runs/baseline.json
"""

from __future__ import annotations

import argparse
import json
import statistics
import subprocess
import sys
import time
from pathlib import Path
from typing import Any

REPO_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_FOLD = REPO_ROOT / "build" / "release" / "src" / "cli" / "fold"
DEFAULT_COMPILE_DB = REPO_ROOT.parent / "curl-source" / "curl" / "build" / "compile_commands.json"
RUNS_DIR = REPO_ROOT / "docs" / "perf-audit" / "runs"

# Metrics surfaced from --profile JSON. Keep in sync with
# include/fold/cli/profile.hpp.
TOP_METRICS = [
    "total_wall_ms",
    "compile_db_load_ms",
    "fanout_orchestration_ms",
    "serial_dispatch_ms",
    "function_fingerprint_ms",
    "project_index_load_ms",
    "project_index_update_ms",
    "call_graph_ms",
    "project_graph_collect_ms",
    "project_graph_merge_ms",
    "project_graph_fallback_ms",
    "project_index_contribution_update_ms",
    "project_index_snapshot_update_ms",
    "security_candidate_ms",
    "analyze_ms",
    "report_ms",
]
DETECTOR_METRICS = [
    "wall_ms",
    "walker_ms",
    "encode_ms",
    "hook_ms",
    "state_merge_ms",
    "control_walk_ms",
    "range_refine_ms",
    "octagon_refine_ms",
    "summary_lookup_ms",
    "z3_queries",
    "stmt_count",
    "control_stmt_count",
    "candidate_sink_count",
    "functions_prechecked_safe",
    "expr_cache_hits",
    "expr_cache_misses",
]


def run_once(fold: Path, compile_db: Path, detectors: str, processes: int,
             tmp_profile: Path, tmp_report: Path) -> dict[str, Any]:
    """Run fold once with --profile. Return the parsed profile JSON."""
    cmd = [
        str(fold), "--quiet", "scan",
        "--compile-db", str(compile_db),
        "--detectors", detectors,
        "--processes", str(processes),
        "--profile",
        "--profile-output", str(tmp_profile),
        "--output", str(tmp_report),
    ]
    started = time.monotonic()
    result = subprocess.run(cmd, capture_output=True, text=True)
    wall_s = time.monotonic() - started
    if result.returncode not in (0, 1):
        sys.stderr.write(f"fold exited {result.returncode}\n")
        sys.stderr.write(result.stderr)
        raise RuntimeError("fold failed; see stderr above")
    if not tmp_profile.exists():
        raise RuntimeError(f"fold did not write profile to {tmp_profile}")
    with tmp_profile.open() as f:
        profile = json.load(f)
    profile["_observed_wall_s"] = wall_s
    return profile


def aggregate(runs: list[dict[str, Any]]) -> dict[str, Any]:
    """Reduce N --profile JSONs to one summary with median+IQR+min+max
    for every metric we care about."""
    summary: dict[str, Any] = {
        "n_runs": len(runs),
        "observed_wall_s": _stats([r["_observed_wall_s"] for r in runs]),
        "scan": {},
        "detectors": {},
    }
    for k in TOP_METRICS:
        vals = [r["scan"].get(k, 0) for r in runs]
        summary["scan"][k] = _stats(vals)
    detector_names = sorted({d["name"] for r in runs for d in r.get("detectors", [])})
    for name in detector_names:
        per_run = []
        for r in runs:
            for d in r.get("detectors", []):
                if d["name"] == name:
                    per_run.append(d)
                    break
        per_metric: dict[str, dict[str, float]] = {}
        for m in DETECTOR_METRICS:
            vals = [d.get(m, 0) for d in per_run]
            per_metric[m] = _stats(vals)
        summary["detectors"][name] = per_metric
    return summary


def _stats(vals: list[float]) -> dict[str, float]:
    if not vals:
        return {"median": 0, "iqr": 0, "min": 0, "max": 0}
    vals = sorted(vals)
    n = len(vals)
    q1 = vals[n // 4]
    q3 = vals[(3 * n) // 4]
    return {
        "median": statistics.median(vals),
        "iqr": q3 - q1,
        "min": vals[0],
        "max": vals[-1],
        "n": n,
    }


def fmt_ms(d: dict[str, float]) -> str:
    """Render a stats dict as a compact "median±iqr (min..max)" string."""
    return f"{int(d['median']):>7}  iqr={int(d['iqr']):>5}  range={int(d['min'])}..{int(d['max'])}"


def print_summary(label: str, summary: dict[str, Any]) -> None:
    print(f"=== {label}  ({summary['n_runs']} runs) ===")
    print(f"  observed_wall_s     {summary['observed_wall_s']['median']:.1f}s  "
          f"iqr={summary['observed_wall_s']['iqr']:.1f}s  "
          f"range={summary['observed_wall_s']['min']:.1f}..{summary['observed_wall_s']['max']:.1f}")
    print()
    print(f"  scan timings (ms):")
    for k in TOP_METRICS:
        d = summary["scan"][k]
        if d["median"] > 0 or d["max"] > 0:
            print(f"    {k:30s} {fmt_ms(d)}")
    print()
    print(f"  detector breakdown (ms):")
    print(f"    {'detector':18s} {'wall':>8s}  {'walker':>8s}  {'encode':>7s}  {'hook':>7s}  {'merge':>5s}  {'queries':>7s}")
    print(f"    {'-'*18} {'-'*8}  {'-'*8}  {'-'*7}  {'-'*7}  {'-'*5}  {'-'*7}")
    for name in sorted(summary["detectors"].keys(),
                       key=lambda n: -summary["detectors"][n]["wall_ms"]["median"]):
        d = summary["detectors"][name]
        row = (
            f"    {name:18s} "
            f"{int(d['wall_ms']['median']):>8}  "
            f"{int(d['walker_ms']['median']):>8}  "
            f"{int(d['encode_ms']['median']):>7}  "
            f"{int(d['hook_ms']['median']):>7}  "
            f"{int(d['state_merge_ms']['median']):>5}  "
            f"{int(d['z3_queries']['median']):>7}"
        )
        print(row)


def print_diff(baseline: dict[str, Any], after: dict[str, Any]) -> None:
    """Side-by-side diff. Flags moves outside median±IQR as "significant"."""
    print()
    print("=== Diff (after vs baseline) ===")
    print(f"{'metric':40s}  {'baseline':>12s}  {'after':>12s}  {'delta':>10s}  {'sig?':>5s}")
    print(f"{'-'*40}  {'-'*12}  {'-'*12}  {'-'*10}  {'-'*5}")

    def diff_one(label: str, b: dict[str, float], a: dict[str, float]) -> None:
        bm, am = b["median"], a["median"]
        delta = am - bm
        # Significant if delta exceeds 1.5 × the larger IQR.
        threshold = 1.5 * max(b["iqr"], a["iqr"], 1)
        sig = "yes" if abs(delta) > threshold else "no"
        pct = f"({delta * 100 / bm:+.1f}%)" if bm else ""
        print(f"{label:40s}  {int(bm):>12}  {int(am):>12}  {int(delta):>+8}{pct:>10s}  {sig:>5}")

    diff_one("scan.total_wall_ms", baseline["scan"]["total_wall_ms"], after["scan"]["total_wall_ms"])
    for k in TOP_METRICS[1:]:
        diff_one(f"scan.{k}", baseline["scan"][k], after["scan"][k])
    for name in sorted(set(baseline["detectors"]) | set(after["detectors"])):
        if name not in baseline["detectors"] or name not in after["detectors"]:
            continue
        b = baseline["detectors"][name]
        a = after["detectors"][name]
        for m in DETECTOR_METRICS:
            diff_one(f"{name}.{m}", b[m], a[m])


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--fold", type=Path, default=DEFAULT_FOLD)
    ap.add_argument("--compile-db", type=Path, default=DEFAULT_COMPILE_DB)
    ap.add_argument("--detectors", default="clone,dead,equiv,null-deref")
    ap.add_argument("--processes", type=int, default=4)
    ap.add_argument("--runs", type=int, default=5,
                    help="number of fold invocations to median over (default 5)")
    ap.add_argument("--label", default="run",
                    help="filename under docs/internal/perf-audit/runs/ to save the summary")
    ap.add_argument("--diff", type=Path,
                    help="path to a baseline summary JSON to diff against")
    args = ap.parse_args()

    if not args.fold.exists():
        sys.stderr.write(f"fold binary not found at {args.fold}\n")
        return 2
    if not args.compile_db.exists():
        sys.stderr.write(f"compile-db not found at {args.compile_db}\n")
        return 2

    RUNS_DIR.mkdir(parents=True, exist_ok=True)
    runs: list[dict[str, Any]] = []
    tmp_profile = Path("/tmp/perf-audit-profile.json")
    tmp_report = Path("/tmp/perf-audit-report.sarif")
    for i in range(args.runs):
        sys.stderr.write(f"[{i + 1}/{args.runs}] running fold...\n")
        try:
            profile = run_once(args.fold, args.compile_db, args.detectors,
                               args.processes, tmp_profile, tmp_report)
        except Exception as e:
            sys.stderr.write(f"run {i + 1} failed: {e}\n")
            return 1
        runs.append(profile)

    summary = aggregate(runs)
    out_path = RUNS_DIR / f"{args.label}.json"
    with out_path.open("w") as f:
        json.dump(summary, f, indent=2)
    sys.stderr.write(f"saved summary to {out_path}\n")

    print_summary(args.label, summary)

    if args.diff:
        if not args.diff.exists():
            sys.stderr.write(f"diff baseline not found at {args.diff}\n")
            return 2
        with args.diff.open() as f:
            baseline = json.load(f)
        print_diff(baseline, summary)

    return 0


if __name__ == "__main__":
    sys.exit(main())