fold/bench/run_security_targets.py

#!/usr/bin/env python3
"""Run Fold source-security benchmark presets on real local projects.

Presets are intentionally optional: if a compile DB is missing, the preset is
reported as skipped rather than failing the whole run.
"""
from __future__ import annotations

import argparse
import json
import statistics
import subprocess
import time
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_FOLD = ROOT / "build" / "release" / "src" / "cli" / "fold"
PRESETS = {
    "yams-wide20-security": {
        "compile_db": ROOT.parent / "yams" / "build" / "release" / "compile_commands.json",
        "files_list": Path("/tmp/fold-yams-wide20.txt"),
        "detectors": "null-deref,div-by-zero,int-overflow,oob",
        "processes": 2,
        "extra": ["--budget-seconds", "10", "--per-query-ms", "150", "--per-tu-timeout", "15"],
    },
    "yams-full-security": {
        "compile_db": ROOT.parent / "yams" / "build" / "release" / "compile_commands.json",
        "files_list": None,
        "detectors": "null-deref,div-by-zero,int-overflow,oob",
        "processes": 4,
        "extra": ["--budget-seconds", "600", "--per-query-ms", "2000"],
    },
    "curl-security": {
        "compile_db": ROOT.parent / "curl-source" / "curl" / "build" / "compile_commands.json",
        "files_list": None,
        "detectors": "null-deref,div-by-zero,int-overflow,oob",
        "processes": 4,
        "extra": ["--budget-seconds", "600", "--per-query-ms", "2000"],
    },
}


def counts_from_sarif(path: Path) -> dict[str, Any]:
    doc = json.loads(path.read_text())
    results = doc.get("runs", [{}])[0].get("results", [])
    by_rule: dict[str, int] = {}
    for r in results:
        by_rule[r.get("ruleId", "")] = by_rule.get(r.get("ruleId", ""), 0) + 1
    return {"total": len(results), "by_rule": dict(sorted(by_rule.items()))}


def run_once(fold: Path, name: str, spec: dict[str, Any], out_dir: Path, repeat: int) -> dict[str, Any]:
    compile_db = Path(spec["compile_db"])
    if not compile_db.exists():
        return {"preset": name, "repeat": repeat, "skipped": f"missing compile DB: {compile_db}"}
    report = out_dir / f"{name}-r{repeat}.sarif"
    profile = out_dir / f"{name}-r{repeat}.profile.json"
    stdout = out_dir / f"{name}-r{repeat}.stdout.log"
    stderr = out_dir / f"{name}-r{repeat}.stderr.log"
    cmd = [
        str(fold), "--quiet", "scan", "--compile-db", str(compile_db),
        "--format", "sarif", "--output", str(report), "--profile", "--profile-output", str(profile),
        "--processes", str(spec["processes"]), "--detectors", spec["detectors"], *spec.get("extra", []),
    ]
    files_list = spec.get("files_list")
    if files_list:
        files_list = Path(files_list)
        if not files_list.exists():
            return {"preset": name, "repeat": repeat, "skipped": f"missing file list: {files_list}"}
        cmd += [line.strip() for line in files_list.read_text().splitlines() if line.strip()]
    started = time.perf_counter()
    proc = subprocess.run(cmd, text=True, capture_output=True, cwd=ROOT)
    elapsed_ms = int((time.perf_counter() - started) * 1000)
    stdout.write_text(proc.stdout)
    stderr.write_text(proc.stderr)
    if proc.returncode not in (0, 1):
        return {"preset": name, "repeat": repeat, "returncode": proc.returncode, "stderr": str(stderr)}
    return {
        "preset": name,
        "repeat": repeat,
        "returncode": proc.returncode,
        "elapsed_ms_external": elapsed_ms,
        "findings": counts_from_sarif(report),
        "profile": json.loads(profile.read_text()),
        "artifacts": {"report": str(report), "profile": str(profile), "stdout": str(stdout), "stderr": str(stderr)},
    }


def median(values: list[float]) -> float | None:
    return statistics.median(values) if values else None


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--fold", type=Path, default=DEFAULT_FOLD)
    ap.add_argument("--preset", choices=["all", *PRESETS.keys()], default="all")
    ap.add_argument("--repeats", type=int, default=1)
    ap.add_argument("--out-dir", type=Path, default=None)
    args = ap.parse_args()
    out_dir = args.out_dir or ROOT / "bench" / "results" / (time.strftime("%Y-%m-%dT%H-%M-%SZ", time.gmtime()) + "_security-targets")
    out_dir.mkdir(parents=True, exist_ok=True)
    selected = PRESETS if args.preset == "all" else {args.preset: PRESETS[args.preset]}
    rows = []
    for name, spec in selected.items():
        for repeat in range(args.repeats):
            row = run_once(args.fold, name, spec, out_dir, repeat)
            rows.append(row)
            status = row.get("skipped") or f"rc={row.get('returncode')} findings={row.get('findings', {}).get('total')}"
            print(f"{name} r{repeat}: {status}", flush=True)
    summary = {"tool": "fold-security-targets", "fold_binary": str(args.fold), "raw": rows, "presets": {}}
    for name in selected:
        subset = [r for r in rows if r.get("preset") == name and not r.get("skipped") and r.get("returncode") in (0, 1)]
        summary["presets"][name] = {
            "runs": len(subset),
            "elapsed_ms_external_median": median([r["elapsed_ms_external"] for r in subset]),
            "findings_last": subset[-1]["findings"] if subset else None,
        }
    (out_dir / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
    print(f"wrote {out_dir / 'summary.json'}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())