2dcbdbc75d
Preserve source, binary, and runtime evidence boundaries while making precision and performance regressions reproducible. - decompose analysis orchestration into bounded modules - add differential, metamorphic, organic, and Yams change-time gates - fix detector, P-Code, fuzz sanitizer, and report identity precision
106 lines
3.3 KiB
Python
Executable file
106 lines
3.3 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run source golden benchmarks through Fold and compare the resulting reports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib.util
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
RUNNER_COMMON_PATH = REPO_ROOT / "scripts" / "benchmarks" / "runner_common.py"
|
|
_spec = importlib.util.spec_from_file_location("runner_common", RUNNER_COMMON_PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError(f"failed to load {RUNNER_COMMON_PATH}")
|
|
common = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(common)
|
|
|
|
DEFAULT_SPEC_DIR = REPO_ROOT / "tests" / "benchmarks" / "source"
|
|
COMPARE = REPO_ROOT / "scripts" / "benchmarks" / "compare_findings.py"
|
|
|
|
|
|
def run_one(
|
|
spec_path: Path, fold_bin: Path, output_dir: Path
|
|
) -> tuple[bool, dict[str, Any]]:
|
|
spec = common.load_json(spec_path)
|
|
name = str(spec.get("name", spec_path.stem))
|
|
report = output_dir / f"{spec_path.stem}.json"
|
|
try:
|
|
compile_db = common.write_compile_db(spec, spec_path, output_dir, REPO_ROOT)
|
|
except Exception as exc: # noqa: BLE001 - benchmark should report setup failure.
|
|
return False, {"name": name, "status": "failed", "reason": str(exc)}
|
|
|
|
cmd = [
|
|
str(fold_bin),
|
|
"scan",
|
|
"--mode",
|
|
"source",
|
|
"--compile-db",
|
|
str(compile_db),
|
|
"--detectors",
|
|
str(spec.get("detectors", "security")),
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
str(report),
|
|
"--processes",
|
|
str(spec.get("processes", 1)),
|
|
]
|
|
scan_args = spec.get("scan_args", [])
|
|
if not isinstance(scan_args, list) or not all(
|
|
isinstance(arg, str) for arg in scan_args
|
|
):
|
|
return False, {
|
|
"name": name,
|
|
"status": "failed",
|
|
"reason": "scan_args must be a list of strings",
|
|
}
|
|
cmd.extend(scan_args)
|
|
return common.scan_and_compare(
|
|
cmd=cmd,
|
|
report=report,
|
|
spec=spec,
|
|
spec_path=spec_path,
|
|
name=name,
|
|
repo_root=REPO_ROOT,
|
|
compare_script=COMPARE,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--build", default="build/release", type=Path, help="Meson build directory"
|
|
)
|
|
parser.add_argument(
|
|
"--fold", type=Path, help="Path to fold binary (default: BUILD/src/cli/fold)"
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir", default=Path("/tmp/fold-source-goldens"), type=Path
|
|
)
|
|
parser.add_argument(
|
|
"specs", nargs="*", type=Path, help="Specific benchmark spec files"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
build = args.build if args.build.is_absolute() else (REPO_ROOT / args.build)
|
|
fold_bin = common.find_fold_binary(REPO_ROOT, build, args.fold)
|
|
if fold_bin is None:
|
|
common.print_missing_fold(args.fold or (build / "src" / "cli" / "fold"))
|
|
return 2
|
|
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
specs = common.specs_from_args(args.specs, [DEFAULT_SPEC_DIR])
|
|
return common.run_suite(
|
|
suite_name="source_goldens",
|
|
specs=specs,
|
|
run_one=lambda spec_path: run_one(
|
|
spec_path, fold_bin, args.output_dir.resolve()
|
|
),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|