119 lines
3.8 KiB
Python
Executable file
119 lines
3.8 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Run binary golden benchmarks through Fold and compare the resulting reports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import importlib.util
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
RUNNER_COMMON_PATH = REPO_ROOT / "scripts" / "benchmarks" / "runner_common.py"
|
|
_spec = importlib.util.spec_from_file_location("runner_common", RUNNER_COMMON_PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError(f"failed to load {RUNNER_COMMON_PATH}")
|
|
common = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(common)
|
|
|
|
DEFAULT_SPEC_DIR = REPO_ROOT / "tests" / "benchmarks" / "binary"
|
|
COMPARE = REPO_ROOT / "scripts" / "benchmarks" / "compare_findings.py"
|
|
|
|
|
|
def binary_for_spec(spec: dict[str, Any], spec_path: Path, output_dir: Path) -> Path:
|
|
if "build_source" in spec:
|
|
return common.build_c_fixture(
|
|
spec,
|
|
spec_path,
|
|
output_dir,
|
|
REPO_ROOT,
|
|
source_key="build_source",
|
|
)
|
|
return common.resolve_path(str(spec["binary"]), spec_path.parent)
|
|
|
|
|
|
def run_one(
|
|
spec_path: Path, fold_bin: Path, output_dir: Path
|
|
) -> tuple[bool, dict[str, Any]]:
|
|
spec = common.load_json(spec_path)
|
|
name = str(spec.get("name", spec_path.stem))
|
|
skip_if_missing = bool(spec.get("skip_if_missing", False))
|
|
try:
|
|
binary = binary_for_spec(spec, spec_path, output_dir)
|
|
except Exception as exc: # noqa: BLE001 - benchmark should report setup failure.
|
|
if skip_if_missing:
|
|
return True, {"name": name, "status": "skipped", "reason": str(exc)}
|
|
return False, {"name": name, "status": "failed", "reason": str(exc)}
|
|
if not binary.exists():
|
|
if skip_if_missing:
|
|
return True, {
|
|
"name": name,
|
|
"status": "skipped",
|
|
"reason": f"missing {binary}",
|
|
}
|
|
return False, {"name": name, "status": "failed", "reason": f"missing {binary}"}
|
|
|
|
report = output_dir / f"{spec_path.stem}.json"
|
|
cmd = [
|
|
str(fold_bin),
|
|
"scan",
|
|
"--mode",
|
|
"binary",
|
|
"--binary",
|
|
str(binary),
|
|
"--detectors",
|
|
str(spec.get("detectors", "security")),
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
str(report),
|
|
]
|
|
if spec.get("binary_max_functions") is not None:
|
|
cmd.extend(["--binary-max-functions", str(spec["binary_max_functions"])])
|
|
return common.scan_and_compare(
|
|
cmd=cmd,
|
|
report=report,
|
|
spec=spec,
|
|
spec_path=spec_path,
|
|
name=name,
|
|
repo_root=REPO_ROOT,
|
|
compare_script=COMPARE,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--build", default="build/release", type=Path, help="Meson build directory"
|
|
)
|
|
parser.add_argument(
|
|
"--fold", type=Path, help="Path to fold binary (default: BUILD/src/cli/fold)"
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir", default=Path("/tmp/fold-binary-goldens"), type=Path
|
|
)
|
|
parser.add_argument(
|
|
"specs", nargs="*", type=Path, help="Specific benchmark spec files"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
build = args.build if args.build.is_absolute() else (REPO_ROOT / args.build)
|
|
fold_bin = common.find_fold_binary(REPO_ROOT, build, args.fold)
|
|
if fold_bin is None:
|
|
common.print_missing_fold(args.fold or (build / "src" / "cli" / "fold"))
|
|
return 2
|
|
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
specs = common.specs_from_args(args.specs, [DEFAULT_SPEC_DIR])
|
|
return common.run_suite(
|
|
suite_name="binary_goldens",
|
|
specs=specs,
|
|
run_one=lambda spec_path: run_one(
|
|
spec_path, fold_bin, args.output_dir.resolve()
|
|
),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|