fold/scripts/benchmarks/run_rust_swift_goldens.py

#!/usr/bin/env python3
"""Run Rust and Swift frontend golden benchmarks through Fold."""

from __future__ import annotations

import argparse
import importlib.util
import os
import shutil
from pathlib import Path
from typing import Any

REPO_ROOT = Path(__file__).resolve().parents[2]
RUNNER_COMMON_PATH = REPO_ROOT / "scripts" / "benchmarks" / "runner_common.py"
_spec = importlib.util.spec_from_file_location("runner_common", RUNNER_COMMON_PATH)
if _spec is None or _spec.loader is None:
    raise ImportError(f"failed to load {RUNNER_COMMON_PATH}")
common = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(common)

DEFAULT_SPEC_DIRS = [
    REPO_ROOT / "tests" / "benchmarks" / "rust",
    REPO_ROOT / "tests" / "benchmarks" / "swift",
]
COMPARE = REPO_ROOT / "scripts" / "benchmarks" / "compare_findings.py"


def locate_rust_frontend() -> Path | None:
    env = os.environ.get("FOLD_RUST_BIN")
    if env and Path(env).exists():
        return Path(env).resolve()
    candidate = REPO_ROOT / "tools" / "fold-rust" / "target" / "release" / "fold-rust"
    if candidate.exists():
        return candidate.resolve()
    path = shutil.which("fold-rust")
    return Path(path).resolve() if path else None


def locate_swift_frontend(build: Path) -> Path | None:
    env = os.environ.get("FOLD_SWIFT_BIN")
    if env and Path(env).exists():
        return Path(env).resolve()
    candidate = build / "tools" / "fold-swift" / "fold-swift"
    if candidate.exists():
        return candidate.resolve()
    path = shutil.which("fold-swift")
    return Path(path).resolve() if path else None


def frontend_env(mode: str, build: Path) -> dict[str, str]:
    env = dict(os.environ)
    if mode == "rust":
        frontend = locate_rust_frontend()
        if frontend is None:
            raise FileNotFoundError("missing fold-rust frontend")
        env["FOLD_RUST_BIN"] = str(frontend)
        return env
    if mode == "swift":
        frontend = locate_swift_frontend(build)
        if frontend is None:
            raise FileNotFoundError("missing fold-swift frontend")
        if shutil.which("swiftc") is None:
            raise FileNotFoundError("missing swiftc")
        env["FOLD_SWIFT_BIN"] = str(frontend)
        return env
    raise ValueError(f"unsupported frontend mode: {mode}")


def prepare_paths(spec: dict[str, Any], spec_path: Path) -> list[Path]:
    paths = [
        common.resolve_path(str(p), spec_path.parent) for p in spec.get("paths", [])
    ]
    if not paths:
        raise ValueError(
            f"frontend spec must contain non-empty paths list: {spec_path}"
        )
    missing = [str(p) for p in paths if not p.exists()]
    if missing:
        raise FileNotFoundError(", ".join(missing))
    return paths


def run_one(
    spec_path: Path, fold_bin: Path, build: Path, output_dir: Path
) -> tuple[bool, dict[str, Any]]:
    spec = common.load_json(spec_path)
    name = str(spec.get("name", spec_path.stem))
    mode = str(spec.get("mode", spec_path.parent.name))
    skip_if_missing = bool(spec.get("skip_if_missing_tools", True))
    try:
        paths = prepare_paths(spec, spec_path)
        env = frontend_env(mode, build)
    except FileNotFoundError as exc:
        if skip_if_missing:
            return True, {"name": name, "status": "skipped", "reason": str(exc)}
        return False, {"name": name, "status": "failed", "reason": str(exc)}
    except Exception as exc:  # noqa: BLE001 - benchmark should report setup failure.
        return False, {"name": name, "status": "failed", "reason": str(exc)}

    report = output_dir / f"{mode}_{spec_path.stem}.json"
    cmd = [str(fold_bin), "scan", "--mode", mode, *map(str, paths)]
    if spec.get("detectors"):
        cmd.extend(["--detectors", str(spec["detectors"])])
    cmd.extend(["--format", "sarif", "--output", str(report)])
    scan_args = spec.get("scan_args", [])
    if not isinstance(scan_args, list) or not all(
        isinstance(arg, str) for arg in scan_args
    ):
        return False, {
            "name": name,
            "status": "failed",
            "reason": "scan_args must be a list of strings",
        }
    cmd.extend(scan_args)
    return common.scan_and_compare(
        cmd=cmd,
        report=report,
        spec=spec,
        spec_path=spec_path,
        name=name,
        repo_root=REPO_ROOT,
        compare_script=COMPARE,
        env=env,
    )


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--build", default="build/release", type=Path, help="Meson build directory"
    )
    parser.add_argument(
        "--fold", type=Path, help="Path to fold binary (default: BUILD/src/cli/fold)"
    )
    parser.add_argument(
        "--output-dir", default=Path("/tmp/fold-rust-swift-goldens"), type=Path
    )
    parser.add_argument(
        "specs", nargs="*", type=Path, help="Specific benchmark spec files"
    )
    args = parser.parse_args()

    build = args.build if args.build.is_absolute() else (REPO_ROOT / args.build)
    fold_bin = common.find_fold_binary(REPO_ROOT, build, args.fold)
    if fold_bin is None:
        common.print_missing_fold(args.fold or (build / "src" / "cli" / "fold"))
        return 2

    args.output_dir.mkdir(parents=True, exist_ok=True)
    specs = common.specs_from_args(args.specs, DEFAULT_SPEC_DIRS)
    return common.run_suite(
        suite_name="rust_swift_goldens",
        specs=specs,
        run_one=lambda spec_path: run_one(
            spec_path, fold_bin, build.resolve(), args.output_dir.resolve()
        ),
    )


if __name__ == "__main__":
    raise SystemExit(main())