fold/scripts/benchmarks/summarize_maturity.py

#!/usr/bin/env python3
"""Run selected Fold maturity benchmark suites and emit one JSON summary."""

from __future__ import annotations

import argparse
import json
import subprocess
import sys
from pathlib import Path
from typing import Any


REPO_ROOT = Path(__file__).resolve().parents[2]
RUN_BINARY = REPO_ROOT / "scripts" / "benchmarks" / "run_binary_goldens.py"
RUN_HYBRID = REPO_ROOT / "scripts" / "benchmarks" / "run_hybrid_goldens.py"
RUN_RUST_SWIFT = REPO_ROOT / "scripts" / "benchmarks" / "run_rust_swift_goldens.py"
RUN_SOURCE = REPO_ROOT / "scripts" / "benchmarks" / "run_source_goldens.py"

SUITES = [
    ("binary_goldens", "binary", RUN_BINARY, "binary"),
    ("hybrid_goldens", "hybrid", RUN_HYBRID, "hybrid"),
    ("rust_swift_goldens", "rust_swift", RUN_RUST_SWIFT, "rust-swift"),
    ("source_goldens", "source", RUN_SOURCE, "source"),
]
BASELINE_FILES = {
    "binary_goldens": "binary.json",
    "hybrid_goldens": "hybrid.json",
    "rust_swift_goldens": "rust_swift.json",
    "source_goldens": "source.json",
}


def run_json(cmd: list[str]) -> tuple[bool, dict[str, Any]]:
    proc = subprocess.run(
        cmd, cwd=REPO_ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
    )
    try:
        loaded = json.loads(proc.stdout)
        data: dict[str, Any] = loaded if isinstance(loaded, dict) else {"raw": loaded}
    except json.JSONDecodeError:
        data = {"raw_output": proc.stdout}
    data["exit_code"] = proc.returncode
    return proc.returncode == 0, data


def suite_enabled(args: argparse.Namespace, attr: str, any_selected: bool) -> bool:
    return bool(args.all or getattr(args, attr) or not any_selected)


def validate_baseline(
    baseline_dir: Path, suite_name: str
) -> tuple[bool, dict[str, Any]]:
    filename = BASELINE_FILES.get(suite_name)
    if not filename:
        return False, {
            "suite": suite_name,
            "status": "failed",
            "reason": "no baseline mapping",
        }
    path = baseline_dir / filename
    if not path.exists():
        return False, {
            "suite": suite_name,
            "status": "failed",
            "reason": "required baseline missing",
            "baseline": str(path),
        }
    try:
        with path.open("r", encoding="utf-8") as f:
            loaded = json.load(f)
    except json.JSONDecodeError as exc:
        return False, {
            "suite": suite_name,
            "status": "failed",
            "reason": f"baseline JSON is malformed: {exc}",
            "baseline": str(path),
        }
    if not isinstance(loaded, dict):
        return False, {
            "suite": suite_name,
            "status": "failed",
            "reason": "baseline root must be a JSON object",
            "baseline": str(path),
        }
    return True, {"suite": suite_name, "baseline": str(path), "status": "ok"}


def run_suite(
    script: Path, build: Path, output_dir: Path
) -> tuple[bool, dict[str, Any]]:
    return run_json(
        [
            str(sys.executable),
            str(script),
            "--build",
            str(build),
            "--output-dir",
            str(output_dir),
        ]
    )


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--build", default="build/release", type=Path)
    parser.add_argument(
        "--all",
        action="store_true",
        help="Run all maturity suites currently implemented",
    )
    parser.add_argument("--binary", action="store_true", help="Run binary golden suite")
    parser.add_argument("--hybrid", action="store_true", help="Run hybrid golden suite")
    parser.add_argument(
        "--rust-swift", action="store_true", help="Run Rust/Swift frontend golden suite"
    )
    parser.add_argument("--source", action="store_true", help="Run source golden suite")
    parser.add_argument("--output-dir", default=Path("/tmp/fold-maturity"), type=Path)
    parser.add_argument(
        "--baseline-dir",
        default=REPO_ROOT / "scripts" / "benchmarks" / "baselines",
        type=Path,
        help="Directory containing maturity baseline JSON files",
    )
    parser.add_argument(
        "--require-baselines",
        action="store_true",
        help="Fail closed if a selected suite's baseline JSON is missing or malformed",
    )
    args = parser.parse_args()

    any_selected = args.all or any(getattr(args, attr) for _, attr, _, _ in SUITES)
    args.output_dir.mkdir(parents=True, exist_ok=True)

    results: list[dict[str, Any]] = []
    ok = True
    implemented = []
    for suite_name, attr, script, subdir in SUITES:
        implemented.append(suite_name)
        if not suite_enabled(args, attr, any_selected):
            continue
        if args.require_baselines:
            baseline_ok, baseline_result = validate_baseline(
                args.baseline_dir, suite_name
            )
            if not baseline_ok:
                ok = False
                results.append(baseline_result)
                continue
        passed, result = run_suite(script, args.build, args.output_dir / subdir)
        ok = ok and passed
        results.append(result)

    summary = {
        "suite": "fold_maturity",
        "status": "ok" if ok else "failed",
        "implemented_suites": implemented,
        "results": results,
    }
    print(json.dumps(summary, indent=2, sort_keys=True))
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())