fold/bench/score.py

#!/usr/bin/env python3
"""Compute precision/recall/F1 of a tool run against a gold standard.

Gold standard schema (bench/gold/<corpus>.json):
    {
        "name": "<corpus-name>",
        "description": "...",
        "expected": [
            {"kind": "clone|equiv|dead|class-dup",
             "files": ["a.cpp", "b.cpp"],   # group that should be reported
             "id": "<stable-key>"}
        ],
        "universe": {
            "files": ["all/evaluated.cpp"],
            "kinds": ["all-evaluated-rules"]
        }
    }

A tool finding "covers" a gold entry if:
    - The kinds match (or the gold entry's kind is "any"), AND
    - Each expected file appears as a `file` in at least one finding location
      sharing the same `id` family (kind prefix match).

When ``universe`` is present, every file/rule pair not listed in ``expected``
is an explicit negative case. This permits honest FP/TN and specificity
measurement without treating arbitrary source locations as negatives.

Output: bench/results/<run>/summary.md with a per-tool precision/recall table.
"""
from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path


REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts" / "benchmarks"))

from fold_eval.scoring import (
    negative_cases_from_universe,
    score_grouped_findings,
    score_labeled_cases,
)


def load_gold(gold_path: Path) -> dict:
    return json.loads(gold_path.read_text())


def score_tool(
    findings: list[dict],
    gold_entries: list[dict],
    negative_entries: list[dict] | None = None,
) -> dict:
    """Compatibility wrapper around the canonical one-to-one evaluator."""
    if negative_entries is not None:
        return score_labeled_cases(findings, gold_entries, negative_entries)
    return score_grouped_findings(findings, gold_entries)


def render(rows: list[dict], gold_name: str) -> str:
    out = [
        f"# fold benchmark — {gold_name}",
        "",
        "| Tool | TP | FP | FN | TN | Findings | Cases | Precision | Recall | Specificity | F1 | Elapsed |",
        "|------|---:|---:|---:|---:|---------:|------:|----------:|-------:|------------:|---:|--------:|",
    ]
    for r in rows:
        score = r["score"]
        true_negative = score.get("true_negative")
        specificity = score.get("specificity")
        out.append(
            f"| {r['tool']} | {score['true_positive']} | "
            f"{score['false_positive']} | {score['false_negative']} | "
            f"{true_negative if true_negative is not None else '-'} | "
            f"{score['finding_groups']} | "
            f"{score.get('positive_total', score.get('gold_total', 0)) + score.get('negative_total', 0)} | "
            f"{score['precision']:.2%} | {score['recall']:.2%} | "
            f"{f'{specificity:.2%}' if specificity is not None else '-'} | "
            f"{score['f1']:.2%} | {r['elapsed_s']:.1f}s |"
        )
    if any(r["score"]["missed_gold_indices"] for r in rows):
        out.append("")
        out.append("## Missed gold entries (by tool)")
        for r in rows:
            missed = r["score"]["missed_gold_indices"]
            if not missed:
                continue
            out.append(f"- **{r['tool']}** missed: {missed}")
    return "\n".join(out) + "\n"


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--run-dir", required=True,
                   help="A bench/results/<stamp>_<name>/ directory")
    p.add_argument("--gold", required=True,
                   help="Path to gold standard JSON")
    args = p.parse_args()

    run_dir = Path(args.run_dir).resolve()
    if not run_dir.is_dir():
        print(f"run-dir not found: {run_dir}", file=sys.stderr)
        return 2

    gold = load_gold(Path(args.gold))
    rows = []
    for tool_json in sorted(run_dir.glob("*.json")):
        # Skip support files written by run_bench.py.
        if tool_json.name in {"meta.json", "compile_commands.json"}:
            continue
        try:
            payload = json.loads(tool_json.read_text())
        except json.JSONDecodeError:
            continue
        # Tool payloads are objects; skip anything that isn't (e.g. the
        # compile DB array if it wasn't filtered above).
        if not isinstance(payload, dict) or "findings" not in payload:
            continue
        negative = (
            negative_cases_from_universe(gold["expected"], gold["universe"])
            if "universe" in gold
            else gold.get("negative")
        )
        score = score_tool(payload.get("findings", []), gold["expected"], negative)
        rows.append({
            "tool": payload["tool"],
            "elapsed_s": payload.get("elapsed_s", 0.0),
            "score": score,
        })

    summary = render(rows, gold.get("name", "unknown"))
    (run_dir / "summary.md").write_text(summary)
    print(summary)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())