fold/bench/gold/_generate_juliet.py

#!/usr/bin/env python3
"""Generate Juliet gold files from the filtered corpus.

Juliet naming varies by flow variant. Some files are explicitly tagged with
good/bad in the filename; others contain both `bad()` and `good*()` entry
points inside the same translation unit. For fold's current scorer we treat a
translation unit as a positive example when it exposes a bad variant.
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
FILTERED_ROOT = ROOT / "corpora" / "juliet" / "filtered"
OUT_ROOT = Path(__file__).resolve().parent

CWE_KIND = {
    "476": "null-deref",
    "369": "div-by-zero",
    "190": "int-overflow",
    "191": "int-overflow",  # integer underflow handled by same detector
    "121": "oob-access",
    "122": "oob-access",
    "124": "oob-access",
    "126": "oob-access",
    "127": "oob-access",
    "194": "sign-conversion",
    "195": "sign-conversion",
    "196": "sign-conversion",
    "197": "int-truncation",
    "561": "dead-branch",
}

CWE_DESC = {
    "476": "Null Pointer Dereference",
    "369": "Divide by Zero",
    "190": "Integer Overflow",
    "191": "Integer Underflow",
    "121": "Stack Buffer Overflow",
    "122": "Heap Buffer Overflow",
    "124": "Buffer Underwrite",
    "126": "Buffer Overread",
    "127": "Buffer Underread",
    "194": "Unexpected Sign Extension",
    "195": "Signed to Unsigned Conversion",
    "196": "Unsigned to Signed Conversion",
    "197": "Numeric Truncation",
    "561": "Dead Code",
}

BAD_NAME_RE = re.compile(r"(?:__bad|_bad_|_bad\.)", re.IGNORECASE)
GOOD_NAME_RE = re.compile(r"(?:__good|_good_|_good\.)", re.IGNORECASE)
BAD_CONTENT_RE = re.compile(r"\bbad\s*\(")
GOOD_CONTENT_RE = re.compile(r"\bgood(?:[A-Z0-9_]+)?\s*\(")


def is_driver_file(path: Path) -> bool:
    lower = path.name.lower()
    return lower.endswith(("_main.c", "_main.cpp"))


def classify_file(path: Path) -> str | None:
    if is_driver_file(path):
        return None

    name = path.name
    if BAD_NAME_RE.search(name):
        return "bad"
    if GOOD_NAME_RE.search(name):
        return "good"

    try:
        text = path.read_text(encoding="utf-8", errors="ignore")
    except OSError:
        return None

    has_bad = bool(BAD_CONTENT_RE.search(text))
    has_good = bool(GOOD_CONTENT_RE.search(text))
    if has_bad:
        return "bad"
    if has_good:
        return "good"
    return None


def kind_for_path(cwe: str, path: Path) -> str:
    """Pick the per-entry kind for a Juliet bad testcase.

    Most CWEs map 1:1 to a single fold detector kind (see CWE_KIND). CWE-561
    spans multiple fold detectors depending on the testcase pattern:
      - `*__return_before_code_*` → dead-statement (CFG-unreachable after return)
      - `*__unused_function_*`    → unused-function (file-static func never called)
      - everything else (`*__main_linux`, testcasesupport stubs) → dead
    """
    base_kind = CWE_KIND[cwe]
    if cwe == "561":
        name = path.name
        if "__return_before_code" in name:
            return "dead-statement"
        if "__unused_function" in name:
            return "unused-function"
        return "dead"
    return base_kind


def build_expected(cwe: str, corpus_dir: Path) -> list[dict]:
    expected = []
    counter = 1
    for path in sorted(corpus_dir.rglob("*")):
        if path.suffix not in {".c", ".cpp"}:
            continue
        classification = classify_file(path)
        if classification != "bad":
            continue
        relative = path.relative_to(corpus_dir).as_posix()
        expected.append(
            {
                "kind": kind_for_path(cwe, path),
                "files": [relative],
                "id": f"juliet-{cwe}-{counter:04d}",
            }
        )
        counter += 1
    return expected


def write_gold(cwe: str, expected: list[dict]) -> Path:
    out_path = OUT_ROOT / f"juliet-{cwe}.json"
    payload = {
        "name": f"juliet-cwe-{cwe}",
        "description": f"Juliet CWE-{cwe} ({CWE_DESC[cwe]}) test cases",
        "expected": expected,
    }
    out_path.write_text(json.dumps(payload, indent=2) + "\n")
    return out_path


def main() -> int:
    if not FILTERED_ROOT.is_dir():
        print(f"filtered Juliet corpus not found at {FILTERED_ROOT}", file=sys.stderr)
        print("Run bench/corpora/juliet/filter_cwes.py first.", file=sys.stderr)
        return 2

    wrote = 0
    for cwe in sorted(CWE_KIND, key=int):
        corpus_dir = FILTERED_ROOT / cwe
        if not corpus_dir.is_dir():
            continue
        expected = build_expected(cwe, corpus_dir)
        out_path = write_gold(cwe, expected)
        print(f"wrote {len(expected)} gold entries -> {out_path}")
        wrote += 1

    if wrote == 0:
        print("No filtered Juliet CWEs found.", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())