#!/usr/bin/env python3 """Generate Juliet gold files from the filtered corpus. Juliet naming varies by flow variant. Some files are explicitly tagged with good/bad in the filename; others contain both `bad()` and `good*()` entry points inside the same translation unit. For fold's current scorer we treat a translation unit as a positive example when it exposes a bad variant. """ from __future__ import annotations import json import re import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent FILTERED_ROOT = ROOT / "corpora" / "juliet" / "filtered" OUT_ROOT = Path(__file__).resolve().parent CWE_KIND = { "476": "null-deref", "369": "div-by-zero", "190": "int-overflow", "191": "int-overflow", # integer underflow handled by same detector "121": "oob-access", "122": "oob-access", "124": "oob-access", "126": "oob-access", "127": "oob-access", "194": "sign-conversion", "195": "sign-conversion", "196": "sign-conversion", "197": "int-truncation", "561": "dead-branch", } CWE_DESC = { "476": "Null Pointer Dereference", "369": "Divide by Zero", "190": "Integer Overflow", "191": "Integer Underflow", "121": "Stack Buffer Overflow", "122": "Heap Buffer Overflow", "124": "Buffer Underwrite", "126": "Buffer Overread", "127": "Buffer Underread", "194": "Unexpected Sign Extension", "195": "Signed to Unsigned Conversion", "196": "Unsigned to Signed Conversion", "197": "Numeric Truncation", "561": "Dead Code", } BAD_NAME_RE = re.compile(r"(?:__bad|_bad_|_bad\.)", re.IGNORECASE) GOOD_NAME_RE = re.compile(r"(?:__good|_good_|_good\.)", re.IGNORECASE) BAD_CONTENT_RE = re.compile(r"\bbad\s*\(") GOOD_CONTENT_RE = re.compile(r"\bgood(?:[A-Z0-9_]+)?\s*\(") def is_driver_file(path: Path) -> bool: lower = path.name.lower() return lower.endswith(("_main.c", "_main.cpp")) def classify_file(path: Path) -> str | None: if is_driver_file(path): return None name = path.name if BAD_NAME_RE.search(name): return "bad" if GOOD_NAME_RE.search(name): return "good" try: text = path.read_text(encoding="utf-8", errors="ignore") except OSError: return None has_bad = bool(BAD_CONTENT_RE.search(text)) has_good = bool(GOOD_CONTENT_RE.search(text)) if has_bad: return "bad" if has_good: return "good" return None def kind_for_path(cwe: str, path: Path) -> str: """Pick the per-entry kind for a Juliet bad testcase. Most CWEs map 1:1 to a single fold detector kind (see CWE_KIND). CWE-561 spans multiple fold detectors depending on the testcase pattern: - `*__return_before_code_*` → dead-statement (CFG-unreachable after return) - `*__unused_function_*` → unused-function (file-static func never called) - everything else (`*__main_linux`, testcasesupport stubs) → dead """ base_kind = CWE_KIND[cwe] if cwe == "561": name = path.name if "__return_before_code" in name: return "dead-statement" if "__unused_function" in name: return "unused-function" return "dead" return base_kind def build_expected(cwe: str, corpus_dir: Path) -> list[dict]: expected = [] counter = 1 for path in sorted(corpus_dir.rglob("*")): if path.suffix not in {".c", ".cpp"}: continue classification = classify_file(path) if classification != "bad": continue relative = path.relative_to(corpus_dir).as_posix() expected.append( { "kind": kind_for_path(cwe, path), "files": [relative], "id": f"juliet-{cwe}-{counter:04d}", } ) counter += 1 return expected def write_gold(cwe: str, expected: list[dict]) -> Path: out_path = OUT_ROOT / f"juliet-{cwe}.json" payload = { "name": f"juliet-cwe-{cwe}", "description": f"Juliet CWE-{cwe} ({CWE_DESC[cwe]}) test cases", "expected": expected, } out_path.write_text(json.dumps(payload, indent=2) + "\n") return out_path def main() -> int: if not FILTERED_ROOT.is_dir(): print(f"filtered Juliet corpus not found at {FILTERED_ROOT}", file=sys.stderr) print("Run bench/corpora/juliet/filter_cwes.py first.", file=sys.stderr) return 2 wrote = 0 for cwe in sorted(CWE_KIND, key=int): corpus_dir = FILTERED_ROOT / cwe if not corpus_dir.is_dir(): continue expected = build_expected(cwe, corpus_dir) out_path = write_gold(cwe, expected) print(f"wrote {len(expected)} gold entries -> {out_path}") wrote += 1 if wrote == 0: print("No filtered Juliet CWEs found.", file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())