3f1234f7bb
Clang LibTooling + Z3 BV-64 encoding. 11 detectors: clone, equiv, class-dup, dead, null-deref, div-by-zero, int-overflow, oob, int-truncation, sign-conversion, taint-track. Interfaces: CLI, MCP (JSON-RPC 2.0), LSP. Output: SARIF 2.1.0, JSON, Markdown.
277 lines
10 KiB
Python
Executable file
277 lines
10 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Drive fold + comparator SAST tools across a corpus and dump SARIF/JSON.
|
|
|
|
Output layout:
|
|
bench/results/<UTC-iso-date>/
|
|
<tool>.json # raw findings keyed by (file, function, kind)
|
|
meta.json # tool versions, elapsed seconds, exit codes
|
|
summary.md # precision/recall table written by score.py
|
|
|
|
Each tool's findings are normalized into a flat list of records:
|
|
{"file": "...", "line": 0, "kind": "clone|equiv|dead|class-dup", "id": "..."}
|
|
which is what score.py compares against the gold standard.
|
|
|
|
POJ-104 / BigCloneBench / SARD fetchers live in bench/corpora/<corpus>/fetch.sh.
|
|
The runner doesn't fetch corpora itself — keep that explicit so the user
|
|
controls when ~GB of source gets downloaded.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
DEFAULT_FOLD = REPO_ROOT / "build" / "release" / "src" / "cli" / "fold"
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H-%M-%SZ")
|
|
|
|
|
|
def write_compile_db(corpus_dir: Path, db_path: Path) -> None:
|
|
"""Generate a compile_commands.json that covers every .cpp under corpus."""
|
|
entries = []
|
|
# Auto-detect include directories for corpora like Juliet that have
|
|
# a testcasesupport/ directory and subdirectories with headers.
|
|
include_dirs = set()
|
|
for p in sorted(corpus_dir.rglob("*")):
|
|
if p.is_dir() and p.name == "testcasesupport":
|
|
include_dirs.add(str(p))
|
|
if p.is_dir() and any(p.glob("*.h")):
|
|
include_dirs.add(str(p))
|
|
inc_flags = " ".join(f"-I{d}" for d in sorted(include_dirs))
|
|
for cpp in sorted(corpus_dir.rglob("*.cpp")):
|
|
entries.append({
|
|
"directory": str(corpus_dir),
|
|
"command": f"clang++ -std=c++23 {inc_flags} -c {cpp}",
|
|
"file": str(cpp),
|
|
})
|
|
for c_file in sorted(corpus_dir.rglob("*.c")):
|
|
entries.append({
|
|
"directory": str(corpus_dir),
|
|
"command": f"clang -std=c99 {inc_flags} -c {c_file}",
|
|
"file": str(c_file),
|
|
})
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
db_path.write_text(json.dumps(entries))
|
|
|
|
|
|
def run_fold(corpus_dir: Path, out_dir: Path, fold_bin: Path,
|
|
detectors: list[str], processes: int) -> dict:
|
|
db = out_dir / "compile_commands.json"
|
|
write_compile_db(corpus_dir, db)
|
|
sarif = out_dir / "fold.sarif"
|
|
args = [
|
|
str(fold_bin), "scan",
|
|
"--compile-db", str(db),
|
|
"--format", "sarif",
|
|
"--output", str(sarif),
|
|
"--min-body-stmts", "1",
|
|
"--processes", str(processes),
|
|
]
|
|
for d in detectors:
|
|
args += ["--detectors", d]
|
|
t0 = time.monotonic()
|
|
rc = subprocess.run(args, capture_output=True, text=True)
|
|
elapsed = time.monotonic() - t0
|
|
findings = []
|
|
if sarif.exists():
|
|
sarif_doc = json.loads(sarif.read_text())
|
|
for run in sarif_doc.get("runs", []):
|
|
for result in run.get("results", []):
|
|
kind = (result.get("ruleId", "fold.unknown")
|
|
.removeprefix("fold."))
|
|
fid = result.get("properties", {}).get("id", "")
|
|
for loc in result.get("locations", []):
|
|
art = loc.get("physicalLocation", {}).get(
|
|
"artifactLocation", {})
|
|
region = loc.get("physicalLocation", {}).get("region", {})
|
|
findings.append({
|
|
"file": art.get("uri", ""),
|
|
"line": region.get("startLine", 0),
|
|
"kind": kind,
|
|
"id": fid,
|
|
})
|
|
return {
|
|
"tool": "fold",
|
|
"version": "0.1.0",
|
|
"elapsed_s": elapsed,
|
|
"exit_code": rc.returncode,
|
|
"stderr_tail": rc.stderr[-2000:] if rc.stderr else "",
|
|
"findings": findings,
|
|
}
|
|
|
|
|
|
def run_clang_tidy(corpus_dir: Path, out_dir: Path) -> dict:
|
|
db = out_dir / "compile_commands.json"
|
|
write_compile_db(corpus_dir, db)
|
|
# Run clang-tidy with a focused redundancy check set. Any check that
|
|
# roughly maps to "fold also reports this" is in the list.
|
|
checks = ",".join([
|
|
"-*",
|
|
"readability-redundant-*",
|
|
"readability-function-size",
|
|
"misc-redundant-expression",
|
|
"misc-unused-*",
|
|
"bugprone-branch-clone",
|
|
])
|
|
findings = []
|
|
t0 = time.monotonic()
|
|
rc_total = 0
|
|
for cpp in sorted(corpus_dir.rglob("*.cpp")):
|
|
rc = subprocess.run(
|
|
["clang-tidy", "-p", str(out_dir), "--checks", checks,
|
|
"--quiet", str(cpp)],
|
|
capture_output=True, text=True)
|
|
rc_total |= rc.returncode
|
|
for line in (rc.stdout or "").splitlines():
|
|
if ": warning: " not in line and ": error: " not in line:
|
|
continue
|
|
try:
|
|
where, _msg = line.split(": ", 1)
|
|
file_, line_no, *_ = where.split(":")
|
|
except ValueError:
|
|
continue
|
|
findings.append({
|
|
"file": file_,
|
|
"line": int(line_no) if line_no.isdigit() else 0,
|
|
"kind": "clang-tidy",
|
|
"id": line.strip(),
|
|
})
|
|
elapsed = time.monotonic() - t0
|
|
return {
|
|
"tool": "clang-tidy",
|
|
"version": subprocess.run(["clang-tidy", "--version"],
|
|
capture_output=True, text=True
|
|
).stdout.splitlines()[0]
|
|
if shutil.which("clang-tidy") else "missing",
|
|
"elapsed_s": elapsed,
|
|
"exit_code": rc_total,
|
|
"findings": findings,
|
|
}
|
|
|
|
|
|
def run_clang_sa(corpus_dir: Path, out_dir: Path) -> dict:
|
|
"""Run Clang Static Analyzer (clang --analyze) per-file."""
|
|
db = out_dir / "compile_commands.json"
|
|
write_compile_db(corpus_dir, db)
|
|
|
|
def map_csa_kind(line: str) -> str:
|
|
"""Map CSA warning to fold's finding kind."""
|
|
low = line.lower()
|
|
if "dereference of null" in low or "null pointer" in low:
|
|
return "null-deref"
|
|
if "division by zero" in low or "divide by zero" in low:
|
|
return "div-by-zero"
|
|
if "overflow" in low and ("integer" in low or "signed" in low):
|
|
return "int-overflow"
|
|
if "array subscript" in low or "out of bound" in low or \
|
|
"out-of-bound" in low or "buffer overflow" in low:
|
|
return "oob-access"
|
|
return "clang-sa"
|
|
|
|
findings = []
|
|
t0 = time.monotonic()
|
|
rc_total = 0
|
|
for cpp in sorted(corpus_dir.rglob("*.cpp")):
|
|
rc = subprocess.run(
|
|
["clang", "--analyze", "-std=c++17",
|
|
"-Xanalyzer", "-analyzer-output=text",
|
|
str(cpp)],
|
|
capture_output=True, text=True)
|
|
rc_total |= rc.returncode
|
|
for line in (rc.stderr or "").splitlines():
|
|
if ": warning:" not in line:
|
|
continue
|
|
try:
|
|
parts = line.split(":", 4)
|
|
file_ = parts[0] if len(parts) > 0 else str(cpp)
|
|
line_no = int(parts[1]) if len(parts) > 1 and parts[1].strip().isdigit() else 0
|
|
kind = map_csa_kind(line)
|
|
except (ValueError, IndexError):
|
|
continue
|
|
findings.append({
|
|
"file": file_,
|
|
"line": line_no,
|
|
"kind": kind,
|
|
"id": line.strip(),
|
|
})
|
|
elapsed = time.monotonic() - t0
|
|
return {
|
|
"tool": "clang-sa",
|
|
"version": subprocess.run(["clang", "--version"],
|
|
capture_output=True, text=True
|
|
).stdout.splitlines()[0]
|
|
if shutil.which("clang") else "missing",
|
|
"elapsed_s": elapsed,
|
|
"exit_code": rc_total,
|
|
"findings": findings,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--corpus", required=True,
|
|
help="Path to corpus directory (containing .cpp files)")
|
|
p.add_argument("--name",
|
|
help="Logical corpus name (default: corpus dir basename)")
|
|
p.add_argument("--tools", default="fold,clang-tidy",
|
|
help="Comma-separated tools to run")
|
|
p.add_argument("--processes", type=int, default=4,
|
|
help="--processes for fold")
|
|
p.add_argument("--detectors", default="clone,equiv,dead",
|
|
help="Comma-separated fold detectors")
|
|
p.add_argument("--fold-bin", default=str(DEFAULT_FOLD))
|
|
p.add_argument("--out-root", default=str(REPO_ROOT / "bench" / "results"))
|
|
args = p.parse_args()
|
|
|
|
corpus = Path(args.corpus).resolve()
|
|
if not corpus.is_dir():
|
|
print(f"corpus path is not a directory: {corpus}", file=sys.stderr)
|
|
return 2
|
|
|
|
name = args.name or corpus.name
|
|
stamp = now_iso()
|
|
out_dir = Path(args.out_root) / f"{stamp}_{name}"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
print(f"writing to {out_dir}")
|
|
|
|
tool_list = [t.strip() for t in args.tools.split(",") if t.strip()]
|
|
results = {}
|
|
if "fold" in tool_list:
|
|
results["fold"] = run_fold(corpus, out_dir, Path(args.fold_bin),
|
|
args.detectors.split(","),
|
|
args.processes)
|
|
if "clang-tidy" in tool_list and shutil.which("clang-tidy"):
|
|
results["clang-tidy"] = run_clang_tidy(corpus, out_dir)
|
|
if "clang-sa" in tool_list and shutil.which("clang"):
|
|
results["clang-sa"] = run_clang_sa(corpus, out_dir)
|
|
|
|
for tool, payload in results.items():
|
|
(out_dir / f"{tool}.json").write_text(
|
|
json.dumps(payload, indent=2, sort_keys=True))
|
|
print(f"{tool}: {len(payload['findings'])} findings "
|
|
f"in {payload['elapsed_s']:.1f}s (rc={payload['exit_code']})")
|
|
|
|
meta = {
|
|
"stamp": stamp,
|
|
"corpus": str(corpus),
|
|
"name": name,
|
|
"tools": list(results.keys()),
|
|
"processes": args.processes,
|
|
"detectors": args.detectors,
|
|
}
|
|
(out_dir / "meta.json").write_text(json.dumps(meta, indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|