612 lines
22 KiB
Python
612 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Run fold --mode=rust against a Rust OSS codebase and record benchmark results.
|
|
|
|
Drives `fold scan --mode=rust` against a well-known Rust codebase (default:
|
|
ripgrep, ~52k LOC). Measures wall time, findings per detector family, and
|
|
IR encoder throughput. Stores results as bench/results/rust-oss-baseline.json
|
|
and compares against the existing curl source/binary benchmark when available.
|
|
|
|
Usage:
|
|
# Clone ripgrep first if not already present
|
|
bench/corpora/rust-oss/fetch.sh
|
|
|
|
# Full run with all Rust detectors
|
|
python3 bench/run_rust_oss.py
|
|
|
|
# Fast smoke run with a detector subset
|
|
python3 bench/run_rust_oss.py --detectors clone,null-deref,rust-command-injection
|
|
|
|
# Custom corpus
|
|
python3 bench/run_rust_oss.py --corpus /path/to/rust/project
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
DEFAULT_FOLD = REPO_ROOT / "build" / "release" / "src" / "cli" / "fold"
|
|
DEFAULT_CORPUS = REPO_ROOT / "bench" / "corpora" / "rust-oss" / "ripgrep"
|
|
RESULTS_ROOT = REPO_ROOT / "bench" / "results"
|
|
FOLD_RUST_TOOLCHAIN = REPO_ROOT / "tools" / "fold-rust" / "rust-toolchain.toml"
|
|
|
|
# Detectors relevant to Rust mode. Mirrors the RustBackend finding-kind map
|
|
# (src/frontend/rust_backend.cpp) plus clone/equiv/dead/class-dup from the
|
|
# shared IR layer. fold-rust runs its own detectors in the cargo-check
|
|
# subprocess; the C++ side filters by --detectors post-hoc.
|
|
RUST_DETECTORS = [
|
|
"clone",
|
|
"equiv",
|
|
"dead",
|
|
"class-dup",
|
|
"complexity",
|
|
"null-deref",
|
|
"div-by-zero",
|
|
"int-overflow",
|
|
"int-truncation",
|
|
"sign-conversion",
|
|
"oob",
|
|
"taint-track",
|
|
"rust-unsafe-api",
|
|
"rust-unchecked-index",
|
|
"rust-path-traversal",
|
|
"rust-command-injection",
|
|
"rust-crypto-misuse",
|
|
]
|
|
|
|
SUCCESS_EXIT_CODES = {0, 1} # fold returns 1 when findings are present.
|
|
|
|
|
|
def now_slug() -> str:
|
|
return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H-%M-%SZ")
|
|
|
|
|
|
def split_csv(value: str) -> list[str]:
|
|
return [item.strip() for item in value.split(",") if item.strip()]
|
|
|
|
|
|
def detect_rust_toolchain() -> str | None:
|
|
"""Read the nightly channel from tools/fold-rust/rust-toolchain.toml."""
|
|
if not FOLD_RUST_TOOLCHAIN.exists():
|
|
return None
|
|
text = FOLD_RUST_TOOLCHAIN.read_text()
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if line.startswith("channel"):
|
|
# channel = "nightly-2026-05-22"
|
|
parts = line.split("=", 1)
|
|
if len(parts) == 2:
|
|
return parts[1].strip().strip('"').strip("'")
|
|
return None
|
|
|
|
|
|
def prep_corpus(corpus_dir: Path) -> dict[str, Any]:
|
|
"""Clean the corpus target dir and collect stats."""
|
|
stats: dict[str, Any] = {"path": str(corpus_dir.resolve())}
|
|
rs_files = sorted(corpus_dir.rglob("*.rs"))
|
|
stats["rs_files"] = len(rs_files)
|
|
loc = 0
|
|
for f in rs_files:
|
|
try:
|
|
loc += len(f.read_text(errors="ignore").splitlines())
|
|
except OSError:
|
|
pass
|
|
stats["rs_loc"] = loc
|
|
|
|
# Clean stale build artifacts so fold-rust gets a clean compilation.
|
|
target_dir = corpus_dir / "target"
|
|
if target_dir.exists():
|
|
shutil.rmtree(target_dir, ignore_errors=True)
|
|
|
|
# Also nuke any stale fold-rust ndjson temp dirs from prior runs.
|
|
tmp = Path("/tmp")
|
|
for d in tmp.glob("fold-rust-ndjson-*"):
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
|
|
return stats
|
|
|
|
|
|
def parse_mixed_output(text: str) -> tuple[dict[str, Any] | None, dict[str, Any] | None, list[str]]:
|
|
"""Parse fold's mixed stdout: log lines + SARIF JSON + profile JSON.
|
|
|
|
Returns (sarif_doc, profile_doc, log_lines).
|
|
"""
|
|
lines = text.splitlines()
|
|
log_lines: list[str] = []
|
|
json_objects: list[dict[str, Any]] = []
|
|
|
|
current_obj: list[str] = []
|
|
in_obj = False
|
|
brace_depth = 0
|
|
for line in lines:
|
|
if not in_obj:
|
|
stripped = line.strip()
|
|
if stripped.startswith("{"):
|
|
in_obj = True
|
|
brace_depth = 0
|
|
else:
|
|
log_lines.append(line)
|
|
continue
|
|
if in_obj:
|
|
current_obj.append(line)
|
|
brace_depth += line.count("{") - line.count("}")
|
|
if brace_depth == 0:
|
|
try:
|
|
json_objects.append(json.loads("\n".join(current_obj)))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
current_obj = []
|
|
in_obj = False
|
|
|
|
sarif_doc = None
|
|
profile_doc = None
|
|
for obj in json_objects:
|
|
if "$schema" in obj and "runs" in obj:
|
|
sarif_doc = obj
|
|
elif "scan" in obj and "timestamp" in obj:
|
|
profile_doc = obj
|
|
|
|
return sarif_doc, profile_doc, log_lines
|
|
|
|
|
|
def extract_ir_encoder_metrics(log_lines: list[str]) -> dict[str, Any]:
|
|
"""Extract IR encoder throughput from log lines like
|
|
'[info] rust shard: IR encoder processed 2670 functions'
|
|
"""
|
|
metrics: dict[str, Any] = {}
|
|
re_encoder = re.compile(r"IR encoder processed (\d+) functions")
|
|
re_emitted = re.compile(r"emitted (\d+) findings total")
|
|
for line in log_lines:
|
|
m = re_encoder.search(line)
|
|
if m:
|
|
metrics["ir_encoder_functions"] = int(m.group(1))
|
|
m = re_emitted.search(line)
|
|
if m:
|
|
metrics["emitted_findings_total"] = int(m.group(1))
|
|
return metrics
|
|
|
|
|
|
def extract_clone_profile(log_lines: list[str]) -> dict[str, Any]:
|
|
"""Extract clone bucket stats from log lines."""
|
|
clone: dict[str, dict[str, Any]] = {}
|
|
re_clone = re.compile(
|
|
r"clone profile \[(\S+)\]: (\d+) buckets, max=(\d+), avg=([\d.]+), total_pairs=(\d+)"
|
|
)
|
|
for line in log_lines:
|
|
m = re_clone.search(line)
|
|
if m:
|
|
variant = m.group(1)
|
|
clone[variant] = {
|
|
"buckets": int(m.group(2)),
|
|
"max_bucket_size": int(m.group(3)),
|
|
"avg_bucket_size": float(m.group(4)),
|
|
"total_pairs": int(m.group(5)),
|
|
}
|
|
return clone
|
|
|
|
|
|
def summarize_findings(sarif_doc: dict[str, Any] | None) -> dict[str, Any]:
|
|
"""Extract per-detector finding counts from a SARIF document."""
|
|
if sarif_doc is None:
|
|
return {"total": 0, "by_rule": {}, "by_detector": {}, "top_findings": []}
|
|
|
|
findings: list[dict[str, Any]] = []
|
|
for run in sarif_doc.get("runs", []):
|
|
for result in run.get("results", []):
|
|
props = result.get("properties", {})
|
|
locations = result.get("locations", [])
|
|
first_loc = locations[0] if locations else {}
|
|
physical = first_loc.get("physicalLocation", {}) if isinstance(first_loc, dict) else {}
|
|
artifact = physical.get("artifactLocation", {})
|
|
region = physical.get("region", {})
|
|
findings.append({
|
|
"id": props.get("id", ""),
|
|
"rule": str(result.get("ruleId", "fold.unknown")).removeprefix("fold."),
|
|
"detector": props.get("detector", props.get("kind", "")),
|
|
"confidence": props.get("confidence", ""),
|
|
"level": result.get("level", ""),
|
|
"score": float(props.get("score", 0.0) or 0.0),
|
|
"message": result.get("message", {}).get("text", ""),
|
|
"file": artifact.get("uri", ""),
|
|
"line": int(region.get("startLine", 0) or 0),
|
|
"backend": props.get("backend", props.get("backend", "")),
|
|
})
|
|
|
|
by_rule = Counter(f["rule"] for f in findings)
|
|
by_detector = Counter(f["detector"] or f["rule"] for f in findings)
|
|
scores = [f["score"] for f in findings]
|
|
|
|
return {
|
|
"total": len(findings),
|
|
"by_rule": dict(sorted(by_rule.items())),
|
|
"by_detector": dict(sorted(by_detector.items())),
|
|
"average_score": sum(scores) / len(scores) if scores else 0.0,
|
|
"top_findings": sorted(findings, key=lambda f: f["score"], reverse=True)[:10],
|
|
}
|
|
|
|
|
|
def run_rust_scan(args: argparse.Namespace, out_dir: Path, corpus_dir: Path,
|
|
corpus_stats: dict[str, Any]) -> dict[str, Any]:
|
|
"""Run fold scan --mode=rust and return structured results."""
|
|
fold_bin = Path(args.fold).expanduser().resolve()
|
|
detectors = split_csv(args.detectors)
|
|
toolchain = args.rust_toolchain or detect_rust_toolchain()
|
|
|
|
command = [
|
|
str(fold_bin),
|
|
"scan",
|
|
"--mode=rust",
|
|
str(corpus_dir.resolve()),
|
|
"--format", "json",
|
|
"--output", str(out_dir / "scan.json"),
|
|
"--profile",
|
|
]
|
|
for detector in detectors:
|
|
command.extend(["--detectors", detector])
|
|
|
|
# Build environment with the correct Rust toolchain.
|
|
env = os.environ.copy()
|
|
if toolchain:
|
|
env["RUSTUP_TOOLCHAIN"] = toolchain
|
|
|
|
started = time.perf_counter()
|
|
proc = subprocess.run(
|
|
command,
|
|
cwd=str(REPO_ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=args.timeout,
|
|
env=env,
|
|
)
|
|
elapsed = time.perf_counter() - started
|
|
|
|
# Save raw outputs.
|
|
(out_dir / "stdout.log").write_text(proc.stdout)
|
|
(out_dir / "stderr.log").write_text(proc.stderr)
|
|
|
|
# Parse mixed stdout: log lines + SARIF JSON + profile JSON.
|
|
sarif_doc, profile_doc, log_lines = parse_mixed_output(proc.stdout)
|
|
|
|
# If scan.json was written via --output, also try parsing that.
|
|
scan_json_path = out_dir / "scan.json"
|
|
if scan_json_path.exists() and sarif_doc is None:
|
|
try:
|
|
sarif_doc = json.loads(scan_json_path.read_text())
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Extract metrics.
|
|
ir_metrics = extract_ir_encoder_metrics(log_lines)
|
|
clone_profile = extract_clone_profile(log_lines)
|
|
findings = summarize_findings(sarif_doc)
|
|
|
|
profile_scan = profile_doc.get("scan", {}) if profile_doc else {}
|
|
|
|
result: dict[str, Any] = {
|
|
"tool": "fold",
|
|
"mode": "rust",
|
|
"version": "0.1.0",
|
|
"command": shlex.join(command),
|
|
"exit_code": proc.returncode,
|
|
"success": proc.returncode in SUCCESS_EXIT_CODES,
|
|
"timed_out": False,
|
|
"elapsed_s": elapsed,
|
|
"corpus": corpus_stats,
|
|
"rust_toolchain": toolchain,
|
|
"profile": {
|
|
"num_functions": profile_scan.get("num_functions", 0),
|
|
"num_tus": profile_scan.get("num_tus", 0),
|
|
"total_wall_ms": profile_scan.get("total_wall_ms", 0),
|
|
"analyze_ms": profile_scan.get("analyze_ms", 0),
|
|
"parse_ms": profile_scan.get("parse_ms", 0),
|
|
"report_ms": profile_scan.get("report_ms", 0),
|
|
},
|
|
"ir_encoder": ir_metrics,
|
|
"clone_profile": clone_profile,
|
|
"findings": findings,
|
|
"log_summary": {
|
|
"ir_encoder_line": next(
|
|
(l for l in log_lines if "IR encoder" in l), ""
|
|
),
|
|
"clone_profile_lines": [l for l in log_lines if "clone profile" in l],
|
|
"shard_lines": [l for l in log_lines if "shard" in l],
|
|
},
|
|
"stderr_tail": proc.stderr[-2000:] if proc.stderr else "",
|
|
}
|
|
|
|
# IR encoder throughput.
|
|
if ir_metrics.get("ir_encoder_functions") and profile_scan.get("analyze_ms"):
|
|
functions = ir_metrics["ir_encoder_functions"]
|
|
ms = profile_scan["analyze_ms"]
|
|
result["ir_encoder"]["throughput_functions_per_sec"] = (
|
|
round(functions / (ms / 1000.0), 1) if ms > 0 else 0.0
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def find_latest_curl_baseline() -> dict[str, Any] | None:
|
|
"""Find the most recent curl A/B benchmark result for comparison."""
|
|
results_dir = RESULTS_ROOT
|
|
if not results_dir.exists():
|
|
return None
|
|
|
|
curl_runs: list[Path] = []
|
|
for d in results_dir.iterdir():
|
|
if d.is_dir() and "curl" in d.name.lower():
|
|
summary = d / "summary.json"
|
|
if summary.exists():
|
|
curl_runs.append(d)
|
|
if not curl_runs:
|
|
return None
|
|
|
|
curl_runs.sort(reverse=True) # most recent first
|
|
latest = curl_runs[0]
|
|
try:
|
|
return json.loads((latest / "summary.json").read_text())
|
|
except (json.JSONDecodeError, OSError):
|
|
return None
|
|
|
|
|
|
def build_comparison(rust_result: dict[str, Any],
|
|
curl_baseline: dict[str, Any] | None) -> dict[str, Any]:
|
|
"""Build comparison section against curl benchmark."""
|
|
comparison: dict[str, Any] = {}
|
|
if curl_baseline is None:
|
|
comparison["available"] = False
|
|
comparison["note"] = "No curl A/B baseline found in bench/results/. Run run_curl_ab.py first."
|
|
return comparison
|
|
|
|
comparison["available"] = True
|
|
comparison["curl_timestamp"] = curl_baseline.get("timestamp", "unknown")
|
|
|
|
# Extract curl source scan metrics.
|
|
curl_source = curl_baseline.get("source", {})
|
|
curl_binary = curl_baseline.get("binary", {})
|
|
|
|
# Rust scan metrics.
|
|
rust_wall_s = rust_result["elapsed_s"]
|
|
rust_findings = rust_result["findings"]["total"]
|
|
rust_functions = rust_result["profile"]["num_functions"]
|
|
rust_throughput = rust_result["ir_encoder"].get("throughput_functions_per_sec", 0)
|
|
|
|
# Curl source scan metrics.
|
|
curl_source_wall = curl_source.get("elapsed_s", 0)
|
|
curl_source_findings = curl_source.get("findings", {}).get("total", 0)
|
|
|
|
comparison["rust"] = {
|
|
"corpus": rust_result["corpus"]["path"],
|
|
"rs_loc": rust_result["corpus"]["rs_loc"],
|
|
"rs_files": rust_result["corpus"]["rs_files"],
|
|
"wall_time_s": rust_wall_s,
|
|
"findings": rust_findings,
|
|
"functions": rust_functions,
|
|
"ir_encoder_throughput_fns_per_sec": rust_throughput,
|
|
"by_detector": rust_result["findings"]["by_detector"],
|
|
}
|
|
comparison["curl_source"] = {
|
|
"wall_time_s": curl_source_wall,
|
|
"findings": curl_source_findings,
|
|
"by_detector": curl_source.get("findings", {}).get("by_detector", {}),
|
|
"skipped": curl_source.get("skipped", False),
|
|
}
|
|
comparison["curl_binary"] = {
|
|
"wall_time_s": curl_binary.get("elapsed_s", 0),
|
|
"findings": curl_binary.get("findings", {}).get("total", 0),
|
|
"by_detector": curl_binary.get("findings", {}).get("by_detector", {}),
|
|
"skipped": curl_binary.get("skipped", False),
|
|
}
|
|
|
|
# Cross-backend notes.
|
|
comparison["notes"] = [
|
|
"Rust scan uses fold-rust HIR frontend + in-process detectors; C++ source scan uses Clang/Z3.",
|
|
"Detector families differ: Rust mode has rust-* detectors; C++ source has CWE-based security detectors.",
|
|
"Throughput numbers are not directly comparable — Rust HIR is pre-lowered; C++ AST is walked symbolically.",
|
|
]
|
|
|
|
return comparison
|
|
|
|
|
|
def markdown_report(baseline: dict[str, Any]) -> str:
|
|
"""Generate a Markdown summary of the Rust OSS benchmark."""
|
|
lines = [
|
|
"# Rust OSS Benchmark — fold --mode=rust",
|
|
"",
|
|
f"- timestamp: `{baseline['timestamp']}`",
|
|
f"- corpus: `{baseline['corpus']['path']}`",
|
|
f"- Rust LOC: `{baseline['corpus']['rs_loc']}`",
|
|
f"- Rust files: `{baseline['corpus']['rs_files']}`",
|
|
f"- Rust toolchain: `{baseline.get('rust_toolchain', 'unknown')}`",
|
|
"",
|
|
"## Scan Results",
|
|
f"- success: `{baseline.get('success')}`",
|
|
f"- exit code: `{baseline.get('exit_code')}`",
|
|
f"- wall time: `{baseline['elapsed_s']:.2f}s`",
|
|
f"- command: `{baseline.get('command', '')}`",
|
|
"",
|
|
"## Profile",
|
|
]
|
|
profile = baseline.get("profile", {})
|
|
lines.append(f"- functions: `{profile.get('num_functions', 0)}`")
|
|
lines.append(f"- TUs: `{profile.get('num_tus', 0)}`")
|
|
lines.append(f"- total wall: `{profile.get('total_wall_ms', 0)}ms`")
|
|
lines.append(f"- analyze: `{profile.get('analyze_ms', 0)}ms`")
|
|
|
|
ir = baseline.get("ir_encoder", {})
|
|
lines.append("")
|
|
lines.append("## IR Encoder")
|
|
lines.append(f"- functions processed: `{ir.get('ir_encoder_functions', 0)}`")
|
|
throughput = ir.get("throughput_functions_per_sec", 0)
|
|
lines.append(f"- throughput: `{throughput:.1f} functions/s`")
|
|
lines.append(f"- emitted findings (total): `{ir.get('emitted_findings_total', 0)}`")
|
|
|
|
clone = baseline.get("clone_profile", {})
|
|
if clone:
|
|
lines.append("")
|
|
lines.append("## Clone Profile")
|
|
lines.append("| Variant | Buckets | Max | Avg | Total Pairs |")
|
|
lines.append("|---|---|---|---|---|")
|
|
for variant, stats in sorted(clone.items()):
|
|
lines.append(
|
|
f"| `{variant}` | {stats['buckets']} | {stats['max_bucket_size']} | "
|
|
f"{stats['avg_bucket_size']:.1f} | {stats['total_pairs']} |"
|
|
)
|
|
|
|
findings = baseline.get("findings", {})
|
|
lines.append("")
|
|
lines.append("## Findings")
|
|
lines.append(f"- total: `{findings.get('total', 0)}`")
|
|
lines.append(f"- average score: `{findings.get('average_score', 0):.2f}`")
|
|
by_detector = findings.get("by_detector", {})
|
|
if by_detector:
|
|
lines.append("")
|
|
lines.append("### Per Detector Family")
|
|
lines.append("| Detector | Count |")
|
|
lines.append("|---|---|")
|
|
for detector, count in sorted(by_detector.items()):
|
|
lines.append(f"| `{detector}` | {count} |")
|
|
|
|
# Comparison section.
|
|
comp = baseline.get("comparison", {})
|
|
lines.append("")
|
|
lines.append("## Comparison with curl A/B")
|
|
if not comp.get("available"):
|
|
lines.append(f"Comparison unavailable: {comp.get('note', 'unknown reason')}")
|
|
else:
|
|
lines.append(f"curl baseline timestamp: `{comp.get('curl_timestamp', '?')}`")
|
|
lines.append("")
|
|
lines.append("| Metric | Rust (ripgrep) | curl Source | curl Binary |")
|
|
lines.append("|---|---|---|---|")
|
|
r = comp.get("rust", {})
|
|
cs = comp.get("curl_source", {})
|
|
cb = comp.get("curl_binary", {})
|
|
lines.append(
|
|
f"| Wall time (s) | {r.get('wall_time_s', 0):.1f} | "
|
|
f"{cs.get('wall_time_s', 0):.1f} | {cb.get('wall_time_s', 0):.1f} |"
|
|
)
|
|
lines.append(
|
|
f"| Findings | {r.get('findings', 0)} | "
|
|
f"{cs.get('findings', 0)} | {cb.get('findings', 0)} |"
|
|
)
|
|
lines.append(
|
|
f"| Functions | {r.get('functions', 0)} | N/A | N/A |"
|
|
)
|
|
lines.append(
|
|
f"| IR throughput (fn/s) | {r.get('ir_encoder_throughput_fns_per_sec', 0):.1f} | N/A | N/A |"
|
|
)
|
|
for note in comp.get("notes", []):
|
|
lines.append(f"\n*{note}*")
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_args(argv: list[str]) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Run fold --mode=rust against a Rust OSS codebase."
|
|
)
|
|
parser.add_argument("--fold", default=str(DEFAULT_FOLD),
|
|
help="Path to fold binary")
|
|
parser.add_argument("--corpus", default=str(DEFAULT_CORPUS),
|
|
help="Path to Rust project directory")
|
|
parser.add_argument("--detectors", default=",".join(RUST_DETECTORS),
|
|
help="Comma-separated fold detectors")
|
|
parser.add_argument("--rust-toolchain", default="",
|
|
help="Rust toolchain for cargo check (auto-detected from "
|
|
"tools/fold-rust/rust-toolchain.toml)")
|
|
parser.add_argument("--timeout", type=int, default=600,
|
|
help="Timeout in seconds for fold scan")
|
|
parser.add_argument("--out-dir", default="",
|
|
help="Output directory (default: bench/results/<timestamp>_rust-oss)")
|
|
parser.add_argument("--skip-scan", action="store_true",
|
|
help="Skip the scan; regenerate report from existing results")
|
|
parser.add_argument("--no-compare", action="store_true",
|
|
help="Skip curl A/B comparison")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
args = parse_args(argv)
|
|
corpus_dir = Path(args.corpus).expanduser().resolve()
|
|
|
|
if not corpus_dir.is_dir():
|
|
print(f"error: corpus directory not found: {corpus_dir}", file=sys.stderr)
|
|
print(" Run: bench/corpora/rust-oss/fetch.sh", file=sys.stderr)
|
|
return 2
|
|
|
|
if not Path(args.fold).expanduser().exists():
|
|
print(f"error: fold binary not found: {args.fold}", file=sys.stderr)
|
|
print(" Run: ninja -C build/release", file=sys.stderr)
|
|
return 2
|
|
|
|
out_dir = Path(args.out_dir).expanduser().resolve() if args.out_dir else (
|
|
RESULTS_ROOT / f"{now_slug()}_rust-oss"
|
|
)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
corpus_stats = prep_corpus(corpus_dir)
|
|
|
|
if args.skip_scan:
|
|
existing = out_dir / "rust-oss-baseline.json"
|
|
if existing.exists():
|
|
baseline = json.loads(existing.read_text())
|
|
else:
|
|
print(f"error: --skip-scan but no existing baseline at {existing}", file=sys.stderr)
|
|
return 2
|
|
else:
|
|
scan_result = run_rust_scan(args, out_dir, corpus_dir, corpus_stats)
|
|
|
|
baseline = {
|
|
"timestamp": now_slug(),
|
|
"inputs": {
|
|
"fold": str(Path(args.fold).expanduser().resolve()),
|
|
"corpus": str(corpus_dir),
|
|
"detectors": split_csv(args.detectors),
|
|
"rust_toolchain": args.rust_toolchain or detect_rust_toolchain(),
|
|
},
|
|
**scan_result,
|
|
}
|
|
|
|
# Add comparison against curl benchmark.
|
|
if not args.no_compare:
|
|
curl_baseline = find_latest_curl_baseline()
|
|
baseline["comparison"] = build_comparison(baseline, curl_baseline)
|
|
|
|
# Write baseline JSON.
|
|
baseline_path = out_dir / "rust-oss-baseline.json"
|
|
baseline_path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n")
|
|
print(f"wrote {baseline_path}")
|
|
|
|
# Write Markdown report.
|
|
md_path = out_dir / "rust-oss-baseline.md"
|
|
md_path.write_text(markdown_report(baseline))
|
|
print(f"wrote {md_path}")
|
|
|
|
# Summary to stdout.
|
|
findings = baseline.get("findings", {})
|
|
profile = baseline.get("profile", {})
|
|
ir = baseline.get("ir_encoder", {})
|
|
print(f"\nRust OSS benchmark ({corpus_dir.name}):")
|
|
print(f" {corpus_stats['rs_loc']} LOC, {corpus_stats['rs_files']} .rs files")
|
|
print(f" wall: {baseline['elapsed_s']:.2f}s")
|
|
print(f" functions: {profile.get('num_functions', 0)}")
|
|
print(f" IR throughput: {ir.get('throughput_functions_per_sec', 0):.1f} fn/s")
|
|
print(f" findings: {findings.get('total', 0)}")
|
|
for detector, count in sorted(findings.get("by_detector", {}).items()):
|
|
print(f" {detector}: {count}")
|
|
|
|
return 0 if baseline.get("success", True) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv[1:]))
|