477 lines
21 KiB
JSON
477 lines
21 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"source_note": "Paperbridge- and primary-source-grounded references for interpreting Fold benchmark results. Numeric entries are only included when they were available from abstracts or report metadata; exact per-tool Juliet tables should be transcribed separately before making direct head-to-head claims.",
|
|
"references": [
|
|
{
|
|
"id": "wagner-2014-juliet-scanners",
|
|
"title": "Using the Juliet Test Suite to Compare Static Security Scanners",
|
|
"authors": ["Andreas Wagner", "Johannes Sametinger"],
|
|
"year": 2014,
|
|
"doi": "10.5220/0005032902440252",
|
|
"venue": "Proceedings of the 11th International Conference on Security and Cryptography",
|
|
"url": "https://doi.org/10.5220/0005032902440252"
|
|
},
|
|
{
|
|
"id": "nist-sate-v-2018",
|
|
"title": "SATE V report: ten years of static analysis tool expositions",
|
|
"authors": ["Aurelien Delaitre", "Bertrand Stivalet", "Paul E. Black", "Vadim Okun", "Athos Ribeiro", "Terry S. Cohen"],
|
|
"year": 2018,
|
|
"doi": "10.6028/nist.sp.500-326",
|
|
"venue": "NIST Special Publication 500-326",
|
|
"url": "https://doi.org/10.6028/nist.sp.500-326"
|
|
},
|
|
{
|
|
"id": "lipp-2022-realworld-c",
|
|
"title": "An empirical study on the effectiveness of static C code analyzers for vulnerability detection",
|
|
"authors": ["Stephan Lipp", "Sebastian Banescu", "Alexander Pretschner"],
|
|
"year": 2022,
|
|
"doi": "10.1145/3533767.3534380",
|
|
"venue": "ISSTA 2022",
|
|
"url": "https://doi.org/10.1145/3533767.3534380"
|
|
},
|
|
{
|
|
"id": "bennett-2024-semgrep-star",
|
|
"title": "Semgrep*: Improving the Limited Performance of Static Application Security Testing (SAST) Tools",
|
|
"authors": ["Gareth Bennett", "Tracy Hall", "Emily Winter", "Steve Counsell"],
|
|
"year": 2024,
|
|
"doi": "10.1145/3661167.3661262",
|
|
"venue": "EASE 2024",
|
|
"url": "https://doi.org/10.1145/3661167.3661262"
|
|
},
|
|
{
|
|
"id": "beyer-2023-svcomp",
|
|
"title": "Competition on Software Verification and Witness Validation: SV-COMP 2023",
|
|
"authors": ["Dirk Beyer"],
|
|
"year": 2023,
|
|
"doi": "10.1007/978-3-031-30820-8_29",
|
|
"venue": "Lecture Notes in Computer Science",
|
|
"url": "https://doi.org/10.1007/978-3-031-30820-8_29"
|
|
},
|
|
{
|
|
"id": "beyer-2022-svcomp",
|
|
"title": "Progress on Software Verification: SV-COMP 2022",
|
|
"authors": ["Dirk Beyer"],
|
|
"year": 2022,
|
|
"doi": "10.1007/978-3-030-99527-0_20",
|
|
"venue": "Lecture Notes in Computer Science",
|
|
"url": "https://doi.org/10.1007/978-3-030-99527-0_20"
|
|
},
|
|
{
|
|
"id": "bhandari-2021-cvefixes",
|
|
"title": "CVEfixes: automated collection of vulnerabilities and their fixes from open-source software",
|
|
"authors": ["Guru Bhandari", "Amara Naseer", "Leon Moonen"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3475960.3475985",
|
|
"venue": "PROMISE 2021",
|
|
"url": "https://doi.org/10.1145/3475960.3475985"
|
|
},
|
|
{
|
|
"id": "chen-2023-diversevul",
|
|
"title": "DiverseVul: A New Vulnerable Source Code Dataset for Deep Learning Based Vulnerability Detection",
|
|
"authors": ["Yizheng Chen", "Zhoujie Ding", "Lamya Alowain", "Xinyun Chen", "David Wagner"],
|
|
"year": 2023,
|
|
"doi": "10.1145/3607199.3607242",
|
|
"venue": "RAID 2023",
|
|
"url": "https://doi.org/10.1145/3607199.3607242"
|
|
},
|
|
{
|
|
"id": "black-2025-sard",
|
|
"title": "The Software Assurance Reference Dataset (SARD)",
|
|
"authors": ["Paul E. Black"],
|
|
"year": 2025,
|
|
"doi": "10.6028/nist.ir.8561",
|
|
"venue": "NIST IR 8561",
|
|
"url": "https://doi.org/10.6028/nist.ir.8561"
|
|
},
|
|
{
|
|
"id": "godefroid-2005-dart",
|
|
"title": "DART: Directed Automated Random Testing",
|
|
"authors": ["Patrice Godefroid", "Nils Klarlund", "Koushik Sen"],
|
|
"year": 2005,
|
|
"doi": "10.1145/1065010.1065036",
|
|
"venue": "PLDI 2005",
|
|
"url": "https://doi.org/10.1145/1065010.1065036"
|
|
},
|
|
{
|
|
"id": "klinger-2019-differential-analyzers",
|
|
"title": "Differential Testing of Soundness and Precision of Static Analyzers",
|
|
"authors": ["Christian Klinger", "Maria Christakis", "Valentin Wüstholz"],
|
|
"year": 2019,
|
|
"doi": "10.1145/3293882.3330553",
|
|
"venue": "ISSTA 2019",
|
|
"url": "https://doi.org/10.1145/3293882.3330553"
|
|
},
|
|
{
|
|
"id": "ferreiro-2025-checkification",
|
|
"title": "Checkification: A Practical Approach for Testing Static Analysis Truths",
|
|
"authors": ["Daniela Ferreiro", "Ignacio Casso", "Jose F. Morales", "Pedro López-García", "Manuel V. Hermenegildo"],
|
|
"year": 2025,
|
|
"doi": "10.1017/S1471068425100069",
|
|
"venue": "Theory and Practice of Logic Programming",
|
|
"url": "https://doi.org/10.1017/S1471068425100069"
|
|
},
|
|
{
|
|
"id": "mansur-2021-metamorphic-datalog",
|
|
"title": "Metamorphic Testing of Datalog Engines",
|
|
"authors": ["Numair Mansur", "Maria Christakis", "Valentin Wüstholz"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3468264.3468573",
|
|
"venue": "ESEC/FSE 2021",
|
|
"url": "https://doi.org/10.1145/3468264.3468573"
|
|
},
|
|
{
|
|
"id": "klees-2018-evaluating-fuzzing",
|
|
"title": "Evaluating Fuzz Testing",
|
|
"authors": ["George Klees", "Andrew Ruef", "Benji Cooper", "Shiyi Wei", "Michael Hicks"],
|
|
"year": 2018,
|
|
"doi": "10.1145/3243734.3243804",
|
|
"venue": "CCS 2018",
|
|
"url": "https://doi.org/10.1145/3243734.3243804"
|
|
},
|
|
{
|
|
"id": "sok-2024-prudent-fuzzing-evaluation",
|
|
"title": "SoK: Prudent Evaluation Practices for Fuzzing",
|
|
"authors": ["Moritz Schloegel", "Nils Bars", "Nico Schiller", "Lukas Bernhard", "Tobias Scharnowski", "Addison Crump", "Arash Ale Ebrahim", "Nicolai Bissantz", "Marius Muench", "Thorsten Holz"],
|
|
"year": 2024,
|
|
"doi": "10.1109/SP54263.2024.00137",
|
|
"venue": "IEEE Symposium on Security and Privacy 2024",
|
|
"url": "https://doi.org/10.1109/SP54263.2024.00137"
|
|
},
|
|
{
|
|
"id": "metzman-2021-fuzzbench",
|
|
"title": "FuzzBench: An Open Fuzzer Benchmarking Platform and Service",
|
|
"authors": ["Jonathan Metzman", "László Szekeres", "Laurent Simon", "Read Sprabery", "Abhishek Arya"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3468264.3473932",
|
|
"venue": "ESEC/FSE 2021",
|
|
"url": "https://doi.org/10.1145/3468264.3473932"
|
|
},
|
|
{
|
|
"id": "hazimeh-2020-magma",
|
|
"title": "Magma: A Ground-Truth Fuzzing Benchmark",
|
|
"authors": ["Ahmad Hazimeh", "Adrian Herrera", "Mathias Payer"],
|
|
"year": 2020,
|
|
"doi": "10.1145/3428334",
|
|
"venue": "Proceedings of the ACM on Measurement and Analysis of Computing Systems",
|
|
"url": "https://doi.org/10.1145/3428334"
|
|
},
|
|
{
|
|
"id": "bundt-2022-synthetic-bugs",
|
|
"title": "Evaluating Synthetic Bugs",
|
|
"authors": ["Joshua Bundt", "Andrew Fasano", "Brendan Dolan-Gavitt", "William Robertson", "Tim Leek"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3433210.3453096",
|
|
"venue": "ESEC/FSE 2021",
|
|
"url": "https://doi.org/10.1145/3433210.3453096"
|
|
},
|
|
{
|
|
"id": "sadowski-2015-tricorder",
|
|
"title": "Tricorder: Building a Program Analysis Ecosystem",
|
|
"authors": ["Caitlin Sadowski", "Jeffrey van Gogh", "Ciera Jaspan", "Emma Söderberg", "Collin Winter"],
|
|
"year": 2015,
|
|
"doi": "10.1109/ICSE.2015.76",
|
|
"venue": "ICSE 2015",
|
|
"url": "https://doi.org/10.1109/ICSE.2015.76"
|
|
},
|
|
{
|
|
"id": "distefano-2019-scaling-static-analysis",
|
|
"title": "Scaling Static Analyses at Facebook",
|
|
"authors": ["Dino Distefano", "Manuel Fähndrich", "Francesco Logozzo", "Peter W. O'Hearn"],
|
|
"year": 2019,
|
|
"doi": "10.1145/3338112",
|
|
"venue": "Communications of the ACM",
|
|
"url": "https://doi.org/10.1145/3338112"
|
|
},
|
|
{
|
|
"id": "ernst-2015-soundiness",
|
|
"title": "In Defense of Soundiness: A Manifesto",
|
|
"authors": ["Michael D. Ernst", "Anders Møller"],
|
|
"year": 2015,
|
|
"doi": "10.1145/2644805",
|
|
"venue": "Communications of the ACM",
|
|
"url": "https://doi.org/10.1145/2644805"
|
|
},
|
|
{
|
|
"id": "li-2024-primevul",
|
|
"title": "Vulnerability Detection with Code Language Models: How Far Are We?",
|
|
"authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim", "Chawin Sitawarin", "Xinyun Chen", "Basel Alomair", "David Wagner", "Baishakhi Ray", "Yizheng Chen"],
|
|
"year": 2024,
|
|
"venue": "arXiv",
|
|
"url": "https://arxiv.org/abs/2403.18624"
|
|
},
|
|
{
|
|
"id": "bae-2021-rudra",
|
|
"title": "Rudra: Finding Memory Safety Bugs in Rust at the Ecosystem Scale",
|
|
"authors": ["Yechan Bae", "Youngsuk Kim", "Ammar Askar", "Jungwon Lim", "Taesoo Kim"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3477132.3483570",
|
|
"venue": "SOSP 2021",
|
|
"url": "https://doi.org/10.1145/3477132.3483570"
|
|
},
|
|
{
|
|
"id": "li-2021-mirchecker",
|
|
"title": "MirChecker: Detecting Bugs in Rust Programs via Static Analysis",
|
|
"authors": ["Zhuohua Li", "Jincheng Wang", "Mingshen Sun", "John C. S. Lui"],
|
|
"year": 2021,
|
|
"doi": "10.1145/3460120.3484541",
|
|
"venue": "CCS 2021",
|
|
"url": "https://doi.org/10.1145/3460120.3484541"
|
|
},
|
|
{
|
|
"id": "cui-2023-safedrop",
|
|
"title": "SafeDrop: Detecting Memory Deallocation Bugs of Rust Programs via Static Data-flow Analysis",
|
|
"authors": ["Mohan Cui", "Chengjun Chen", "Hui Xu", "Yangfan Zhou"],
|
|
"year": 2023,
|
|
"doi": "10.1145/3542948",
|
|
"venue": "ACM Transactions on Software Engineering and Methodology",
|
|
"url": "https://doi.org/10.1145/3542948"
|
|
},
|
|
{
|
|
"id": "zhuang-2025-dont-panic",
|
|
"title": "Don't Panic! Finding Bugs Hidden Behind Rust Runtime Safety Checks",
|
|
"authors": ["Zeyang Zhuang", "Zilun Wang", "Wei Meng"],
|
|
"year": 2025,
|
|
"doi": "10.1145/3719027.3765142",
|
|
"venue": "Proceedings of the ACM on Software Engineering",
|
|
"url": "https://doi.org/10.1145/3719027.3765142"
|
|
},
|
|
{
|
|
"id": "johns-2011-scanstud",
|
|
"title": "Scanstud: A Methodology for Systematic, Fine-Grained Evaluation of Static Analysis Tools",
|
|
"authors": ["Martin Johns", "Moritz Jodeit"],
|
|
"year": 2011,
|
|
"doi": "10.1109/icstw.2011.32",
|
|
"venue": "IEEE ICST Workshops 2011",
|
|
"url": "https://doi.org/10.1109/icstw.2011.32"
|
|
},
|
|
{
|
|
"id": "braga-2017-crypto-sast-benchmark",
|
|
"title": "Practical Evaluation of Static Analysis Tools for Cryptography: Benchmarking Method and Case Study",
|
|
"authors": ["Alexandre Braga", "Ricardo Dahab", "Nuno Antunes"],
|
|
"year": 2017,
|
|
"doi": "10.1109/issre.2017.27",
|
|
"venue": "IEEE ISSRE 2017",
|
|
"url": "https://doi.org/10.1109/issre.2017.27"
|
|
},
|
|
{
|
|
"id": "lenarduzzi-2021-six-sast",
|
|
"title": "A Critical Comparison on Six Static Analysis Tools: Detection, Agreement, and Precision",
|
|
"authors": ["Valentina Lenarduzzi", "Savanna Lujan", "Nyyti Saarimäki", "Fabio Palomba"],
|
|
"year": 2021,
|
|
"venue": "arXiv",
|
|
"url": "https://arxiv.org/abs/2101.08832"
|
|
},
|
|
{
|
|
"id": "hewson-2015-perf-regression-oracles",
|
|
"title": "Performance Regression Testing on the Java Virtual Machine Using Statistical Test Oracles",
|
|
"authors": ["Fergus Hewson", "Jens Dietrich", "Stephen Marsland"],
|
|
"year": 2015,
|
|
"doi": "10.1109/aswec.2015.14",
|
|
"venue": "ASWEC 2015",
|
|
"url": "https://doi.org/10.1109/aswec.2015.14"
|
|
},
|
|
{
|
|
"id": "mordahl-2023-configurable-sast-benchmarking",
|
|
"title": "Automatic Testing and Benchmarking for Configurable Static Analysis Tools",
|
|
"authors": ["Austin Mordahl", "Dakota Soles", "Miao Liu", "Zenong Zhang", "Shiyi Wei"],
|
|
"year": 2023,
|
|
"doi": "10.1145/3597926.3605232",
|
|
"venue": "ISSTA 2023",
|
|
"url": "https://doi.org/10.1145/3597926.3605232"
|
|
}
|
|
],
|
|
"baselines": [
|
|
{
|
|
"id": "juliet-scanner-comparison",
|
|
"reference_id": "wagner-2014-juliet-scanners",
|
|
"benchmark": "Juliet Test Suite",
|
|
"tool_scope": "static security scanners",
|
|
"comparability": "same synthetic benchmark family as Fold's Juliet slices; exact per-tool result tables still need manual transcription",
|
|
"metrics": []
|
|
},
|
|
{
|
|
"id": "sate-v-methodology",
|
|
"reference_id": "nist-sate-v-2018",
|
|
"benchmark": "NIST SATE / SAMATE exposition corpus",
|
|
"tool_scope": "multi-year static analyzer exposition",
|
|
"comparability": "methodology and cautionary baseline, not a direct precision-recall target for the current Fold Juliet slices",
|
|
"metrics": []
|
|
},
|
|
{
|
|
"id": "realworld-c-static-analyzers",
|
|
"reference_id": "lipp-2022-realworld-c",
|
|
"benchmark": "27 C projects, 1.15 MLOC, 192 CVE-ground-truth vulnerabilities",
|
|
"tool_scope": "five free/open-source analyzers plus one commercial analyzer",
|
|
"comparability": "real-world CVE benchmark; use as an external recall sanity band rather than direct Juliet equivalence",
|
|
"metrics": [
|
|
{"name": "single_tool_false_negative_rate", "value_min": 0.47, "value_max": 0.80, "unit": "ratio"},
|
|
{"name": "combined_tool_false_negative_rate", "value_min": 0.30, "value_max": 0.69, "unit": "ratio"},
|
|
{"name": "projects", "value": 27, "unit": "count"},
|
|
{"name": "vulnerabilities", "value": 192, "unit": "count"},
|
|
{"name": "source_lines", "value": 1150000, "unit": "count"}
|
|
]
|
|
},
|
|
{
|
|
"id": "production-sast-detection",
|
|
"reference_id": "bennett-2024-semgrep-star",
|
|
"benchmark": "manually curated production-code vulnerabilities",
|
|
"tool_scope": "four SAST tools plus Semgrep* rule augmentation",
|
|
"comparability": "production-code detection-rate band; useful for positioning real-world expectations",
|
|
"metrics": [
|
|
{"name": "single_tool_detection_rate", "value_min": 0.112, "value_max": 0.265, "unit": "ratio"},
|
|
{"name": "four_tool_combined_detection_rate", "value": 0.388, "unit": "ratio"},
|
|
{"name": "semgrep_star_detection_rate", "value": 0.447, "unit": "ratio"},
|
|
{"name": "semgrep_star_relative_improvement", "value": 1.81, "unit": "ratio"}
|
|
]
|
|
},
|
|
{
|
|
"id": "svcomp-2023-scale",
|
|
"reference_id": "beyer-2023-svcomp",
|
|
"benchmark": "SV-COMP 2023",
|
|
"tool_scope": "automatic software verifiers",
|
|
"comparability": "task-level verification benchmark; Fold maps only selected properties today",
|
|
"metrics": [
|
|
{"name": "verification_systems", "value": 52, "unit": "count"},
|
|
{"name": "c_tasks", "value": 23805, "unit": "count"},
|
|
{"name": "java_tasks", "value": 586, "unit": "count"}
|
|
]
|
|
},
|
|
{
|
|
"id": "svcomp-2022-scale",
|
|
"reference_id": "beyer-2022-svcomp",
|
|
"benchmark": "SV-COMP 2022",
|
|
"tool_scope": "automatic software verifiers",
|
|
"comparability": "task-level verification benchmark; use for expansion after BV-width stability is fixed",
|
|
"metrics": [
|
|
{"name": "verification_systems", "value": 47, "unit": "count"},
|
|
{"name": "c_tasks", "value": 15648, "unit": "count"},
|
|
{"name": "java_tasks", "value": 586, "unit": "count"}
|
|
]
|
|
},
|
|
{
|
|
"id": "cvefixes-dataset-scale",
|
|
"reference_id": "bhandari-2021-cvefixes",
|
|
"benchmark": "CVEfixes vulnerability-fix corpus",
|
|
"tool_scope": "real-world vulnerability/fix dataset",
|
|
"comparability": "candidate real-world Fold benchmark after repository checkout and compile database synthesis",
|
|
"metrics": [
|
|
{"name": "cve_records", "value": 5365, "unit": "count"},
|
|
{"name": "projects", "value": 1754, "unit": "count"},
|
|
{"name": "fixing_commits", "value": 5495, "unit": "count"}
|
|
]
|
|
},
|
|
{
|
|
"id": "diversevul-dataset-scale",
|
|
"reference_id": "chen-2023-diversevul",
|
|
"benchmark": "DiverseVul",
|
|
"tool_scope": "C/C++ vulnerable and non-vulnerable functions",
|
|
"comparability": "candidate triage corpus for detector coverage, but function labels are weaker than path-specific SAST ground truth",
|
|
"metrics": [
|
|
{"name": "vulnerable_functions", "value": 18945, "unit": "count"},
|
|
{"name": "non_vulnerable_functions", "value": 330492, "unit": "count"},
|
|
{"name": "cwes", "value": 150, "unit": "count"},
|
|
{"name": "commits", "value": 7514, "unit": "count"}
|
|
]
|
|
},
|
|
{
|
|
"id": "sard-scale",
|
|
"reference_id": "black-2025-sard",
|
|
"benchmark": "NIST SARD",
|
|
"tool_scope": "buggy reference programs across multiple languages and weakness classes",
|
|
"comparability": "natural expansion target beyond the current Juliet filters",
|
|
"metrics": [
|
|
{"name": "buggy_programs", "value": 450000, "unit": "count", "qualifier": "over"},
|
|
{"name": "languages", "value": 5, "unit": "count"},
|
|
{"name": "weakness_classes", "value": 150, "unit": "count", "qualifier": "over"}
|
|
]
|
|
}
|
|
],
|
|
"benchmark_targets": [
|
|
{
|
|
"id": "juliet-expanded-fold",
|
|
"status": "ready",
|
|
"runner": "bench/run_summary_ab.py or bench/run_fold_on_juliet.py",
|
|
"scope": "Juliet CWEs with checked-in gold files: 121, 122, 124, 126, 127, 190, 191, 194, 195, 196, 197, 369, 476, 561",
|
|
"next_action": "Run per-CWE summary A/B with file-level scoring and compare ceiling gaps."
|
|
},
|
|
{
|
|
"id": "clang-sa-juliet",
|
|
"status": "ready-if-scan-build-installed",
|
|
"runner": "bench/run_csa_on_juliet.py",
|
|
"scope": "local Clang Static Analyzer head-to-head for null deref, divide-by-zero, integer overflow availability, and array bounds",
|
|
"next_action": "Run after confirming LLVM scan-build path."
|
|
},
|
|
{
|
|
"id": "esbmc-juliet",
|
|
"status": "blocked",
|
|
"runner": "bench/run_esbmc_on_juliet.py",
|
|
"scope": "bounded-model-checking head-to-head, currently wired for CWE-476 null dereference",
|
|
"next_action": "Install ESBMC or set ESBMC_BIN."
|
|
},
|
|
{
|
|
"id": "svcomp-fold",
|
|
"status": "blocked",
|
|
"runner": "bench/run_svcomp.py",
|
|
"scope": "SV-COMP MemSafety valid-deref, NoOverflows, and reachability proxy categories",
|
|
"next_action": "Fix Fold BV-width sort mismatches before trusting broad SV-COMP runs."
|
|
},
|
|
{
|
|
"id": "sard-samate",
|
|
"status": "candidate",
|
|
"runner": "new fetcher under bench/corpora/sard-*",
|
|
"scope": "NIST SARD/SAMATE CWE slices, starting with C dead-code and memory-safety subsets",
|
|
"next_action": "Add fetchers that preserve SARD metadata IDs as gold labels."
|
|
},
|
|
{
|
|
"id": "realworld-cve-corpus",
|
|
"status": "candidate",
|
|
"runner": "new CVEfixes or curated-CVE harness",
|
|
"scope": "repository-level C/C++ CVE fixes with generated compile databases",
|
|
"next_action": "Select a small reproducible project subset before scaling."
|
|
}
|
|
],
|
|
"scaffolding_implications": [
|
|
{
|
|
"group": "profiling-eval-2026-07",
|
|
"note": "Paperbridge pass of 2026-07-27 grounding the manifest-driven project evaluation scaffolding (scripts/benchmarks/run_project_eval.py + fold_eval.project). Implications are one-line engineering decisions, not head-to-head metrics.",
|
|
"entries": [
|
|
{
|
|
"reference_id": "bae-2021-rudra",
|
|
"implication": "Rust analyzer credibility requires ecosystem-scale corpora plus manual per-finding FP review; the Rust lane needs disposition baselines, not a single OSS smoke run."
|
|
},
|
|
{
|
|
"reference_id": "li-2021-mirchecker",
|
|
"implication": "MIR-level property detection is evaluated per bug class with crate provenance; Rust gold expectations should be keyed per detector family with crate/version pins."
|
|
},
|
|
{
|
|
"reference_id": "cui-2023-safedrop",
|
|
"implication": "Rust data-flow tools report TP/FP counts from manual inspection on real crates; our Rust manifests must support reviewed dispositions identical to the source lane."
|
|
},
|
|
{
|
|
"reference_id": "zhuang-2025-dont-panic",
|
|
"implication": "Rust bug classes differ from C++; gold expectations for Rust should use Rust-native oracles (panic/index/unwrap surfaces) rather than transplanted CWE slices."
|
|
},
|
|
{
|
|
"reference_id": "johns-2011-scanstud",
|
|
"implication": "Fine-grained methodology-first evaluation: versioned per-project manifests with per-case metadata instead of ad-hoc script flags."
|
|
},
|
|
{
|
|
"reference_id": "braga-2017-crypto-sast-benchmark",
|
|
"implication": "Repeatable benchmarking method beats tool count: manifests pin tool revision, configuration fingerprint, corpus hash, and budgets per run."
|
|
},
|
|
{
|
|
"reference_id": "lenarduzzi-2021-six-sast",
|
|
"implication": "Inter-tool agreement is low; comparator lanes stay differential evidence and never become ground truth for project evaluation."
|
|
},
|
|
{
|
|
"reference_id": "hewson-2015-perf-regression-oracles",
|
|
"implication": "Performance gates need statistical oracles over repeated samples, not single medians; project runs reuse fold_eval paired_comparison/bootstrap intervals."
|
|
},
|
|
{
|
|
"reference_id": "mordahl-2023-configurable-sast-benchmarking",
|
|
"implication": "Configuration materially changes analyzer results; every project run records the configuration fingerprint alongside findings and profile."
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|