{
  "schema_version": 1,
  "source_note": "Paperbridge- and primary-source-grounded references for interpreting Fold benchmark results. Numeric entries are only included when they were available from abstracts or report metadata; exact per-tool Juliet tables should be transcribed separately before making direct head-to-head claims.",
  "references": [
    {
      "id": "wagner-2014-juliet-scanners",
      "title": "Using the Juliet Test Suite to Compare Static Security Scanners",
      "authors": ["Andreas Wagner", "Johannes Sametinger"],
      "year": 2014,
      "doi": "10.5220/0005032902440252",
      "venue": "Proceedings of the 11th International Conference on Security and Cryptography",
      "url": "https://doi.org/10.5220/0005032902440252"
    },
    {
      "id": "nist-sate-v-2018",
      "title": "SATE V report: ten years of static analysis tool expositions",
      "authors": ["Aurelien Delaitre", "Bertrand Stivalet", "Paul E. Black", "Vadim Okun", "Athos Ribeiro", "Terry S. Cohen"],
      "year": 2018,
      "doi": "10.6028/nist.sp.500-326",
      "venue": "NIST Special Publication 500-326",
      "url": "https://doi.org/10.6028/nist.sp.500-326"
    },
    {
      "id": "lipp-2022-realworld-c",
      "title": "An empirical study on the effectiveness of static C code analyzers for vulnerability detection",
      "authors": ["Stephan Lipp", "Sebastian Banescu", "Alexander Pretschner"],
      "year": 2022,
      "doi": "10.1145/3533767.3534380",
      "venue": "ISSTA 2022",
      "url": "https://doi.org/10.1145/3533767.3534380"
    },
    {
      "id": "bennett-2024-semgrep-star",
      "title": "Semgrep*: Improving the Limited Performance of Static Application Security Testing (SAST) Tools",
      "authors": ["Gareth Bennett", "Tracy Hall", "Emily Winter", "Steve Counsell"],
      "year": 2024,
      "doi": "10.1145/3661167.3661262",
      "venue": "EASE 2024",
      "url": "https://doi.org/10.1145/3661167.3661262"
    },
    {
      "id": "beyer-2023-svcomp",
      "title": "Competition on Software Verification and Witness Validation: SV-COMP 2023",
      "authors": ["Dirk Beyer"],
      "year": 2023,
      "doi": "10.1007/978-3-031-30820-8_29",
      "venue": "Lecture Notes in Computer Science",
      "url": "https://doi.org/10.1007/978-3-031-30820-8_29"
    },
    {
      "id": "beyer-2022-svcomp",
      "title": "Progress on Software Verification: SV-COMP 2022",
      "authors": ["Dirk Beyer"],
      "year": 2022,
      "doi": "10.1007/978-3-030-99527-0_20",
      "venue": "Lecture Notes in Computer Science",
      "url": "https://doi.org/10.1007/978-3-030-99527-0_20"
    },
    {
      "id": "bhandari-2021-cvefixes",
      "title": "CVEfixes: automated collection of vulnerabilities and their fixes from open-source software",
      "authors": ["Guru Bhandari", "Amara Naseer", "Leon Moonen"],
      "year": 2021,
      "doi": "10.1145/3475960.3475985",
      "venue": "PROMISE 2021",
      "url": "https://doi.org/10.1145/3475960.3475985"
    },
    {
      "id": "chen-2023-diversevul",
      "title": "DiverseVul: A New Vulnerable Source Code Dataset for Deep Learning Based Vulnerability Detection",
      "authors": ["Yizheng Chen", "Zhoujie Ding", "Lamya Alowain", "Xinyun Chen", "David Wagner"],
      "year": 2023,
      "doi": "10.1145/3607199.3607242",
      "venue": "RAID 2023",
      "url": "https://doi.org/10.1145/3607199.3607242"
    },
    {
      "id": "black-2025-sard",
      "title": "The Software Assurance Reference Dataset (SARD)",
      "authors": ["Paul E. Black"],
      "year": 2025,
      "doi": "10.6028/nist.ir.8561",
      "venue": "NIST IR 8561",
      "url": "https://doi.org/10.6028/nist.ir.8561"
    },
    {
      "id": "godefroid-2005-dart",
      "title": "DART: Directed Automated Random Testing",
      "authors": ["Patrice Godefroid", "Nils Klarlund", "Koushik Sen"],
      "year": 2005,
      "doi": "10.1145/1065010.1065036",
      "venue": "PLDI 2005",
      "url": "https://doi.org/10.1145/1065010.1065036"
    },
    {
      "id": "klinger-2019-differential-analyzers",
      "title": "Differential Testing of Soundness and Precision of Static Analyzers",
      "authors": ["Christian Klinger", "Maria Christakis", "Valentin Wüstholz"],
      "year": 2019,
      "doi": "10.1145/3293882.3330553",
      "venue": "ISSTA 2019",
      "url": "https://doi.org/10.1145/3293882.3330553"
    },
    {
      "id": "ferreiro-2025-checkification",
      "title": "Checkification: A Practical Approach for Testing Static Analysis Truths",
      "authors": ["Daniela Ferreiro", "Ignacio Casso", "Jose F. Morales", "Pedro López-García", "Manuel V. Hermenegildo"],
      "year": 2025,
      "doi": "10.1017/S1471068425100069",
      "venue": "Theory and Practice of Logic Programming",
      "url": "https://doi.org/10.1017/S1471068425100069"
    },
    {
      "id": "mansur-2021-metamorphic-datalog",
      "title": "Metamorphic Testing of Datalog Engines",
      "authors": ["Numair Mansur", "Maria Christakis", "Valentin Wüstholz"],
      "year": 2021,
      "doi": "10.1145/3468264.3468573",
      "venue": "ESEC/FSE 2021",
      "url": "https://doi.org/10.1145/3468264.3468573"
    },
    {
      "id": "klees-2018-evaluating-fuzzing",
      "title": "Evaluating Fuzz Testing",
      "authors": ["George Klees", "Andrew Ruef", "Benji Cooper", "Shiyi Wei", "Michael Hicks"],
      "year": 2018,
      "doi": "10.1145/3243734.3243804",
      "venue": "CCS 2018",
      "url": "https://doi.org/10.1145/3243734.3243804"
    },
    {
      "id": "sok-2024-prudent-fuzzing-evaluation",
      "title": "SoK: Prudent Evaluation Practices for Fuzzing",
      "authors": ["Moritz Schloegel", "Nils Bars", "Nico Schiller", "Lukas Bernhard", "Tobias Scharnowski", "Addison Crump", "Arash Ale Ebrahim", "Nicolai Bissantz", "Marius Muench", "Thorsten Holz"],
      "year": 2024,
      "doi": "10.1109/SP54263.2024.00137",
      "venue": "IEEE Symposium on Security and Privacy 2024",
      "url": "https://doi.org/10.1109/SP54263.2024.00137"
    },
    {
      "id": "metzman-2021-fuzzbench",
      "title": "FuzzBench: An Open Fuzzer Benchmarking Platform and Service",
      "authors": ["Jonathan Metzman", "László Szekeres", "Laurent Simon", "Read Sprabery", "Abhishek Arya"],
      "year": 2021,
      "doi": "10.1145/3468264.3473932",
      "venue": "ESEC/FSE 2021",
      "url": "https://doi.org/10.1145/3468264.3473932"
    },
    {
      "id": "hazimeh-2020-magma",
      "title": "Magma: A Ground-Truth Fuzzing Benchmark",
      "authors": ["Ahmad Hazimeh", "Adrian Herrera", "Mathias Payer"],
      "year": 2020,
      "doi": "10.1145/3428334",
      "venue": "Proceedings of the ACM on Measurement and Analysis of Computing Systems",
      "url": "https://doi.org/10.1145/3428334"
    },
    {
      "id": "bundt-2022-synthetic-bugs",
      "title": "Evaluating Synthetic Bugs",
      "authors": ["Joshua Bundt", "Andrew Fasano", "Brendan Dolan-Gavitt", "William Robertson", "Tim Leek"],
      "year": 2021,
      "doi": "10.1145/3433210.3453096",
      "venue": "ESEC/FSE 2021",
      "url": "https://doi.org/10.1145/3433210.3453096"
    },
    {
      "id": "sadowski-2015-tricorder",
      "title": "Tricorder: Building a Program Analysis Ecosystem",
      "authors": ["Caitlin Sadowski", "Jeffrey van Gogh", "Ciera Jaspan", "Emma Söderberg", "Collin Winter"],
      "year": 2015,
      "doi": "10.1109/ICSE.2015.76",
      "venue": "ICSE 2015",
      "url": "https://doi.org/10.1109/ICSE.2015.76"
    },
    {
      "id": "distefano-2019-scaling-static-analysis",
      "title": "Scaling Static Analyses at Facebook",
      "authors": ["Dino Distefano", "Manuel Fähndrich", "Francesco Logozzo", "Peter W. O'Hearn"],
      "year": 2019,
      "doi": "10.1145/3338112",
      "venue": "Communications of the ACM",
      "url": "https://doi.org/10.1145/3338112"
    },
    {
      "id": "ernst-2015-soundiness",
      "title": "In Defense of Soundiness: A Manifesto",
      "authors": ["Michael D. Ernst", "Anders Møller"],
      "year": 2015,
      "doi": "10.1145/2644805",
      "venue": "Communications of the ACM",
      "url": "https://doi.org/10.1145/2644805"
    },
    {
      "id": "li-2024-primevul",
      "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
      "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim", "Chawin Sitawarin", "Xinyun Chen", "Basel Alomair", "David Wagner", "Baishakhi Ray", "Yizheng Chen"],
      "year": 2024,
      "venue": "arXiv",
      "url": "https://arxiv.org/abs/2403.18624"
    },
    {
      "id": "bae-2021-rudra",
      "title": "Rudra: Finding Memory Safety Bugs in Rust at the Ecosystem Scale",
      "authors": ["Yechan Bae", "Youngsuk Kim", "Ammar Askar", "Jungwon Lim", "Taesoo Kim"],
      "year": 2021,
      "doi": "10.1145/3477132.3483570",
      "venue": "SOSP 2021",
      "url": "https://doi.org/10.1145/3477132.3483570"
    },
    {
      "id": "li-2021-mirchecker",
      "title": "MirChecker: Detecting Bugs in Rust Programs via Static Analysis",
      "authors": ["Zhuohua Li", "Jincheng Wang", "Mingshen Sun", "John C. S. Lui"],
      "year": 2021,
      "doi": "10.1145/3460120.3484541",
      "venue": "CCS 2021",
      "url": "https://doi.org/10.1145/3460120.3484541"
    },
    {
      "id": "cui-2023-safedrop",
      "title": "SafeDrop: Detecting Memory Deallocation Bugs of Rust Programs via Static Data-flow Analysis",
      "authors": ["Mohan Cui", "Chengjun Chen", "Hui Xu", "Yangfan Zhou"],
      "year": 2023,
      "doi": "10.1145/3542948",
      "venue": "ACM Transactions on Software Engineering and Methodology",
      "url": "https://doi.org/10.1145/3542948"
    },
    {
      "id": "zhuang-2025-dont-panic",
      "title": "Don't Panic! Finding Bugs Hidden Behind Rust Runtime Safety Checks",
      "authors": ["Zeyang Zhuang", "Zilun Wang", "Wei Meng"],
      "year": 2025,
      "doi": "10.1145/3719027.3765142",
      "venue": "Proceedings of the ACM on Software Engineering",
      "url": "https://doi.org/10.1145/3719027.3765142"
    },
    {
      "id": "johns-2011-scanstud",
      "title": "Scanstud: A Methodology for Systematic, Fine-Grained Evaluation of Static Analysis Tools",
      "authors": ["Martin Johns", "Moritz Jodeit"],
      "year": 2011,
      "doi": "10.1109/icstw.2011.32",
      "venue": "IEEE ICST Workshops 2011",
      "url": "https://doi.org/10.1109/icstw.2011.32"
    },
    {
      "id": "braga-2017-crypto-sast-benchmark",
      "title": "Practical Evaluation of Static Analysis Tools for Cryptography: Benchmarking Method and Case Study",
      "authors": ["Alexandre Braga", "Ricardo Dahab", "Nuno Antunes"],
      "year": 2017,
      "doi": "10.1109/issre.2017.27",
      "venue": "IEEE ISSRE 2017",
      "url": "https://doi.org/10.1109/issre.2017.27"
    },
    {
      "id": "lenarduzzi-2021-six-sast",
      "title": "A Critical Comparison on Six Static Analysis Tools: Detection, Agreement, and Precision",
      "authors": ["Valentina Lenarduzzi", "Savanna Lujan", "Nyyti Saarimäki", "Fabio Palomba"],
      "year": 2021,
      "venue": "arXiv",
      "url": "https://arxiv.org/abs/2101.08832"
    },
    {
      "id": "hewson-2015-perf-regression-oracles",
      "title": "Performance Regression Testing on the Java Virtual Machine Using Statistical Test Oracles",
      "authors": ["Fergus Hewson", "Jens Dietrich", "Stephen Marsland"],
      "year": 2015,
      "doi": "10.1109/aswec.2015.14",
      "venue": "ASWEC 2015",
      "url": "https://doi.org/10.1109/aswec.2015.14"
    },
    {
      "id": "mordahl-2023-configurable-sast-benchmarking",
      "title": "Automatic Testing and Benchmarking for Configurable Static Analysis Tools",
      "authors": ["Austin Mordahl", "Dakota Soles", "Miao Liu", "Zenong Zhang", "Shiyi Wei"],
      "year": 2023,
      "doi": "10.1145/3597926.3605232",
      "venue": "ISSTA 2023",
      "url": "https://doi.org/10.1145/3597926.3605232"
    }
  ],
  "baselines": [
    {
      "id": "juliet-scanner-comparison",
      "reference_id": "wagner-2014-juliet-scanners",
      "benchmark": "Juliet Test Suite",
      "tool_scope": "static security scanners",
      "comparability": "same synthetic benchmark family as Fold's Juliet slices; exact per-tool result tables still need manual transcription",
      "metrics": []
    },
    {
      "id": "sate-v-methodology",
      "reference_id": "nist-sate-v-2018",
      "benchmark": "NIST SATE / SAMATE exposition corpus",
      "tool_scope": "multi-year static analyzer exposition",
      "comparability": "methodology and cautionary baseline, not a direct precision-recall target for the current Fold Juliet slices",
      "metrics": []
    },
    {
      "id": "realworld-c-static-analyzers",
      "reference_id": "lipp-2022-realworld-c",
      "benchmark": "27 C projects, 1.15 MLOC, 192 CVE-ground-truth vulnerabilities",
      "tool_scope": "five free/open-source analyzers plus one commercial analyzer",
      "comparability": "real-world CVE benchmark; use as an external recall sanity band rather than direct Juliet equivalence",
      "metrics": [
        {"name": "single_tool_false_negative_rate", "value_min": 0.47, "value_max": 0.80, "unit": "ratio"},
        {"name": "combined_tool_false_negative_rate", "value_min": 0.30, "value_max": 0.69, "unit": "ratio"},
        {"name": "projects", "value": 27, "unit": "count"},
        {"name": "vulnerabilities", "value": 192, "unit": "count"},
        {"name": "source_lines", "value": 1150000, "unit": "count"}
      ]
    },
    {
      "id": "production-sast-detection",
      "reference_id": "bennett-2024-semgrep-star",
      "benchmark": "manually curated production-code vulnerabilities",
      "tool_scope": "four SAST tools plus Semgrep* rule augmentation",
      "comparability": "production-code detection-rate band; useful for positioning real-world expectations",
      "metrics": [
        {"name": "single_tool_detection_rate", "value_min": 0.112, "value_max": 0.265, "unit": "ratio"},
        {"name": "four_tool_combined_detection_rate", "value": 0.388, "unit": "ratio"},
        {"name": "semgrep_star_detection_rate", "value": 0.447, "unit": "ratio"},
        {"name": "semgrep_star_relative_improvement", "value": 1.81, "unit": "ratio"}
      ]
    },
    {
      "id": "svcomp-2023-scale",
      "reference_id": "beyer-2023-svcomp",
      "benchmark": "SV-COMP 2023",
      "tool_scope": "automatic software verifiers",
      "comparability": "task-level verification benchmark; Fold maps only selected properties today",
      "metrics": [
        {"name": "verification_systems", "value": 52, "unit": "count"},
        {"name": "c_tasks", "value": 23805, "unit": "count"},
        {"name": "java_tasks", "value": 586, "unit": "count"}
      ]
    },
    {
      "id": "svcomp-2022-scale",
      "reference_id": "beyer-2022-svcomp",
      "benchmark": "SV-COMP 2022",
      "tool_scope": "automatic software verifiers",
      "comparability": "task-level verification benchmark; use for expansion after BV-width stability is fixed",
      "metrics": [
        {"name": "verification_systems", "value": 47, "unit": "count"},
        {"name": "c_tasks", "value": 15648, "unit": "count"},
        {"name": "java_tasks", "value": 586, "unit": "count"}
      ]
    },
    {
      "id": "cvefixes-dataset-scale",
      "reference_id": "bhandari-2021-cvefixes",
      "benchmark": "CVEfixes vulnerability-fix corpus",
      "tool_scope": "real-world vulnerability/fix dataset",
      "comparability": "candidate real-world Fold benchmark after repository checkout and compile database synthesis",
      "metrics": [
        {"name": "cve_records", "value": 5365, "unit": "count"},
        {"name": "projects", "value": 1754, "unit": "count"},
        {"name": "fixing_commits", "value": 5495, "unit": "count"}
      ]
    },
    {
      "id": "diversevul-dataset-scale",
      "reference_id": "chen-2023-diversevul",
      "benchmark": "DiverseVul",
      "tool_scope": "C/C++ vulnerable and non-vulnerable functions",
      "comparability": "candidate triage corpus for detector coverage, but function labels are weaker than path-specific SAST ground truth",
      "metrics": [
        {"name": "vulnerable_functions", "value": 18945, "unit": "count"},
        {"name": "non_vulnerable_functions", "value": 330492, "unit": "count"},
        {"name": "cwes", "value": 150, "unit": "count"},
        {"name": "commits", "value": 7514, "unit": "count"}
      ]
    },
    {
      "id": "sard-scale",
      "reference_id": "black-2025-sard",
      "benchmark": "NIST SARD",
      "tool_scope": "buggy reference programs across multiple languages and weakness classes",
      "comparability": "natural expansion target beyond the current Juliet filters",
      "metrics": [
        {"name": "buggy_programs", "value": 450000, "unit": "count", "qualifier": "over"},
        {"name": "languages", "value": 5, "unit": "count"},
        {"name": "weakness_classes", "value": 150, "unit": "count", "qualifier": "over"}
      ]
    }
  ],
  "benchmark_targets": [
    {
      "id": "juliet-expanded-fold",
      "status": "ready",
      "runner": "bench/run_summary_ab.py or bench/run_fold_on_juliet.py",
      "scope": "Juliet CWEs with checked-in gold files: 121, 122, 124, 126, 127, 190, 191, 194, 195, 196, 197, 369, 476, 561",
      "next_action": "Run per-CWE summary A/B with file-level scoring and compare ceiling gaps."
    },
    {
      "id": "clang-sa-juliet",
      "status": "ready-if-scan-build-installed",
      "runner": "bench/run_csa_on_juliet.py",
      "scope": "local Clang Static Analyzer head-to-head for null deref, divide-by-zero, integer overflow availability, and array bounds",
      "next_action": "Run after confirming LLVM scan-build path."
    },
    {
      "id": "esbmc-juliet",
      "status": "blocked",
      "runner": "bench/run_esbmc_on_juliet.py",
      "scope": "bounded-model-checking head-to-head, currently wired for CWE-476 null dereference",
      "next_action": "Install ESBMC or set ESBMC_BIN."
    },
    {
      "id": "svcomp-fold",
      "status": "blocked",
      "runner": "bench/run_svcomp.py",
      "scope": "SV-COMP MemSafety valid-deref, NoOverflows, and reachability proxy categories",
      "next_action": "Fix Fold BV-width sort mismatches before trusting broad SV-COMP runs."
    },
    {
      "id": "sard-samate",
      "status": "candidate",
      "runner": "new fetcher under bench/corpora/sard-*",
      "scope": "NIST SARD/SAMATE CWE slices, starting with C dead-code and memory-safety subsets",
      "next_action": "Add fetchers that preserve SARD metadata IDs as gold labels."
    },
    {
      "id": "realworld-cve-corpus",
      "status": "candidate",
      "runner": "new CVEfixes or curated-CVE harness",
      "scope": "repository-level C/C++ CVE fixes with generated compile databases",
      "next_action": "Select a small reproducible project subset before scaling."
    }
  ],
  "scaffolding_implications": [
    {
      "group": "profiling-eval-2026-07",
      "note": "Paperbridge pass of 2026-07-27 grounding the manifest-driven project evaluation scaffolding (scripts/benchmarks/run_project_eval.py + fold_eval.project). Implications are one-line engineering decisions, not head-to-head metrics.",
      "entries": [
        {
          "reference_id": "bae-2021-rudra",
          "implication": "Rust analyzer credibility requires ecosystem-scale corpora plus manual per-finding FP review; the Rust lane needs disposition baselines, not a single OSS smoke run."
        },
        {
          "reference_id": "li-2021-mirchecker",
          "implication": "MIR-level property detection is evaluated per bug class with crate provenance; Rust gold expectations should be keyed per detector family with crate/version pins."
        },
        {
          "reference_id": "cui-2023-safedrop",
          "implication": "Rust data-flow tools report TP/FP counts from manual inspection on real crates; our Rust manifests must support reviewed dispositions identical to the source lane."
        },
        {
          "reference_id": "zhuang-2025-dont-panic",
          "implication": "Rust bug classes differ from C++; gold expectations for Rust should use Rust-native oracles (panic/index/unwrap surfaces) rather than transplanted CWE slices."
        },
        {
          "reference_id": "johns-2011-scanstud",
          "implication": "Fine-grained methodology-first evaluation: versioned per-project manifests with per-case metadata instead of ad-hoc script flags."
        },
        {
          "reference_id": "braga-2017-crypto-sast-benchmark",
          "implication": "Repeatable benchmarking method beats tool count: manifests pin tool revision, configuration fingerprint, corpus hash, and budgets per run."
        },
        {
          "reference_id": "lenarduzzi-2021-six-sast",
          "implication": "Inter-tool agreement is low; comparator lanes stay differential evidence and never become ground truth for project evaluation."
        },
        {
          "reference_id": "hewson-2015-perf-regression-oracles",
          "implication": "Performance gates need statistical oracles over repeated samples, not single medians; project runs reuse fold_eval paired_comparison/bootstrap intervals."
        },
        {
          "reference_id": "mordahl-2023-configurable-sast-benchmarking",
          "implication": "Configuration materially changes analyzer results; every project run records the configuration fingerprint alongside findings and profile."
        }
      ]
    }
  ]
}