trevon/fold
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
fold/bench/corpora/ctf-challenges/fold-results.json
trvon 9161ed2b98
 eval: CTF binary scan — 3.7% binary detection vs 66.7% source 
- Binary scan: 1/27 findings (only gets() import detected)
- Source scan: 18/27 findings (66.7% TP rate)
- Fuzz harnesses: generated but cov:2 (Swift ABI issue)
- Binary backend needs significant improvement
2026-05-29 20:03:07 -04:00

773 lines
No EOL
24 KiB
JSON
Raw Permalink Blame History

{
"meta": {
"tool": "fold",
"version": "0.1.0",
"scan_date": "2026-05-29",
"mode": "binary",
"architecture": "arm64",
"platform": "darwin"
},
"summary": {
"total_binaries": 10,
"total_ground_truth_vulnerabilities": 27,
"binary_scan_total_findings": 1,
"source_scan_total_findings": 37,
"source_scan_vulnerabilities_detected": 18,
"source_scan_false_negatives": 9,
"source_tp_rate_pct": 66.7,
"total_fuzz_harnesses": 40,
"total_fuzzable_functions": 40,
"total_exported_functions": 44,
"fuzzable_discovery_pct": 90.9
},
"binary_results": {
"libbof_overflow": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libbof_overflow.dylib",
"ground_truth": {
"total_vulnerabilities": 4,
"vulnerabilities": [
{
"cwe": "CWE-242",
"type": "use-of-deprecated-api",
"func": "vulnerable_gets",
"desc": "gets() - no bounds checking"
},
{
"cwe": "CWE-121",
"type": "stack-buffer-overflow",
"func": "vulnerable_strcpy",
"desc": "strcpy without length check"
},
{
"cwe": "CWE-134",
"type": "format-string",
"func": "vulnerable_sprintf",
"desc": "sprintf with user format string"
},
{
"cwe": "CWE-121",
"type": "stack-buffer-overflow",
"func": "vulnerable_sprintf",
"desc": "sprintf buffer overflow"
}
],
"total_functions": 4,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 1,
"findings": [
{
"rule": "fold.taint-track",
"confidence": "heuristic",
"message": "Function calls potentially unsafe routine 'bl sym.imp.gets'"
}
]
},
"source_scan": {
"findings_count": 11,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `strcpy` is always true in `vulnerable_strcpy`"
},
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `sprintf` is always true in `vulnerable_sprintf`"
},
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `strncpy` is always true in `safe_version`"
},
{
"rule": "fold.deprecated-api",
"confidence": "syntactic",
"message": "Call to deprecated `gets`: This function is provided for compatibility reasons only. Due to security concerns inherent in the design of gets(3), it i"
},
{
"rule": "fold.oob-access",
"confidence": "heuristic",
"message": "Symbolic array index may exceed length 64 under reachable paths"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `vulnerable_gets`"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `vulnerable_gets`"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `vulnerable_strcpy`"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `vulnerable_sprintf`"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `safe_version`"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `buf` may be read before being written in `safe_version`"
}
],
"vulnerabilities_detected": 4,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 12,
"strings_found": 2
},
"fuzz_prep": {
"harnesses_generated": 4,
"corpus_seeds": 12,
"fuzzable_functions_discovered": 4
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 28.6,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 100.0,
"binary_findings": 1,
"source_findings": 11
}
},
"libcmd_injection": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libcmd_injection.dylib",
"ground_truth": {
"total_vulnerabilities": 3,
"vulnerabilities": [
{
"cwe": "CWE-78",
"type": "command-injection",
"func": "command_injection",
"desc": "system() with user input"
},
{
"cwe": "CWE-798",
"type": "hardcoded-credential",
"func": "hardcoded_credential_check",
"desc": "Hardcoded password 'supersecret123'"
},
{
"cwe": "CWE-798",
"type": "hardcoded-credential",
"func": "hardcoded_api_key",
"desc": "Hardcoded API key 'sk-1234567890abcdef'"
}
],
"total_functions": 3,
"exported_functions": 3
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 3,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `sprintf` is always true in `command_injection`"
},
{
"rule": "fold.hardcoded-credential",
"confidence": "syntactic",
"message": "Hardcoded API key (sk-\u2026) in string literal"
},
{
"rule": "fold.uninit-read",
"confidence": "proved",
"message": "Variable `cmd` may be read before being written in `command_injection`"
}
],
"vulnerabilities_detected": 2,
"vulnerabilities_missed": 1
},
"binary_extraction": {
"exported_symbols_found": 3,
"exported_symbols_total": 3,
"all_symbols": 9,
"strings_found": 5
},
"fuzz_prep": {
"harnesses_generated": 3,
"corpus_seeds": 9,
"fuzzable_functions_discovered": 3
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 83.3,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 66.7,
"binary_findings": 0,
"source_findings": 3
}
},
"libdiv_by_zero": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libdiv_by_zero.dylib",
"ground_truth": {
"total_vulnerabilities": 3,
"vulnerabilities": [
{
"cwe": "CWE-369",
"type": "divide-by-zero",
"func": "div_by_zero_direct",
"desc": "Literal 0 divisor"
},
{
"cwe": "CWE-369",
"type": "divide-by-zero",
"func": "div_by_zero_variable",
"desc": "Variable zero divisor"
},
{
"cwe": "CWE-369",
"type": "divide-by-zero",
"func": "mod_by_zero",
"desc": "Modulo by zero"
}
],
"total_functions": 4,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 6,
"findings": [
{
"rule": "fold.clone",
"confidence": "syntactic",
"message": "Likely duplicate (exact) across: div_by_zero_direct, mod_by_zero"
},
{
"rule": "fold.clone",
"confidence": "syntactic",
"message": "Likely duplicate (exact) across: div_by_zero_variable, safe_division"
},
{
"rule": "fold.div-by-zero",
"confidence": "proved",
"message": "Possible division by zero at div_by_zero.c:14 (e.g., when divisor=0)"
},
{
"rule": "fold.div-by-zero",
"confidence": "proved",
"message": "Possible division by zero at div_by_zero.c:3 (e.g., when divisor=0)"
},
{
"rule": "fold.div-by-zero",
"confidence": "proved",
"message": "Possible division by zero at div_by_zero.c:8 (e.g., when y=0)"
},
{
"rule": "fold.int-overflow",
"confidence": "heuristic",
"message": "Signed integer overflow possible at `/` (e.g., when x=2147483648, y=4294967295)"
}
],
"vulnerabilities_detected": 3,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 4,
"strings_found": 1
},
"fuzz_prep": {
"harnesses_generated": 4,
"corpus_seeds": 12,
"fuzzable_functions_discovered": 4
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 16.7,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 100.0,
"binary_findings": 0,
"source_findings": 6
}
},
"libformat_string": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libformat_string.dylib",
"ground_truth": {
"total_vulnerabilities": 3,
"vulnerabilities": [
{
"cwe": "CWE-134",
"type": "format-string",
"func": "fmt_string_direct",
"desc": "printf with user-controlled format"
},
{
"cwe": "CWE-134",
"type": "format-string",
"func": "fmt_string_fprintf",
"desc": "fprintf with user-controlled format"
},
{
"cwe": "CWE-134",
"type": "format-string",
"func": "fmt_string_sprintf",
"desc": "sprintf with user-controlled format"
}
],
"total_functions": 4,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 1,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `sprintf` is always true in `fmt_string_sprintf`"
}
],
"vulnerabilities_detected": 0,
"vulnerabilities_missed": 3
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 7,
"strings_found": 1
},
"fuzz_prep": {
"harnesses_generated": 4,
"corpus_seeds": 12,
"fuzzable_functions_discovered": 4
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 16.7,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 0.0,
"binary_findings": 0,
"source_findings": 1
}
},
"libint_overflow": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libint_overflow.dylib",
"ground_truth": {
"total_vulnerabilities": 4,
"vulnerabilities": [
{
"cwe": "CWE-190",
"type": "integer-overflow",
"func": "vulnerable_add",
"desc": "Signed add overflow"
},
{
"cwe": "CWE-190",
"type": "integer-overflow",
"func": "vulnerable_mul",
"desc": "Signed multiply overflow"
},
{
"cwe": "CWE-190",
"type": "integer-overflow",
"func": "vulnerable_alloc",
"desc": "count*elem_size overflow"
},
{
"cwe": "CWE-122",
"type": "heap-buffer-overflow",
"func": "heap_overflow_via_int_overflow",
"desc": "Overflow via int arithmetic"
}
],
"total_functions": 4,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 7,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Condition `buf` is always true in `heap_overflow_via_int_overflow`"
},
{
"rule": "fold.clone",
"confidence": "syntactic",
"message": "Likely duplicate (exact) across: vulnerable_add, vulnerable_mul"
},
{
"rule": "fold.int-overflow",
"confidence": "heuristic",
"message": "Signed integer overflow possible at `*` (e.g., when a=2148007937, b=2147483644)"
},
{
"rule": "fold.int-overflow",
"confidence": "heuristic",
"message": "Signed integer overflow possible at `*` (e.g., when count=2505381202, elem_size=1063039790)"
},
{
"rule": "fold.int-overflow",
"confidence": "heuristic",
"message": "Signed integer overflow possible at `+` (e.g., when a=3221225471, b=2147483651)"
},
{
"rule": "fold.memory-leak",
"confidence": "proved",
"message": "Memory leak: heap allocation at int_overflow.c:23:24 in function `heap_overflow_via_int_overflow` is never freed on at least one execution path"
},
{
"rule": "fold.redundant-store",
"confidence": "proved",
"message": "Redundant store: value assigned to `buf` is never read before being overwritten in `heap_overflow_via_int_overflow`"
}
],
"vulnerabilities_detected": 4,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 7,
"strings_found": 1
},
"fuzz_prep": {
"harnesses_generated": 4,
"corpus_seeds": 12,
"fuzzable_functions_discovered": 4
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 14.3,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 100.0,
"binary_findings": 0,
"source_findings": 7
}
},
"libnull_deref": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libnull_deref.dylib",
"ground_truth": {
"total_vulnerabilities": 3,
"vulnerabilities": [
{
"cwe": "CWE-476",
"type": "null-deref",
"func": "null_deref_direct",
"desc": "Literal NULL deref"
},
{
"cwe": "CWE-476",
"type": "null-deref",
"func": "null_deref_conditional",
"desc": "Conditional NULL deref when flag==0xDEAD"
},
{
"cwe": "CWE-476",
"type": "null-deref",
"func": "null_deref_after_allocation_failure",
"desc": "No null check after malloc"
}
],
"total_functions": 4,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 5,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Condition `!p` is always false in `safe_after_malloc`"
},
{
"rule": "fold.dead-branch",
"confidence": "proved",
"message": "Unreachable branch in `safe_after_malloc`"
},
{
"rule": "fold.null-deref",
"confidence": "proved",
"message": "Null pointer dereference: `p` is nullable (e.g., when p=0x0)"
},
{
"rule": "fold.null-deref",
"confidence": "proved",
"message": "Null pointer dereference: `p` is nullable (e.g., when p=0x0)"
},
{
"rule": "fold.redundant-store",
"confidence": "proved",
"message": "Redundant store: value assigned to `p` is never read before being overwritten in `null_deref_conditional`"
}
],
"vulnerabilities_detected": 3,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 6,
"strings_found": 1
},
"fuzz_prep": {
"harnesses_generated": 4,
"corpus_seeds": 12,
"fuzzable_functions_discovered": 4
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 16.7,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 100.0,
"binary_findings": 0,
"source_findings": 5
}
},
"libuaf_doublefree": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libuaf_doublefree.dylib",
"ground_truth": {
"total_vulnerabilities": 4,
"vulnerabilities": [
{
"cwe": "CWE-415",
"type": "double-free",
"func": "double_free_bug",
"desc": "Two frees without reallocation"
},
{
"cwe": "CWE-416",
"type": "use-after-free",
"func": "use_after_free_bug",
"desc": "strcpy to freed pointer"
},
{
"cwe": "CWE-416",
"type": "use-after-free",
"func": "use_global_after_free",
"desc": "Use after free via global"
},
{
"cwe": "CWE-401",
"type": "memory-leak",
"func": "set_global_and_free",
"desc": "Allocation never freed"
}
],
"total_functions": 5,
"exported_functions": 4
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 4,
"findings": [
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `strcpy` is always true in `use_after_free_bug`"
},
{
"rule": "fold.always-condition",
"confidence": "proved",
"message": "Ternary condition `strcpy` is always true in `use_global_after_free`"
},
{
"rule": "fold.memory-leak",
"confidence": "proved",
"message": "Memory leak: heap allocation at uaf_doublefree.c:20:25 in function `set_global_and_free` is never freed on at least one execution path"
},
{
"rule": "fold.oob-access",
"confidence": "proved",
"message": "Double free on tracked heap region"
}
],
"vulnerabilities_detected": 2,
"vulnerabilities_missed": 2
},
"binary_extraction": {
"exported_symbols_found": 4,
"exported_symbols_total": 4,
"all_symbols": 8,
"strings_found": 2
},
"fuzz_prep": {
"harnesses_generated": 5,
"corpus_seeds": 15,
"fuzzable_functions_discovered": 5
},
"coverage_metrics": {
"symbol_extraction_pct": 80.0,
"string_extraction_pct": 28.6,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 50.0,
"binary_findings": 0,
"source_findings": 4
}
},
"libvuln_demo": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libvuln_demo.dylib",
"ground_truth": {
"total_vulnerabilities": 3,
"vulnerabilities": [
{
"cwe": "CWE-121",
"type": "stack-buffer-overflow",
"func": "vuln_buffer_overflow",
"desc": "strcpy without bounds"
},
{
"cwe": "CWE-134",
"type": "format-string",
"func": "vuln_format_string",
"desc": "printf with user format string"
},
{
"cwe": "CWE-416",
"type": "use-after-free",
"func": "vuln_use_after_free",
"desc": "Deref after free"
}
],
"total_functions": 4,
"exported_functions": 1
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 0,
"findings": [],
"vulnerabilities_detected": 0,
"vulnerabilities_missed": 3
},
"binary_extraction": {
"exported_symbols_found": 1,
"exported_symbols_total": 1,
"all_symbols": 13,
"strings_found": 14
},
"fuzz_prep": {
"harnesses_generated": 1,
"corpus_seeds": 3,
"fuzzable_functions_discovered": 1
},
"coverage_metrics": {
"symbol_extraction_pct": 25.0,
"string_extraction_pct": 100,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 0.0,
"binary_findings": 0,
"source_findings": 0
}
},
"libFuzzMeSwift": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libFuzzMeSwift.dylib",
"ground_truth": {
"total_vulnerabilities": 0,
"vulnerabilities": [],
"total_functions": 15,
"exported_functions": 15
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 0,
"findings": [],
"vulnerabilities_detected": 0,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 15,
"exported_symbols_total": 15,
"all_symbols": 68,
"strings_found": 13
},
"fuzz_prep": {
"harnesses_generated": 10,
"corpus_seeds": 18,
"fuzzable_functions_discovered": 10
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 100,
"fuzzable_discovery_pct": 66.7,
"source_tp_rate_pct": 100.0,
"binary_findings": 0,
"source_findings": 0
}
},
"libtest_fuzz": {
"binary_path": "/Users/trevon/work/tools/fold/bench/corpora/ctf-challenges/libtest_fuzz.dylib",
"ground_truth": {
"total_vulnerabilities": 0,
"vulnerabilities": [],
"total_functions": 1,
"exported_functions": 1
},
"binary_scan": {
"findings_count": 0,
"findings": []
},
"source_scan": {
"findings_count": 0,
"findings": [],
"vulnerabilities_detected": 0,
"vulnerabilities_missed": 0
},
"binary_extraction": {
"exported_symbols_found": 1,
"exported_symbols_total": 1,
"all_symbols": 1,
"strings_found": 1
},
"fuzz_prep": {
"harnesses_generated": 1,
"corpus_seeds": 3,
"fuzzable_functions_discovered": 1
},
"coverage_metrics": {
"symbol_extraction_pct": 100.0,
"string_extraction_pct": 33.3,
"fuzzable_discovery_pct": 100.0,
"source_tp_rate_pct": 100.0,
"binary_findings": 0,
"source_findings": 0
}
}
}
}
Reference in a new issue View git blame Copy permalink