strawberry/tests/source_invariants.py
trvon 2e94ae1485
Some checks failed
Build and Test / build (debug, clang) (push) Has been cancelled
Build and Test / build (debug, gcc) (push) Has been cancelled
Build and Test / build (release, clang) (push) Has been cancelled
Build and Test / build (release, gcc) (push) Has been cancelled
Build and Test / macos (push) Has been cancelled
Build and Test / docker (push) Has been cancelled
Privacy: tracker list feeds (advisory only, never block)
First piece of tracker attribution (docs/tracker-attribution.md): which
programs reach advertising and tracking domains, built with the
measurement rules of the CarolinaCon 2026 privacy talk (bridgescope):
a listed domain is a circumstantial label, never proof.

- Feed kind "trackers": hosts lines, plain domains and adblock
  whole-domain rules (||host^, ||host^$third-party). Cosmetic rules,
  exceptions, paths and other options are skipped; the comment cut used
  to turn "site.example##.ad" into a domain.
- TrackerSet: suffix matching (a.b.t.example matches t.example, never
  nott.example, an IP literal or a bare TLD); first feed reported.
- List parsing moves from rule_updater.cpp to ioc_list.cpp (no network
  code) so the query-side core can use it.
- Source invariant R11: tracker lists never meet exec-guard or OpenSnitch
  code. They are not in the OpenSnitch blocklist sync.

Tests: T1-T3 (test_tracker_set), trackers in the list-parsing property
and the list fuzzer (60 s clean), R11 self-test fixture.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 05:25:53 -06:00

249 lines
12 KiB
Python

#!/usr/bin/env python3
"""Static source invariants for Strawberry (run by `meson test source_invariants`).
These encode lessons that unit tests, property tests and fuzzers missed. The
2026-10-25 host freeze started with an uncaught exception from an iostream read
of /proc (std::istreambuf_iterator -> ios_base::failure on ESRCH), which aborted
the daemon while it held fanotify exec-permission events.
Rules (each violation prints file:line and the rule id):
R1 istreambuf std::istreambuf_iterator anywhere (throws from the
stream buffer on read errors; nothing catches it)
R2 ifstream std::ifstream / std::fstream in src/ or include/
unless the line carries `invariant-allow(ifstream): <reason>`
R3 proc-stream a "/proc" or "/sys" literal on a line that also uses a stream
R4 c-callback-noexcept a function passed to a C API (libbpf ring buffer,
YARA scan/compiler callbacks, curl *FUNCTION options,
sigaction, set_terminate) that is not declared noexcept
R5 thread-guarded std::thread / std::jthread constructed without run_guarded(
R6 sto std::stoi/stol/stoll/stoul/stoull/stof/stod in src/ (throws;
use from_chars helpers) unless `invariant-allow(sto): <reason>`
R7 empty-catch a catch block with no statements (comments do not count):
handle it, or record it with STRAWBERRY_NOTE_SWALLOWED()
(strawberry/swallowed.hpp); clang-tidy: bugprone-empty-catch
R8 file-node-id a "file_" / "file_path_" id literal in src/ or include/
outside strawberry/node_ids.hpp (three modules once built
file ids three different ways; use file_node_id())
R9 json-dump-utf8 a nlohmann .dump( in src/ or include/ without
error_handler_t::replace on the same line: host strings
are bytes, and the default handler throws on invalid UTF-8
R10 learning-response a translation unit couples learning types to exec-guard,
containment or policy APIs (daemon_app.cpp and main.cpp
are orchestration boundaries requiring manual review)
R11 tracker-response a translation unit uses privacy tracker lists (tracker_set.hpp)
together with exec-guard, OpenSnitch or containment APIs:
tracker contacts are advisory and never block
(docs/tracker-attribution.md; daemon_app.cpp and main.cpp
are reviewed by hand)
Usage: source_invariants.py <repo-root> check the tree
source_invariants.py --self-test <repo-root> check that every rule fires on
tests/fixtures/invariants/bad_*.cpp
"""
import re
import sys
from pathlib import Path
SCAN_DIRS = ["src", "include", "mcp", "app"]
STRICT_DIRS = ["src", "include"] # R2/R6 apply here
EXTS = {".cpp", ".hpp", ".h", ".cc"}
C_CALLBACK_APIS = {
"ring_buffer__new": 1, # (map_fd, sample_cb, ctx, opts)
"ring_buffer__add": 2, # (rb, map_fd, sample_cb, ctx)
"perf_buffer__new": 2,
"yr_rules_scan_mem": 4, # (rules, data, size, flags, callback, ...)
"yr_rules_scan_file": 3,
"yr_rules_scan_fd": 3,
"yr_rules_scan_proc": 3,
"yr_compiler_set_callback": 1,
"std::set_terminate": 0,
"libbpf_set_print": 0,
"curl_easy_setopt": 2, # only CURLOPT_*FUNCTION (checked below)
}
RE_ISTREAMBUF = re.compile(r"istreambuf_iterator")
RE_IFSTREAM = re.compile(r"\bstd::(?:ifstream|fstream)\b")
RE_STREAM_USE = re.compile(r"\b(?:std::)?(?:ifstream|fstream|istream|getline)\b")
RE_PROC_LIT = re.compile(r'"/(?:proc|sys)\b')
RE_THREAD = re.compile(r"\bstd::j?thread\s*\(|=\s*std::j?thread\s*\(")
RE_STO = re.compile(r"\bstd::sto(?:i|l|ll|ul|ull|f|d|ld)\s*\(")
RE_EMPTY_CATCH = re.compile(r"\bcatch\s*\([^)]*\)\s*\{\s*\}")
RE_DUMP = re.compile(r"\.dump\(")
RE_FILE_ID_LIT = re.compile(r'"file_(?:path_)?"')
NODE_IDS_HEADER = "include/strawberry/node_ids.hpp"
RE_SIGACTION_HANDLER = re.compile(r"\.sa_(?:handler|sigaction)\s*=\s*([A-Za-z_][\w:]*)")
def strip_comment(line: str) -> str:
# good enough for this codebase: drop // comments not inside a string
out, in_str = [], False
i = 0
while i < len(line):
c = line[i]
if c == '"' and (i == 0 or line[i - 1] != "\\"):
in_str = not in_str
if not in_str and line.startswith("//", i):
break
out.append(c)
i += 1
return "".join(out)
def split_args(text: str, start: int):
"""Split call arguments starting after '(' at text[start]; returns list."""
depth, args, cur = 0, [], []
i = start
while i < len(text):
c = text[i]
if c in "([{":
depth += 1
elif c in ")]}":
if depth == 0:
args.append("".join(cur).strip())
return args
depth -= 1
elif c == "," and depth == 0:
args.append("".join(cur).strip())
cur = []
i += 1
continue
cur.append(c)
i += 1
return args
def function_is_noexcept(name: str, sources: dict) -> bool:
short = name.split("::")[-1].lstrip("&")
decl = re.compile(r"\b" + re.escape(short) + r"\s*\([^;{]*\)\s*(?:const\s*)?noexcept\b")
return any(decl.search(text) for text in sources.values())
def check_file(path: Path, rel: str, text: str, sources: dict, violations: list):
strict = rel.split("/")[0] in STRICT_DIRS
lines = text.splitlines()
thread_vectors = set(re.findall(r"std::vector<\s*std::j?thread\s*>\s+(\w+)", text)) if strict else set()
for no, raw in enumerate(lines, 1):
line = strip_comment(raw)
if RE_ISTREAMBUF.search(line):
violations.append((rel, no, "R1 istreambuf", raw.strip()))
if strict and RE_IFSTREAM.search(line) and "invariant-allow(ifstream)" not in raw:
violations.append((rel, no, "R2 ifstream", raw.strip()))
if RE_PROC_LIT.search(line) and RE_STREAM_USE.search(line):
violations.append((rel, no, "R3 proc-stream", raw.strip()))
if strict and RE_THREAD.search(line) and "run_guarded(" not in line:
# allow the constructor call to continue on the next line
nxt = strip_comment(lines[no]) if no < len(lines) else ""
if "run_guarded(" not in nxt:
violations.append((rel, no, "R5 thread-guarded", raw.strip()))
if strict and rel != NODE_IDS_HEADER and RE_FILE_ID_LIT.search(line):
violations.append((rel, no, "R8 file-node-id", raw.strip()))
if strict and RE_DUMP.search(line) and "error_handler_t::replace" not in line:
violations.append((rel, no, "R9 json-dump-utf8", raw.strip()))
if strict and RE_STO.search(line) and "invariant-allow(sto)" not in raw:
violations.append((rel, no, "R6 sto", raw.strip()))
for vec in thread_vectors:
if re.search(r"\b" + re.escape(vec) + r"\s*\.\s*(?:emplace_back|push_back)\s*\(", line) and "run_guarded(" not in line:
violations.append((rel, no, "R5 thread-guarded", raw.strip()))
m = RE_SIGACTION_HANDLER.search(line)
if m and m.group(1) not in ("SIG_DFL", "SIG_IGN") and not function_is_noexcept(m.group(1), sources):
violations.append((rel, no, "R4 c-callback-noexcept", raw.strip()))
flat = strip_comment_block(text)
if strict and rel not in {"src/daemon_app.cpp", "src/main.cpp"}:
learning = re.search(r'(?:#include[^\n]*[/\"]learn/|\blearn::|\b(?:InspectionDecision|InspectionOutcome|ShadowScorer)\b)', flat)
response = re.search(r'#include[^\n]*(?:exec_guard|opensnitch|policy)\.hpp', flat)
if learning and response:
violations.append((rel, flat.count("\n", 0, learning.start()) + 1,
"R10 learning-response", "learning code may not depend on response APIs"))
if strict and rel not in {"src/daemon_app.cpp", "src/main.cpp"}:
tracker = re.search(r'(?:#include[^\n]*tracker_set\.hpp|\bTrackerSet\b|\bTrackerMatch\b)', flat)
response = re.search(r'#include[^\n]*(?:exec_guard|opensnitch)\.hpp', flat)
if tracker and response:
violations.append((rel, flat.count("\n", 0, tracker.start()) + 1,
"R11 tracker-response", "tracker lists may not reach blocking or containment"))
for m in RE_EMPTY_CATCH.finditer(flat):
line_no = flat.count("\n", 0, m.start()) + 1
violations.append((rel, line_no, "R7 empty-catch", " ".join(m.group(0).split())))
for api, idx in C_CALLBACK_APIS.items():
for m in re.finditer(re.escape(api) + r"\s*\(", flat):
args = split_args(flat, m.end())
if len(args) <= idx:
continue
cb = args[idx].strip().lstrip("+").strip()
if api == "curl_easy_setopt" and not args[1].strip().endswith("FUNCTION"):
continue
if cb.startswith("[") or cb in ("nullptr", "NULL", "0"):
if cb.startswith("[") and "noexcept" not in cb.split("{")[0]:
line_no = flat.count("\n", 0, m.start()) + 1
violations.append((rel, line_no, "R4 c-callback-noexcept", f"{api}(... lambda ...)"))
continue
ident = re.sub(r"^&", "", cb).split("(")[0].strip()
if re.fullmatch(r"[A-Za-z_][\w:]*", ident) and not function_is_noexcept(ident, sources):
line_no = flat.count("\n", 0, m.start()) + 1
violations.append((rel, line_no, "R4 c-callback-noexcept", f"{api}(..., {ident}, ...)"))
def strip_comment_block(text: str) -> str:
text = re.sub(r"/\*.*?\*/", lambda m: "\n" * m.group(0).count("\n"), text, flags=re.S)
return "\n".join(strip_comment(line) for line in text.splitlines())
def collect(root: Path, dirs):
files = {}
for d in dirs:
base = root / d
if not base.is_dir():
continue
for p in sorted(base.rglob("*")):
if p.suffix in EXTS and p.is_file():
files[p.relative_to(root).as_posix()] = p.read_text(errors="replace")
return files
def run(root: Path, files: dict) -> list:
violations = []
for rel, text in files.items():
check_file(root / rel, rel, text, files, violations)
return violations
def main(argv):
if len(argv) == 3 and argv[1] == "--self-test":
root = Path(argv[2])
fixtures = root / "tests" / "fixtures" / "invariants"
failed = False
for bad in sorted(fixtures.glob("bad_*.cpp")):
rule = bad.stem.split("_", 1)[1].upper() # bad_r1_... -> R1_...
rule_id = rule.split("_")[0]
files = {"src/" + bad.name: bad.read_text()}
found = {v[2].split()[0] for v in run(root, files)}
if rule_id not in found:
print(f"SELF-TEST FAIL: {bad.name} did not trigger {rule_id} (got {sorted(found)})")
failed = True
good = fixtures / "good.cpp"
if good.exists():
v = run(root, {"src/good.cpp": good.read_text()})
if v:
print("SELF-TEST FAIL: good.cpp flagged:", v)
failed = True
print("self-test:", "FAIL" if failed else "ok")
return 1 if failed else 0
root = Path(argv[1] if len(argv) > 1 else ".")
violations = run(root, collect(root, SCAN_DIRS))
allowed = []
for rel, text in collect(root, STRICT_DIRS).items():
for no, line in enumerate(text.splitlines(), 1):
if "invariant-allow(" in line:
allowed.append(f"{rel}:{no}: {line.strip()}")
for rel, no, rule, text in violations:
print(f"{rel}:{no}: {rule}: {text}")
print(f"\n{len(violations)} violation(s); {len(allowed)} documented exception(s)")
for a in allowed:
print(" allowed:", a)
return 1 if violations else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))