#!/usr/bin/env python3 """Check threat-grounded trace files against the pre-registered rules. docs/evaluation/stage4-threat-traces-preregistration.md (2026-10-28) fixes how threats-train.trace and threats-eval.trace are collected. This checker enforces the mechanical part: * every trace has the required `# meta` keys, with valid values * split rules: eval families were first reported on/after 2025-07-01 and on/before 2026-10-28 and carry a bucket (new-family | new-variant); train/dev families were first reported before 2025-07-01; in each training file every fifth family (first-report order) is `dev` * no family appears in both files (family-level split: variants must not leak) * network indicators are documentation-only: IPv4 only in RFC 5737 ranges, domains only `.example` * every trace compiles with the replay parser (builddir/tests/strawberry-trace-check), when built * eval file size: at most 30 traces; training at most 40 families (+ at most 10 atomic) --sha256 prints the file digest used to freeze the evaluation list (record it in an amendment + YAMS before the first evaluation; the file is never edited afterwards). usage: scripts/threat_traces.py [--sha256] [--trace-check BIN] FILE... exit 0 = all files pass. """ import hashlib import ipaddress import os import re import subprocess import sys from datetime import date CUTOFF = date(2025, 7, 1) EVAL_END = date(2026, 10, 28) REQUIRED = ["family", "campaign", "first_reported", "source_url", "source_type", "attack", "split", "normalized", "inferred", "license"] DOC_NETS = [ipaddress.ip_network(n) for n in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24")] IP_RE = re.compile(r"\b(\d{1,3}(?:\.\d{1,3}){3})\b") DOMAIN_RE = re.compile(r"\b((?:[a-z0-9-]+\.)+[a-z]{2,})\b", re.I) # Hostname-shaped words that are file names or tool names, not domains. NOT_DOMAINS = {"desktop", "service", "timer", "socket", "conf", "sh", "py", "so", "json", "yaml", "yml", "txt", "log", "tar", "gz", "xz", "zip", "bin", "elf", "rc", "d", "local", "cache", "tmp", "pid", "lock", "key", "pub", "pem", "crt", "deb", "rpm", "apk", "img", "iso", "dat", "db", "list", "repo", "rules", "path", "mount", "target", "bashrc", "profile", "ko", "o", "c", "js", "php", "pl", "rb", "go", "jar", "war", "class", "xml", "html", "htm", "cgi", "exe", "dll", "lnk", "bat", "ps1", "vbs", "plist", "dylib", "app", "pkg", "dmg"} def parse(path): """Yield (meta dict, trace name, body lines, first line number) per trace.""" traces, meta, name, body, start = [], {}, None, [], 0 with open(path, encoding="utf-8") as f: for no, raw in enumerate(f, 1): line = raw.rstrip("\n") m = re.match(r"#\s*meta\s+([a-z_]+)\s*[:=]\s*(.*)$", line) if m: if name is not None: # meta after a trace body starts the next trace traces.append((meta, name, body, start)) meta, name, body = {}, None, [] meta[m.group(1)] = m.group(2).strip() continue t = re.match(r"trace\s+(\S+)\s*$", line) if t: if name is not None: traces.append((meta, name, body, start)) meta, body = {}, [] name, start = t.group(1), no continue if name is not None and line.strip() and not line.lstrip().startswith("#"): body.append((no, line)) if name is not None: traces.append((meta, name, body, start)) return traces def check_file(path, trace_check): errors, warnings = [], [] traces = parse(path) is_eval = "eval" in os.path.basename(path) if not traces: errors.append("no traces") families = {} for meta, name, body, start in traces: where = f"{path}:{start} trace {name}" for key in REQUIRED + (["bucket"] if is_eval else []): if key not in meta or not meta[key]: errors.append(f"{where}: missing meta '{key}'") try: first = date.fromisoformat(meta.get("first_reported", "")) except ValueError: errors.append(f"{where}: first_reported must be YYYY-MM-DD") first = None split = meta.get("split", "") if is_eval: if split != "eval": errors.append(f"{where}: split must be eval in an eval file") if meta.get("bucket") not in ("new-family", "new-variant"): errors.append(f"{where}: bucket must be new-family or new-variant") if first and not (CUTOFF <= first <= EVAL_END): errors.append(f"{where}: eval family first reported {first}, outside {CUTOFF}..{EVAL_END}") if meta.get("source_type") != "writeup": errors.append(f"{where}: eval traces come from write-ups only") else: if split not in ("train", "dev"): errors.append(f"{where}: split must be train or dev") if first and first >= CUTOFF: errors.append(f"{where}: training family first reported {first}, on/after cutoff {CUTOFF}") if meta.get("source_type") not in ("writeup", "atomic"): errors.append(f"{where}: source_type must be writeup or atomic") if not str(meta.get("source_url", "")).startswith(("http://", "https://")): errors.append(f"{where}: source_url must be a URL") fam = meta.get("family", "").strip().lower() if fam: families.setdefault(fam, []).append((first, split, meta.get("source_type"), where)) for no, line in body: for ip in IP_RE.findall(line): try: addr = ipaddress.ip_address(ip) except ValueError: continue if not any(addr in n for n in DOC_NETS) and not addr.is_loopback: errors.append(f"{path}:{no}: non-documentation IP {ip} (use RFC 5737)") for dom in DOMAIN_RE.findall(line): tld = dom.rsplit(".", 1)[-1].lower() if tld in NOT_DOMAINS or IP_RE.fullmatch(dom): continue if line[: line.find(dom)].rstrip().endswith("/"): # a path component, e.g. /usr/lib/x.so continue if not dom.lower().endswith(".example"): warnings.append(f"{path}:{no}: '{dom}' looks like a domain; domains must be .example") # Every fifth family in first-report order is dev (training files, write-up families). if not is_eval: fam_order = sorted(((min(d for d, *_ in v if d) if any(d for d, *_ in v) else date.max), f) for f, v in families.items() if any(st == "writeup" for _, _, st, _ in v)) for idx, (_, fam) in enumerate(fam_order, 1): want = "dev" if idx % 5 == 0 else "train" got = {s for _, s, st, _ in families[fam] if st == "writeup"} if got != {want}: errors.append(f"family '{fam}' is #{idx} in first-report order: split must be {want}, got {sorted(got)}") writeups = sum(1 for v in families.values() if any(st == "writeup" for _, _, st, _ in v)) atomic_traces = sum(1 for meta, *_ in traces if meta.get("source_type") == "atomic") if writeups > 40: errors.append(f"{writeups} training families (max 40)") if atomic_traces > 10: errors.append(f"{atomic_traces} atomic traces (max 10)") elif len(traces) > 30: errors.append(f"{len(traces)} eval traces (max 30)") if trace_check and os.access(trace_check, os.X_OK): r = subprocess.run([trace_check, path], capture_output=True, text=True) if r.returncode != 0: errors.append(f"does not compile: {r.stderr.strip()}") else: warnings.append("trace parser not checked (build builddir/tests/strawberry-trace-check)") return traces, families, errors, warnings def main(argv): show_sha = "--sha256" in argv argv = [a for a in argv if a != "--sha256"] trace_check = "builddir/tests/strawberry-trace-check" if "--trace-check" in argv: i = argv.index("--trace-check") trace_check = argv[i + 1] del argv[i:i + 2] if not argv: print(__doc__) return 2 ok = True fam_by_file = {} for path in argv: traces, families, errors, warnings = check_file(path, trace_check) fam_by_file[path] = set(families) for w in warnings: print(f"warning: {w}") for e in errors: print(f"ERROR: {e}") ok = ok and not errors status = "OK" if not errors else "FAILED" print(f"{path}: {len(traces)} traces, {len(families)} families: {status}") if show_sha: with open(path, "rb") as f: print(f"sha256 {hashlib.sha256(f.read()).hexdigest()} {path}") paths = list(fam_by_file) for i in range(len(paths)): for j in range(i + 1, len(paths)): both = fam_by_file[paths[i]] & fam_by_file[paths[j]] if both: print(f"ERROR: families in both {paths[i]} and {paths[j]}: {sorted(both)}") ok = False return 0 if ok else 1 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))