strawberry/scripts/threat_traces.py
trvon 3fd5d23b77
threat traces checker + fleet gate in the dockerized Salt test
- scripts/threat_traces.py: enforces the mechanical rules of the 2026-10-28
  threat-trace pre-registration: required # meta keys; eval families first
  reported 2025-07-01..2026-10-28 with a new-family/new-variant bucket and
  write-up sources only; training families before the cutoff with every
  fifth family (first-report order) in dev; no family in both files;
  RFC 5737 addresses and .example domains only; size caps; and that every
  file compiles with replay's parser. --sha256 prints the freeze digest.
- tests/trace_check.cpp: strawberry-trace-check dev tool (replay's parser).
- strawberry.gate: optional gate.warmup_hours (unset = the pre-registered
  12 h; TEST pillars only).
- deploy/salt/test/fleet_run.sh: gate phase - tops sensor history past the
  10 min injection floor, runs orch.strawberry_gate, asserts 3 gate models
  were published, 9 pooled cells over 3 sensors with positives, sensors keep
  their reports, and still no promoted model anywhere.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 09:36:24 -07:00

191 lines
9.2 KiB
Python
Executable file

#!/usr/bin/env python3
"""Check threat-grounded trace files against the pre-registered rules.
docs/evaluation/stage4-threat-traces-preregistration.md (2026-10-28) fixes how threats-train.trace and
threats-eval.trace are collected. This checker enforces the mechanical part:
* every trace has the required `# meta` keys, with valid values
* split rules: eval families were first reported on/after 2025-07-01 and on/before 2026-10-28 and
carry a bucket (new-family | new-variant); train/dev families were first reported before
2025-07-01; in each training file every fifth family (first-report order) is `dev`
* no family appears in both files (family-level split: variants must not leak)
* network indicators are documentation-only: IPv4 only in RFC 5737 ranges, domains only `.example`
* every trace compiles with the replay parser (builddir/tests/strawberry-trace-check), when built
* eval file size: at most 30 traces; training at most 40 families (+ at most 10 atomic)
--sha256 prints the file digest used to freeze the evaluation list (record it in an amendment +
YAMS before the first evaluation; the file is never edited afterwards).
usage: scripts/threat_traces.py [--sha256] [--trace-check BIN] FILE...
exit 0 = all files pass.
"""
import hashlib
import ipaddress
import os
import re
import subprocess
import sys
from datetime import date
CUTOFF = date(2025, 7, 1)
EVAL_END = date(2026, 10, 28)
REQUIRED = ["family", "campaign", "first_reported", "source_url", "source_type", "attack", "split",
"normalized", "inferred", "license"]
DOC_NETS = [ipaddress.ip_network(n) for n in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24")]
IP_RE = re.compile(r"\b(\d{1,3}(?:\.\d{1,3}){3})\b")
DOMAIN_RE = re.compile(r"\b((?:[a-z0-9-]+\.)+[a-z]{2,})\b", re.I)
# Hostname-shaped words that are file names or tool names, not domains.
NOT_DOMAINS = {"desktop", "service", "timer", "socket", "conf", "sh", "py", "so", "json", "yaml", "yml",
"txt", "log", "tar", "gz", "xz", "zip", "bin", "elf", "rc", "d", "local", "cache", "tmp",
"pid", "lock", "key", "pub", "pem", "crt", "deb", "rpm", "apk", "img", "iso", "dat", "db",
"list", "repo", "rules", "path", "mount", "target", "bashrc", "profile", "ko", "o", "c",
"js", "php", "pl", "rb", "go", "jar", "war", "class", "xml", "html", "htm", "cgi", "exe",
"dll", "lnk", "bat", "ps1", "vbs", "plist", "dylib", "app", "pkg", "dmg"}
def parse(path):
"""Yield (meta dict, trace name, body lines, first line number) per trace."""
traces, meta, name, body, start = [], {}, None, [], 0
with open(path, encoding="utf-8") as f:
for no, raw in enumerate(f, 1):
line = raw.rstrip("\n")
m = re.match(r"#\s*meta\s+([a-z_]+)\s*[:=]\s*(.*)$", line)
if m:
if name is not None: # meta after a trace body starts the next trace
traces.append((meta, name, body, start))
meta, name, body = {}, None, []
meta[m.group(1)] = m.group(2).strip()
continue
t = re.match(r"trace\s+(\S+)\s*$", line)
if t:
if name is not None:
traces.append((meta, name, body, start))
meta, body = {}, []
name, start = t.group(1), no
continue
if name is not None and line.strip() and not line.lstrip().startswith("#"):
body.append((no, line))
if name is not None:
traces.append((meta, name, body, start))
return traces
def check_file(path, trace_check):
errors, warnings = [], []
traces = parse(path)
is_eval = "eval" in os.path.basename(path)
if not traces:
errors.append("no traces")
families = {}
for meta, name, body, start in traces:
where = f"{path}:{start} trace {name}"
for key in REQUIRED + (["bucket"] if is_eval else []):
if key not in meta or not meta[key]:
errors.append(f"{where}: missing meta '{key}'")
try:
first = date.fromisoformat(meta.get("first_reported", ""))
except ValueError:
errors.append(f"{where}: first_reported must be YYYY-MM-DD")
first = None
split = meta.get("split", "")
if is_eval:
if split != "eval":
errors.append(f"{where}: split must be eval in an eval file")
if meta.get("bucket") not in ("new-family", "new-variant"):
errors.append(f"{where}: bucket must be new-family or new-variant")
if first and not (CUTOFF <= first <= EVAL_END):
errors.append(f"{where}: eval family first reported {first}, outside {CUTOFF}..{EVAL_END}")
if meta.get("source_type") != "writeup":
errors.append(f"{where}: eval traces come from write-ups only")
else:
if split not in ("train", "dev"):
errors.append(f"{where}: split must be train or dev")
if first and first >= CUTOFF:
errors.append(f"{where}: training family first reported {first}, on/after cutoff {CUTOFF}")
if meta.get("source_type") not in ("writeup", "atomic"):
errors.append(f"{where}: source_type must be writeup or atomic")
if not str(meta.get("source_url", "")).startswith(("http://", "https://")):
errors.append(f"{where}: source_url must be a URL")
fam = meta.get("family", "").strip().lower()
if fam:
families.setdefault(fam, []).append((first, split, meta.get("source_type"), where))
for no, line in body:
for ip in IP_RE.findall(line):
try:
addr = ipaddress.ip_address(ip)
except ValueError:
continue
if not any(addr in n for n in DOC_NETS) and not addr.is_loopback:
errors.append(f"{path}:{no}: non-documentation IP {ip} (use RFC 5737)")
for dom in DOMAIN_RE.findall(line):
tld = dom.rsplit(".", 1)[-1].lower()
if tld in NOT_DOMAINS or IP_RE.fullmatch(dom):
continue
if line[: line.find(dom)].rstrip().endswith("/"): # a path component, e.g. /usr/lib/x.so
continue
if not dom.lower().endswith(".example"):
warnings.append(f"{path}:{no}: '{dom}' looks like a domain; domains must be .example")
# Every fifth family in first-report order is dev (training files, write-up families).
if not is_eval:
fam_order = sorted(((min(d for d, *_ in v if d) if any(d for d, *_ in v) else date.max), f)
for f, v in families.items() if any(st == "writeup" for _, _, st, _ in v))
for idx, (_, fam) in enumerate(fam_order, 1):
want = "dev" if idx % 5 == 0 else "train"
got = {s for _, s, st, _ in families[fam] if st == "writeup"}
if got != {want}:
errors.append(f"family '{fam}' is #{idx} in first-report order: split must be {want}, got {sorted(got)}")
writeups = sum(1 for v in families.values() if any(st == "writeup" for _, _, st, _ in v))
atomic_traces = sum(1 for meta, *_ in traces if meta.get("source_type") == "atomic")
if writeups > 40:
errors.append(f"{writeups} training families (max 40)")
if atomic_traces > 10:
errors.append(f"{atomic_traces} atomic traces (max 10)")
elif len(traces) > 30:
errors.append(f"{len(traces)} eval traces (max 30)")
if trace_check and os.access(trace_check, os.X_OK):
r = subprocess.run([trace_check, path], capture_output=True, text=True)
if r.returncode != 0:
errors.append(f"does not compile: {r.stderr.strip()}")
else:
warnings.append("trace parser not checked (build builddir/tests/strawberry-trace-check)")
return traces, families, errors, warnings
def main(argv):
show_sha = "--sha256" in argv
argv = [a for a in argv if a != "--sha256"]
trace_check = "builddir/tests/strawberry-trace-check"
if "--trace-check" in argv:
i = argv.index("--trace-check")
trace_check = argv[i + 1]
del argv[i:i + 2]
if not argv:
print(__doc__)
return 2
ok = True
fam_by_file = {}
for path in argv:
traces, families, errors, warnings = check_file(path, trace_check)
fam_by_file[path] = set(families)
for w in warnings:
print(f"warning: {w}")
for e in errors:
print(f"ERROR: {e}")
ok = ok and not errors
status = "OK" if not errors else "FAILED"
print(f"{path}: {len(traces)} traces, {len(families)} families: {status}")
if show_sha:
with open(path, "rb") as f:
print(f"sha256 {hashlib.sha256(f.read()).hexdigest()} {path}")
paths = list(fam_by_file)
for i in range(len(paths)):
for j in range(i + 1, len(paths)):
both = fam_by_file[paths[i]] & fam_by_file[paths[j]]
if both:
print(f"ERROR: families in both {paths[i]} and {paths[j]}: {sorted(both)}")
ok = False
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))