- scripts/threat_traces.py: enforces the mechanical rules of the 2026-10-28 threat-trace pre-registration: required # meta keys; eval families first reported 2025-07-01..2026-10-28 with a new-family/new-variant bucket and write-up sources only; training families before the cutoff with every fifth family (first-report order) in dev; no family in both files; RFC 5737 addresses and .example domains only; size caps; and that every file compiles with replay's parser. --sha256 prints the freeze digest. - tests/trace_check.cpp: strawberry-trace-check dev tool (replay's parser). - strawberry.gate: optional gate.warmup_hours (unset = the pre-registered 12 h; TEST pillars only). - deploy/salt/test/fleet_run.sh: gate phase - tops sensor history past the 10 min injection floor, runs orch.strawberry_gate, asserts 3 gate models were published, 9 pooled cells over 3 sensors with positives, sensors keep their reports, and still no promoted model anywhere. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
191 lines
9.2 KiB
Python
Executable file
191 lines
9.2 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Check threat-grounded trace files against the pre-registered rules.
|
|
|
|
docs/evaluation/stage4-threat-traces-preregistration.md (2026-10-28) fixes how threats-train.trace and
|
|
threats-eval.trace are collected. This checker enforces the mechanical part:
|
|
|
|
* every trace has the required `# meta` keys, with valid values
|
|
* split rules: eval families were first reported on/after 2025-07-01 and on/before 2026-10-28 and
|
|
carry a bucket (new-family | new-variant); train/dev families were first reported before
|
|
2025-07-01; in each training file every fifth family (first-report order) is `dev`
|
|
* no family appears in both files (family-level split: variants must not leak)
|
|
* network indicators are documentation-only: IPv4 only in RFC 5737 ranges, domains only `.example`
|
|
* every trace compiles with the replay parser (builddir/tests/strawberry-trace-check), when built
|
|
* eval file size: at most 30 traces; training at most 40 families (+ at most 10 atomic)
|
|
|
|
--sha256 prints the file digest used to freeze the evaluation list (record it in an amendment +
|
|
YAMS before the first evaluation; the file is never edited afterwards).
|
|
|
|
usage: scripts/threat_traces.py [--sha256] [--trace-check BIN] FILE...
|
|
exit 0 = all files pass.
|
|
"""
|
|
import hashlib
|
|
import ipaddress
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from datetime import date
|
|
|
|
CUTOFF = date(2025, 7, 1)
|
|
EVAL_END = date(2026, 10, 28)
|
|
REQUIRED = ["family", "campaign", "first_reported", "source_url", "source_type", "attack", "split",
|
|
"normalized", "inferred", "license"]
|
|
DOC_NETS = [ipaddress.ip_network(n) for n in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24")]
|
|
IP_RE = re.compile(r"\b(\d{1,3}(?:\.\d{1,3}){3})\b")
|
|
DOMAIN_RE = re.compile(r"\b((?:[a-z0-9-]+\.)+[a-z]{2,})\b", re.I)
|
|
# Hostname-shaped words that are file names or tool names, not domains.
|
|
NOT_DOMAINS = {"desktop", "service", "timer", "socket", "conf", "sh", "py", "so", "json", "yaml", "yml",
|
|
"txt", "log", "tar", "gz", "xz", "zip", "bin", "elf", "rc", "d", "local", "cache", "tmp",
|
|
"pid", "lock", "key", "pub", "pem", "crt", "deb", "rpm", "apk", "img", "iso", "dat", "db",
|
|
"list", "repo", "rules", "path", "mount", "target", "bashrc", "profile", "ko", "o", "c",
|
|
"js", "php", "pl", "rb", "go", "jar", "war", "class", "xml", "html", "htm", "cgi", "exe",
|
|
"dll", "lnk", "bat", "ps1", "vbs", "plist", "dylib", "app", "pkg", "dmg"}
|
|
|
|
|
|
def parse(path):
|
|
"""Yield (meta dict, trace name, body lines, first line number) per trace."""
|
|
traces, meta, name, body, start = [], {}, None, [], 0
|
|
with open(path, encoding="utf-8") as f:
|
|
for no, raw in enumerate(f, 1):
|
|
line = raw.rstrip("\n")
|
|
m = re.match(r"#\s*meta\s+([a-z_]+)\s*[:=]\s*(.*)$", line)
|
|
if m:
|
|
if name is not None: # meta after a trace body starts the next trace
|
|
traces.append((meta, name, body, start))
|
|
meta, name, body = {}, None, []
|
|
meta[m.group(1)] = m.group(2).strip()
|
|
continue
|
|
t = re.match(r"trace\s+(\S+)\s*$", line)
|
|
if t:
|
|
if name is not None:
|
|
traces.append((meta, name, body, start))
|
|
meta, body = {}, []
|
|
name, start = t.group(1), no
|
|
continue
|
|
if name is not None and line.strip() and not line.lstrip().startswith("#"):
|
|
body.append((no, line))
|
|
if name is not None:
|
|
traces.append((meta, name, body, start))
|
|
return traces
|
|
|
|
|
|
def check_file(path, trace_check):
|
|
errors, warnings = [], []
|
|
traces = parse(path)
|
|
is_eval = "eval" in os.path.basename(path)
|
|
if not traces:
|
|
errors.append("no traces")
|
|
families = {}
|
|
for meta, name, body, start in traces:
|
|
where = f"{path}:{start} trace {name}"
|
|
for key in REQUIRED + (["bucket"] if is_eval else []):
|
|
if key not in meta or not meta[key]:
|
|
errors.append(f"{where}: missing meta '{key}'")
|
|
try:
|
|
first = date.fromisoformat(meta.get("first_reported", ""))
|
|
except ValueError:
|
|
errors.append(f"{where}: first_reported must be YYYY-MM-DD")
|
|
first = None
|
|
split = meta.get("split", "")
|
|
if is_eval:
|
|
if split != "eval":
|
|
errors.append(f"{where}: split must be eval in an eval file")
|
|
if meta.get("bucket") not in ("new-family", "new-variant"):
|
|
errors.append(f"{where}: bucket must be new-family or new-variant")
|
|
if first and not (CUTOFF <= first <= EVAL_END):
|
|
errors.append(f"{where}: eval family first reported {first}, outside {CUTOFF}..{EVAL_END}")
|
|
if meta.get("source_type") != "writeup":
|
|
errors.append(f"{where}: eval traces come from write-ups only")
|
|
else:
|
|
if split not in ("train", "dev"):
|
|
errors.append(f"{where}: split must be train or dev")
|
|
if first and first >= CUTOFF:
|
|
errors.append(f"{where}: training family first reported {first}, on/after cutoff {CUTOFF}")
|
|
if meta.get("source_type") not in ("writeup", "atomic"):
|
|
errors.append(f"{where}: source_type must be writeup or atomic")
|
|
if not str(meta.get("source_url", "")).startswith(("http://", "https://")):
|
|
errors.append(f"{where}: source_url must be a URL")
|
|
fam = meta.get("family", "").strip().lower()
|
|
if fam:
|
|
families.setdefault(fam, []).append((first, split, meta.get("source_type"), where))
|
|
for no, line in body:
|
|
for ip in IP_RE.findall(line):
|
|
try:
|
|
addr = ipaddress.ip_address(ip)
|
|
except ValueError:
|
|
continue
|
|
if not any(addr in n for n in DOC_NETS) and not addr.is_loopback:
|
|
errors.append(f"{path}:{no}: non-documentation IP {ip} (use RFC 5737)")
|
|
for dom in DOMAIN_RE.findall(line):
|
|
tld = dom.rsplit(".", 1)[-1].lower()
|
|
if tld in NOT_DOMAINS or IP_RE.fullmatch(dom):
|
|
continue
|
|
if line[: line.find(dom)].rstrip().endswith("/"): # a path component, e.g. /usr/lib/x.so
|
|
continue
|
|
if not dom.lower().endswith(".example"):
|
|
warnings.append(f"{path}:{no}: '{dom}' looks like a domain; domains must be .example")
|
|
# Every fifth family in first-report order is dev (training files, write-up families).
|
|
if not is_eval:
|
|
fam_order = sorted(((min(d for d, *_ in v if d) if any(d for d, *_ in v) else date.max), f)
|
|
for f, v in families.items() if any(st == "writeup" for _, _, st, _ in v))
|
|
for idx, (_, fam) in enumerate(fam_order, 1):
|
|
want = "dev" if idx % 5 == 0 else "train"
|
|
got = {s for _, s, st, _ in families[fam] if st == "writeup"}
|
|
if got != {want}:
|
|
errors.append(f"family '{fam}' is #{idx} in first-report order: split must be {want}, got {sorted(got)}")
|
|
writeups = sum(1 for v in families.values() if any(st == "writeup" for _, _, st, _ in v))
|
|
atomic_traces = sum(1 for meta, *_ in traces if meta.get("source_type") == "atomic")
|
|
if writeups > 40:
|
|
errors.append(f"{writeups} training families (max 40)")
|
|
if atomic_traces > 10:
|
|
errors.append(f"{atomic_traces} atomic traces (max 10)")
|
|
elif len(traces) > 30:
|
|
errors.append(f"{len(traces)} eval traces (max 30)")
|
|
if trace_check and os.access(trace_check, os.X_OK):
|
|
r = subprocess.run([trace_check, path], capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
errors.append(f"does not compile: {r.stderr.strip()}")
|
|
else:
|
|
warnings.append("trace parser not checked (build builddir/tests/strawberry-trace-check)")
|
|
return traces, families, errors, warnings
|
|
|
|
|
|
def main(argv):
|
|
show_sha = "--sha256" in argv
|
|
argv = [a for a in argv if a != "--sha256"]
|
|
trace_check = "builddir/tests/strawberry-trace-check"
|
|
if "--trace-check" in argv:
|
|
i = argv.index("--trace-check")
|
|
trace_check = argv[i + 1]
|
|
del argv[i:i + 2]
|
|
if not argv:
|
|
print(__doc__)
|
|
return 2
|
|
ok = True
|
|
fam_by_file = {}
|
|
for path in argv:
|
|
traces, families, errors, warnings = check_file(path, trace_check)
|
|
fam_by_file[path] = set(families)
|
|
for w in warnings:
|
|
print(f"warning: {w}")
|
|
for e in errors:
|
|
print(f"ERROR: {e}")
|
|
ok = ok and not errors
|
|
status = "OK" if not errors else "FAILED"
|
|
print(f"{path}: {len(traces)} traces, {len(families)} families: {status}")
|
|
if show_sha:
|
|
with open(path, "rb") as f:
|
|
print(f"sha256 {hashlib.sha256(f.read()).hexdigest()} {path}")
|
|
paths = list(fam_by_file)
|
|
for i in range(len(paths)):
|
|
for j in range(i + 1, len(paths)):
|
|
both = fam_by_file[paths[i]] & fam_by_file[paths[j]]
|
|
if both:
|
|
print(f"ERROR: families in both {paths[i]} and {paths[j]}: {sorted(both)}")
|
|
ok = False
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|