The verifier treated every negative link status as "Caldera never ran it", but
Caldera's states are not signed that way: HIGH_VIZ=-5, UNTRUSTED=-4,
EXECUTE=-3, DISCARD=-2, PAUSE=-1, SUCCESS=0, ERROR=1, TIMEOUT=124. EXECUTE is
the normal in-flight state, so a healthy incident was failed on its first poll
and the range reported zero verified incidents while the marker really ran.
Links are now classified explicitly: EXECUTE and PAUSE keep polling, SUCCESS
still requires a completion timestamp, and DISCARD/UNTRUSTED/HIGH_VIZ/TIMEOUT
or a nonzero exit are terminal failures.
Reconcile also treated a missing operation as pending forever, so an incident
whose operation disappeared (Caldera recreated after the OOM) was retried
indefinitely. A 404 is now terminal ("operation no longer exists"), while any
other read error still stays pending and never becomes "executed". The Caldera
client turns a malformed JSON body into a RuntimeError instead of leaking a
decode error from deep inside the client.
Verified against the live range: a queued link (-3) polled to SUCCESS with a
finish timestamp and was recorded verified with the executing agent; synthetic
cases cover every state code, SUCCESS without a finish, 404 and 500.
554 lines
26 KiB
Python
Executable file
554 lines
26 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Validation scheduler for the local Caldera range: drives Caldera so each range sensor records a
|
|
labeled incident, used to produce promotion-gate evidence for the Strawberry detector.
|
|
|
|
This is test orchestration for an isolated lab (Caldera on loopback, sensors on a private slirp
|
|
network). It never touches anything outside the range. Each run through Caldera drops and executes an
|
|
inert marker binary (a copy of /bin/sleep with a per-incident marker string appended) so the detector
|
|
has something to record; there is no real payload.
|
|
|
|
Design for travel: the cadence is a persisted wall-clock due time (state field "next_due"), so
|
|
suspends and reboots pause it and a restart resumes the interval instead of firing immediately. State:
|
|
RANGE_DIR/state/attack_state.json. That file is the record of what actually ran: a file that is present
|
|
but unreadable or malformed stops the scheduler (it is never silently replaced, and marker ids are
|
|
never reused); only an absent file means "no incidents yet".
|
|
|
|
Cadence: every 30-90 min, one incident runs on the next sensor (round robin), via a fresh Caldera
|
|
ability + operation so each lands in its own replay window. The marker for each incident is NOT known
|
|
to the sensors' rules at the time, so the detector does not flag it at exec. Later (after
|
|
LABEL_DELAY_MIN) the scheduler pushes an updated rule file that recognizes the accumulated markers to
|
|
every sensor and reloads rules; the detector's retro-hunt then labels the earlier windows after the
|
|
fact. Those delayed labels are exactly the "extra positives" the gate measures.
|
|
|
|
An incident counts as executed only once Caldera's own operation/link record shows the ability link
|
|
finished successfully on an agent of the target sensor group; the observed execution time comes from
|
|
that record. Until then the incident stays explicitly unverified: it is never aged into the rule push
|
|
and never advances labeled_through. The incident id, marker and ability are persisted before Caldera is
|
|
called, so a crash or timeout leaves a pending record to reconcile instead of a reused identity.
|
|
"""
|
|
import http.client
|
|
import json
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
RANGE_DIR = Path(os.environ.get("SB_RANGE_DIR", "/var/tmp/sb-range"))
|
|
STATE_DIR = RANGE_DIR / "state"
|
|
STATE = STATE_DIR / "attack_state.json"
|
|
KEY = "ADMIN123"
|
|
CALDERA_HOST = "127.0.0.1" # Caldera stays on the host's loopback for the whole range
|
|
CALDERA_PORT = 8888
|
|
SENSORS = int(os.environ.get("SB_RANGE_SENSORS", "3"))
|
|
MIN_GAP_S = int(os.environ.get("SB_RANGE_MIN_GAP_S", "1800")) # 30 min
|
|
MAX_GAP_S = int(os.environ.get("SB_RANGE_MAX_GAP_S", "5400")) # 90 min
|
|
LABEL_DELAY_MIN = int(os.environ.get("SB_RANGE_LABEL_DELAY_MIN", "120"))
|
|
SSHK = os.environ.get("SB_RANGE_SSHK", str(HERE / "../vm/id_vm"))
|
|
SCHEMA = 2 # state layout; 1 was the unverified creation-time layout
|
|
RETRY_GAP_S = 300 # retry an attempt that could not be armed (no agent yet) in ~5 min
|
|
RECONCILE_WINDOW_S = int(os.environ.get("SB_RANGE_RECONCILE_WINDOW_S", "3600"))
|
|
VERIFY_TRIES = int(os.environ.get("SB_RANGE_VERIFY_TRIES", "12"))
|
|
VERIFY_DELAY_S = int(os.environ.get("SB_RANGE_VERIFY_DELAY_S", "10"))
|
|
VERIFY_TIMEOUT_S = int(os.environ.get("SB_RANGE_VERIFY_TIMEOUT_S", "10"))
|
|
# Caldera link states (app/objects/secondclass/c_link.py): negative does NOT mean failure.
|
|
# EXECUTE/PAUSE are in-flight, SUCCESS is the only success, the rest are terminal failures.
|
|
LINK_HIGH_VIZ = -5
|
|
LINK_UNTRUSTED = -4
|
|
LINK_EXECUTE = -3
|
|
LINK_DISCARD = -2
|
|
LINK_PAUSE = -1
|
|
LINK_TIMEOUT = 124
|
|
|
|
# Incident shapes: each is a distinct benign-looking exec tree a detector should rank as unusual. They
|
|
# use only the inert marker binary and documentation addresses. Templated per incident id (IID) and the
|
|
# per-incident marker (MARK). Caldera runs the command string as the agent user.
|
|
SHAPES = [
|
|
# download-like fetch then exec of a dropped marker
|
|
"cp /bin/sleep /var/tmp/.cache/m_@IID@ && printf @MARK@ >> /var/tmp/.cache/m_@IID@ && /var/tmp/.cache/m_@IID@ 2",
|
|
# shell pipeline that writes then runs
|
|
"sh -c 'cp /bin/sleep /var/tmp/.cache/s_@IID@; printf @MARK@ >> /var/tmp/.cache/s_@IID@; /var/tmp/.cache/s_@IID@ 2'",
|
|
# user-systemd-like persistence path plus exec
|
|
"mkdir -p $HOME/.local/bin && cp /bin/sleep $HOME/.local/bin/u_@IID@ && printf @MARK@ >> $HOME/.local/bin/u_@IID@ && $HOME/.local/bin/u_@IID@ 2",
|
|
# cron-like staging dir plus exec
|
|
"mkdir -p /var/tmp/.cache/c_@IID@ && cp /bin/sleep /var/tmp/.cache/c_@IID@/r && printf @MARK@ >> /var/tmp/.cache/c_@IID@/r && /var/tmp/.cache/c_@IID@/r 2",
|
|
]
|
|
|
|
|
|
class StateError(RuntimeError):
|
|
"""The state file is present but cannot be trusted; it is preserved, never overwritten."""
|
|
|
|
|
|
def api(method, path, body=None, timeout=30):
|
|
"""Caldera REST call, pinned to the range's fixed loopback origin: the connection goes to
|
|
CALDERA_HOST:CALDERA_PORT and only /api/v2/ paths are allowed, so no request can be aimed at another
|
|
host, scheme or endpoint. Non-2xx raises, as before."""
|
|
if not path.startswith("/api/v2/") or ".." in path:
|
|
raise RuntimeError(f"refusing non-range Caldera path: {path}")
|
|
payload = json.dumps(body).encode() if body is not None else None
|
|
conn = http.client.HTTPConnection(CALDERA_HOST, CALDERA_PORT, timeout=timeout)
|
|
try:
|
|
conn.request(method, path, body=payload,
|
|
headers={"KEY": KEY, "Content-Type": "application/json"})
|
|
resp = conn.getresponse()
|
|
raw = resp.read()
|
|
status, reason = resp.status, resp.reason
|
|
finally:
|
|
conn.close()
|
|
if status >= 400:
|
|
raise RuntimeError(f"caldera {method} {path} -> {status} {reason}")
|
|
if not raw.strip():
|
|
return None
|
|
try:
|
|
return json.loads(raw)
|
|
except ValueError as e:
|
|
# A truncated/non-JSON body must surface as a Caldera error the caller
|
|
# handles, never as an uncaught exception from deep inside the client.
|
|
raise RuntimeError(f"caldera {method} {path} -> malformed JSON: {e}") from e
|
|
|
|
|
|
def default_state():
|
|
return {"schema": SCHEMA, "incidents": [], "next_sensor": 1, "labeled_through": 0, "iid": 0,
|
|
"next_due": 0.0, "labels_pushed_at": None}
|
|
|
|
|
|
def last_incident_at(st):
|
|
"""Newest timestamp known for a recorded incident: observed execution, else the launch attempt."""
|
|
times = []
|
|
for inc in st.get("incidents") or []:
|
|
if not isinstance(inc, dict):
|
|
continue
|
|
for key in ("at", "started"):
|
|
value = inc.get(key)
|
|
if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0:
|
|
times.append(float(value))
|
|
return max(times) if times else None
|
|
|
|
|
|
def next_due_from_last(st):
|
|
"""Next cadence point derived from the newest recorded incident, so a restart with recent history
|
|
waits out the remaining interval instead of firing immediately. A state with no incidents at all
|
|
keeps 0.0: the first incident of a fresh range is meant to start right away."""
|
|
last = last_incident_at(st)
|
|
return last + MIN_GAP_S if last else 0.0
|
|
|
|
|
|
def migrate(st):
|
|
"""Fill in current-schema fields. Pre-verification files (schema 1: the layout had no "schema" key)
|
|
recorded an incident as done at operation-creation time with no Caldera/link evidence, so those stay
|
|
unverified history: ids and markers remain consumed, but they never enter label aging. Their ability
|
|
id is derivable from the incident id, so reconciliation can still confirm them from Caldera's own
|
|
link data. Any other schema version is left untouched for validate() to refuse."""
|
|
version = st.get("schema", 1)
|
|
if version != SCHEMA:
|
|
if version != 1:
|
|
return st
|
|
for inc in st.get("incidents") or []:
|
|
if not isinstance(inc, dict):
|
|
continue
|
|
if "started" not in inc:
|
|
inc["started"] = inc.get("at") if isinstance(inc.get("at"), (int, float)) else None
|
|
if inc.get("verified") is not True:
|
|
inc["verified"] = False
|
|
inc["at"] = None
|
|
inc["at_source"] = None
|
|
if not inc.get("note"):
|
|
inc["note"] = "legacy record: execution never verified"
|
|
if isinstance(inc.get("iid"), int) and "ability" not in inc:
|
|
inc["ability"] = f"sbrange-{inc['iid']:08x}"
|
|
st["schema"] = SCHEMA
|
|
if not isinstance(st.get("next_due"), (int, float)) or st["next_due"] <= 0:
|
|
st["next_due"] = next_due_from_last(st)
|
|
st.setdefault("labels_pushed_at", None)
|
|
return st
|
|
|
|
|
|
def validate(st):
|
|
"""Reject a state file whose contents cannot be trusted, before anything writes to it."""
|
|
def bad(msg):
|
|
raise StateError(f"{STATE}: {msg}")
|
|
|
|
if st.get("schema") != SCHEMA:
|
|
bad(f"unsupported schema {st.get('schema')!r}")
|
|
iid = st.get("iid")
|
|
if not isinstance(iid, int) or isinstance(iid, bool) or iid < 0:
|
|
bad(f"iid {iid!r} is not a non-negative integer")
|
|
sensor = st.get("next_sensor")
|
|
if not isinstance(sensor, int) or isinstance(sensor, bool) or sensor < 1:
|
|
bad(f"next_sensor {sensor!r} is not a sensor number")
|
|
if sensor > SENSORS: # SENSORS is an env knob: a smaller fleet wraps, it does not invalidate history
|
|
st["next_sensor"] = (sensor - 1) % SENSORS + 1
|
|
labeled = st.get("labeled_through")
|
|
if not isinstance(labeled, int) or isinstance(labeled, bool) or labeled < 0:
|
|
bad(f"labeled_through {labeled!r} is not a non-negative integer")
|
|
next_due = st.get("next_due")
|
|
if not isinstance(next_due, (int, float)) or isinstance(next_due, bool) or next_due < 0:
|
|
bad(f"next_due {next_due!r} is not a timestamp")
|
|
pushed_at = st.get("labels_pushed_at")
|
|
if pushed_at is not None and not isinstance(pushed_at, (int, float)):
|
|
bad(f"labels_pushed_at {pushed_at!r} is not a timestamp")
|
|
incidents = st.get("incidents")
|
|
if not isinstance(incidents, list):
|
|
bad("incidents is not a list")
|
|
seen = set()
|
|
for inc in incidents:
|
|
if not isinstance(inc, dict):
|
|
bad("an incident record is not an object")
|
|
n = inc.get("iid")
|
|
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
|
|
bad(f"incident {inc!r} has no valid iid")
|
|
if n in seen:
|
|
bad(f"incident id {n} appears twice (its marker id would be reused)")
|
|
seen.add(n)
|
|
if not isinstance(inc.get("sensor"), int) or inc["sensor"] < 1:
|
|
bad(f"incident {n} has no valid sensor")
|
|
if not isinstance(inc.get("mark"), str) or not inc["mark"]:
|
|
bad(f"incident {n} has no marker string")
|
|
if not isinstance(inc.get("verified"), bool):
|
|
bad(f"incident {n} has no verified flag")
|
|
at = inc.get("at")
|
|
if inc["verified"]:
|
|
if not isinstance(at, (int, float)) or at <= 0:
|
|
bad(f"incident {n} is marked verified without an observed execution time")
|
|
if not isinstance(inc.get("op"), str) or not inc["op"]:
|
|
bad(f"incident {n} is marked verified without a Caldera operation id")
|
|
elif at is not None:
|
|
bad(f"incident {n} is unverified but carries an execution time")
|
|
if seen and iid < max(seen):
|
|
bad(f"iid {iid} is behind incident {max(seen)} (identity would be reused)")
|
|
if labeled > len(incidents):
|
|
bad(f"labeled_through {labeled} exceeds the {len(incidents)} recorded incident(s)")
|
|
return st
|
|
|
|
|
|
def load():
|
|
"""State, or the empty default only when the file is genuinely absent. A present file that cannot
|
|
be read, parsed or validated raises StateError instead of being replaced: overwriting it would
|
|
reuse marker ids that may already have executed and would silently drop evidence."""
|
|
if not STATE.exists():
|
|
return default_state()
|
|
try:
|
|
raw = STATE.read_text()
|
|
except OSError as e:
|
|
raise StateError(f"{STATE}: cannot be read ({e})") from e
|
|
try:
|
|
st = json.loads(raw)
|
|
except ValueError as e:
|
|
raise StateError(f"{STATE}: not valid JSON ({e})") from e
|
|
if not isinstance(st, dict):
|
|
raise StateError(f"{STATE}: top level is not a JSON object")
|
|
return validate(migrate(st))
|
|
|
|
|
|
def save(st):
|
|
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
|
tmp = STATE.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(st, indent=2))
|
|
tmp.replace(STATE)
|
|
|
|
|
|
def agent_paw(group):
|
|
"""Newest live agent in the sensor's group, or None."""
|
|
agents = api("GET", "/api/v2/agents") or []
|
|
live = [a for a in agents if a.get("group") == group and a.get("trusted", True)]
|
|
live.sort(key=lambda a: a.get("last_seen", ""), reverse=True)
|
|
return live[0]["paw"] if live else None
|
|
|
|
|
|
def _epoch(value):
|
|
"""Caldera reports link timestamps as epoch numbers; accept an ISO-8601 string as well."""
|
|
if isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, (int, float)) and value > 0:
|
|
return float(value)
|
|
if isinstance(value, str) and value.strip():
|
|
try:
|
|
return datetime.fromisoformat(value.strip().replace("Z", "+00:00")).timestamp()
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def operation_links(op_id):
|
|
"""The operation's ability links. The operation object carries its chain; fall back to the
|
|
operation list for Caldera builds that omit the chain from the single-operation view."""
|
|
detail = api("GET", f"/api/v2/operations/{op_id}", timeout=VERIFY_TIMEOUT_S) or {}
|
|
if isinstance(detail, dict):
|
|
links = detail.get("chain") or detail.get("links")
|
|
if links:
|
|
return detail, links
|
|
ops = api("GET", "/api/v2/operations", timeout=VERIFY_TIMEOUT_S) or []
|
|
for cand in ops if isinstance(ops, list) else []:
|
|
if isinstance(cand, dict) and cand.get("id") == op_id:
|
|
links = cand.get("chain") or cand.get("links")
|
|
if links:
|
|
return cand, links
|
|
return (detail if isinstance(detail, dict) else {}), []
|
|
|
|
|
|
def link_for(links, ability_id):
|
|
for link in links:
|
|
if not isinstance(link, dict):
|
|
continue
|
|
ability = link.get("ability")
|
|
nested = ability.get("ability_id") if isinstance(ability, dict) else None
|
|
if (link.get("ability_id") or nested) == ability_id:
|
|
return link
|
|
return None
|
|
|
|
|
|
def check_operation(inc):
|
|
"""One bounded read of Caldera's record for an incident's operation: did its ability link actually
|
|
run, on which agent, and when did it finish?
|
|
|
|
Returns (state, note) with state "verified" | "pending" | "failed". Only "verified" proves
|
|
execution: the link must carry a completion time and a success status, and an operation that
|
|
finished without the link proves the opposite. A "verified" incident receives the observed
|
|
execution time (`at`) from the link's own finish timestamp; pending/failed ones keep `at` None and
|
|
stay out of label aging.
|
|
"""
|
|
if not inc.get("op"):
|
|
return "pending", "no operation recorded yet"
|
|
try:
|
|
detail, links = operation_links(inc["op"])
|
|
except Exception as e: # a Caldera hiccup must never turn into "executed"
|
|
# A 404 means the operation is gone (Caldera recreated/reset): it can never be
|
|
# verified, so it must not stay pending forever.
|
|
if "404" in str(e):
|
|
return "failed", "operation no longer exists in Caldera (404)"
|
|
return "pending", f"operation read failed: {e}"
|
|
op_state = detail.get("state")
|
|
inc["op_state"] = op_state
|
|
if not links:
|
|
if op_state == "finished":
|
|
return "failed", "operation finished with no links"
|
|
return "pending", "operation has no links yet"
|
|
link = link_for(links, inc.get("ability"))
|
|
if link is None:
|
|
if op_state == "finished":
|
|
return "failed", "operation finished without a link for the incident ability"
|
|
return "pending", "operation has no link for the incident ability yet"
|
|
status = link.get("status")
|
|
if isinstance(status, bool) or not isinstance(status, (int, float)):
|
|
return "pending", "link status not reported"
|
|
# Caldera's link states are NOT "negative means bad": EXECUTE(-3) and PAUSE(-1) are
|
|
# in-flight, SUCCESS(0) is the only success. A queued link must keep polling instead
|
|
# of being reported as "never ran".
|
|
code = int(status)
|
|
if code in (LINK_EXECUTE, LINK_PAUSE):
|
|
return "pending", f"link still queued (state {code})"
|
|
if code == LINK_DISCARD:
|
|
return "failed", "link discarded (state -2): Caldera will not run it"
|
|
if code == LINK_UNTRUSTED:
|
|
return "failed", "link untrusted (state -4): the agent is not trusted for it"
|
|
if code == LINK_HIGH_VIZ:
|
|
return "failed", "link held for high-visibility approval (state -5)"
|
|
if code == LINK_TIMEOUT:
|
|
return "failed", "link timed out (state 124) on the agent"
|
|
if code != 0:
|
|
return "failed", f"link exit status {code}: the marker command failed"
|
|
# A link that has not run yet also reports status 0, so completion must come from its timestamps:
|
|
# `finish` is set when the agent reported the result, `collect` when the agent picked it up.
|
|
at, at_source = _epoch(link.get("finish")), "caldera-finish"
|
|
if at is None:
|
|
collected = _epoch(link.get("collect"))
|
|
if collected is not None and op_state == "finished":
|
|
at, at_source = collected, "caldera-collect"
|
|
else:
|
|
return "pending", "link has no finish timestamp yet"
|
|
ran_paw = link.get("paw")
|
|
if not ran_paw:
|
|
return "failed", "link reports no agent paw (which agent ran it is unknown)"
|
|
inc["ran_paw"] = ran_paw
|
|
inc["at"] = at
|
|
inc["at_source"] = at_source
|
|
inc["verified"] = True
|
|
inc["note"] = None
|
|
note = f"link ran on {ran_paw}, finished {time.strftime('%FT%T', time.localtime(at))} ({at_source})"
|
|
if ran_paw != inc.get("paw"):
|
|
note += f"; selected agent was {inc.get('paw')} (same sensor group)"
|
|
return "verified", note
|
|
|
|
|
|
def verify_operation(inc, tries=VERIFY_TRIES, delay=VERIFY_DELAY_S):
|
|
"""Poll the operation record, bounded by tries * (delay + timeout). A "pending" answer is retried;
|
|
a "failed" answer, or a poll budget that runs out, leaves the incident explicitly unverified."""
|
|
state, note = "pending", "not checked"
|
|
for attempt in range(max(1, tries)):
|
|
try:
|
|
state, note = check_operation(inc)
|
|
except Exception as e:
|
|
state, note = "pending", f"operation check failed: {e}"
|
|
print(f" incident {inc['iid']} poll {attempt + 1}/{tries}: {state}: {note}", flush=True)
|
|
if state != "pending":
|
|
break
|
|
if attempt + 1 < tries:
|
|
time.sleep(delay)
|
|
return state, note
|
|
|
|
|
|
def reconcile(st):
|
|
"""Re-check incidents left unverified by a crash, a timeout or a discarded link: once per cycle,
|
|
only while they are recent. Reconciliation can only add evidence; the id/marker of an unverified
|
|
incident stays consumed, so nothing is relaunched under an identity that may already have run."""
|
|
now = time.time()
|
|
changed = False
|
|
for inc in st["incidents"]:
|
|
if inc.get("verified") or not inc.get("op") or not inc.get("ability"):
|
|
continue
|
|
started = inc.get("started")
|
|
if not isinstance(started, (int, float)) or now - started > RECONCILE_WINDOW_S:
|
|
continue
|
|
before = (inc.get("verified"), inc.get("note"), inc.get("op_state"))
|
|
state, note = check_operation(inc)
|
|
if state != "verified":
|
|
inc["note"] = f"{state}: {note}"
|
|
if (inc.get("verified"), inc.get("note"), inc.get("op_state")) != before:
|
|
changed = True
|
|
print(time.strftime("%FT%T"), f"incident {inc['iid']} reconciled: {state}: {note}", flush=True)
|
|
if changed:
|
|
save(st)
|
|
return changed
|
|
|
|
|
|
def run_incident(st):
|
|
"""Launch one incident on the next sensor and return a one-line report.
|
|
|
|
The incident id, marker and ability are saved to the state file before Caldera is called, so a
|
|
crash mid-launch leaves a pending record to reconcile rather than a silently reused identity. The
|
|
record only becomes `verified` (and only then eligible for label aging) when Caldera's link data
|
|
shows the command actually ran.
|
|
"""
|
|
sensor = st["next_sensor"]
|
|
group = f"sensor-{sensor}"
|
|
paw = agent_paw(group)
|
|
if not paw:
|
|
st["next_due"] = time.time() + RETRY_GAP_S # no identity consumed yet
|
|
save(st)
|
|
return f"no live agent in {group}; retrying in {RETRY_GAP_S}s"
|
|
iid = st["iid"] + 1
|
|
mark = f"SB-RANGE-MARK-{iid:04d}"
|
|
shape = SHAPES[iid % len(SHAPES)]
|
|
cmd = shape.replace("@IID@", str(iid)).replace("@MARK@", mark)
|
|
ability_id = f"sbrange-{iid:08x}"
|
|
inc = {"iid": iid, "sensor": sensor, "mark": mark, "paw": paw, "ability": ability_id,
|
|
"op": None, "started": time.time(), "verified": False, "at": None, "at_source": None,
|
|
"op_state": None, "ran_paw": None, "note": "pending: operation not launched"}
|
|
st["iid"] = iid
|
|
st["next_sensor"] = sensor % SENSORS + 1
|
|
st["incidents"].append(inc)
|
|
st["next_due"] = inc["started"] + random.randint(MIN_GAP_S, MAX_GAP_S)
|
|
save(st) # identity is durable before the Caldera call
|
|
ability = {
|
|
"ability_id": ability_id, "name": f"sb range incident {iid}", "tactic": "execution",
|
|
"technique_id": "T1204", "technique_name": "User Execution",
|
|
"executors": [{"platform": "linux", "name": "sh", "command": cmd}],
|
|
}
|
|
adv_id = f"sbrange-adv-{iid:08x}"
|
|
op = {"name": f"sbrange-op-{iid}", "adversary": {"adversary_id": adv_id},
|
|
"group": group, "planner": {"id": "aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a"},
|
|
"source": {"id": "ed32b9c3-9593-4c33-b0db-e2007315096b"}, "auto_close": True, "state": "running"}
|
|
try:
|
|
api("PUT", f"/api/v2/abilities/{ability_id}", ability)
|
|
api("PUT", f"/api/v2/adversaries/{adv_id}",
|
|
{"adversary_id": adv_id, "name": f"sb range {iid}", "atomic_ordering": [ability_id]})
|
|
created = api("POST", "/api/v2/operations", op)
|
|
except Exception as e:
|
|
inc["note"] = f"launch failed: {e}"
|
|
save(st)
|
|
return f"incident {iid} on {group}: launch failed ({e}); recorded unverified, id stays consumed"
|
|
inc["op"] = (created or {}).get("id")
|
|
if not inc["op"]:
|
|
inc["note"] = "Caldera returned no operation id"
|
|
save(st)
|
|
return f"incident {iid} on {group}: no operation id returned; recorded unverified"
|
|
inc["note"] = "pending: operation created, link not verified"
|
|
save(st)
|
|
state, note = verify_operation(inc)
|
|
inc["note"] = None if state == "verified" else f"{state}: {note}"
|
|
save(st)
|
|
if state == "verified":
|
|
return f"incident {iid} on {group} verified (mark {mark}): {note}"
|
|
return (f"incident {iid} on {group} NOT verified ({state}: {note}); kept pending, "
|
|
f"not aged into labels")
|
|
|
|
|
|
def ripe_incidents(st, now):
|
|
"""Incidents eligible for the delayed rule push: verified ones whose observed execution is at least
|
|
LABEL_DELAY_MIN old. Unverified incidents are excluded, so unverified work is never presented as a
|
|
completed incident and cannot pull a window into the label set."""
|
|
cutoff = now - LABEL_DELAY_MIN * 60
|
|
return [i for i in st["incidents"]
|
|
if i.get("verified") is True and isinstance(i.get("at"), (int, float)) and i["at"] <= cutoff]
|
|
|
|
|
|
def push_labels(st):
|
|
"""Rules recognizing every marker up to the newest ripe verified incident go out to all sensors; the
|
|
detector's retro-hunt then labels the earlier windows. `labeled_through` counts *verified incidents
|
|
whose markers were pushed and reloaded everywhere*, not labels this scheduler observed: whether the
|
|
retro-hunt actually labeled a window is what the gate measures."""
|
|
ripe = ripe_incidents(st, time.time())
|
|
if len(ripe) <= st["labeled_through"]:
|
|
return None
|
|
marks = sorted({i["mark"] for i in ripe})
|
|
rule = "rule SB_Range_Marker {\n meta:\n severity = \"high\"\n strings:\n"
|
|
rule += "".join(f" $m{n} = \"{m}\"\n" for n, m in enumerate(marks))
|
|
rule += " condition:\n any of them\n}\n"
|
|
sensors = {i["sensor"] for i in ripe}
|
|
pushed = []
|
|
for s in sensors:
|
|
port = 2240 + s
|
|
ssh = ["ssh", "-i", SSHK, "-p", str(port), "-o", "StrictHostKeyChecking=no",
|
|
"-o", "UserKnownHostsFile=/dev/null", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
|
|
"arch@127.0.0.1"]
|
|
try:
|
|
subprocess.run(ssh + ["sudo tee /etc/strawberry/rules/sb_range_markers.yar >/dev/null && "
|
|
"sudo strawberry --config /etc/strawberry/strawberry.conf rules reload"],
|
|
input=rule.encode(), timeout=60, check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
pushed.append(s)
|
|
except Exception as e: # a suspended / unreachable sensor is retried next cycle
|
|
print(f" label push to sensor-{s} failed: {e}", flush=True)
|
|
if set(pushed) >= sensors:
|
|
st["labeled_through"] = len(ripe)
|
|
st["labels_pushed_at"] = time.time()
|
|
save(st)
|
|
return f"rule push for {len(marks)} verified marker(s) -> sensors {sorted(pushed)}"
|
|
|
|
|
|
def main():
|
|
try:
|
|
st = load()
|
|
except StateError as e:
|
|
print(f"range attack scheduler refusing to start: {e}", flush=True)
|
|
print(" the state file is preserved as-is; repair it or move it aside by hand", flush=True)
|
|
raise SystemExit(2) from None
|
|
save(st) # persist the migration / derived cadence before anything else
|
|
unverified = sum(1 for i in st["incidents"] if not i.get("verified"))
|
|
due = (time.strftime("%FT%T", time.localtime(st["next_due"])) if st["next_due"] else "now")
|
|
print(f"range attack scheduler up (incidents {st['iid']}, {unverified} unverified, next due {due})",
|
|
flush=True)
|
|
while True:
|
|
now = time.time()
|
|
if st["next_due"] > now + MAX_GAP_S: # clock moved backwards: keep the cadence sane
|
|
st["next_due"] = now + MIN_GAP_S
|
|
save(st)
|
|
reconcile(st)
|
|
msg = push_labels(st)
|
|
if msg:
|
|
print(time.strftime("%FT%T"), msg, flush=True)
|
|
if time.time() >= st["next_due"]:
|
|
try:
|
|
info = run_incident(st)
|
|
except Exception as e:
|
|
info = f"incident error: {e}"
|
|
st["next_due"] = time.time() + RETRY_GAP_S
|
|
save(st)
|
|
print(time.strftime("%FT%T"), info, flush=True)
|
|
time.sleep(60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|