#!/usr/bin/env python3 """Validation scheduler for the local Caldera range: drives Caldera so each range sensor records a labeled incident, used to produce promotion-gate evidence for the Strawberry detector. This is test orchestration for an isolated lab (Caldera on loopback, sensors on a private slirp network). It never touches anything outside the range. Each run through Caldera drops and executes an inert marker binary (a copy of /bin/sleep with a per-incident marker string appended) so the detector has something to record; there is no real payload. Design for travel: the cadence is a persisted wall-clock due time (state field "next_due"), so suspends and reboots pause it and a restart resumes the interval instead of firing immediately. State: RANGE_DIR/state/attack_state.json. That file is the record of what actually ran: a file that is present but unreadable or malformed stops the scheduler (it is never silently replaced, and marker ids are never reused); only an absent file means "no incidents yet". Cadence: every 30-90 min, one incident runs on the next sensor (round robin), via a fresh Caldera ability + operation so each lands in its own replay window. The marker for each incident is NOT known to the sensors' rules at the time, so the detector does not flag it at exec. Later (after LABEL_DELAY_MIN) the scheduler pushes an updated rule file that recognizes the accumulated markers to every sensor and reloads rules; the detector's retro-hunt then labels the earlier windows after the fact. Those delayed labels are exactly the "extra positives" the gate measures. An incident counts as executed only once Caldera's own operation/link record shows the ability link finished successfully on an agent of the target sensor group; the observed execution time comes from that record. Until then the incident stays explicitly unverified: it is never aged into the rule push and never advances labeled_through. The incident id, marker and ability are persisted before Caldera is called, so a crash or timeout leaves a pending record to reconcile instead of a reused identity. """ import http.client import json import os import random import subprocess import time from datetime import datetime from pathlib import Path HERE = Path(__file__).resolve().parent RANGE_DIR = Path(os.environ.get("SB_RANGE_DIR", "/var/tmp/sb-range")) STATE_DIR = RANGE_DIR / "state" STATE = STATE_DIR / "attack_state.json" KEY = "ADMIN123" CALDERA_HOST = "127.0.0.1" # Caldera stays on the host's loopback for the whole range CALDERA_PORT = 8888 SENSORS = int(os.environ.get("SB_RANGE_SENSORS", "3")) MIN_GAP_S = int(os.environ.get("SB_RANGE_MIN_GAP_S", "1800")) # 30 min MAX_GAP_S = int(os.environ.get("SB_RANGE_MAX_GAP_S", "5400")) # 90 min LABEL_DELAY_MIN = int(os.environ.get("SB_RANGE_LABEL_DELAY_MIN", "120")) SSHK = os.environ.get("SB_RANGE_SSHK", str(HERE / "../vm/id_vm")) SCHEMA = 2 # state layout; 1 was the unverified creation-time layout RETRY_GAP_S = 300 # retry an attempt that could not be armed (no agent yet) in ~5 min RECONCILE_WINDOW_S = int(os.environ.get("SB_RANGE_RECONCILE_WINDOW_S", "3600")) VERIFY_TRIES = int(os.environ.get("SB_RANGE_VERIFY_TRIES", "12")) VERIFY_DELAY_S = int(os.environ.get("SB_RANGE_VERIFY_DELAY_S", "10")) VERIFY_TIMEOUT_S = int(os.environ.get("SB_RANGE_VERIFY_TIMEOUT_S", "10")) # Caldera link states (app/objects/secondclass/c_link.py): negative does NOT mean failure. # EXECUTE/PAUSE are in-flight, SUCCESS is the only success, the rest are terminal failures. LINK_HIGH_VIZ = -5 LINK_UNTRUSTED = -4 LINK_EXECUTE = -3 LINK_DISCARD = -2 LINK_PAUSE = -1 LINK_TIMEOUT = 124 # Incident shapes: each is a distinct benign-looking exec tree a detector should rank as unusual. They # use only the inert marker binary and documentation addresses. Templated per incident id (IID) and the # per-incident marker (MARK). Caldera runs the command string as the agent user. SHAPES = [ # download-like fetch then exec of a dropped marker "cp /bin/sleep /var/tmp/.cache/m_@IID@ && printf @MARK@ >> /var/tmp/.cache/m_@IID@ && /var/tmp/.cache/m_@IID@ 2", # shell pipeline that writes then runs "sh -c 'cp /bin/sleep /var/tmp/.cache/s_@IID@; printf @MARK@ >> /var/tmp/.cache/s_@IID@; /var/tmp/.cache/s_@IID@ 2'", # user-systemd-like persistence path plus exec "mkdir -p $HOME/.local/bin && cp /bin/sleep $HOME/.local/bin/u_@IID@ && printf @MARK@ >> $HOME/.local/bin/u_@IID@ && $HOME/.local/bin/u_@IID@ 2", # cron-like staging dir plus exec "mkdir -p /var/tmp/.cache/c_@IID@ && cp /bin/sleep /var/tmp/.cache/c_@IID@/r && printf @MARK@ >> /var/tmp/.cache/c_@IID@/r && /var/tmp/.cache/c_@IID@/r 2", ] class StateError(RuntimeError): """The state file is present but cannot be trusted; it is preserved, never overwritten.""" def api(method, path, body=None, timeout=30): """Caldera REST call, pinned to the range's fixed loopback origin: the connection goes to CALDERA_HOST:CALDERA_PORT and only /api/v2/ paths are allowed, so no request can be aimed at another host, scheme or endpoint. Non-2xx raises, as before.""" if not path.startswith("/api/v2/") or ".." in path: raise RuntimeError(f"refusing non-range Caldera path: {path}") payload = json.dumps(body).encode() if body is not None else None conn = http.client.HTTPConnection(CALDERA_HOST, CALDERA_PORT, timeout=timeout) try: conn.request(method, path, body=payload, headers={"KEY": KEY, "Content-Type": "application/json"}) resp = conn.getresponse() raw = resp.read() status, reason = resp.status, resp.reason finally: conn.close() if status >= 400: raise RuntimeError(f"caldera {method} {path} -> {status} {reason}") if not raw.strip(): return None try: return json.loads(raw) except ValueError as e: # A truncated/non-JSON body must surface as a Caldera error the caller # handles, never as an uncaught exception from deep inside the client. raise RuntimeError(f"caldera {method} {path} -> malformed JSON: {e}") from e def default_state(): return {"schema": SCHEMA, "incidents": [], "next_sensor": 1, "labeled_through": 0, "iid": 0, "next_due": 0.0, "labels_pushed_at": None} def last_incident_at(st): """Newest timestamp known for a recorded incident: observed execution, else the launch attempt.""" times = [] for inc in st.get("incidents") or []: if not isinstance(inc, dict): continue for key in ("at", "started"): value = inc.get(key) if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0: times.append(float(value)) return max(times) if times else None def next_due_from_last(st): """Next cadence point derived from the newest recorded incident, so a restart with recent history waits out the remaining interval instead of firing immediately. A state with no incidents at all keeps 0.0: the first incident of a fresh range is meant to start right away.""" last = last_incident_at(st) return last + MIN_GAP_S if last else 0.0 def migrate(st): """Fill in current-schema fields. Pre-verification files (schema 1: the layout had no "schema" key) recorded an incident as done at operation-creation time with no Caldera/link evidence, so those stay unverified history: ids and markers remain consumed, but they never enter label aging. Their ability id is derivable from the incident id, so reconciliation can still confirm them from Caldera's own link data. Any other schema version is left untouched for validate() to refuse.""" version = st.get("schema", 1) if version != SCHEMA: if version != 1: return st for inc in st.get("incidents") or []: if not isinstance(inc, dict): continue if "started" not in inc: inc["started"] = inc.get("at") if isinstance(inc.get("at"), (int, float)) else None if inc.get("verified") is not True: inc["verified"] = False inc["at"] = None inc["at_source"] = None if not inc.get("note"): inc["note"] = "legacy record: execution never verified" if isinstance(inc.get("iid"), int) and "ability" not in inc: inc["ability"] = f"sbrange-{inc['iid']:08x}" st["schema"] = SCHEMA if not isinstance(st.get("next_due"), (int, float)) or st["next_due"] <= 0: st["next_due"] = next_due_from_last(st) st.setdefault("labels_pushed_at", None) return st def validate(st): """Reject a state file whose contents cannot be trusted, before anything writes to it.""" def bad(msg): raise StateError(f"{STATE}: {msg}") if st.get("schema") != SCHEMA: bad(f"unsupported schema {st.get('schema')!r}") iid = st.get("iid") if not isinstance(iid, int) or isinstance(iid, bool) or iid < 0: bad(f"iid {iid!r} is not a non-negative integer") sensor = st.get("next_sensor") if not isinstance(sensor, int) or isinstance(sensor, bool) or sensor < 1: bad(f"next_sensor {sensor!r} is not a sensor number") if sensor > SENSORS: # SENSORS is an env knob: a smaller fleet wraps, it does not invalidate history st["next_sensor"] = (sensor - 1) % SENSORS + 1 labeled = st.get("labeled_through") if not isinstance(labeled, int) or isinstance(labeled, bool) or labeled < 0: bad(f"labeled_through {labeled!r} is not a non-negative integer") next_due = st.get("next_due") if not isinstance(next_due, (int, float)) or isinstance(next_due, bool) or next_due < 0: bad(f"next_due {next_due!r} is not a timestamp") pushed_at = st.get("labels_pushed_at") if pushed_at is not None and not isinstance(pushed_at, (int, float)): bad(f"labels_pushed_at {pushed_at!r} is not a timestamp") incidents = st.get("incidents") if not isinstance(incidents, list): bad("incidents is not a list") seen = set() for inc in incidents: if not isinstance(inc, dict): bad("an incident record is not an object") n = inc.get("iid") if not isinstance(n, int) or isinstance(n, bool) or n < 1: bad(f"incident {inc!r} has no valid iid") if n in seen: bad(f"incident id {n} appears twice (its marker id would be reused)") seen.add(n) if not isinstance(inc.get("sensor"), int) or inc["sensor"] < 1: bad(f"incident {n} has no valid sensor") if not isinstance(inc.get("mark"), str) or not inc["mark"]: bad(f"incident {n} has no marker string") if not isinstance(inc.get("verified"), bool): bad(f"incident {n} has no verified flag") at = inc.get("at") if inc["verified"]: if not isinstance(at, (int, float)) or at <= 0: bad(f"incident {n} is marked verified without an observed execution time") if not isinstance(inc.get("op"), str) or not inc["op"]: bad(f"incident {n} is marked verified without a Caldera operation id") elif at is not None: bad(f"incident {n} is unverified but carries an execution time") if seen and iid < max(seen): bad(f"iid {iid} is behind incident {max(seen)} (identity would be reused)") if labeled > len(incidents): bad(f"labeled_through {labeled} exceeds the {len(incidents)} recorded incident(s)") return st def load(): """State, or the empty default only when the file is genuinely absent. A present file that cannot be read, parsed or validated raises StateError instead of being replaced: overwriting it would reuse marker ids that may already have executed and would silently drop evidence.""" if not STATE.exists(): return default_state() try: raw = STATE.read_text() except OSError as e: raise StateError(f"{STATE}: cannot be read ({e})") from e try: st = json.loads(raw) except ValueError as e: raise StateError(f"{STATE}: not valid JSON ({e})") from e if not isinstance(st, dict): raise StateError(f"{STATE}: top level is not a JSON object") return validate(migrate(st)) def save(st): STATE_DIR.mkdir(parents=True, exist_ok=True) tmp = STATE.with_suffix(".tmp") tmp.write_text(json.dumps(st, indent=2)) tmp.replace(STATE) def agent_paw(group): """Newest live agent in the sensor's group, or None.""" agents = api("GET", "/api/v2/agents") or [] live = [a for a in agents if a.get("group") == group and a.get("trusted", True)] live.sort(key=lambda a: a.get("last_seen", ""), reverse=True) return live[0]["paw"] if live else None def _epoch(value): """Caldera reports link timestamps as epoch numbers; accept an ISO-8601 string as well.""" if isinstance(value, bool): return None if isinstance(value, (int, float)) and value > 0: return float(value) if isinstance(value, str) and value.strip(): try: return datetime.fromisoformat(value.strip().replace("Z", "+00:00")).timestamp() except ValueError: return None return None def operation_links(op_id): """The operation's ability links. The operation object carries its chain; fall back to the operation list for Caldera builds that omit the chain from the single-operation view.""" detail = api("GET", f"/api/v2/operations/{op_id}", timeout=VERIFY_TIMEOUT_S) or {} if isinstance(detail, dict): links = detail.get("chain") or detail.get("links") if links: return detail, links ops = api("GET", "/api/v2/operations", timeout=VERIFY_TIMEOUT_S) or [] for cand in ops if isinstance(ops, list) else []: if isinstance(cand, dict) and cand.get("id") == op_id: links = cand.get("chain") or cand.get("links") if links: return cand, links return (detail if isinstance(detail, dict) else {}), [] def link_for(links, ability_id): for link in links: if not isinstance(link, dict): continue ability = link.get("ability") nested = ability.get("ability_id") if isinstance(ability, dict) else None if (link.get("ability_id") or nested) == ability_id: return link return None def check_operation(inc): """One bounded read of Caldera's record for an incident's operation: did its ability link actually run, on which agent, and when did it finish? Returns (state, note) with state "verified" | "pending" | "failed". Only "verified" proves execution: the link must carry a completion time and a success status, and an operation that finished without the link proves the opposite. A "verified" incident receives the observed execution time (`at`) from the link's own finish timestamp; pending/failed ones keep `at` None and stay out of label aging. """ if not inc.get("op"): return "pending", "no operation recorded yet" try: detail, links = operation_links(inc["op"]) except Exception as e: # a Caldera hiccup must never turn into "executed" # A 404 means the operation is gone (Caldera recreated/reset): it can never be # verified, so it must not stay pending forever. if "404" in str(e): return "failed", "operation no longer exists in Caldera (404)" return "pending", f"operation read failed: {e}" op_state = detail.get("state") inc["op_state"] = op_state if not links: if op_state == "finished": return "failed", "operation finished with no links" return "pending", "operation has no links yet" link = link_for(links, inc.get("ability")) if link is None: if op_state == "finished": return "failed", "operation finished without a link for the incident ability" return "pending", "operation has no link for the incident ability yet" status = link.get("status") if isinstance(status, bool) or not isinstance(status, (int, float)): return "pending", "link status not reported" # Caldera's link states are NOT "negative means bad": EXECUTE(-3) and PAUSE(-1) are # in-flight, SUCCESS(0) is the only success. A queued link must keep polling instead # of being reported as "never ran". code = int(status) if code in (LINK_EXECUTE, LINK_PAUSE): return "pending", f"link still queued (state {code})" if code == LINK_DISCARD: return "failed", "link discarded (state -2): Caldera will not run it" if code == LINK_UNTRUSTED: return "failed", "link untrusted (state -4): the agent is not trusted for it" if code == LINK_HIGH_VIZ: return "failed", "link held for high-visibility approval (state -5)" if code == LINK_TIMEOUT: return "failed", "link timed out (state 124) on the agent" if code != 0: return "failed", f"link exit status {code}: the marker command failed" # A link that has not run yet also reports status 0, so completion must come from its timestamps: # `finish` is set when the agent reported the result, `collect` when the agent picked it up. at, at_source = _epoch(link.get("finish")), "caldera-finish" if at is None: collected = _epoch(link.get("collect")) if collected is not None and op_state == "finished": at, at_source = collected, "caldera-collect" else: return "pending", "link has no finish timestamp yet" ran_paw = link.get("paw") if not ran_paw: return "failed", "link reports no agent paw (which agent ran it is unknown)" inc["ran_paw"] = ran_paw inc["at"] = at inc["at_source"] = at_source inc["verified"] = True inc["note"] = None note = f"link ran on {ran_paw}, finished {time.strftime('%FT%T', time.localtime(at))} ({at_source})" if ran_paw != inc.get("paw"): note += f"; selected agent was {inc.get('paw')} (same sensor group)" return "verified", note def verify_operation(inc, tries=VERIFY_TRIES, delay=VERIFY_DELAY_S): """Poll the operation record, bounded by tries * (delay + timeout). A "pending" answer is retried; a "failed" answer, or a poll budget that runs out, leaves the incident explicitly unverified.""" state, note = "pending", "not checked" for attempt in range(max(1, tries)): try: state, note = check_operation(inc) except Exception as e: state, note = "pending", f"operation check failed: {e}" print(f" incident {inc['iid']} poll {attempt + 1}/{tries}: {state}: {note}", flush=True) if state != "pending": break if attempt + 1 < tries: time.sleep(delay) return state, note def reconcile(st): """Re-check incidents left unverified by a crash, a timeout or a discarded link: once per cycle, only while they are recent. Reconciliation can only add evidence; the id/marker of an unverified incident stays consumed, so nothing is relaunched under an identity that may already have run.""" now = time.time() changed = False for inc in st["incidents"]: if inc.get("verified") or not inc.get("op") or not inc.get("ability"): continue started = inc.get("started") if not isinstance(started, (int, float)) or now - started > RECONCILE_WINDOW_S: continue before = (inc.get("verified"), inc.get("note"), inc.get("op_state")) state, note = check_operation(inc) if state != "verified": inc["note"] = f"{state}: {note}" if (inc.get("verified"), inc.get("note"), inc.get("op_state")) != before: changed = True print(time.strftime("%FT%T"), f"incident {inc['iid']} reconciled: {state}: {note}", flush=True) if changed: save(st) return changed def run_incident(st): """Launch one incident on the next sensor and return a one-line report. The incident id, marker and ability are saved to the state file before Caldera is called, so a crash mid-launch leaves a pending record to reconcile rather than a silently reused identity. The record only becomes `verified` (and only then eligible for label aging) when Caldera's link data shows the command actually ran. """ sensor = st["next_sensor"] group = f"sensor-{sensor}" paw = agent_paw(group) if not paw: st["next_due"] = time.time() + RETRY_GAP_S # no identity consumed yet save(st) return f"no live agent in {group}; retrying in {RETRY_GAP_S}s" iid = st["iid"] + 1 mark = f"SB-RANGE-MARK-{iid:04d}" shape = SHAPES[iid % len(SHAPES)] cmd = shape.replace("@IID@", str(iid)).replace("@MARK@", mark) ability_id = f"sbrange-{iid:08x}" inc = {"iid": iid, "sensor": sensor, "mark": mark, "paw": paw, "ability": ability_id, "op": None, "started": time.time(), "verified": False, "at": None, "at_source": None, "op_state": None, "ran_paw": None, "note": "pending: operation not launched"} st["iid"] = iid st["next_sensor"] = sensor % SENSORS + 1 st["incidents"].append(inc) st["next_due"] = inc["started"] + random.randint(MIN_GAP_S, MAX_GAP_S) save(st) # identity is durable before the Caldera call ability = { "ability_id": ability_id, "name": f"sb range incident {iid}", "tactic": "execution", "technique_id": "T1204", "technique_name": "User Execution", "executors": [{"platform": "linux", "name": "sh", "command": cmd}], } adv_id = f"sbrange-adv-{iid:08x}" op = {"name": f"sbrange-op-{iid}", "adversary": {"adversary_id": adv_id}, "group": group, "planner": {"id": "aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a"}, "source": {"id": "ed32b9c3-9593-4c33-b0db-e2007315096b"}, "auto_close": True, "state": "running"} try: api("PUT", f"/api/v2/abilities/{ability_id}", ability) api("PUT", f"/api/v2/adversaries/{adv_id}", {"adversary_id": adv_id, "name": f"sb range {iid}", "atomic_ordering": [ability_id]}) created = api("POST", "/api/v2/operations", op) except Exception as e: inc["note"] = f"launch failed: {e}" save(st) return f"incident {iid} on {group}: launch failed ({e}); recorded unverified, id stays consumed" inc["op"] = (created or {}).get("id") if not inc["op"]: inc["note"] = "Caldera returned no operation id" save(st) return f"incident {iid} on {group}: no operation id returned; recorded unverified" inc["note"] = "pending: operation created, link not verified" save(st) state, note = verify_operation(inc) inc["note"] = None if state == "verified" else f"{state}: {note}" save(st) if state == "verified": return f"incident {iid} on {group} verified (mark {mark}): {note}" return (f"incident {iid} on {group} NOT verified ({state}: {note}); kept pending, " f"not aged into labels") def ripe_incidents(st, now): """Incidents eligible for the delayed rule push: verified ones whose observed execution is at least LABEL_DELAY_MIN old. Unverified incidents are excluded, so unverified work is never presented as a completed incident and cannot pull a window into the label set.""" cutoff = now - LABEL_DELAY_MIN * 60 return [i for i in st["incidents"] if i.get("verified") is True and isinstance(i.get("at"), (int, float)) and i["at"] <= cutoff] def push_labels(st): """Rules recognizing every marker up to the newest ripe verified incident go out to all sensors; the detector's retro-hunt then labels the earlier windows. `labeled_through` counts *verified incidents whose markers were pushed and reloaded everywhere*, not labels this scheduler observed: whether the retro-hunt actually labeled a window is what the gate measures.""" ripe = ripe_incidents(st, time.time()) if len(ripe) <= st["labeled_through"]: return None marks = sorted({i["mark"] for i in ripe}) rule = "rule SB_Range_Marker {\n meta:\n severity = \"high\"\n strings:\n" rule += "".join(f" $m{n} = \"{m}\"\n" for n, m in enumerate(marks)) rule += " condition:\n any of them\n}\n" sensors = {i["sensor"] for i in ripe} pushed = [] for s in sensors: port = 2240 + s ssh = ["ssh", "-i", SSHK, "-p", str(port), "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "arch@127.0.0.1"] try: subprocess.run(ssh + ["sudo tee /etc/strawberry/rules/sb_range_markers.yar >/dev/null && " "sudo strawberry --config /etc/strawberry/strawberry.conf rules reload"], input=rule.encode(), timeout=60, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) pushed.append(s) except Exception as e: # a suspended / unreachable sensor is retried next cycle print(f" label push to sensor-{s} failed: {e}", flush=True) if set(pushed) >= sensors: st["labeled_through"] = len(ripe) st["labels_pushed_at"] = time.time() save(st) return f"rule push for {len(marks)} verified marker(s) -> sensors {sorted(pushed)}" def main(): try: st = load() except StateError as e: print(f"range attack scheduler refusing to start: {e}", flush=True) print(" the state file is preserved as-is; repair it or move it aside by hand", flush=True) raise SystemExit(2) from None save(st) # persist the migration / derived cadence before anything else unverified = sum(1 for i in st["incidents"] if not i.get("verified")) due = (time.strftime("%FT%T", time.localtime(st["next_due"])) if st["next_due"] else "now") print(f"range attack scheduler up (incidents {st['iid']}, {unverified} unverified, next due {due})", flush=True) while True: now = time.time() if st["next_due"] > now + MAX_GAP_S: # clock moved backwards: keep the cadence sane st["next_due"] = now + MIN_GAP_S save(st) reconcile(st) msg = push_labels(st) if msg: print(time.strftime("%FT%T"), msg, flush=True) if time.time() >= st["next_due"]: try: info = run_incident(st) except Exception as e: info = f"incident error: {e}" st["next_due"] = time.time() + RETRY_GAP_S save(st) print(time.strftime("%FT%T"), info, flush=True) time.sleep(60) if __name__ == "__main__": main()