strawberry/deploy/validation/range/range_attack.py
trvon aa7143e005
range: read Caldera link states correctly, treat a vanished operation as failed
The verifier treated every negative link status as "Caldera never ran it", but
Caldera's states are not signed that way: HIGH_VIZ=-5, UNTRUSTED=-4,
EXECUTE=-3, DISCARD=-2, PAUSE=-1, SUCCESS=0, ERROR=1, TIMEOUT=124. EXECUTE is
the normal in-flight state, so a healthy incident was failed on its first poll
and the range reported zero verified incidents while the marker really ran.

Links are now classified explicitly: EXECUTE and PAUSE keep polling, SUCCESS
still requires a completion timestamp, and DISCARD/UNTRUSTED/HIGH_VIZ/TIMEOUT
or a nonzero exit are terminal failures.

Reconcile also treated a missing operation as pending forever, so an incident
whose operation disappeared (Caldera recreated after the OOM) was retried
indefinitely. A 404 is now terminal ("operation no longer exists"), while any
other read error still stays pending and never becomes "executed". The Caldera
client turns a malformed JSON body into a RuntimeError instead of leaking a
decode error from deep inside the client.

Verified against the live range: a queued link (-3) polled to SUCCESS with a
finish timestamp and was recorded verified with the executing agent; synthetic
cases cover every state code, SUCCESS without a finish, 404 and 500.
2026-11-17 16:16:23 -07:00

554 lines
26 KiB
Python
Executable file

#!/usr/bin/env python3
"""Validation scheduler for the local Caldera range: drives Caldera so each range sensor records a
labeled incident, used to produce promotion-gate evidence for the Strawberry detector.
This is test orchestration for an isolated lab (Caldera on loopback, sensors on a private slirp
network). It never touches anything outside the range. Each run through Caldera drops and executes an
inert marker binary (a copy of /bin/sleep with a per-incident marker string appended) so the detector
has something to record; there is no real payload.
Design for travel: the cadence is a persisted wall-clock due time (state field "next_due"), so
suspends and reboots pause it and a restart resumes the interval instead of firing immediately. State:
RANGE_DIR/state/attack_state.json. That file is the record of what actually ran: a file that is present
but unreadable or malformed stops the scheduler (it is never silently replaced, and marker ids are
never reused); only an absent file means "no incidents yet".
Cadence: every 30-90 min, one incident runs on the next sensor (round robin), via a fresh Caldera
ability + operation so each lands in its own replay window. The marker for each incident is NOT known
to the sensors' rules at the time, so the detector does not flag it at exec. Later (after
LABEL_DELAY_MIN) the scheduler pushes an updated rule file that recognizes the accumulated markers to
every sensor and reloads rules; the detector's retro-hunt then labels the earlier windows after the
fact. Those delayed labels are exactly the "extra positives" the gate measures.
An incident counts as executed only once Caldera's own operation/link record shows the ability link
finished successfully on an agent of the target sensor group; the observed execution time comes from
that record. Until then the incident stays explicitly unverified: it is never aged into the rule push
and never advances labeled_through. The incident id, marker and ability are persisted before Caldera is
called, so a crash or timeout leaves a pending record to reconcile instead of a reused identity.
"""
import http.client
import json
import os
import random
import subprocess
import time
from datetime import datetime
from pathlib import Path
HERE = Path(__file__).resolve().parent
RANGE_DIR = Path(os.environ.get("SB_RANGE_DIR", "/var/tmp/sb-range"))
STATE_DIR = RANGE_DIR / "state"
STATE = STATE_DIR / "attack_state.json"
KEY = "ADMIN123"
CALDERA_HOST = "127.0.0.1" # Caldera stays on the host's loopback for the whole range
CALDERA_PORT = 8888
SENSORS = int(os.environ.get("SB_RANGE_SENSORS", "3"))
MIN_GAP_S = int(os.environ.get("SB_RANGE_MIN_GAP_S", "1800")) # 30 min
MAX_GAP_S = int(os.environ.get("SB_RANGE_MAX_GAP_S", "5400")) # 90 min
LABEL_DELAY_MIN = int(os.environ.get("SB_RANGE_LABEL_DELAY_MIN", "120"))
SSHK = os.environ.get("SB_RANGE_SSHK", str(HERE / "../vm/id_vm"))
SCHEMA = 2 # state layout; 1 was the unverified creation-time layout
RETRY_GAP_S = 300 # retry an attempt that could not be armed (no agent yet) in ~5 min
RECONCILE_WINDOW_S = int(os.environ.get("SB_RANGE_RECONCILE_WINDOW_S", "3600"))
VERIFY_TRIES = int(os.environ.get("SB_RANGE_VERIFY_TRIES", "12"))
VERIFY_DELAY_S = int(os.environ.get("SB_RANGE_VERIFY_DELAY_S", "10"))
VERIFY_TIMEOUT_S = int(os.environ.get("SB_RANGE_VERIFY_TIMEOUT_S", "10"))
# Caldera link states (app/objects/secondclass/c_link.py): negative does NOT mean failure.
# EXECUTE/PAUSE are in-flight, SUCCESS is the only success, the rest are terminal failures.
LINK_HIGH_VIZ = -5
LINK_UNTRUSTED = -4
LINK_EXECUTE = -3
LINK_DISCARD = -2
LINK_PAUSE = -1
LINK_TIMEOUT = 124
# Incident shapes: each is a distinct benign-looking exec tree a detector should rank as unusual. They
# use only the inert marker binary and documentation addresses. Templated per incident id (IID) and the
# per-incident marker (MARK). Caldera runs the command string as the agent user.
SHAPES = [
# download-like fetch then exec of a dropped marker
"cp /bin/sleep /var/tmp/.cache/m_@IID@ && printf @MARK@ >> /var/tmp/.cache/m_@IID@ && /var/tmp/.cache/m_@IID@ 2",
# shell pipeline that writes then runs
"sh -c 'cp /bin/sleep /var/tmp/.cache/s_@IID@; printf @MARK@ >> /var/tmp/.cache/s_@IID@; /var/tmp/.cache/s_@IID@ 2'",
# user-systemd-like persistence path plus exec
"mkdir -p $HOME/.local/bin && cp /bin/sleep $HOME/.local/bin/u_@IID@ && printf @MARK@ >> $HOME/.local/bin/u_@IID@ && $HOME/.local/bin/u_@IID@ 2",
# cron-like staging dir plus exec
"mkdir -p /var/tmp/.cache/c_@IID@ && cp /bin/sleep /var/tmp/.cache/c_@IID@/r && printf @MARK@ >> /var/tmp/.cache/c_@IID@/r && /var/tmp/.cache/c_@IID@/r 2",
]
class StateError(RuntimeError):
"""The state file is present but cannot be trusted; it is preserved, never overwritten."""
def api(method, path, body=None, timeout=30):
"""Caldera REST call, pinned to the range's fixed loopback origin: the connection goes to
CALDERA_HOST:CALDERA_PORT and only /api/v2/ paths are allowed, so no request can be aimed at another
host, scheme or endpoint. Non-2xx raises, as before."""
if not path.startswith("/api/v2/") or ".." in path:
raise RuntimeError(f"refusing non-range Caldera path: {path}")
payload = json.dumps(body).encode() if body is not None else None
conn = http.client.HTTPConnection(CALDERA_HOST, CALDERA_PORT, timeout=timeout)
try:
conn.request(method, path, body=payload,
headers={"KEY": KEY, "Content-Type": "application/json"})
resp = conn.getresponse()
raw = resp.read()
status, reason = resp.status, resp.reason
finally:
conn.close()
if status >= 400:
raise RuntimeError(f"caldera {method} {path} -> {status} {reason}")
if not raw.strip():
return None
try:
return json.loads(raw)
except ValueError as e:
# A truncated/non-JSON body must surface as a Caldera error the caller
# handles, never as an uncaught exception from deep inside the client.
raise RuntimeError(f"caldera {method} {path} -> malformed JSON: {e}") from e
def default_state():
return {"schema": SCHEMA, "incidents": [], "next_sensor": 1, "labeled_through": 0, "iid": 0,
"next_due": 0.0, "labels_pushed_at": None}
def last_incident_at(st):
"""Newest timestamp known for a recorded incident: observed execution, else the launch attempt."""
times = []
for inc in st.get("incidents") or []:
if not isinstance(inc, dict):
continue
for key in ("at", "started"):
value = inc.get(key)
if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0:
times.append(float(value))
return max(times) if times else None
def next_due_from_last(st):
"""Next cadence point derived from the newest recorded incident, so a restart with recent history
waits out the remaining interval instead of firing immediately. A state with no incidents at all
keeps 0.0: the first incident of a fresh range is meant to start right away."""
last = last_incident_at(st)
return last + MIN_GAP_S if last else 0.0
def migrate(st):
"""Fill in current-schema fields. Pre-verification files (schema 1: the layout had no "schema" key)
recorded an incident as done at operation-creation time with no Caldera/link evidence, so those stay
unverified history: ids and markers remain consumed, but they never enter label aging. Their ability
id is derivable from the incident id, so reconciliation can still confirm them from Caldera's own
link data. Any other schema version is left untouched for validate() to refuse."""
version = st.get("schema", 1)
if version != SCHEMA:
if version != 1:
return st
for inc in st.get("incidents") or []:
if not isinstance(inc, dict):
continue
if "started" not in inc:
inc["started"] = inc.get("at") if isinstance(inc.get("at"), (int, float)) else None
if inc.get("verified") is not True:
inc["verified"] = False
inc["at"] = None
inc["at_source"] = None
if not inc.get("note"):
inc["note"] = "legacy record: execution never verified"
if isinstance(inc.get("iid"), int) and "ability" not in inc:
inc["ability"] = f"sbrange-{inc['iid']:08x}"
st["schema"] = SCHEMA
if not isinstance(st.get("next_due"), (int, float)) or st["next_due"] <= 0:
st["next_due"] = next_due_from_last(st)
st.setdefault("labels_pushed_at", None)
return st
def validate(st):
"""Reject a state file whose contents cannot be trusted, before anything writes to it."""
def bad(msg):
raise StateError(f"{STATE}: {msg}")
if st.get("schema") != SCHEMA:
bad(f"unsupported schema {st.get('schema')!r}")
iid = st.get("iid")
if not isinstance(iid, int) or isinstance(iid, bool) or iid < 0:
bad(f"iid {iid!r} is not a non-negative integer")
sensor = st.get("next_sensor")
if not isinstance(sensor, int) or isinstance(sensor, bool) or sensor < 1:
bad(f"next_sensor {sensor!r} is not a sensor number")
if sensor > SENSORS: # SENSORS is an env knob: a smaller fleet wraps, it does not invalidate history
st["next_sensor"] = (sensor - 1) % SENSORS + 1
labeled = st.get("labeled_through")
if not isinstance(labeled, int) or isinstance(labeled, bool) or labeled < 0:
bad(f"labeled_through {labeled!r} is not a non-negative integer")
next_due = st.get("next_due")
if not isinstance(next_due, (int, float)) or isinstance(next_due, bool) or next_due < 0:
bad(f"next_due {next_due!r} is not a timestamp")
pushed_at = st.get("labels_pushed_at")
if pushed_at is not None and not isinstance(pushed_at, (int, float)):
bad(f"labels_pushed_at {pushed_at!r} is not a timestamp")
incidents = st.get("incidents")
if not isinstance(incidents, list):
bad("incidents is not a list")
seen = set()
for inc in incidents:
if not isinstance(inc, dict):
bad("an incident record is not an object")
n = inc.get("iid")
if not isinstance(n, int) or isinstance(n, bool) or n < 1:
bad(f"incident {inc!r} has no valid iid")
if n in seen:
bad(f"incident id {n} appears twice (its marker id would be reused)")
seen.add(n)
if not isinstance(inc.get("sensor"), int) or inc["sensor"] < 1:
bad(f"incident {n} has no valid sensor")
if not isinstance(inc.get("mark"), str) or not inc["mark"]:
bad(f"incident {n} has no marker string")
if not isinstance(inc.get("verified"), bool):
bad(f"incident {n} has no verified flag")
at = inc.get("at")
if inc["verified"]:
if not isinstance(at, (int, float)) or at <= 0:
bad(f"incident {n} is marked verified without an observed execution time")
if not isinstance(inc.get("op"), str) or not inc["op"]:
bad(f"incident {n} is marked verified without a Caldera operation id")
elif at is not None:
bad(f"incident {n} is unverified but carries an execution time")
if seen and iid < max(seen):
bad(f"iid {iid} is behind incident {max(seen)} (identity would be reused)")
if labeled > len(incidents):
bad(f"labeled_through {labeled} exceeds the {len(incidents)} recorded incident(s)")
return st
def load():
"""State, or the empty default only when the file is genuinely absent. A present file that cannot
be read, parsed or validated raises StateError instead of being replaced: overwriting it would
reuse marker ids that may already have executed and would silently drop evidence."""
if not STATE.exists():
return default_state()
try:
raw = STATE.read_text()
except OSError as e:
raise StateError(f"{STATE}: cannot be read ({e})") from e
try:
st = json.loads(raw)
except ValueError as e:
raise StateError(f"{STATE}: not valid JSON ({e})") from e
if not isinstance(st, dict):
raise StateError(f"{STATE}: top level is not a JSON object")
return validate(migrate(st))
def save(st):
STATE_DIR.mkdir(parents=True, exist_ok=True)
tmp = STATE.with_suffix(".tmp")
tmp.write_text(json.dumps(st, indent=2))
tmp.replace(STATE)
def agent_paw(group):
"""Newest live agent in the sensor's group, or None."""
agents = api("GET", "/api/v2/agents") or []
live = [a for a in agents if a.get("group") == group and a.get("trusted", True)]
live.sort(key=lambda a: a.get("last_seen", ""), reverse=True)
return live[0]["paw"] if live else None
def _epoch(value):
"""Caldera reports link timestamps as epoch numbers; accept an ISO-8601 string as well."""
if isinstance(value, bool):
return None
if isinstance(value, (int, float)) and value > 0:
return float(value)
if isinstance(value, str) and value.strip():
try:
return datetime.fromisoformat(value.strip().replace("Z", "+00:00")).timestamp()
except ValueError:
return None
return None
def operation_links(op_id):
"""The operation's ability links. The operation object carries its chain; fall back to the
operation list for Caldera builds that omit the chain from the single-operation view."""
detail = api("GET", f"/api/v2/operations/{op_id}", timeout=VERIFY_TIMEOUT_S) or {}
if isinstance(detail, dict):
links = detail.get("chain") or detail.get("links")
if links:
return detail, links
ops = api("GET", "/api/v2/operations", timeout=VERIFY_TIMEOUT_S) or []
for cand in ops if isinstance(ops, list) else []:
if isinstance(cand, dict) and cand.get("id") == op_id:
links = cand.get("chain") or cand.get("links")
if links:
return cand, links
return (detail if isinstance(detail, dict) else {}), []
def link_for(links, ability_id):
for link in links:
if not isinstance(link, dict):
continue
ability = link.get("ability")
nested = ability.get("ability_id") if isinstance(ability, dict) else None
if (link.get("ability_id") or nested) == ability_id:
return link
return None
def check_operation(inc):
"""One bounded read of Caldera's record for an incident's operation: did its ability link actually
run, on which agent, and when did it finish?
Returns (state, note) with state "verified" | "pending" | "failed". Only "verified" proves
execution: the link must carry a completion time and a success status, and an operation that
finished without the link proves the opposite. A "verified" incident receives the observed
execution time (`at`) from the link's own finish timestamp; pending/failed ones keep `at` None and
stay out of label aging.
"""
if not inc.get("op"):
return "pending", "no operation recorded yet"
try:
detail, links = operation_links(inc["op"])
except Exception as e: # a Caldera hiccup must never turn into "executed"
# A 404 means the operation is gone (Caldera recreated/reset): it can never be
# verified, so it must not stay pending forever.
if "404" in str(e):
return "failed", "operation no longer exists in Caldera (404)"
return "pending", f"operation read failed: {e}"
op_state = detail.get("state")
inc["op_state"] = op_state
if not links:
if op_state == "finished":
return "failed", "operation finished with no links"
return "pending", "operation has no links yet"
link = link_for(links, inc.get("ability"))
if link is None:
if op_state == "finished":
return "failed", "operation finished without a link for the incident ability"
return "pending", "operation has no link for the incident ability yet"
status = link.get("status")
if isinstance(status, bool) or not isinstance(status, (int, float)):
return "pending", "link status not reported"
# Caldera's link states are NOT "negative means bad": EXECUTE(-3) and PAUSE(-1) are
# in-flight, SUCCESS(0) is the only success. A queued link must keep polling instead
# of being reported as "never ran".
code = int(status)
if code in (LINK_EXECUTE, LINK_PAUSE):
return "pending", f"link still queued (state {code})"
if code == LINK_DISCARD:
return "failed", "link discarded (state -2): Caldera will not run it"
if code == LINK_UNTRUSTED:
return "failed", "link untrusted (state -4): the agent is not trusted for it"
if code == LINK_HIGH_VIZ:
return "failed", "link held for high-visibility approval (state -5)"
if code == LINK_TIMEOUT:
return "failed", "link timed out (state 124) on the agent"
if code != 0:
return "failed", f"link exit status {code}: the marker command failed"
# A link that has not run yet also reports status 0, so completion must come from its timestamps:
# `finish` is set when the agent reported the result, `collect` when the agent picked it up.
at, at_source = _epoch(link.get("finish")), "caldera-finish"
if at is None:
collected = _epoch(link.get("collect"))
if collected is not None and op_state == "finished":
at, at_source = collected, "caldera-collect"
else:
return "pending", "link has no finish timestamp yet"
ran_paw = link.get("paw")
if not ran_paw:
return "failed", "link reports no agent paw (which agent ran it is unknown)"
inc["ran_paw"] = ran_paw
inc["at"] = at
inc["at_source"] = at_source
inc["verified"] = True
inc["note"] = None
note = f"link ran on {ran_paw}, finished {time.strftime('%FT%T', time.localtime(at))} ({at_source})"
if ran_paw != inc.get("paw"):
note += f"; selected agent was {inc.get('paw')} (same sensor group)"
return "verified", note
def verify_operation(inc, tries=VERIFY_TRIES, delay=VERIFY_DELAY_S):
"""Poll the operation record, bounded by tries * (delay + timeout). A "pending" answer is retried;
a "failed" answer, or a poll budget that runs out, leaves the incident explicitly unverified."""
state, note = "pending", "not checked"
for attempt in range(max(1, tries)):
try:
state, note = check_operation(inc)
except Exception as e:
state, note = "pending", f"operation check failed: {e}"
print(f" incident {inc['iid']} poll {attempt + 1}/{tries}: {state}: {note}", flush=True)
if state != "pending":
break
if attempt + 1 < tries:
time.sleep(delay)
return state, note
def reconcile(st):
"""Re-check incidents left unverified by a crash, a timeout or a discarded link: once per cycle,
only while they are recent. Reconciliation can only add evidence; the id/marker of an unverified
incident stays consumed, so nothing is relaunched under an identity that may already have run."""
now = time.time()
changed = False
for inc in st["incidents"]:
if inc.get("verified") or not inc.get("op") or not inc.get("ability"):
continue
started = inc.get("started")
if not isinstance(started, (int, float)) or now - started > RECONCILE_WINDOW_S:
continue
before = (inc.get("verified"), inc.get("note"), inc.get("op_state"))
state, note = check_operation(inc)
if state != "verified":
inc["note"] = f"{state}: {note}"
if (inc.get("verified"), inc.get("note"), inc.get("op_state")) != before:
changed = True
print(time.strftime("%FT%T"), f"incident {inc['iid']} reconciled: {state}: {note}", flush=True)
if changed:
save(st)
return changed
def run_incident(st):
"""Launch one incident on the next sensor and return a one-line report.
The incident id, marker and ability are saved to the state file before Caldera is called, so a
crash mid-launch leaves a pending record to reconcile rather than a silently reused identity. The
record only becomes `verified` (and only then eligible for label aging) when Caldera's link data
shows the command actually ran.
"""
sensor = st["next_sensor"]
group = f"sensor-{sensor}"
paw = agent_paw(group)
if not paw:
st["next_due"] = time.time() + RETRY_GAP_S # no identity consumed yet
save(st)
return f"no live agent in {group}; retrying in {RETRY_GAP_S}s"
iid = st["iid"] + 1
mark = f"SB-RANGE-MARK-{iid:04d}"
shape = SHAPES[iid % len(SHAPES)]
cmd = shape.replace("@IID@", str(iid)).replace("@MARK@", mark)
ability_id = f"sbrange-{iid:08x}"
inc = {"iid": iid, "sensor": sensor, "mark": mark, "paw": paw, "ability": ability_id,
"op": None, "started": time.time(), "verified": False, "at": None, "at_source": None,
"op_state": None, "ran_paw": None, "note": "pending: operation not launched"}
st["iid"] = iid
st["next_sensor"] = sensor % SENSORS + 1
st["incidents"].append(inc)
st["next_due"] = inc["started"] + random.randint(MIN_GAP_S, MAX_GAP_S)
save(st) # identity is durable before the Caldera call
ability = {
"ability_id": ability_id, "name": f"sb range incident {iid}", "tactic": "execution",
"technique_id": "T1204", "technique_name": "User Execution",
"executors": [{"platform": "linux", "name": "sh", "command": cmd}],
}
adv_id = f"sbrange-adv-{iid:08x}"
op = {"name": f"sbrange-op-{iid}", "adversary": {"adversary_id": adv_id},
"group": group, "planner": {"id": "aaa7c857-37a0-4c4a-85f7-4e9f7f30e31a"},
"source": {"id": "ed32b9c3-9593-4c33-b0db-e2007315096b"}, "auto_close": True, "state": "running"}
try:
api("PUT", f"/api/v2/abilities/{ability_id}", ability)
api("PUT", f"/api/v2/adversaries/{adv_id}",
{"adversary_id": adv_id, "name": f"sb range {iid}", "atomic_ordering": [ability_id]})
created = api("POST", "/api/v2/operations", op)
except Exception as e:
inc["note"] = f"launch failed: {e}"
save(st)
return f"incident {iid} on {group}: launch failed ({e}); recorded unverified, id stays consumed"
inc["op"] = (created or {}).get("id")
if not inc["op"]:
inc["note"] = "Caldera returned no operation id"
save(st)
return f"incident {iid} on {group}: no operation id returned; recorded unverified"
inc["note"] = "pending: operation created, link not verified"
save(st)
state, note = verify_operation(inc)
inc["note"] = None if state == "verified" else f"{state}: {note}"
save(st)
if state == "verified":
return f"incident {iid} on {group} verified (mark {mark}): {note}"
return (f"incident {iid} on {group} NOT verified ({state}: {note}); kept pending, "
f"not aged into labels")
def ripe_incidents(st, now):
"""Incidents eligible for the delayed rule push: verified ones whose observed execution is at least
LABEL_DELAY_MIN old. Unverified incidents are excluded, so unverified work is never presented as a
completed incident and cannot pull a window into the label set."""
cutoff = now - LABEL_DELAY_MIN * 60
return [i for i in st["incidents"]
if i.get("verified") is True and isinstance(i.get("at"), (int, float)) and i["at"] <= cutoff]
def push_labels(st):
"""Rules recognizing every marker up to the newest ripe verified incident go out to all sensors; the
detector's retro-hunt then labels the earlier windows. `labeled_through` counts *verified incidents
whose markers were pushed and reloaded everywhere*, not labels this scheduler observed: whether the
retro-hunt actually labeled a window is what the gate measures."""
ripe = ripe_incidents(st, time.time())
if len(ripe) <= st["labeled_through"]:
return None
marks = sorted({i["mark"] for i in ripe})
rule = "rule SB_Range_Marker {\n meta:\n severity = \"high\"\n strings:\n"
rule += "".join(f" $m{n} = \"{m}\"\n" for n, m in enumerate(marks))
rule += " condition:\n any of them\n}\n"
sensors = {i["sensor"] for i in ripe}
pushed = []
for s in sensors:
port = 2240 + s
ssh = ["ssh", "-i", SSHK, "-p", str(port), "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
"arch@127.0.0.1"]
try:
subprocess.run(ssh + ["sudo tee /etc/strawberry/rules/sb_range_markers.yar >/dev/null && "
"sudo strawberry --config /etc/strawberry/strawberry.conf rules reload"],
input=rule.encode(), timeout=60, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
pushed.append(s)
except Exception as e: # a suspended / unreachable sensor is retried next cycle
print(f" label push to sensor-{s} failed: {e}", flush=True)
if set(pushed) >= sensors:
st["labeled_through"] = len(ripe)
st["labels_pushed_at"] = time.time()
save(st)
return f"rule push for {len(marks)} verified marker(s) -> sensors {sorted(pushed)}"
def main():
try:
st = load()
except StateError as e:
print(f"range attack scheduler refusing to start: {e}", flush=True)
print(" the state file is preserved as-is; repair it or move it aside by hand", flush=True)
raise SystemExit(2) from None
save(st) # persist the migration / derived cadence before anything else
unverified = sum(1 for i in st["incidents"] if not i.get("verified"))
due = (time.strftime("%FT%T", time.localtime(st["next_due"])) if st["next_due"] else "now")
print(f"range attack scheduler up (incidents {st['iid']}, {unverified} unverified, next due {due})",
flush=True)
while True:
now = time.time()
if st["next_due"] > now + MAX_GAP_S: # clock moved backwards: keep the cadence sane
st["next_due"] = now + MIN_GAP_S
save(st)
reconcile(st)
msg = push_labels(st)
if msg:
print(time.strftime("%FT%T"), msg, flush=True)
if time.time() >= st["next_due"]:
try:
info = run_incident(st)
except Exception as e:
info = f"incident error: {e}"
st["next_due"] = time.time() + RETRY_GAP_S
save(st)
print(time.strftime("%FT%T"), info, flush=True)
time.sleep(60)
if __name__ == "__main__":
main()