strawberry/deploy/validation/range/range_status.sh
trvon 39c5bc4efe
range: host-safety preflight, verified teardown, per-range scheduler identity
Host safety: range_up refuses to start below 10 GiB MemAvailable or 20 GiB free
disk, range_status prints the headroom, and the README documents both floors
and the 8 GiB pause a monitor should enforce. The 2026-11-17 global OOM was
caused by fifteen concurrent compilers plus the range, not the range alone, but
starting blind is what made it easy.

Teardown: sensor ids come from the vms/ contents rather than SB_RANGE_SENSORS,
so a lowered count or missing pid file cannot leave a VM behind; vm_pid accepts
a PID only when its command line still names that sensor's QEMU and disk, so a
recycled PID is never counted as live or signalled; a /proc sweep covers QEMUs
whose pid file was lost; --destroy refuses while any of them survives and
deletes only the sensor-N disks and seeds this range created.

Scheduler: range_up writes a per-range pid file and up/down/status act only on
a PID whose command line runs this repo's range_attack.py with SB_RANGE_DIR
resolving here, replacing the host-wide pkill that could kill another range.

Status: agents are reported live/stale/unknown per sensor group instead of a
raw registration total (six registrations for three VMs after restarts), and
the scheduler line now separates verified incidents from those without
execution proof, reporting "rule-pushed" rather than implying observed labels.
2026-11-17 13:45:22 -07:00

133 lines
5.4 KiB
Bash
Executable file

#!/usr/bin/env bash
# One-line health of the range: Caldera, agent liveness per sensor group, each sensor VM (ssh, daemon
# backend, detections, store size), host headroom, and the attack scheduler's progress. Read-only.
set -uo pipefail
. "$(dirname "${BASH_SOURCE[0]}")/range_common.sh"
# Caldera's agent endpoint lists every paw that ever registered, so a total counts agent restarts too
# (three VMs restarted once showed six). Report live vs stale per sensor group, and say "unknown"
# rather than print a total that reads like a VM count.
agent_report() {
python3 -c '
import json, sys, time
from datetime import datetime, timezone
sensors, stale_s = int(sys.argv[1]), int(sys.argv[2])
try:
agents = json.load(sys.stdin)
except Exception as exc:
print("agents: liveness unknown (agent list is not JSON: %s)" % exc)
raise SystemExit(0)
if not isinstance(agents, list):
print("agents: liveness unknown (unexpected agent payload)")
raise SystemExit(0)
def beacon_age(agent):
"""Seconds since the last beacon, or None when the field is missing/unreadable."""
raw = agent.get("last_seen")
if not raw:
return None
try:
seen = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
except ValueError:
return None
if seen.tzinfo is None:
seen = seen.replace(tzinfo=timezone.utc)
return time.time() - seen.timestamp()
groups = {}
for agent in agents:
if isinstance(agent, dict):
groups.setdefault(agent.get("group") or "ungrouped", []).append(agent)
parts, live, stale, unknown = [], 0, 0, 0
for group in sorted(groups):
gl = gs = gu = 0
for agent in groups[group]:
age = beacon_age(agent)
if age is None:
gu += 1
elif age <= stale_s:
gl += 1
else:
gs += 1
live, stale, unknown = live + gl, stale + gs, unknown + gu
bits = []
if gl:
bits.append("%d live" % gl)
if gs:
bits.append("%d stale (no beacon for >%ds)" % (gs, stale_s))
if gu:
bits.append("%d liveness unknown (no readable last_seen)" % gu)
parts.append("%s: %s" % (group, ", ".join(bits)))
for i in range(1, sensors + 1):
if "sensor-%d" % i not in groups:
parts.append("sensor-%d: no agent registered" % i)
print("agents: " + ("; ".join(parts) if parts else "none registered"))
print("agents: %d registration(s) = %d live, %d stale, %d unknown" % (len(agents), live, stale, unknown))
' "$SENSORS" "$AGENT_STALE_S"
}
if docker ps --format '{{.Names}}' | grep -qx "$CALDERA_NAME"; then
echo "caldera: up"
agents=$(caldera "$CALDERA_URL/api/v2/agents" 2>/dev/null || true)
if [ -n "$agents" ]; then
printf '%s' "$agents" | agent_report || echo "agents: liveness unknown (agent list unreadable)"
else
echo "agents: liveness unknown (agent list unavailable)"
fi
else
echo "caldera: DOWN (agent liveness unknown)"
fi
for i in $(seq 1 "$SENSORS"); do
if [ -z "$(vm_pid "$i" || true)" ]; then
echo "sensor-$i: VM not running"
continue
fi
if ! ssh_vm "$i" true 2>/dev/null; then
echo "sensor-$i: VM up, ssh not ready"
continue
fi
line=$(ssh_vm "$i" 'sudo strawberry --config /etc/strawberry/strawberry.conf status --json 2>/dev/null' \
| python3 -c 'import json,sys
try:
d=json.load(sys.stdin)["data"]
except Exception:
print("status unavailable"); raise SystemExit
m=d["monitor"]; p=d["pipeline"]
print("backend=%s detections=%s scanned=%s events=%s" % (m.get("backend"), p.get("detections"), p.get("scanned"), p.get("events")))' 2>/dev/null || echo "status unavailable")
store=$(ssh_vm "$i" 'du -sh /var/lib/strawberry/evidence 2>/dev/null | cut -f1' 2>/dev/null || echo "?")
echo "sensor-$i: $line store=$store"
done
st=$RANGE_DIR/state/attack_state.json
printf 'host: MemAvailable %s MiB, free %s MiB on %s\n' \
"$(( $(mem_available_kb) / 1024 ))" "$(( $(disk_available_kb) / 1024 ))" "$RANGE_DIR"
if [ -s "$st" ]; then
# `labeled_through` is a rules-pushed counter over VERIFIED incidents, not an observation of
# retro-hunt/store labels: the gate measures those later. Legacy (schema 1) incidents carry no
# `verified` field and were recorded without execution proof, so they are reported separately.
python3 -c 'import json,sys
raw=open(sys.argv[1],"rb").read()
try:
d=json.loads(raw)
except Exception as e:
print("scheduler: state unreadable (%s) - incident/label counts unknown" % type(e).__name__)
raise SystemExit(0)
inc=d.get("incidents")
if not isinstance(inc,list):
print("scheduler: state has no incident list - counts unknown")
raise SystemExit(0)
verified=[i for i in inc if isinstance(i,dict) and i.get("verified") is True]
legacy=[i for i in inc if isinstance(i,dict) and "verified" not in i]
print("scheduler: %d incident(s), %d verified, %d without execution proof%s, %d rule-pushed (labels observed by the gate, not asserted here), next sensor-%d" % (
len(inc), len(verified), len(legacy),
" (legacy schema-1)" if legacy else "",
d.get("labeled_through",0) or 0, d.get("next_sensor",1)))' "$st"
else
echo "scheduler: no incidents yet"
fi
if [ -n "$(sched_pids)" ]; then
echo "scheduler process: running (pid $(sched_pids | tr '\n' ' '))"
elif [ -e "$SCHED_PID_FILE" ]; then
echo "scheduler process: NOT running (stale $SCHED_PID_FILE; range_up.sh restarts it)"
else
echo "scheduler process: NOT running (range_up.sh restarts it)"
fi