Host safety: range_up refuses to start below 10 GiB MemAvailable or 20 GiB free disk, range_status prints the headroom, and the README documents both floors and the 8 GiB pause a monitor should enforce. The 2026-11-17 global OOM was caused by fifteen concurrent compilers plus the range, not the range alone, but starting blind is what made it easy. Teardown: sensor ids come from the vms/ contents rather than SB_RANGE_SENSORS, so a lowered count or missing pid file cannot leave a VM behind; vm_pid accepts a PID only when its command line still names that sensor's QEMU and disk, so a recycled PID is never counted as live or signalled; a /proc sweep covers QEMUs whose pid file was lost; --destroy refuses while any of them survives and deletes only the sensor-N disks and seeds this range created. Scheduler: range_up writes a per-range pid file and up/down/status act only on a PID whose command line runs this repo's range_attack.py with SB_RANGE_DIR resolving here, replacing the host-wide pkill that could kill another range. Status: agents are reported live/stale/unknown per sensor group instead of a raw registration total (six registrations for three VMs after restarts), and the scheduler line now separates verified incidents from those without execution proof, reporting "rule-pushed" rather than implying observed labels.
133 lines
5.4 KiB
Bash
Executable file
133 lines
5.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# One-line health of the range: Caldera, agent liveness per sensor group, each sensor VM (ssh, daemon
|
|
# backend, detections, store size), host headroom, and the attack scheduler's progress. Read-only.
|
|
set -uo pipefail
|
|
. "$(dirname "${BASH_SOURCE[0]}")/range_common.sh"
|
|
|
|
# Caldera's agent endpoint lists every paw that ever registered, so a total counts agent restarts too
|
|
# (three VMs restarted once showed six). Report live vs stale per sensor group, and say "unknown"
|
|
# rather than print a total that reads like a VM count.
|
|
agent_report() {
|
|
python3 -c '
|
|
import json, sys, time
|
|
from datetime import datetime, timezone
|
|
sensors, stale_s = int(sys.argv[1]), int(sys.argv[2])
|
|
try:
|
|
agents = json.load(sys.stdin)
|
|
except Exception as exc:
|
|
print("agents: liveness unknown (agent list is not JSON: %s)" % exc)
|
|
raise SystemExit(0)
|
|
if not isinstance(agents, list):
|
|
print("agents: liveness unknown (unexpected agent payload)")
|
|
raise SystemExit(0)
|
|
def beacon_age(agent):
|
|
"""Seconds since the last beacon, or None when the field is missing/unreadable."""
|
|
raw = agent.get("last_seen")
|
|
if not raw:
|
|
return None
|
|
try:
|
|
seen = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
if seen.tzinfo is None:
|
|
seen = seen.replace(tzinfo=timezone.utc)
|
|
return time.time() - seen.timestamp()
|
|
groups = {}
|
|
for agent in agents:
|
|
if isinstance(agent, dict):
|
|
groups.setdefault(agent.get("group") or "ungrouped", []).append(agent)
|
|
parts, live, stale, unknown = [], 0, 0, 0
|
|
for group in sorted(groups):
|
|
gl = gs = gu = 0
|
|
for agent in groups[group]:
|
|
age = beacon_age(agent)
|
|
if age is None:
|
|
gu += 1
|
|
elif age <= stale_s:
|
|
gl += 1
|
|
else:
|
|
gs += 1
|
|
live, stale, unknown = live + gl, stale + gs, unknown + gu
|
|
bits = []
|
|
if gl:
|
|
bits.append("%d live" % gl)
|
|
if gs:
|
|
bits.append("%d stale (no beacon for >%ds)" % (gs, stale_s))
|
|
if gu:
|
|
bits.append("%d liveness unknown (no readable last_seen)" % gu)
|
|
parts.append("%s: %s" % (group, ", ".join(bits)))
|
|
for i in range(1, sensors + 1):
|
|
if "sensor-%d" % i not in groups:
|
|
parts.append("sensor-%d: no agent registered" % i)
|
|
print("agents: " + ("; ".join(parts) if parts else "none registered"))
|
|
print("agents: %d registration(s) = %d live, %d stale, %d unknown" % (len(agents), live, stale, unknown))
|
|
' "$SENSORS" "$AGENT_STALE_S"
|
|
}
|
|
|
|
if docker ps --format '{{.Names}}' | grep -qx "$CALDERA_NAME"; then
|
|
echo "caldera: up"
|
|
agents=$(caldera "$CALDERA_URL/api/v2/agents" 2>/dev/null || true)
|
|
if [ -n "$agents" ]; then
|
|
printf '%s' "$agents" | agent_report || echo "agents: liveness unknown (agent list unreadable)"
|
|
else
|
|
echo "agents: liveness unknown (agent list unavailable)"
|
|
fi
|
|
else
|
|
echo "caldera: DOWN (agent liveness unknown)"
|
|
fi
|
|
|
|
for i in $(seq 1 "$SENSORS"); do
|
|
if [ -z "$(vm_pid "$i" || true)" ]; then
|
|
echo "sensor-$i: VM not running"
|
|
continue
|
|
fi
|
|
if ! ssh_vm "$i" true 2>/dev/null; then
|
|
echo "sensor-$i: VM up, ssh not ready"
|
|
continue
|
|
fi
|
|
line=$(ssh_vm "$i" 'sudo strawberry --config /etc/strawberry/strawberry.conf status --json 2>/dev/null' \
|
|
| python3 -c 'import json,sys
|
|
try:
|
|
d=json.load(sys.stdin)["data"]
|
|
except Exception:
|
|
print("status unavailable"); raise SystemExit
|
|
m=d["monitor"]; p=d["pipeline"]
|
|
print("backend=%s detections=%s scanned=%s events=%s" % (m.get("backend"), p.get("detections"), p.get("scanned"), p.get("events")))' 2>/dev/null || echo "status unavailable")
|
|
store=$(ssh_vm "$i" 'du -sh /var/lib/strawberry/evidence 2>/dev/null | cut -f1' 2>/dev/null || echo "?")
|
|
echo "sensor-$i: $line store=$store"
|
|
done
|
|
|
|
st=$RANGE_DIR/state/attack_state.json
|
|
printf 'host: MemAvailable %s MiB, free %s MiB on %s\n' \
|
|
"$(( $(mem_available_kb) / 1024 ))" "$(( $(disk_available_kb) / 1024 ))" "$RANGE_DIR"
|
|
if [ -s "$st" ]; then
|
|
# `labeled_through` is a rules-pushed counter over VERIFIED incidents, not an observation of
|
|
# retro-hunt/store labels: the gate measures those later. Legacy (schema 1) incidents carry no
|
|
# `verified` field and were recorded without execution proof, so they are reported separately.
|
|
python3 -c 'import json,sys
|
|
raw=open(sys.argv[1],"rb").read()
|
|
try:
|
|
d=json.loads(raw)
|
|
except Exception as e:
|
|
print("scheduler: state unreadable (%s) - incident/label counts unknown" % type(e).__name__)
|
|
raise SystemExit(0)
|
|
inc=d.get("incidents")
|
|
if not isinstance(inc,list):
|
|
print("scheduler: state has no incident list - counts unknown")
|
|
raise SystemExit(0)
|
|
verified=[i for i in inc if isinstance(i,dict) and i.get("verified") is True]
|
|
legacy=[i for i in inc if isinstance(i,dict) and "verified" not in i]
|
|
print("scheduler: %d incident(s), %d verified, %d without execution proof%s, %d rule-pushed (labels observed by the gate, not asserted here), next sensor-%d" % (
|
|
len(inc), len(verified), len(legacy),
|
|
" (legacy schema-1)" if legacy else "",
|
|
d.get("labeled_through",0) or 0, d.get("next_sensor",1)))' "$st"
|
|
else
|
|
echo "scheduler: no incidents yet"
|
|
fi
|
|
if [ -n "$(sched_pids)" ]; then
|
|
echo "scheduler process: running (pid $(sched_pids | tr '\n' ' '))"
|
|
elif [ -e "$SCHED_PID_FILE" ]; then
|
|
echo "scheduler process: NOT running (stale $SCHED_PID_FILE; range_up.sh restarts it)"
|
|
else
|
|
echo "scheduler process: NOT running (range_up.sh restarts it)"
|
|
fi
|