#!/usr/bin/env bash # One-line health of the range: Caldera, agent liveness per sensor group, each sensor VM (ssh, daemon # backend, detections, store size), host headroom, and the attack scheduler's progress. Read-only. set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/range_common.sh" # Caldera's agent endpoint lists every paw that ever registered, so a total counts agent restarts too # (three VMs restarted once showed six). Report live vs stale per sensor group, and say "unknown" # rather than print a total that reads like a VM count. agent_report() { python3 -c ' import json, sys, time from datetime import datetime, timezone sensors, stale_s = int(sys.argv[1]), int(sys.argv[2]) try: agents = json.load(sys.stdin) except Exception as exc: print("agents: liveness unknown (agent list is not JSON: %s)" % exc) raise SystemExit(0) if not isinstance(agents, list): print("agents: liveness unknown (unexpected agent payload)") raise SystemExit(0) def beacon_age(agent): """Seconds since the last beacon, or None when the field is missing/unreadable.""" raw = agent.get("last_seen") if not raw: return None try: seen = datetime.fromisoformat(str(raw).replace("Z", "+00:00")) except ValueError: return None if seen.tzinfo is None: seen = seen.replace(tzinfo=timezone.utc) return time.time() - seen.timestamp() groups = {} for agent in agents: if isinstance(agent, dict): groups.setdefault(agent.get("group") or "ungrouped", []).append(agent) parts, live, stale, unknown = [], 0, 0, 0 for group in sorted(groups): gl = gs = gu = 0 for agent in groups[group]: age = beacon_age(agent) if age is None: gu += 1 elif age <= stale_s: gl += 1 else: gs += 1 live, stale, unknown = live + gl, stale + gs, unknown + gu bits = [] if gl: bits.append("%d live" % gl) if gs: bits.append("%d stale (no beacon for >%ds)" % (gs, stale_s)) if gu: bits.append("%d liveness unknown (no readable last_seen)" % gu) parts.append("%s: %s" % (group, ", ".join(bits))) for i in range(1, sensors + 1): if "sensor-%d" % i not in groups: parts.append("sensor-%d: no agent registered" % i) print("agents: " + ("; ".join(parts) if parts else "none registered")) print("agents: %d registration(s) = %d live, %d stale, %d unknown" % (len(agents), live, stale, unknown)) ' "$SENSORS" "$AGENT_STALE_S" } if docker ps --format '{{.Names}}' | grep -qx "$CALDERA_NAME"; then echo "caldera: up" agents=$(caldera "$CALDERA_URL/api/v2/agents" 2>/dev/null || true) if [ -n "$agents" ]; then printf '%s' "$agents" | agent_report || echo "agents: liveness unknown (agent list unreadable)" else echo "agents: liveness unknown (agent list unavailable)" fi else echo "caldera: DOWN (agent liveness unknown)" fi for i in $(seq 1 "$SENSORS"); do if [ -z "$(vm_pid "$i" || true)" ]; then echo "sensor-$i: VM not running" continue fi if ! ssh_vm "$i" true 2>/dev/null; then echo "sensor-$i: VM up, ssh not ready" continue fi line=$(ssh_vm "$i" 'sudo strawberry --config /etc/strawberry/strawberry.conf status --json 2>/dev/null' \ | python3 -c 'import json,sys try: d=json.load(sys.stdin)["data"] except Exception: print("status unavailable"); raise SystemExit m=d["monitor"]; p=d["pipeline"] print("backend=%s detections=%s scanned=%s events=%s" % (m.get("backend"), p.get("detections"), p.get("scanned"), p.get("events")))' 2>/dev/null || echo "status unavailable") store=$(ssh_vm "$i" 'du -sh /var/lib/strawberry/evidence 2>/dev/null | cut -f1' 2>/dev/null || echo "?") echo "sensor-$i: $line store=$store" done st=$RANGE_DIR/state/attack_state.json printf 'host: MemAvailable %s MiB, free %s MiB on %s\n' \ "$(( $(mem_available_kb) / 1024 ))" "$(( $(disk_available_kb) / 1024 ))" "$RANGE_DIR" if [ -s "$st" ]; then # `labeled_through` is a rules-pushed counter over VERIFIED incidents, not an observation of # retro-hunt/store labels: the gate measures those later. Legacy (schema 1) incidents carry no # `verified` field and were recorded without execution proof, so they are reported separately. python3 -c 'import json,sys raw=open(sys.argv[1],"rb").read() try: d=json.loads(raw) except Exception as e: print("scheduler: state unreadable (%s) - incident/label counts unknown" % type(e).__name__) raise SystemExit(0) inc=d.get("incidents") if not isinstance(inc,list): print("scheduler: state has no incident list - counts unknown") raise SystemExit(0) verified=[i for i in inc if isinstance(i,dict) and i.get("verified") is True] legacy=[i for i in inc if isinstance(i,dict) and "verified" not in i] print("scheduler: %d incident(s), %d verified, %d without execution proof%s, %d rule-pushed (labels observed by the gate, not asserted here), next sensor-%d" % ( len(inc), len(verified), len(legacy), " (legacy schema-1)" if legacy else "", d.get("labeled_through",0) or 0, d.get("next_sensor",1)))' "$st" else echo "scheduler: no incidents yet" fi if [ -n "$(sched_pids)" ]; then echo "scheduler process: running (pid $(sched_pids | tr '\n' ' '))" elif [ -e "$SCHED_PID_FILE" ]; then echo "scheduler process: NOT running (stale $SCHED_PID_FILE; range_up.sh restarts it)" else echo "scheduler process: NOT running (range_up.sh restarts it)" fi