strawberry/deploy/salt/test/fleet_run.sh
trvon 3fd5d23b77
threat traces checker + fleet gate in the dockerized Salt test
- scripts/threat_traces.py: enforces the mechanical rules of the 2026-10-28
  threat-trace pre-registration: required # meta keys; eval families first
  reported 2025-07-01..2026-10-28 with a new-family/new-variant bucket and
  write-up sources only; training families before the cutoff with every
  fifth family (first-report order) in dev; no family in both files;
  RFC 5737 addresses and .example domains only; size caps; and that every
  file compiles with replay's parser. --sha256 prints the freeze digest.
- tests/trace_check.cpp: strawberry-trace-check dev tool (replay's parser).
- strawberry.gate: optional gate.warmup_hours (unset = the pre-registered
  12 h; TEST pillars only).
- deploy/salt/test/fleet_run.sh: gate phase - tops sensor history past the
  10 min injection floor, runs orch.strawberry_gate, asserts 3 gate models
  were published, 9 pooled cells over 3 sensors with positives, sensors keep
  their reports, and still no promoted model anywhere.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 09:36:24 -07:00

279 lines
20 KiB
Bash
Executable file

#!/usr/bin/env bash
# Level B fleet test: real Salt states, real release tarball, real daemons, real cp.push, one
# orchestrated cycle (canary-gated) and a second incremental cycle. Idempotent: tears the fleet
# down first (fleet_down.sh) and brings it up again (fleet_up.sh). Prints PASS/FAIL per
# assertion; exit 1 if any FAIL. Logs: /var/tmp/sb-fleet/logs/run-*.
#
# FLEET_WORKLOAD_SECS=180 (default) FLEET_KEEP=1 keeps the fleet up afterwards
#
# NOT validated here (rootless docker): eBPF exec tracing and fanotify exec prevention. The
# daemon runs in its procfs-polling + YARA mode, which is what every export below is made from.
set -euo pipefail
# shellcheck source=fleet_common.sh
. "$(dirname "${BASH_SOURCE[0]}")/fleet_common.sh"
WORK_SECS=${FLEET_WORKLOAD_SECS:-180}
GATE_WORK_SECS=${FLEET_GATE_WORKLOAD_SECS:-660} # tops sensor history past the 10 min trace-injection floor
L=$FLEET_LOGS
J() { python3 -I -c "$@"; }
RESULTS=()
FAILS=0
pass() { RESULTS+=("PASS $1"); echo "PASS $1"; }
fail() { RESULTS+=("FAIL $1"); echo "FAIL $1" >&2; FAILS=$((FAILS + 1)); }
chk() { local name=$1; shift; if "$@" >>"$L/run-checks.log" 2>&1; then pass "$name"; else fail "$name"; fi; }
salt_json() { docker exec "$FLEET_CENTRAL" salt --timeout=900 --out=json --static "$@"; }
status_field() { # <container> <rules.cache|rules.load_ms|running|...>
docker exec "$1" strawberry status --json | J 'import json,sys
d=json.load(sys.stdin)["data"]
for k in sys.argv[1].split("."): d=d[k]
print(d)' "$2"
}
all_ok() { # <json file>: every minion's every state result true, none rendered as a list/error
J 'import json,sys
d=json.load(open(sys.argv[1])); bad=0
assert len(d)>=4, "minions answered: %d"%len(d)
for m,r in d.items():
if not isinstance(r,dict): print(m,r); bad+=1; continue
for k,v in r.items():
if not v["result"]: print(m,k,v["comment"][:200]); bad+=1
sys.exit(1 if bad else 0)' "$1"
}
changes_of() { # <json>: number of states that reported changes
J 'import json,sys
d=json.load(open(sys.argv[1])); print(sum(1 for r in d.values() for v in r.values() if v.get("changes")))' "$1"
}
: >"$L/run-checks.log"
echo "== fleet down/up"
"$FLEET_HERE/fleet_down.sh" >"$L/run-00-down.log" 2>&1
"$FLEET_HERE/fleet_up.sh" >"$L/run-01-up.log" 2>&1 || { tail -20 "$L/run-01-up.log" >&2; exit 2; }
head12=$(git -C "$FLEET_REPO" rev-parse --short=12 HEAD)
inspect_bin=$FLEET_DIR/src-$head12/builddir/tests/strawberry-fleet-inspect
for c in "${FLEET_SENSORS[@]}"; do
docker cp "$FLEET_HERE/workload.sh" "$c:/opt/workload.sh"
[ -x "$inspect_bin" ] && docker cp "$inspect_bin" "$c:/opt/strawberry-fleet-inspect"
done
[ -x "$inspect_bin" ] && docker cp "$inspect_bin" "$FLEET_CENTRAL:/opt/strawberry-fleet-inspect"
# total windows across the given bundle paths on the central node (fleet-inspect prints windows=N)
windows_of() { docker exec "$FLEET_CENTRAL" bash -c 'for f in "$@"; do /opt/strawberry-fleet-inspect "$f"; done' _ "$@" |
sed -n 's/.* windows=\([0-9]*\).*/\1/p' | awk '{s+=$1} END{print s+0}'; }
pass "fleet up: systemd as PID 1 (no --privileged), salt-master + 4 minions answering"
# ---- a. state.apply strawberry ---------------------------------------------------------------
echo "== a. state.apply strawberry"
salt_json '*' state.apply strawberry >"$L/run-02-apply1.json" 2>"$L/run-02-apply1.err" || true
chk "state.apply strawberry: every state on all 4 minions succeeded" all_ok "$L/run-02-apply1.json"
salt_json '*' state.apply strawberry >"$L/run-03-apply2.json" 2>&1 || true
idem_ok() { all_ok "$1" && [ "$(changes_of "$1")" = 0 ]; }
chk "state.apply strawberry is idempotent (2nd run: 0 changes, 0 failures)" idem_ok "$L/run-03-apply2.json"
for c in "${FLEET_SENSORS[@]}"; do
chk "$c: strawberry daemon active, status --json running" \
bash -c "[ \"\$(docker exec $c systemctl is-active strawberry)\" = active ] && [ \"\$(docker exec $c strawberry status --json | python3 -I -c 'import json,sys; print(json.load(sys.stdin)[\"data\"][\"running\"])')\" = True ]"
done
chk "central: strawberry daemon + central-cycle timer active" \
bash -c "docker exec $FLEET_CENTRAL systemctl is-active strawberry strawberry-central-cycle.timer | grep -vc '^active\$' | grep -qx 0"
chk "sensors: strawberry-export.timer active" \
bash -c "for c in ${FLEET_SENSORS[*]}; do [ \"\$(docker exec \$c systemctl is-active strawberry-export.timer)\" = active ] || exit 1; done"
chk "config: redaction on, rule_auto_fetch off, node_id = minion id" \
bash -c "for c in ${FLEET_SENSORS[*]}; do docker exec \$c grep -qx 'capture_redact_cmdline=true' /etc/strawberry/strawberry.conf && docker exec \$c grep -qx 'rule_auto_fetch=false' /etc/strawberry/strawberry.conf && docker exec \$c grep -qx \"node_id=sensor-\${c##*-}\" /etc/strawberry/strawberry.conf || exit 1; done"
# R2: first start of each daemon is a compiled-rules cache miss (read from its journal)
declare -A MISS_MS HIT_MS
for c in "${FLEET_SENSORS[@]}"; do
line=$(docker exec "$c" journalctl -u strawberry --no-pager | grep 'Loaded .* rules from' | head -1)
echo "$c first start: $line" | tee -a "$L/run-r2.log"
MISS_MS[$c]=$(sed -n 's/.*cache miss, \([0-9]*\) ms.*/\1/p' <<<"$line")
[ -n "${MISS_MS[$c]}" ] && pass "R2 $c: first start = cache miss (${MISS_MS[$c]} ms)" || fail "R2 $c: first start was not a cache miss: $line"
done
# harmless local test rule + a planted binary on sensor-2 (labels a window when it runs)
docker exec -i sbfleet-sensor-2 bash -c 'cat >/etc/strawberry/rules/fleet_test.yar && chmod 600 /etc/strawberry/rules/fleet_test.yar' <<'EOF'
rule Fleet_Test_Marker {
meta:
description = "Fleet harness test marker (harmless)"
severity = "high"
strings:
$m = "SB-FLEET-TEST-MARKER-d41d8c" ascii
condition:
uint32(0) == 0x464c457f and $m
}
EOF
docker exec sbfleet-sensor-2 bash -c "cp /bin/sleep /usr/local/bin/sb-evil && printf 'SB-FLEET-TEST-MARKER-d41d8c' >> /usr/local/bin/sb-evil"
# R2: restart every daemon. sensor-1/3: unchanged sources -> hit. sensor-2: its rule set changed -> miss, then hit.
for c in "${FLEET_SENSORS[@]}"; do docker exec "$c" systemctl restart strawberry; done
sleep 8
for c in sbfleet-sensor-1 sbfleet-sensor-3; do
st=$(status_field "$c" rules.cache); ms=$(status_field "$c" rules.load_ms); HIT_MS[$c]=$ms
echo "$c restart: cache=$st load_ms=$ms" | tee -a "$L/run-r2.log"
[ "$st" = hit ] && pass "R2 $c: restart = cache hit (${ms} ms vs ${MISS_MS[$c]} ms miss)" || fail "R2 $c: restart cache=$st"
done
st=$(status_field sbfleet-sensor-2 rules.cache); ms=$(status_field sbfleet-sensor-2 rules.load_ms)
echo "sbfleet-sensor-2 restart after adding a rule file: cache=$st load_ms=$ms" | tee -a "$L/run-r2.log"
[ "$st" = miss ] && pass "R2 sensor-2: changed rule sources invalidate the cache (miss, ${ms} ms)" || fail "R2 sensor-2: expected miss after rule change, got $st"
docker exec sbfleet-sensor-2 systemctl restart strawberry; sleep 8
st=$(status_field sbfleet-sensor-2 rules.cache); ms=$(status_field sbfleet-sensor-2 rules.load_ms); HIT_MS[sbfleet-sensor-2]=$ms
echo "sbfleet-sensor-2 second restart: cache=$st load_ms=$ms" | tee -a "$L/run-r2.log"
[ "$st" = hit ] && pass "R2 sensor-2: next restart = cache hit (${ms} ms)" || fail "R2 sensor-2: second restart cache=$st"
# ---- b. workload -----------------------------------------------------------------------------
echo "== b. workload ${WORK_SECS}s on 3 sensors"
pids=()
for c in "${FLEET_SENSORS[@]}"; do
n=${c##*-}
timeout $((WORK_SECS + 120)) docker exec "$c" bash /opt/workload.sh "$n" "$WORK_SECS" >"$L/run-04-workload-$n.log" 2>&1 &
pids+=($!)
done
sleep 10
# YARA-detectable execs, early in the run: `fleet train` refuses fewer than 5 positive windows in its
# TRAINING split, and the default split is temporal (newest 20% held out), so label a dozen distinct windows early.
# A replay window is a root exec + its descendants for 60 s: children of ONE docker-exec shell are one causal
# tree (one window, one label; seen 2026-11-14). Each planted exec therefore gets its own docker exec root.
for i in $(seq 1 12); do docker exec sbfleet-sensor-2 bash -c '/usr/local/bin/sb-evil 1' >/dev/null 2>&1 || true; sleep 2; done
for p in "${pids[@]}"; do wait "$p" || true; done
sleep 3
det=$(status_field sbfleet-sensor-2 pipeline.detections)
[ "${det:-0}" -ge 1 ] && pass "sensor-2: daemon recorded the YARA detection (detections=$det)" || fail "sensor-2: no detection recorded"
for c in "${FLEET_SENSORS[@]}"; do
w=$(docker exec "$c" strawberry export --out /tmp/probe.sbx --json | J 'import json,sys; print(json.load(sys.stdin)["windows"])')
echo "$c probe export windows=$w" | tee -a "$L/run-checks.log"
[ "$w" -ge 20 ] && pass "$c: real telemetry produced $w windows (live, no manual seal needed)" || fail "$c: only $w windows"
done
if [ -x "$inspect_bin" ]; then
lab=$(docker exec sbfleet-sensor-2 /opt/strawberry-fleet-inspect /tmp/probe.sbx | sed -n 's/.* labels=\([0-9]*\).*/\1/p')
[ "${lab:-0}" -ge 1 ] && pass "sensor-2: export carries $lab labeled window(s)" || fail "sensor-2: no labeled window in export"
else
fail "fleet-inspect binary missing ($inspect_bin)"
fi
# ---- d. orchestrated cycle 1 (canary held) ---------------------------------------------------
echo "== d. orch cycle 1 (canary_ok=false)"
BUNDLES=/var/lib/strawberry/fleet/bundles
DIST=/srv/salt/strawberry/dist
docker exec "$FLEET_CENTRAL" salt-run state.orchestrate orch.strawberry_cycle >"$L/run-05-orch1.log" 2>&1 && rc=0 || rc=$?
tail -15 "$L/run-05-orch1.log"
[ "$rc" = 0 ] && pass "orch.strawberry_cycle (canary_ok=false) succeeded" || fail "orch cycle 1 exit $rc (see run-05-orch1.log)"
nb=$(docker exec "$FLEET_CENTRAL" bash -c "ls $BUNDLES/export-*.sbx 2>/dev/null | wc -l")
[ "$nb" = 3 ] && pass "bundle_dir holds one bundle per sensor ($nb) after cp.push + collect" || fail "bundle_dir has $nb bundles (want 3)"
docker exec "$FLEET_CENTRAL" bash -c "ls -l $BUNDLES/" >"$L/run-bundles1.txt"
R1_FILES=$(docker exec "$FLEET_CENTRAL" bash -c "ls $BUNDLES/export-*.sbx" | sort)
R1_SIZE=$(docker exec "$FLEET_CENTRAL" bash -c "stat -c %s $BUNDLES/export-*.sbx" | awk '{s+=$1} END{print s}')
# shellcheck disable=SC2086
R1_WIN=$(windows_of $R1_FILES)
chk "dist has shadow.sbm + fleet-overlay.sbo + .sha256 sidecars" \
bash -c "docker exec $FLEET_CENTRAL bash -c 'cd $DIST && sha256sum -c --quiet shadow.sbm.sha256 fleet-overlay.sbo.sha256'"
has() { docker exec "$1" test -s "/var/lib/strawberry/model/$2"; }
has_both() { has "$1" shadow.sbm && has "$1" fleet-overlay.sbo; }
has_none() { ! has "$1" shadow.sbm && ! has "$1" fleet-overlay.sbo; }
chk "canary sensor-1 received shadow.sbm + fleet-overlay.sbo" has_both sbfleet-sensor-1
chk "sensor-2 did NOT receive them (canary hold)" has_none sbfleet-sensor-2
chk "sensor-3 did NOT receive them (canary hold)" has_none sbfleet-sensor-3
chk "canary files are 0600 root" \
bash -c "[ \"\$(docker exec sbfleet-sensor-1 stat -c '%a %U' /var/lib/strawberry/model/shadow.sbm /var/lib/strawberry/model/fleet-overlay.sbo | sort -u)\" = '600 root' ]"
TRAIN1=$(docker exec "$FLEET_CENTRAL" bash -c "ls -d /var/lib/strawberry/fleet/work/run-* | sort | tail -1")
docker exec "$FLEET_CENTRAL" cat "$TRAIN1/train-seed1.json" >"$L/run-train1.json"
J 'import json; d=json.load(open("/var/tmp/sb-fleet/logs/run-train1.json")); print("train1:",{k:d[k] for k in ("nodes","bundles_read","windows","positives","duplicate_windows") if k in d})' | tee -a "$L/run-checks.log"
chk "cycle 1: fleet train merged 3 nodes from 3 bundles" \
J 'import json,sys; d=json.load(open("/var/tmp/sb-fleet/logs/run-train1.json")); sys.exit(0 if d["nodes"]==3 and d["bundles_read"]==3 and d["bundles_failed"]==0 else 1)'
# ---- d2. canary_ok=true -> fleet distribute --------------------------------------------------
echo "== d2. orch with canary_ok=true"
docker exec "$FLEET_CENTRAL" salt-run state.orchestrate orch.strawberry_cycle \
pillar='{"strawberry":{"distribute":{"canary_ok":true}}}' >"$L/run-06-orch1b.log" 2>&1 && rc=0 || rc=$?
tail -12 "$L/run-06-orch1b.log"
[ "$rc" = 0 ] && pass "orch.strawberry_cycle (canary_ok=true override) succeeded" || fail "orch canary_ok=true exit $rc (see run-06-orch1b.log)"
ALL_NOW=$(docker exec "$FLEET_CENTRAL" bash -c "ls $BUNDLES/export-*.sbx" | sort)
D2_FILES=$(comm -13 <(echo "$R1_FILES") <(echo "$ALL_NOW"))
nd2=$(grep -c . <<<"$D2_FILES" || true)
D2_SIZE=$(docker exec "$FLEET_CENTRAL" stat -c %s $D2_FILES | awk '{s+=$1} END{print s}')
echo "cycle 1b (re-export right after cycle 1, no new activity): $nd2 new bundles, $D2_SIZE bytes (cycle 1: $R1_SIZE)" | tee -a "$L/run-checks.log"
[ "$nd2" = 3 ] && [ "$D2_SIZE" -lt "$R1_SIZE" ] && pass "cycle 1b: 3 incremental bundles, smaller than cycle 1 ($D2_SIZE B < $R1_SIZE B)" || fail "cycle 1b: $nd2 bundles, $D2_SIZE B vs $R1_SIZE B"
for c in "${FLEET_SENSORS[@]}"; do chk "after canary_ok=true: $c has shadow.sbm + fleet-overlay.sbo" has_both "$c"; done
chk "promoted.sbm exists nowhere (all nodes, dist, work dirs)" \
bash -c "for c in ${FLEET_ALL[*]}; do [ -z \"\$(docker exec \$c find /var/lib/strawberry /srv/salt -name 'promoted.sbm' 2>/dev/null)\" ] || exit 1; done"
chk "fleet-wide hashes match the central dist sidecars" \
bash -c "want=\$(docker exec $FLEET_CENTRAL cat $DIST/shadow.sbm.sha256 | cut -d' ' -f1); for c in ${FLEET_SENSORS[*]}; do [ \"\$(docker exec \$c sha256sum /var/lib/strawberry/model/shadow.sbm | cut -d' ' -f1)\" = \"\$want\" ] || exit 1; done"
# ---- e. round 2 -----------------------------------------------------------------------------
echo "== e. round 2: fleet bits + incremental cycle"
if [ -x "$inspect_bin" ]; then
docker exec sbfleet-sensor-1 strawberry export --out /tmp/r2.sbx >/dev/null
out=$(docker exec sbfleet-sensor-1 /opt/strawberry-fleet-inspect /tmp/r2.sbx)
echo "sensor-1 round-2 export: $out" | tee -a "$L/run-checks.log"
ff=$(sed -n 's/.*fleet_feat=\([0-9]*\).*/\1/p' <<<"$out")
[ "${ff:-0}" -gt 0 ] && pass "sensor-1 round-2 export carries fleet bits (fleet_feat=$ff)" || fail "sensor-1 round-2 export has no fleet bits: $out"
fi
# new activity, then a second orchestrated cycle (exports use --since auto)
pids=()
for c in "${FLEET_SENSORS[@]}"; do
timeout 240 docker exec "$c" bash /opt/workload.sh "${c##*-}" 60 >"$L/run-07-workload2-${c##*-}.log" 2>&1 &
pids+=($!)
done
for p in "${pids[@]}"; do wait "$p" || true; done
docker exec "$FLEET_CENTRAL" salt-run state.orchestrate orch.strawberry_cycle \
pillar='{"strawberry":{"distribute":{"canary_ok":true}}}' >"$L/run-08-orch2.log" 2>&1 && rc=0 || rc=$?
tail -8 "$L/run-08-orch2.log"
[ "$rc" = 0 ] && pass "second orchestrated cycle succeeded" || fail "orch cycle 2 exit $rc (see run-08-orch2.log)"
R2_FILES=$(docker exec "$FLEET_CENTRAL" bash -c "ls $BUNDLES/export-*.sbx" | sort | comm -13 <(printf "%s\n%s\n" "$R1_FILES" "$D2_FILES" | sort) -)
n2=$(grep -c . <<<"$R2_FILES" || true)
R2_SIZE=$(docker exec "$FLEET_CENTRAL" stat -c %s $R2_FILES | awk '{s+=$1} END{print s}')
echo "round 1: 3 bundles, $R1_SIZE bytes; round 2: $n2 new bundles, $R2_SIZE bytes" | tee -a "$L/run-checks.log"
[ "$n2" = 3 ] && pass "round 2 added one new bundle per sensor" || fail "round 2 added $n2 bundles (want 3)"
# Incremental = fewer windows than a FULL export of the same store at the same moment (bytes are the wrong
# yardstick: fixed-size rarity sketches + all labeled windows ride in every bundle). Round-1 bundles are not a
# fair baseline either: the 120 s cursor overlap re-ships windows that may still have been open, and this
# test's whole history is only a few minutes (in a 6 h cadence the overlap is <1%).
for c in "${FLEET_SENSORS[@]}"; do
newest=$(docker exec "$c" bash -c 'ls -t /var/lib/strawberry/export/export-*.sbx | head -1')
inc=$(docker exec "$c" /opt/strawberry-fleet-inspect "$newest" | sed -n 's/.* windows=\([0-9]*\).*/\1/p')
docker exec "$c" strawberry export --out /tmp/full-now.sbx >/dev/null
full=$(docker exec "$c" /opt/strawberry-fleet-inspect /tmp/full-now.sbx | sed -n 's/.* windows=\([0-9]*\).*/\1/p')
echo "$c: incremental bundle $inc windows, full export now $full" | tee -a "$L/run-checks.log"
[ "${inc:-0}" -gt 0 ] && [ "$inc" -lt "$full" ] && pass "$c: --since auto ships fewer windows than a full export ($inc < $full)" \
|| fail "$c: incremental $inc not fewer than full $full"
done
chk "sensors keep a cursor (.export-cursor) in export_dir" \
bash -c "for c in ${FLEET_SENSORS[*]}; do docker exec \$c test -s /var/lib/strawberry/export/.export-cursor || exit 1; done"
TRAIN2=$(docker exec "$FLEET_CENTRAL" bash -c "ls -d /var/lib/strawberry/fleet/work/run-* | sort | tail -1")
docker exec "$FLEET_CENTRAL" cat "$TRAIN2/train-seed1.json" >"$L/run-train2.json"
docker exec "$FLEET_CENTRAL" cat "$TRAIN2/overlay.json" >"$L/run-overlay2.json"
J 'import json; d=json.load(open("/var/tmp/sb-fleet/logs/run-train2.json")); print("train2:",{k:d[k] for k in ("nodes","bundles_read","windows","positives","duplicate_windows") if k in d})' | tee -a "$L/run-checks.log"
chk "cycle 2: train dedups re-shipped windows (duplicate_windows > 0), 3 nodes" \
J 'import json,sys; d=json.load(open("/var/tmp/sb-fleet/logs/run-train2.json")); sys.exit(0 if d["duplicate_windows"]>0 and d["nodes"]==3 else 1)'
chk "cycle 2: overlay uses the newest bundle per node (3 superseded)" \
J 'import json,sys; d=json.load(open("/var/tmp/sb-fleet/logs/run-overlay2.json")); sys.exit(0 if d["nodes"]==3 and d["bundles_superseded"]>=3 else 1)'
# ---- e2. fleet promotion gate (orch.strawberry_gate) ---------------------------------------------
# Needs > 10 min of ranked history per sensor for trace injection; top the history up first.
echo "== e2. fleet gate: extra workload, then orch.strawberry_gate"
pids=()
for c in "${FLEET_SENSORS[@]}"; do
timeout $((GATE_WORK_SECS + 120)) docker exec "$c" bash /opt/workload.sh "${c##*-}" "$GATE_WORK_SECS" >"$L/run-09-workload3-${c##*-}.log" 2>&1 &
pids+=($!)
done
for p in "${pids[@]}"; do wait "$p" || true; done
docker exec "$FLEET_CENTRAL" bash -c 'ls /srv/salt/strawberry/dist/gate/model-seed*.sbm' >"$L/run-10-gate-models.log" 2>&1 \
&& pass "cycle published 3 gate models (dist/gate/model-seed{1,2,3}.sbm)" || fail "no gate models in dist/gate (see run-10-gate-models.log)"
docker exec "$FLEET_CENTRAL" salt-run state.orchestrate orch.strawberry_gate >"$L/run-11-orch-gate.log" 2>&1 && rc=0 || rc=$?
tail -6 "$L/run-11-orch-gate.log"
[ "$rc" = 0 ] && pass "orch.strawberry_gate succeeded (a failing verdict is not an orchestration error)" || fail "orch.strawberry_gate exit $rc (see run-11-orch-gate.log)"
VERDICT=$(docker exec "$FLEET_CENTRAL" bash -c "ls -d /var/lib/strawberry/fleet/work/gate-* | sort | tail -1")
docker exec "$FLEET_CENTRAL" cat "$VERDICT/verdict.json" >"$L/run-gate-verdict.json" 2>/dev/null || true
chk "fleet gate verdict: 9 cells (3 seeds x 3 periods), each pooled over all 3 sensors, with positives" \
J 'import json,sys; v=json.load(open("/var/tmp/sb-fleet/logs/run-gate-verdict.json")); c=v["cells"]; sys.exit(0 if v["reports"]==3 and len(c)==9 and all(x["nodes"]==3 and x["extra_positives"]>0 for x in c) else 1)'
J 'import json; v=json.load(open("/var/tmp/sb-fleet/logs/run-gate-verdict.json")); print("gate verdict passed=%s; positives per cell %s" % (v["passed"], sorted({x["extra_positives"] for x in v["cells"]})))' | tee -a "$L/run-checks.log"
chk "sensors keep their own gate reports; promoted.sbm still nowhere" \
bash -c "for c in ${FLEET_SENSORS[*]}; do docker exec \$c test -s /var/lib/strawberry/gate/report.json || exit 1; docker exec \$c test ! -e /var/lib/strawberry/model/promoted.sbm || exit 1; done"
# ---- f. summary -----------------------------------------------------------------------------
echo
echo "================ SUMMARY ================"
printf '%s\n' "${RESULTS[@]}"
echo "-----------------------------------------"
echo "R2 compiled-rules cache (load_ms): miss ${MISS_MS[*]} | hit ${HIT_MS[*]}"
echo "bundles: round1 $R1_SIZE B total (3), round2 $R2_SIZE B total ($n2)"
echo "FAIL count: $FAILS"
printf '%s\n' "${RESULTS[@]}" >"$L/run-summary.txt"
if [ "${FLEET_KEEP:-0}" != 1 ]; then "$FLEET_HERE/fleet_down.sh" >/dev/null 2>&1 || true; fi
[ "$FAILS" = 0 ]