strawberry/deploy/salt/test/fleet_up.sh
trvon 39ffa02ae6
Salt: dockerized multi-node fleet test (master/central + 3 sensors) + state fixes
deploy/salt/test/: builds a release tarball from HEAD in ubuntu:24.04 (CI path),
boots systemd containers (rootless, no --privileged), master==central + 3
sensors with the host's Salt onedir mounted read-only, applies the real states,
runs a workload with planted YARA detections, then two orchestrated cycles.
43 checks (install/config/idempotence, real daemons + telemetry, R2 cache
miss/hit, cp.push collect, 3-seed cycle, canary hold then fleet rollout, no
promoted model anywhere, fleet bits in round 2, --since cursor, dedup).

State fixes found by running them for real:
- init.sls no longer includes strawberry.distribute: a plain state.apply on
  sensors shipped models fleet-wide and bypassed the canary; only the
  orchestration distributes now.
- install.sls: CI tarballs carry a +g<sha> suffix that 'strawberry version'
  does not print, so install.sh re-ran on every apply; record the installed
  release and compare base version + marker.
- orch collect: mv -f (re-collected bundles must not be skipped).
- central cycle: builds the overlay first; a training refusal for too few
  teacher labels (normal on a young fleet) publishes the overlay only and
  keeps the sensors' shadow, any other error still fails.
- distribute.sls: serves only what the cycle published (cp.hash_file; the
  cp.list_master file-list cache can miss a just-published file).
Render check gains overlay-only and full publish cases (20 checks).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 12:16:31 -07:00

150 lines
6.6 KiB
Bash
Executable file

#!/usr/bin/env bash
# Bring up the dockerized fleet: sbfleet-central (salt-master + salt-minion id strawberry-central)
# and sbfleet-sensor-1..3 (salt-minion ids sensor-1..3), each a systemd-as-PID-1 container.
#
# Salt comes from the host onedir (/opt/saltstack, mounted read-only). The release tarball built
# by build_tarball.sh is served to the minions via salt://strawberry/release (a writable dir that
# also holds salt://strawberry/dist, where the central cycle publishes).
#
# systemd in ROOTLESS docker: the container cgroup2 is read-only and docker rejects
# writable-cgroups under rootless, so each container gets --cgroupns=host plus a private,
# delegated cgroup directory bind-mounted at /sys/fs/cgroup (no --privileged). Because that
# moves systemd out of docker's own scope, the --cpus/--memory caps are applied to that
# directory directly (cpu.max / memory.max).
#
# The sensor unit's CapabilityBoundingSet/AmbientCapabilities need the caps in the container
# (rootless: they are scoped to the container's user namespace, not the host).
#
# TEST ONLY: the master runs with auto_accept: True.
set -euo pipefail
# shellcheck source=fleet_common.sh
. "$(dirname "${BASH_SOURCE[0]}")/fleet_common.sh"
CPUS=${FLEET_CPUS:-1} # cpus per container
MEM=${FLEET_MEM:-2g}
# 1. release tarball (cached per HEAD) + pin
"$FLEET_HERE/build_tarball.sh"
head12=$(git -C "$FLEET_REPO" rev-parse --short=12 HEAD)
tgz=$(ls -t "$FLEET_DIR"/dist/strawberry-*+g"${head12}"-noble-amd64.tgz | head -1)
base=$(basename "$tgz")
ver=${base#strawberry-}; ver=${ver%-noble-amd64.tgz}
sha=$(cut -d' ' -f1 "$tgz.sha256")
echo "release $ver sha256 $sha"
# 2. generated master state: release pillar + served tarball (+ empty dist)
find "$FLEET_GEN" -mindepth 1 -delete 2>/dev/null || true
mkdir -p "$FLEET_GEN/pillar" "$FLEET_GEN/srv-salt/strawberry/release/v$ver" "$FLEET_GEN/srv-salt/strawberry/dist"
command cp -f "$tgz" "$FLEET_GEN/srv-salt/strawberry/release/v$ver/$base"
cat >"$FLEET_GEN/pillar/strawberry_release.sls" <<EOT
# generated by fleet_up.sh
strawberry:
release:
version: '$ver'
artifact_sha256: '$sha'
url_base: 'salt://strawberry/release'
EOT
# 3. image + network
docker image inspect "$FLEET_IMAGE" >/dev/null 2>&1 || \
nice -n 19 docker build --cpu-quota=200000 --cpu-period=100000 --memory=4g \
-t "$FLEET_IMAGE" -f "$FLEET_HERE/Dockerfile.node" "$FLEET_HERE" >"$FLEET_LOGS/02-image.log" 2>&1
docker network inspect "$FLEET_NET" >/dev/null 2>&1 || docker network create "$FLEET_NET" >/dev/null
# 4. containers
start_node() {
local c=$1; shift
docker rm -f "$c" >/dev/null 2>&1 || true
local cg=$CG_ROOT/$c
mkdir -p "$cg"
# cap the whole subtree (systemd moves itself out of docker's scope, see the header)
echo "+cpu +memory +pids" >"$CG_ROOT/cgroup.subtree_control" 2>/dev/null || true
echo "$((CPUS * 100000)) 100000" >"$cg/cpu.max"
case "$MEM" in *g) echo $(( ${MEM%g} * 1024 * 1024 * 1024 )) >"$cg/memory.max" ;; *m) echo $(( ${MEM%m} * 1024 * 1024 )) >"$cg/memory.max" ;; esac
docker run -d --name "$c" --hostname "$c" --network "$FLEET_NET" \
--cpus="$CPUS" --memory="$MEM" \
--cgroupns=host -v "$cg":/sys/fs/cgroup:rw \
--tmpfs /run --tmpfs /run/lock --tmpfs /tmp \
--stop-signal SIGRTMIN+3 \
--cap-add SYS_ADMIN --cap-add BPF --cap-add PERFMON --cap-add SYS_RESOURCE \
--cap-add SYS_PTRACE --cap-add DAC_READ_SEARCH --cap-add KILL \
-v /opt/saltstack:/opt/saltstack:ro \
"$@" "$FLEET_IMAGE" >/dev/null
}
start_node "$FLEET_CENTRAL" \
-v "$FLEET_GEN/srv-salt":/srv/salt:rw \
-v "$FLEET_GEN":/opt/fleet-gen:ro \
-v "$FLEET_REPO/deploy":/opt/repo/deploy:ro
for c in "${FLEET_SENSORS[@]}"; do start_node "$c"; done
for c in "${FLEET_ALL[@]}"; do
for _ in $(seq 1 60); do
st=$(docker exec "$c" systemctl is-system-running 2>/dev/null || true)
case "$st" in running|degraded) break ;; esac
sleep 1
done
echo "$c systemd: $st"
case "$st" in running|degraded) ;; *) echo "systemd did not boot in $c" >&2; exit 1 ;; esac
done
# Name resolution without Docker's embedded DNS (127.0.0.11 stopped answering inside the systemd
# containers on 2026-11-14 while IP connectivity was fine): pin every fleet name in /etc/hosts.
hosts=""
for c in "${FLEET_ALL[@]}"; do
ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$FLEET_NET\").IPAddress}}" "$c")
[ -n "$ip" ] || { echo "no IP for $c on $FLEET_NET" >&2; exit 1; }
hosts+="$ip $c"$'\n'
done
for c in "${FLEET_ALL[@]}"; do printf '%s' "$hosts" | docker exec -i "$c" bash -c 'cat >>/etc/hosts'; done
docker exec "${FLEET_SENSORS[0]}" getent hosts "$FLEET_CENTRAL" >/dev/null || { echo "hosts pinning failed" >&2; exit 1; }
# 5. salt master (central) + minions
docker exec -i "$FLEET_CENTRAL" bash -c 'mkdir -p /etc/salt/master.d && cat >/etc/salt/master.d/fleet.conf' <<'EOT'
interface: 0.0.0.0
worker_threads: 3
file_recv: True # lets minions cp.push their bundles
auto_accept: True # TEST ONLY
file_roots:
base:
- /srv/salt # strawberry/dist (written by the cycle), strawberry/release (tarball)
- /opt/repo/deploy/salt/salt # salt://strawberry/*, salt://orch/*
- /opt/repo/deploy/salt # salt://files/*
pillar_roots:
base:
- /opt/fleet-gen/pillar # generated release pin
- /opt/repo/deploy/salt/test/pillar # test overrides
- /opt/repo/deploy/salt/pillar
EOT
unit() { # <container> <name> <exec>
docker exec -i "$1" bash -c "cat >/etc/systemd/system/$2.service" <<EOT
[Unit]
Description=$2 (fleet test)
After=network.target
[Service]
ExecStart=$3
Restart=on-failure
LimitNOFILE=100000
[Install]
WantedBy=multi-user.target
EOT
}
unit "$FLEET_CENTRAL" salt-master /opt/saltstack/salt/salt-master
for c in "${FLEET_ALL[@]}"; do
docker exec -i "$c" bash -c 'mkdir -p /etc/salt/minion.d && cat >/etc/salt/minion.d/fleet.conf' <<EOT
id: $(minion_id "$c")
master: $FLEET_CENTRAL
# systemd-run --scope fails with the delegated-cgroup trick (see header); run units plainly
systemd.scope: False
EOT
unit "$c" salt-minion /opt/saltstack/salt/salt-minion
done
dxb "$FLEET_CENTRAL" 'systemctl daemon-reload && systemctl enable --now salt-master'
for c in "${FLEET_ALL[@]}"; do dxb "$c" 'systemctl daemon-reload && systemctl enable --now salt-minion'; done
for _ in $(seq 1 60); do
n=$(docker exec "$FLEET_CENTRAL" salt --timeout=5 --out=txt '*' test.ping 2>/dev/null | grep -c 'True' || true)
[ "${n:-0}" -ge 4 ] && break
sleep 3
done
[ "${n:-0}" -ge 4 ] || { echo "only ${n:-0}/4 minions answered test.ping" >&2; exit 1; }
echo "fleet up: master+central, sensor-1..3 (4/4 minions up), release $ver"