deploy/salt/test/: builds a release tarball from HEAD in ubuntu:24.04 (CI path), boots systemd containers (rootless, no --privileged), master==central + 3 sensors with the host's Salt onedir mounted read-only, applies the real states, runs a workload with planted YARA detections, then two orchestrated cycles. 43 checks (install/config/idempotence, real daemons + telemetry, R2 cache miss/hit, cp.push collect, 3-seed cycle, canary hold then fleet rollout, no promoted model anywhere, fleet bits in round 2, --since cursor, dedup). State fixes found by running them for real: - init.sls no longer includes strawberry.distribute: a plain state.apply on sensors shipped models fleet-wide and bypassed the canary; only the orchestration distributes now. - install.sls: CI tarballs carry a +g<sha> suffix that 'strawberry version' does not print, so install.sh re-ran on every apply; record the installed release and compare base version + marker. - orch collect: mv -f (re-collected bundles must not be skipped). - central cycle: builds the overlay first; a training refusal for too few teacher labels (normal on a young fleet) publishes the overlay only and keeps the sensors' shadow, any other error still fails. - distribute.sls: serves only what the cycle published (cp.hash_file; the cp.list_master file-list cache can miss a just-published file). Render check gains overlay-only and full publish cases (20 checks). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
150 lines
6.6 KiB
Bash
Executable file
150 lines
6.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Bring up the dockerized fleet: sbfleet-central (salt-master + salt-minion id strawberry-central)
|
|
# and sbfleet-sensor-1..3 (salt-minion ids sensor-1..3), each a systemd-as-PID-1 container.
|
|
#
|
|
# Salt comes from the host onedir (/opt/saltstack, mounted read-only). The release tarball built
|
|
# by build_tarball.sh is served to the minions via salt://strawberry/release (a writable dir that
|
|
# also holds salt://strawberry/dist, where the central cycle publishes).
|
|
#
|
|
# systemd in ROOTLESS docker: the container cgroup2 is read-only and docker rejects
|
|
# writable-cgroups under rootless, so each container gets --cgroupns=host plus a private,
|
|
# delegated cgroup directory bind-mounted at /sys/fs/cgroup (no --privileged). Because that
|
|
# moves systemd out of docker's own scope, the --cpus/--memory caps are applied to that
|
|
# directory directly (cpu.max / memory.max).
|
|
#
|
|
# The sensor unit's CapabilityBoundingSet/AmbientCapabilities need the caps in the container
|
|
# (rootless: they are scoped to the container's user namespace, not the host).
|
|
#
|
|
# TEST ONLY: the master runs with auto_accept: True.
|
|
set -euo pipefail
|
|
# shellcheck source=fleet_common.sh
|
|
. "$(dirname "${BASH_SOURCE[0]}")/fleet_common.sh"
|
|
|
|
CPUS=${FLEET_CPUS:-1} # cpus per container
|
|
MEM=${FLEET_MEM:-2g}
|
|
|
|
# 1. release tarball (cached per HEAD) + pin
|
|
"$FLEET_HERE/build_tarball.sh"
|
|
head12=$(git -C "$FLEET_REPO" rev-parse --short=12 HEAD)
|
|
tgz=$(ls -t "$FLEET_DIR"/dist/strawberry-*+g"${head12}"-noble-amd64.tgz | head -1)
|
|
base=$(basename "$tgz")
|
|
ver=${base#strawberry-}; ver=${ver%-noble-amd64.tgz}
|
|
sha=$(cut -d' ' -f1 "$tgz.sha256")
|
|
echo "release $ver sha256 $sha"
|
|
|
|
# 2. generated master state: release pillar + served tarball (+ empty dist)
|
|
find "$FLEET_GEN" -mindepth 1 -delete 2>/dev/null || true
|
|
mkdir -p "$FLEET_GEN/pillar" "$FLEET_GEN/srv-salt/strawberry/release/v$ver" "$FLEET_GEN/srv-salt/strawberry/dist"
|
|
command cp -f "$tgz" "$FLEET_GEN/srv-salt/strawberry/release/v$ver/$base"
|
|
cat >"$FLEET_GEN/pillar/strawberry_release.sls" <<EOT
|
|
# generated by fleet_up.sh
|
|
strawberry:
|
|
release:
|
|
version: '$ver'
|
|
artifact_sha256: '$sha'
|
|
url_base: 'salt://strawberry/release'
|
|
EOT
|
|
|
|
# 3. image + network
|
|
docker image inspect "$FLEET_IMAGE" >/dev/null 2>&1 || \
|
|
nice -n 19 docker build --cpu-quota=200000 --cpu-period=100000 --memory=4g \
|
|
-t "$FLEET_IMAGE" -f "$FLEET_HERE/Dockerfile.node" "$FLEET_HERE" >"$FLEET_LOGS/02-image.log" 2>&1
|
|
docker network inspect "$FLEET_NET" >/dev/null 2>&1 || docker network create "$FLEET_NET" >/dev/null
|
|
|
|
# 4. containers
|
|
start_node() {
|
|
local c=$1; shift
|
|
docker rm -f "$c" >/dev/null 2>&1 || true
|
|
local cg=$CG_ROOT/$c
|
|
mkdir -p "$cg"
|
|
# cap the whole subtree (systemd moves itself out of docker's scope, see the header)
|
|
echo "+cpu +memory +pids" >"$CG_ROOT/cgroup.subtree_control" 2>/dev/null || true
|
|
echo "$((CPUS * 100000)) 100000" >"$cg/cpu.max"
|
|
case "$MEM" in *g) echo $(( ${MEM%g} * 1024 * 1024 * 1024 )) >"$cg/memory.max" ;; *m) echo $(( ${MEM%m} * 1024 * 1024 )) >"$cg/memory.max" ;; esac
|
|
docker run -d --name "$c" --hostname "$c" --network "$FLEET_NET" \
|
|
--cpus="$CPUS" --memory="$MEM" \
|
|
--cgroupns=host -v "$cg":/sys/fs/cgroup:rw \
|
|
--tmpfs /run --tmpfs /run/lock --tmpfs /tmp \
|
|
--stop-signal SIGRTMIN+3 \
|
|
--cap-add SYS_ADMIN --cap-add BPF --cap-add PERFMON --cap-add SYS_RESOURCE \
|
|
--cap-add SYS_PTRACE --cap-add DAC_READ_SEARCH --cap-add KILL \
|
|
-v /opt/saltstack:/opt/saltstack:ro \
|
|
"$@" "$FLEET_IMAGE" >/dev/null
|
|
}
|
|
start_node "$FLEET_CENTRAL" \
|
|
-v "$FLEET_GEN/srv-salt":/srv/salt:rw \
|
|
-v "$FLEET_GEN":/opt/fleet-gen:ro \
|
|
-v "$FLEET_REPO/deploy":/opt/repo/deploy:ro
|
|
for c in "${FLEET_SENSORS[@]}"; do start_node "$c"; done
|
|
|
|
for c in "${FLEET_ALL[@]}"; do
|
|
for _ in $(seq 1 60); do
|
|
st=$(docker exec "$c" systemctl is-system-running 2>/dev/null || true)
|
|
case "$st" in running|degraded) break ;; esac
|
|
sleep 1
|
|
done
|
|
echo "$c systemd: $st"
|
|
case "$st" in running|degraded) ;; *) echo "systemd did not boot in $c" >&2; exit 1 ;; esac
|
|
done
|
|
|
|
# Name resolution without Docker's embedded DNS (127.0.0.11 stopped answering inside the systemd
|
|
# containers on 2026-11-14 while IP connectivity was fine): pin every fleet name in /etc/hosts.
|
|
hosts=""
|
|
for c in "${FLEET_ALL[@]}"; do
|
|
ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$FLEET_NET\").IPAddress}}" "$c")
|
|
[ -n "$ip" ] || { echo "no IP for $c on $FLEET_NET" >&2; exit 1; }
|
|
hosts+="$ip $c"$'\n'
|
|
done
|
|
for c in "${FLEET_ALL[@]}"; do printf '%s' "$hosts" | docker exec -i "$c" bash -c 'cat >>/etc/hosts'; done
|
|
docker exec "${FLEET_SENSORS[0]}" getent hosts "$FLEET_CENTRAL" >/dev/null || { echo "hosts pinning failed" >&2; exit 1; }
|
|
|
|
# 5. salt master (central) + minions
|
|
docker exec -i "$FLEET_CENTRAL" bash -c 'mkdir -p /etc/salt/master.d && cat >/etc/salt/master.d/fleet.conf' <<'EOT'
|
|
interface: 0.0.0.0
|
|
worker_threads: 3
|
|
file_recv: True # lets minions cp.push their bundles
|
|
auto_accept: True # TEST ONLY
|
|
file_roots:
|
|
base:
|
|
- /srv/salt # strawberry/dist (written by the cycle), strawberry/release (tarball)
|
|
- /opt/repo/deploy/salt/salt # salt://strawberry/*, salt://orch/*
|
|
- /opt/repo/deploy/salt # salt://files/*
|
|
pillar_roots:
|
|
base:
|
|
- /opt/fleet-gen/pillar # generated release pin
|
|
- /opt/repo/deploy/salt/test/pillar # test overrides
|
|
- /opt/repo/deploy/salt/pillar
|
|
EOT
|
|
unit() { # <container> <name> <exec>
|
|
docker exec -i "$1" bash -c "cat >/etc/systemd/system/$2.service" <<EOT
|
|
[Unit]
|
|
Description=$2 (fleet test)
|
|
After=network.target
|
|
[Service]
|
|
ExecStart=$3
|
|
Restart=on-failure
|
|
LimitNOFILE=100000
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOT
|
|
}
|
|
unit "$FLEET_CENTRAL" salt-master /opt/saltstack/salt/salt-master
|
|
for c in "${FLEET_ALL[@]}"; do
|
|
docker exec -i "$c" bash -c 'mkdir -p /etc/salt/minion.d && cat >/etc/salt/minion.d/fleet.conf' <<EOT
|
|
id: $(minion_id "$c")
|
|
master: $FLEET_CENTRAL
|
|
# systemd-run --scope fails with the delegated-cgroup trick (see header); run units plainly
|
|
systemd.scope: False
|
|
EOT
|
|
unit "$c" salt-minion /opt/saltstack/salt/salt-minion
|
|
done
|
|
dxb "$FLEET_CENTRAL" 'systemctl daemon-reload && systemctl enable --now salt-master'
|
|
for c in "${FLEET_ALL[@]}"; do dxb "$c" 'systemctl daemon-reload && systemctl enable --now salt-minion'; done
|
|
|
|
for _ in $(seq 1 60); do
|
|
n=$(docker exec "$FLEET_CENTRAL" salt --timeout=5 --out=txt '*' test.ping 2>/dev/null | grep -c 'True' || true)
|
|
[ "${n:-0}" -ge 4 ] && break
|
|
sleep 3
|
|
done
|
|
[ "${n:-0}" -ge 4 ] || { echo "only ${n:-0}/4 minions answered test.ping" >&2; exit 1; }
|
|
echo "fleet up: master+central, sensor-1..3 (4/4 minions up), release $ver"
|