strawberry/deploy/validation/range/range_up.sh
trvon 0e3d89d918
range: export the shared constants and keep shellcheck quiet
range_common.sh is a sourced library: range_up.sh, range_down.sh and
range_status.sh consume its constants, which shellcheck cannot see when it
analyses the file alone, so it reported every one as unused. Exporting them
states that interface, and the two $(cd ...) assignments are split from their
export so a failing cd is not masked.
2026-11-19 11:49:41 -07:00

170 lines
8.8 KiB
Bash
Executable file

#!/usr/bin/env bash
# Bring the real-sensor range up (idempotent: rerun after a reboot / travel and it resumes).
# Caldera (rootless Docker, capped) + SENSORS Arch VMs (qemu/KVM, 2 vCPU, 2 GiB each), each running the
# real Strawberry daemon (eBPF + fanotify exec guard), a background workload and a Caldera agent, all as
# systemd units inside the VM so they come back after a VM reboot. Disks persist under RANGE_DIR/vms.
# Then starts the attack scheduler (range_attack.py) detached; it resumes from its state file.
set -euo pipefail
. "$(dirname "${BASH_SOURCE[0]}")/range_common.sh"
L=$RANGE_DIR/logs
# ---- host safety ------------------------------------------------------------------------------------
# The range needs ~7 GiB for the VMs + Caldera; refuse a start that cannot fit, rather than
# repeating the 2026-11-17 global OOM (15 concurrent compilers + the range exhausted RAM+swap).
avail_kb=$(mem_available_kb)
disk_kb=$(disk_available_kb)
if [ "$avail_kb" -lt "$RANGE_MIN_AVAIL_KB" ]; then
echo "refusing to start the range: ${avail_kb} KiB MemAvailable < ${RANGE_MIN_AVAIL_KB} KiB" >&2
echo "pause other memory-heavy work (builds, browsers, research jobs) first" >&2
exit 1
fi
if [ "$disk_kb" -lt "$RANGE_MIN_DISK_KB" ]; then
echo "refusing to start the range: ${disk_kb} KiB free < ${RANGE_MIN_DISK_KB} KiB" >&2
exit 1
fi
echo "range preflight: MemAvailable ${avail_kb} KiB, free ${disk_kb} KiB"
# ---- payload: the CURRENT build (Arch host build runs on the Arch guest) -------------------------------
BIN=$REPO/builddir/src/strawberry
BPF=$(find "$REPO/builddir" -name process_monitor.bpf.o | head -1)
[ -x "$BIN" ] && [ -s "$BPF" ] || { echo "build first (builddir/src/strawberry + process_monitor.bpf.o)" >&2; exit 1; }
P=$RANGE_DIR/payload
install -m 0755 "$BIN" "$P/strawberry"
install -m 0644 "$BPF" "$P/process_monitor.bpf.o"
mkdir -p "$P/rules" "$P/units"
command cp -f "$REPO/packaging/rules-snapshot/yara-forge-core/yara-rules-core.yar" "$P/rules/" # NOT the range markers
cat > "$P/node.conf" <<'EOF'
yara_rules_path=/etc/strawberry/rules
db_path=/var/lib/strawberry/evidence
model_dir=/var/lib/strawberry/model
control_socket_path=/run/strawberry/control.sock
enable_bpf_monitoring=true
rule_auto_fetch=false
retro_hunt=true
capture_redact_cmdline=true
model_shadow=true
scan_profile=balanced
export_dir=/var/lib/strawberry/export
EOF
cat > "$P/units/strawberry.service" <<'EOF'
[Unit]
Description=Strawberry daemon (range sensor)
After=network-online.target
[Service]
RuntimeDirectory=strawberry
RuntimeDirectoryMode=0755
Environment=STRAWBERRY_BPF_OBJECT=/usr/local/bin/process_monitor.bpf.o
ExecStart=/usr/local/bin/strawberry --config /etc/strawberry/strawberry.conf daemon run
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
command cp -f "$RANGE_HERE/range_workload.sh" "$P/range_workload.sh"
cat > "$P/units/range-workload.service" <<'EOF'
[Unit]
Description=Range background workload (benign host activity)
After=strawberry.service
[Service]
User=arch
ExecStart=/usr/local/bin/range_workload.sh
Restart=always
RestartSec=10
Nice=10
[Install]
WantedBy=multi-user.target
EOF
# ---- Caldera ---------------------------------------------------------------------------------------------
if ! docker ps --format '{{.Names}}' | grep -qx "$CALDERA_NAME"; then
if docker ps -a --format '{{.Names}}' | grep -qx "$CALDERA_NAME"; then
docker start "$CALDERA_NAME" >/dev/null
else
docker run -d --name "$CALDERA_NAME" --cpus=2 --memory=3g -p 127.0.0.1:8888:8888 caldera:slim -E default >/dev/null
fi
fi
for _ in $(seq 1 90); do caldera "$CALDERA_URL/api/v2/health" >/dev/null 2>&1 && break; sleep 2; done
caldera "$CALDERA_URL/api/v2/health" >/dev/null || { echo "caldera did not come up" >&2; docker logs --tail 20 "$CALDERA_NAME" >&2; exit 1; }
[ -s "$P/sandcat" ] || curl -sf -X POST -H "KEY: $CALDERA_KEY" -H "file:sandcat.go" -H "platform:linux" \
"$CALDERA_URL/file/download" -o "$P/sandcat"
chmod 0755 "$P/sandcat"
echo "caldera up"
# ---- sensors ---------------------------------------------------------------------------------------------
for i in $(seq 1 "$SENSORS"); do
name=sensor-$i
disk=$RANGE_DIR/vms/$name.qcow2
seed=$RANGE_DIR/vms/$name-seed.iso
[ -f "$disk" ] || qemu-img create -f qcow2 -b "$BASE_IMG" -F qcow2 "$disk" 20G >/dev/null
if [ ! -f "$seed" ]; then
sd=$(mktemp -d "$RANGE_DIR/vms/seed.XXXX")
printf 'instance-id: sb-range-%s\nlocal-hostname: %s\n' "$name" "$name" >"$sd/meta-data"
sed "s/^hostname: .*/hostname: $name/" "$RANGE_HERE/user-data.tmpl" \
| sed "s|@SSH_KEY@|$(cat "$SSHK.pub")|" >"$sd/user-data"
cloud-localds "$seed" "$sd/user-data" "$sd/meta-data"
find "$sd" -delete
fi
if [ -z "$(vm_pid "$i" || true)" ]; then
nice -n 10 qemu-system-x86_64 -enable-kvm -m "$VM_MEM_MB" -smp 2 -cpu host -name "$name" \
-drive "file=$disk,if=virtio,format=qcow2" -drive "file=$seed,if=virtio,format=raw" \
-netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$(port_of "$i")-:22" -device virtio-net-pci,netdev=n0 \
-display none -serial "file:$L/$name-serial.log" -pidfile "$RANGE_DIR/vms/$name.pid" -daemonize
echo "$name booting (ssh :$(port_of "$i"))"
fi
done
for i in $(seq 1 "$SENSORS"); do
for _ in $(seq 1 120); do ssh_vm "$i" true 2>/dev/null && break; sleep 3; done
ssh_vm "$i" true || { echo "sensor-$i: no ssh" >&2; exit 1; }
for _ in $(seq 1 200); do ssh_vm "$i" test -f /run/strawberry-vm-ready -o -f /etc/strawberry/.range-provisioned 2>/dev/null && break; sleep 3; done
# Keep the unit current on resume too. In particular RuntimeDirectory
# recreates /run/strawberry after a guest reboot, so the control socket
# and delayed-label reloads keep working.
scp_vm "$i" "$P/units/strawberry.service" /tmp/strawberry.service
ssh_vm "$i" "sudo install -m0644 /tmp/strawberry.service /etc/systemd/system/strawberry.service
sudo systemctl daemon-reload"
if ! ssh_vm "$i" test -f /etc/strawberry/.range-provisioned; then
scp_vm "$i" "$P" /tmp/payload
ssh_vm "$i" "set -e; sudo install -m755 /tmp/payload/strawberry /usr/local/bin/strawberry
sudo install -m644 /tmp/payload/process_monitor.bpf.o /usr/local/bin/process_monitor.bpf.o
sudo install -m755 /tmp/payload/range_workload.sh /usr/local/bin/range_workload.sh
sudo install -m755 /tmp/payload/sandcat /usr/local/bin/sandcat
sudo mkdir -p /etc/strawberry/rules /var/lib/strawberry/evidence /var/lib/strawberry/model /var/lib/strawberry/export /run/strawberry
sudo cp /tmp/payload/rules/* /etc/strawberry/rules/
sudo cp /tmp/payload/node.conf /etc/strawberry/strawberry.conf
echo node_id=sensor-$i | sudo tee -a /etc/strawberry/strawberry.conf >/dev/null
sudo cp /tmp/payload/units/*.service /etc/systemd/system/
printf '[Unit]\nDescription=Caldera agent (range)\nAfter=network-online.target\n[Service]\nUser=arch\nExecStart=/usr/local/bin/sandcat -server $GUEST_CALDERA -group sensor-$i\nRestart=always\nRestartSec=15\n[Install]\nWantedBy=multi-user.target\n' | sudo tee /etc/systemd/system/range-agent.service >/dev/null
sudo mkdir -p /var/tmp/.cache && sudo chmod 1777 /var/tmp/.cache
sudo systemctl daemon-reload
sudo systemctl enable --now strawberry.service range-workload.service range-agent.service
sudo touch /etc/strawberry/.range-provisioned"
echo "sensor-$i provisioned"
else
ssh_vm "$i" "sudo systemctl restart strawberry.service
sudo systemctl start range-workload.service range-agent.service"
fi
done
# ---- scheduler -------------------------------------------------------------------------------------------
# One scheduler per range. The pid file records the process range_up.sh started; sched_pids() also
# rescans /proc and only counts a PID whose command line runs this repo's range_attack.py with
# SB_RANGE_DIR resolving to this RANGE_DIR, so another range's scheduler is neither adopted nor killed.
if [ -n "$(sched_pids)" ]; then
running=$(sched_pids | tr '\n' ' ')
echo "attack scheduler already running (pid ${running% })"
else
mkdir -p "$RANGE_DIR/state"
rm -f "$SCHED_PID_FILE"
# shellcheck disable=SC2016 # $$, $1 and $2 expand in the detached child, not here
SB_RANGE_DIR="$RANGE_DIR" setsid -f bash -c 'echo $$ >"$1"; exec python3 -I "$2"' _ \
"$SCHED_PID_FILE" "$SCHED_SCRIPT" </dev/null >>"$L/attack.log" 2>&1
for _ in $(seq 1 25); do [ -s "$SCHED_PID_FILE" ] && break; sleep 0.2; done
started=$(sched_pids | tr '\n' ' ')
if [ -n "$started" ]; then
echo "attack scheduler started (pid ${started% }, log $L/attack.log)"
else
echo "attack scheduler did not start; see $L/attack.log" >&2
exit 1
fi
fi
"$RANGE_HERE/range_status.sh"