fleet train advertised --json but printed text, so the central cycle's train-seedN.json evidence was not JSON; it now emits one object incl. duplicate_windows. fleet overlay gains --json (bundles_superseded, overlays_skipped). Both commands read only *.sbx (no dotfiles) from the bundle dir, so a cursor or stray file is never fed to the merge. scripts/fleet_e2e.sh drives the real binary + central cycle script over N synthetic sensors: --since auto export chain, Salt-pull simulation, 3-seed train + overlay, sha256 publish, determinism, overlay distribution and a round-2 export carrying fleet rarity features. 6/6 checks pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
6.5 KiB
Bash
Executable file
112 lines
6.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Fleet end-to-end (Level A: one host, no Docker, no root). Drives the REAL binary and the REAL central
|
|
# cycle script through a full hub-and-spoke round with N synthetic sensor nodes:
|
|
#
|
|
# seed N stores (distinct node_id) -> export x2 per node into an egress dir (--since auto chain)
|
|
# -> "Salt pull" (copy) every bundle into one central bundle_dir (+ a stray file and an overlay)
|
|
# -> strawberry-central-cycle.sh (fleet train x3 seeds + fleet overlay + sha256 sidecars + publish)
|
|
# -> distribute fleet-overlay.sbo into each node's model_dir -> round-2 export carries fleet_bits
|
|
#
|
|
# Asserts: every node is merged; duplicate windows from the repeated export are deduped (not retrained);
|
|
# the overlay takes one (newest) bundle per node; non-bundle files are ignored; the cycle never promotes;
|
|
# training is deterministic per seed; round-1 features have no fleet bits and round-2 features do.
|
|
#
|
|
# Usage: scripts/fleet_e2e.sh [builddir] (default builddir). Exit 0 = all passed.
|
|
set -euo pipefail
|
|
|
|
BD="${1:-builddir}"
|
|
BIN="$PWD/$BD/src/strawberry"
|
|
SEED_TOOL="$PWD/$BD/tests/strawberry-fleet-seed"
|
|
INSPECT="$PWD/$BD/tests/strawberry-fleet-inspect"
|
|
CYCLE="$PWD/deploy/salt/files/strawberry-central-cycle.sh"
|
|
NODES="${FLEET_NODES:-3}"
|
|
for f in "$BIN" "$SEED_TOOL" "$INSPECT" "$CYCLE"; do [ -x "$f" ] || { echo "missing $f (build first)" >&2; exit 1; }; done
|
|
|
|
work=$(mktemp -d "${TMPDIR:-/var/tmp}/strawberry-fleet-e2e.XXXXXX")
|
|
pass=0
|
|
ok() { echo "ok $*"; pass=$((pass + 1)); }
|
|
fail() { echo "FAIL $*" >&2; echo "work dir kept: $work" >&2; exit 1; }
|
|
jget() { python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$1" "$2"; }
|
|
nice_run() { nice -n 19 "$@"; }
|
|
|
|
central="$work/central"
|
|
bundles="$central/bundles"
|
|
mkdir -p "$bundles" "$central/work" "$central/dist"
|
|
|
|
# ---- sensors: seed + two exports each (full, then incremental) ----------------------------------
|
|
for i in $(seq 1 "$NODES"); do
|
|
n="$work/node$i"
|
|
mkdir -p "$n/model" "$n/egress"
|
|
cat >"$n/strawberry.conf" <<EOF
|
|
node_id=e2e-node-$i
|
|
model_dir=$n/model
|
|
export_dir=$n/egress
|
|
export_retention_hours=0
|
|
export_max_mb=0
|
|
control_socket_path=
|
|
EOF
|
|
nice_run "$SEED_TOOL" "$n/store" --node "node$i" --execs $((300 + 50 * i)) --seed "$i" --model-dir "$n/model" >/dev/null
|
|
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --since auto --json >"$n/export1.json"
|
|
[ -f "$n/egress/.export-cursor" ] || fail "node$i: cursor not written after export 1"
|
|
sleep 0.01 # distinct created_unix_ns for the second bundle
|
|
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --since auto --json >"$n/export2.json"
|
|
w1=$(jget "$n/export1.json" windows); w2=$(jget "$n/export2.json" windows)
|
|
[ "$w1" -gt 0 ] || fail "node$i: export 1 has no windows"
|
|
[ "$w2" -lt "$w1" ] || fail "node$i: incremental export ($w2) not smaller than full ($w1)"
|
|
[ "$(find "$n/egress" -name 'export-*.sbx' | wc -l)" -eq 2 ] || fail "node$i: expected 2 bundles in egress"
|
|
# Salt pull (simulated): bundles only; the cursor stays on the node.
|
|
command cp -f "$n"/egress/export-*.sbx "$bundles/"
|
|
done
|
|
ok "$NODES sensors exported a full + an incremental bundle each (cursor chain)"
|
|
|
|
# Noise a real bundle_dir accumulates: a stray file and an old overlay; both must be ignored.
|
|
echo "not a bundle" >"$bundles/README.txt"
|
|
echo "cursor" >"$bundles/.export-cursor"
|
|
|
|
# ---- central cycle (the script Salt runs) --------------------------------------------------------
|
|
STRAWBERRY_BIN="$BIN" STRAWBERRY_CYCLE_ALLOW_NONROOT=1 EPOCHS="${FLEET_EPOCHS:-10}" MAX_NODE_SHARE=1 \
|
|
nice_run "$CYCLE" "$bundles" "$central/work" "$central/dist" >"$work/cycle.log" 2>&1 \
|
|
|| { cat "$work/cycle.log" >&2; fail "central cycle failed"; }
|
|
run=$(find "$central/work" -maxdepth 1 -name 'run-*' | sort | tail -1)
|
|
t1="$run/train-seed1.json"
|
|
[ "$(jget "$t1" nodes)" -eq "$NODES" ] || fail "train merged $(jget "$t1" nodes) nodes, want $NODES"
|
|
[ "$(jget "$t1" bundles_read)" -eq $((2 * NODES)) ] || fail "train read $(jget "$t1" bundles_read) bundles"
|
|
[ "$(jget "$t1" bundles_failed)" -eq 0 ] || fail "stray files were fed to the merge"
|
|
[ "$(jget "$t1" duplicate_windows)" -gt 0 ] || fail "incremental re-shipped windows were not deduped"
|
|
ok "fleet train: $NODES nodes, $(jget "$t1" windows) windows, $(jget "$t1" duplicate_windows) duplicates deduped, 0 stray inputs"
|
|
ov="$run/overlay.json"
|
|
[ "$(jget "$ov" nodes)" -eq "$NODES" ] || fail "overlay merged $(jget "$ov" nodes) nodes"
|
|
[ "$(jget "$ov" bundles_superseded)" -eq "$NODES" ] || fail "overlay superseded $(jget "$ov" bundles_superseded), want $NODES"
|
|
ok "fleet overlay: newest bundle per node ($NODES superseded)"
|
|
for f in shadow.sbm fleet-overlay.sbo; do
|
|
(cd "$central/dist" && sha256sum -c --quiet "$f.sha256") || fail "dist sidecar mismatch: $f"
|
|
done
|
|
[ ! -e "$central/dist/promoted.sbm" ] || fail "the cycle produced a promoted model"
|
|
ok "dist: shadow + overlay published with valid sha256 sidecars, nothing promoted"
|
|
|
|
# Determinism: a second cycle over the same bundles gives the same seed-1 model weights.
|
|
sleep 1 # distinct run stamp
|
|
STRAWBERRY_BIN="$BIN" STRAWBERRY_CYCLE_ALLOW_NONROOT=1 EPOCHS="${FLEET_EPOCHS:-10}" MAX_NODE_SHARE=1 \
|
|
nice_run "$CYCLE" "$bundles" "$central/work" "$central/dist2" >/dev/null 2>&1 || fail "second cycle failed"
|
|
run2=$(find "$central/work" -maxdepth 1 -name 'run-*' | sort | tail -1)
|
|
[ "$run2" != "$run" ] || fail "second cycle reused the run dir"
|
|
for k in windows positives train test model_auc; do
|
|
[ "$(jget "$run/train-seed1.json" $k)" = "$(jget "$run2/train-seed1.json" $k)" ] || fail "seed 1 not deterministic ($k)"
|
|
done
|
|
cmp -s "$central/dist/fleet-overlay.sbo" "$central/dist2/fleet-overlay.sbo" || fail "overlay not deterministic"
|
|
ok "deterministic: seed-1 training metrics and the overlay bytes repeat"
|
|
|
|
# ---- distribute (Salt push, simulated) + round 2 -------------------------------------------------
|
|
for i in $(seq 1 "$NODES"); do
|
|
n="$work/node$i"
|
|
r1=$("$INSPECT" "$(find "$n/egress" -name 'export-*.sbx' | sort | head -1)")
|
|
case "$r1" in *" fleet_feat=0"*) ;; *) fail "node$i round 1 already has fleet bits: $r1" ;; esac
|
|
install -m 0600 "$central/dist/fleet-overlay.sbo" "$n/model/fleet-overlay.sbo"
|
|
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --out "$n/round2.sbx" >/dev/null
|
|
r2=$("$INSPECT" "$n/round2.sbx")
|
|
ff=$(echo "$r2" | sed -n 's/.*fleet_feat=\([0-9]*\).*/\1/p')
|
|
[ "${ff:-0}" -gt 0 ] || fail "node$i round 2 has no fleet bits: $r2"
|
|
done
|
|
ok "round 2: every node's export carries fleet rarity features from the distributed overlay"
|
|
|
|
echo "fleet e2e: ALL PASSED ($pass checks, $NODES nodes)"
|