strawberry/scripts/fleet_e2e.sh
trvon f4adc12a74
gate: judge the model against a calibrated rarity comparator
The rarity-greedy comparator acted on every evaluated window, so it spent
each hour's budget on the first ~15 windows and scored 0 hits in every
cell. Criteria 2 and 3 of the Stage 4 gate were void against it.

rarity_calibrated keeps the same rarity value, chooser, costs and budget,
and adds a per-node threshold frozen before the evaluated span: the m-th
largest value over the trailing 24 h of label-free history between the
warm-up and the --eval-after bound, with m set so expected spend matches
the hourly budget. Without that history the comparator is uncalibrated
and the gate fails by name; it never falls back to another rule.

rarity_greedy (the old first-come rule) and rarity_hindsight (per-hour
ordering, not causal) are reported as diagnostics and never judged.
fleet gate pools counts, keeps thresholds per node, and fails a pooled
cell when any node is uncalibrated or lacks the comparator.

Contract change: in `fleet gate --json`, rarity_hits and
rarity_cpu_seconds now describe the calibrated comparator; the old
numbers are rarity_greedy_hits and rarity_greedy_cpu_seconds.

Pre-registration: docs/evaluation/stage4-rarity-baseline-preregistration.md,
sha256 4916c95cd2581e3749acfe46c3ccf827aa5ce98c06150b49b978fc70da013f95,
indexed in YAMS at 2026-11-20T18:55:48Z, before the code. This commit is
dated after the registration time in that document (2026-11-21T00:00Z);
the YAMS record is what predates it.

Checked: meson suite 58/58, scripts/fleet_e2e.sh, clang-tidy and the
strict analyzer on the changed TUs. Not run: fold on the changed TUs,
make fuzz, the live Docker tests and the perf KPI. The comparator has
not been run on validation-range data.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-20 19:20:55 -07:00

204 lines
12 KiB
Bash
Executable file

#!/usr/bin/env bash
# Fleet end-to-end (Level A: one host, no Docker, no root). Drives the REAL binary and the REAL central
# cycle script through a full hub-and-spoke round with N synthetic sensor nodes:
#
# seed N stores (distinct node_id) -> export x2 per node into an egress dir (--since auto chain)
# -> "Salt pull" (copy) every bundle into one central bundle_dir (+ a stray file and an overlay)
# -> strawberry-central-cycle.sh (fleet train x3 seeds + fleet overlay + sha256 sidecars + publish)
# -> distribute fleet-overlay.sbo into each node's model_dir -> round-2 export carries fleet_bits
#
# Asserts: every node is merged; duplicate windows from the repeated export are deduped (not retrained);
# the overlay takes one (newest) bundle per node; non-bundle files are ignored; the cycle never promotes;
# training is deterministic per seed; round-1 features have no fleet bits and round-2 features do.
# The gate step asserts five checks per cell (promotion_gate's own count), the model digest and
# training range, and that every cell publishes `in_sample`. First it asserts that an in-sample
# evidence set (replaying the same synthetic history the models trained on) never passes; then it
# appends strictly later windows and replays with `--eval-after auto`, asserting that all 9 cells
# evaluate out-of-sample (`in_sample=false`, `eval_first_ns > training_last_ns`) against a calibrated
# rarity comparator (uncalibrated without a bound, calibrated on every node with one).
#
# Usage: scripts/fleet_e2e.sh [builddir] (default builddir). Exit 0 = all passed.
set -euo pipefail
BD="${1:-builddir}"
BIN="$PWD/$BD/src/strawberry"
SEED_TOOL="$PWD/$BD/tests/strawberry-fleet-seed"
INSPECT="$PWD/$BD/tests/strawberry-fleet-inspect"
CYCLE="$PWD/deploy/salt/files/strawberry-central-cycle.sh"
NODES="${FLEET_NODES:-3}"
for f in "$BIN" "$SEED_TOOL" "$INSPECT" "$CYCLE"; do [ -x "$f" ] || { echo "missing $f (build first)" >&2; exit 1; }; done
work=$(mktemp -d "${TMPDIR:-/var/tmp}/strawberry-fleet-e2e.XXXXXX")
pass=0
ok() { echo "ok $*"; pass=$((pass + 1)); }
fail() { echo "FAIL $*" >&2; echo "work dir kept: $work" >&2; exit 1; }
jget() { python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$1" "$2"; }
nice_run() { nice -n 19 "$@"; }
central="$work/central"
bundles="$central/bundles"
mkdir -p "$bundles" "$central/work" "$central/dist"
# ---- sensors: seed + two exports each (full, then incremental) ----------------------------------
for i in $(seq 1 "$NODES"); do
n="$work/node$i"
mkdir -p "$n/model" "$n/egress"
cat >"$n/strawberry.conf" <<EOF
node_id=e2e-node-$i
model_dir=$n/model
export_dir=$n/egress
export_retention_hours=0
export_max_mb=0
control_socket_path=
EOF
nice_run "$SEED_TOOL" "$n/store" --node "node$i" --execs $((3600 + 100 * i)) --seed "$i" --model-dir "$n/model" >/dev/null
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --since auto --json >"$n/export1.json"
[ -f "$n/egress/.export-cursor" ] || fail "node$i: cursor not written after export 1"
sleep 0.01 # distinct created_unix_ns for the second bundle
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --since auto --json >"$n/export2.json"
w1=$(jget "$n/export1.json" windows); w2=$(jget "$n/export2.json" windows)
[ "$w1" -gt 0 ] || fail "node$i: export 1 has no windows"
[ "$w2" -lt "$w1" ] || fail "node$i: incremental export ($w2) not smaller than full ($w1)"
[ "$(find "$n/egress" -name 'export-*.sbx' | wc -l)" -eq 2 ] || fail "node$i: expected 2 bundles in egress"
# Salt pull (simulated): bundles only; the cursor stays on the node.
command cp -f "$n"/egress/export-*.sbx "$bundles/"
done
ok "$NODES sensors exported a full + an incremental bundle each (cursor chain)"
# Noise a real bundle_dir accumulates: a stray file and an old overlay; both must be ignored.
echo "not a bundle" >"$bundles/README.txt"
echo "cursor" >"$bundles/.export-cursor"
# ---- central cycle (the script Salt runs) --------------------------------------------------------
STRAWBERRY_BIN="$BIN" STRAWBERRY_CYCLE_ALLOW_NONROOT=1 EPOCHS="${FLEET_EPOCHS:-10}" MAX_NODE_SHARE=1 \
nice_run "$CYCLE" "$bundles" "$central/work" "$central/dist" >"$work/cycle.log" 2>&1 \
|| { cat "$work/cycle.log" >&2; fail "central cycle failed"; }
run=$(find "$central/work" -maxdepth 1 -name 'run-*' | sort | tail -1)
t1="$run/train-seed1.json"
[ "$(jget "$t1" nodes)" -eq "$NODES" ] || fail "train merged $(jget "$t1" nodes) nodes, want $NODES"
[ "$(jget "$t1" bundles_read)" -eq $((2 * NODES)) ] || fail "train read $(jget "$t1" bundles_read) bundles"
[ "$(jget "$t1" bundles_failed)" -eq 0 ] || fail "stray files were fed to the merge"
[ "$(jget "$t1" duplicate_windows)" -gt 0 ] || fail "incremental re-shipped windows were not deduped"
ok "fleet train: $NODES nodes, $(jget "$t1" windows) windows, $(jget "$t1" duplicate_windows) duplicates deduped, 0 stray inputs"
ov="$run/overlay.json"
[ "$(jget "$ov" nodes)" -eq "$NODES" ] || fail "overlay merged $(jget "$ov" nodes) nodes"
[ "$(jget "$ov" bundles_superseded)" -eq "$NODES" ] || fail "overlay superseded $(jget "$ov" bundles_superseded), want $NODES"
ok "fleet overlay: newest bundle per node ($NODES superseded)"
for f in shadow.sbm fleet-overlay.sbo; do
(cd "$central/dist" && sha256sum -c --quiet "$f.sha256") || fail "dist sidecar mismatch: $f"
done
[ ! -e "$central/dist/promoted.sbm" ] || fail "the cycle produced a promoted model"
ok "dist: shadow + overlay published with valid sha256 sidecars, nothing promoted"
# Determinism: a second cycle over the same bundles gives the same seed-1 model weights.
sleep 1 # distinct run stamp
STRAWBERRY_BIN="$BIN" STRAWBERRY_CYCLE_ALLOW_NONROOT=1 EPOCHS="${FLEET_EPOCHS:-10}" MAX_NODE_SHARE=1 \
nice_run "$CYCLE" "$bundles" "$central/work" "$central/dist2" >/dev/null 2>&1 || fail "second cycle failed"
run2=$(find "$central/work" -maxdepth 1 -name 'run-*' | sort | tail -1)
[ "$run2" != "$run" ] || fail "second cycle reused the run dir"
for k in windows positives train test model_auc; do
[ "$(jget "$run/train-seed1.json" $k)" = "$(jget "$run2/train-seed1.json" $k)" ] || fail "seed 1 not deterministic ($k)"
done
cmp -s "$central/dist/fleet-overlay.sbo" "$central/dist2/fleet-overlay.sbo" || fail "overlay not deterministic"
ok "deterministic: seed-1 training metrics and the overlay bytes repeat"
# ---- distribute (Salt push, simulated) + round 2 -------------------------------------------------
for i in $(seq 1 "$NODES"); do
n="$work/node$i"
r1=$("$INSPECT" "$(find "$n/egress" -name 'export-*.sbx' | sort | head -1)")
case "$r1" in *" fleet_feat=0"*) ;; *) fail "node$i round 1 already has fleet bits: $r1" ;; esac
install -m 0600 "$central/dist/fleet-overlay.sbo" "$n/model/fleet-overlay.sbo"
nice_run "$BIN" --config "$n/strawberry.conf" export "$n/store" --out "$n/round2.sbx" >/dev/null
r2=$("$INSPECT" "$n/round2.sbx")
ff=$(echo "$r2" | sed -n 's/.*fleet_feat=\([0-9]*\).*/\1/p')
[ "${ff:-0}" -gt 0 ] || fail "node$i round 2 has no fleet bits: $r2"
done
ok "round 2: every node's export carries fleet rarity features from the distributed overlay"
# ---- fleet promotion gate: each node evaluates the 3 seed models on its own raw history ------------
reports="$central/gate-reports"
mkdir -p "$reports"
for i in $(seq 1 "$NODES"); do
n="$work/node$i"
eval_args=()
for seed in 1 2 3; do eval_args+=(--eval-model "$run/model-seed$seed.sbm"); done
nice_run "$BIN" --config "$n/strawberry.conf" replay "$n/store" --policy-eval "${eval_args[@]}" --periods 3 \
--inject "$PWD/config/traces/standard.trace" --inject-copies 6 --warmup 0 --json >"$reports/node$i.json" \
|| fail "node$i: gate replay failed"
done
rc=0
nice_run "$BIN" fleet gate "$reports" --json >"$work/gate.json" || rc=$?
cells=$(jget "$work/gate.json" cells | python3 -I -c 'import ast,sys; print(len(ast.literal_eval(sys.stdin.read())))')
[ "$cells" -eq 9 ] || fail "fleet gate produced $cells cells, want 9 (3 seeds x 3 periods)"
python3 -I - "$work/gate.json" "$NODES" <<'PY' || fail "gate cells not pooled over every node / no positives / evidence overstated"
import json, sys
g = json.load(open(sys.argv[1])); n = int(sys.argv[2])
assert g["reports"] == n, g["reports"]
assert g["refused"] is False, g["errors"]
for c in g["cells"]:
assert c["nodes"] == n and c["extra_positives"] > 0, c
# promotion_gate() emits five checks per cell (extra positives, baseline healthy, efficiency,
# recall, out-of-sample). Assert at least those, not a frozen count, so a lane that adds a
# completeness/mixed-set check does not silently break this harness.
assert len(c["checks"]) >= 5, c["checks"]
assert c["model_sha256"] and c["training_range_known"], c
# Every cell must publish whether it evaluated windows inside the model's training range, so an
# in-sample set is visible in the evidence instead of being hidden behind a verdict.
assert isinstance(c["in_sample"], bool), c
# Today's e2e trains every seed on the same synthetic history it replays, so every cell is
# in-sample. The gate must not pass such a set, and no in-sample cell may pass - that is the
# honest-evidence invariant, and it survives the eval-bound lane, whose point is that a cell can
# then be out-of-sample and (if the criteria hold) legitimately pass.
in_sample = [c for c in g["cells"] if c["in_sample"]]
assert in_sample, "no in-sample cell reported: the e2e replays the history it trained on"
assert g["passed"] is False, "gate passed an in-sample evidence set"
for c in in_sample:
assert c["passed"] is False, c
assert any(ch.startswith("FAIL: out-of-sample") for ch in c["checks"]), c["checks"]
# No evaluation bound means no label-free history to calibrate the comparator on.
assert c["baseline_uncalibrated_nodes"] == c["nodes"], c
assert any(ch.startswith("FAIL: baseline uncalibrated") for ch in c["checks"]), c["checks"]
PY
verdict=$(jget "$work/gate.json" passed)
if [ "$verdict" = "True" ]; then [ "$rc" -eq 0 ] || fail "gate passed but exit $rc"; else [ "$rc" -ne 0 ] || fail "gate failed but exit 0"; fi
ok "fleet gate: 3 seeds x 3 periods pooled over $NODES nodes, verdict passed=$verdict (exit $rc matches; in-sample cells are reported, not hidden)"
# ---- out-of-sample gate: append strictly later windows per node, replay with --eval-after auto ---
oos_reports="$central/gate-oos-reports"
mkdir -p "$oos_reports"
for i in $(seq 1 "$NODES"); do
n="$work/node$i"
nice_run "$SEED_TOOL" "$n/store" --node "node$i" --execs $((3600 + 100 * i)) --start-offset-s 4500 --seed $((100 + i)) >/dev/null
eval_args=()
for seed in 1 2 3; do eval_args+=(--eval-model "$run/model-seed$seed.sbm"); done
nice_run "$BIN" --config "$n/strawberry.conf" replay "$n/store" --policy-eval "${eval_args[@]}" --eval-after auto --periods 3 \
--inject "$PWD/config/traces/standard.trace" --inject-copies 6 --warmup 0 --json >"$oos_reports/node$i.json" \
|| fail "node$i: out-of-sample gate replay failed"
done
oos_rc=0
nice_run "$BIN" fleet gate "$oos_reports" --json >"$work/gate-oos.json" || oos_rc=$?
python3 -I - "$work/gate-oos.json" "$NODES" <<'PY' || fail "out-of-sample gate provenance check failed"
import json, sys
g = json.load(open(sys.argv[1])); n = int(sys.argv[2])
assert g["reports"] == n, g["reports"]
assert g["complete"] is True, g
assert g["refused"] is False, g["errors"]
assert len(g["cells"]) == 9, len(g["cells"])
for c in g["cells"]:
assert c["nodes"] == n and c["extra_positives"] > 0, c
assert c["training_range_known"] is True and c["provenance_incomplete"] is False, c
assert c["in_sample"] is False, c
assert c["eval_first_ns"] > c["training_last_ns"], c
assert any(ch.startswith("ok: out-of-sample:") for ch in c["checks"]), c["checks"]
# The comparator is the calibrated rarity policy, frozen per node from the history before the
# bound; the first-come rarity_greedy is reported beside it and never judged.
assert c["baseline"] == "rarity_calibrated", c
assert c["baseline_uncalibrated_nodes"] == 0 and c["baseline_calibration_windows"] > 0, c
assert not any("baseline uncalibrated" in ch for ch in c["checks"]), c["checks"]
assert "rarity_greedy_hits" in c and "rarity_hindsight_hits" in c, c
PY
oos_verdict=$(jget "$work/gate-oos.json" passed)
if [ "$oos_verdict" = "True" ]; then [ "$oos_rc" -eq 0 ] || fail "oos gate passed but exit $oos_rc"; else [ "$oos_rc" -ne 0 ] || fail "oos gate failed but exit 0"; fi
ok "fleet gate (out-of-sample): 3 seeds x 3 disjoint periods over $NODES nodes, in_sample=false, passed=$oos_verdict (exit $oos_rc matches)"
echo "fleet e2e: ALL PASSED ($pass checks, $NODES nodes)"