The pre-registered gate (docs/evaluation/stage4-promotion-gate.md) asks for the criteria to hold on 3 non-overlapping periods x 3 training seeds; until now the tool reported a single run and the operator recorded the rest. This produces the 9 results for a fleet, with the criteria unchanged: - replay --policy-eval --eval-model FILE (repeatable) --periods N: one pass evaluates every model on every equal time slice of the ranked history; each (model, period) cell simulates the model and rarity-greedy with their own hourly budgets exactly as policy_eval does (policy_eval.cells). With one model equal to --model and one period the cell equals the single-model evaluation (test R11b); periods partition windows and positives. - strawberry fleet gate DIR: pools the cells of every node report (windows, positives, inspections, CPU-s, hits, refusals) per (model, period) and applies learn::promotion_gate unchanged to each; passes only if all pass. Malformed / cell-less reports are skipped and listed. - Evaluation stays on the node (raw telemetry never leaves it): the central cycle now also publishes all three seed models to dist/gate/; strawberry.gate (sensors) evaluates them with the pre-registered evaluation traces; the operator-run orch.strawberry_gate pushes the reports back and pools them; evidence under central.work_dir/gate-<stamp>/. Nothing promotes. - docs/evaluation/stage4-fleet-gate.md: the procedure (new dated document; criteria and numbers unchanged). Validated: test_fleet_gate (pooling sums, pooling reaching 10 positives neither node has alone, one failing cell fails the fleet, malformed reports skipped), R11b, Salt render checks (no models -> refusal; 3 models x 3 periods; orchestration pools), and scripts/fleet_e2e.sh end to end: 3 nodes -> 9 pooled cells with 28-50 positives each; the untrained synthetic models correctly do not pass (exit code matches the verdict). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
5 KiB
Bash
Executable file
113 lines
5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Strawberry central learning cycle (runs on the central node, as root, niced).
|
|
#
|
|
# strawberry-central-cycle.sh <bundle_dir> <work_dir> <dist_dir>
|
|
#
|
|
# bundle_dir directory of pulled sensor bundles (export-*.sbx)
|
|
# work_dir per-run outputs are kept here: <work_dir>/run-<UTC stamp>/
|
|
# dist_dir the Salt master's file_roots path (salt://strawberry/dist/) that
|
|
# strawberry.distribute serves to the sensors
|
|
#
|
|
# Env (optional): STRAWBERRY_BIN (default /usr/local/bin/strawberry), EPOCHS (30),
|
|
# MAX_NODE_SHARE (0.25).
|
|
#
|
|
# It trains with three seeds (1,2,3) and keeps all three outputs as EVIDENCE for the
|
|
# operator; seed 1's model is distributed as shadow.sbm (no auto-selection). It builds
|
|
# the fleet rarity overlay, writes <file>.sha256 sidecars and copies the pair into
|
|
# dist_dir. It NEVER runs the model-promotion verb: that stays an operator step.
|
|
set -euo pipefail
|
|
|
|
if [ "$(id -u)" -ne 0 ] && [ "${STRAWBERRY_CYCLE_ALLOW_NONROOT:-}" != "1" ]; then
|
|
echo "run as root (reads root-only model state)" >&2
|
|
exit 2
|
|
fi
|
|
bundle_dir=${1:?usage: $0 <bundle_dir> <work_dir> <dist_dir>}
|
|
work_dir=${2:?usage: $0 <bundle_dir> <work_dir> <dist_dir>}
|
|
dist_dir=${3:?usage: $0 <bundle_dir> <work_dir> <dist_dir>}
|
|
bin=${STRAWBERRY_BIN:-/usr/local/bin/strawberry}
|
|
epochs=${EPOCHS:-30}
|
|
share=${MAX_NODE_SHARE:-0.25}
|
|
|
|
[ -x "$bin" ] || { echo "missing $bin" >&2; exit 1; }
|
|
[ -d "$bundle_dir" ] || { echo "no bundle dir: $bundle_dir" >&2; exit 1; }
|
|
if ! compgen -G "$bundle_dir/export-*.sbx" >/dev/null && ! compgen -G "$bundle_dir/*.sbx" >/dev/null; then
|
|
echo "no bundles (*.sbx) in $bundle_dir" >&2
|
|
exit 1
|
|
fi
|
|
|
|
umask 077
|
|
stamp=$(date -u +%Y%m%dT%H%M%SZ)
|
|
run="$work_dir/run-$stamp"
|
|
mkdir -p "$run" "$dist_dir"
|
|
|
|
nicer=(nice -n 19)
|
|
command -v ionice >/dev/null 2>&1 && nicer=(ionice -c3 nice -n 19)
|
|
|
|
# The overlay needs no labels: build it first so a label-starved fleet still gets fleet rarity.
|
|
"${nicer[@]}" "$bin" fleet overlay "$bundle_dir" --out "$run/fleet-overlay.sbo" --json >"$run/overlay.json"
|
|
test -s "$run/fleet-overlay.sbo" || { echo "overlay missing" >&2; exit 1; }
|
|
|
|
# Training refuses with too few teacher labels (a young fleet): that is a normal outcome, not a
|
|
# failure. Then no model is published (the sensors keep their current shadow) and only the overlay
|
|
# goes out. Any other training error fails the cycle.
|
|
model=trained
|
|
for seed in 1 2 3; do
|
|
if ! "${nicer[@]}" "$bin" fleet train "$bundle_dir" --out "$run/model-seed$seed.sbm" --seed "$seed" --epochs "$epochs" --max-node-share "$share" --json >"$run/train-seed$seed.json" 2>"$run/train-seed$seed.err"; then
|
|
if grep -q "not enough teacher labels" "$run/train-seed$seed.err"; then
|
|
model=refused
|
|
echo "training refused (seed $seed): $(cat "$run/train-seed$seed.err")" >&2
|
|
break
|
|
fi
|
|
cat "$run/train-seed$seed.err" >&2
|
|
exit 1
|
|
fi
|
|
test -s "$run/model-seed$seed.sbm" || { echo "seed $seed produced no model" >&2; exit 1; }
|
|
done
|
|
|
|
# Seed 1 is the distributed shadow model; seeds 2 and 3 are comparison evidence only.
|
|
publish=(fleet-overlay.sbo)
|
|
if [ "$model" = trained ]; then
|
|
cp "$run/model-seed1.sbm" "$run/shadow.sbm"
|
|
publish=(shadow.sbm fleet-overlay.sbo)
|
|
fi
|
|
|
|
{
|
|
printf '{"run":"%s","bundle_dir":"%s","epochs":%s,"max_node_share":%s,' "$stamp" "$bundle_dir" "$epochs" "$share"
|
|
if [ "$model" = trained ]; then printf '"model":"trained","distributed":"shadow.sbm=seed1",'
|
|
else printf '"model":"refused (not enough teacher labels)","distributed":"overlay only",'; fi
|
|
printf '"seeds":['
|
|
for seed in 1 2 3; do
|
|
[ "$seed" -gt 1 ] && printf ','
|
|
printf '{"seed":%s,"model":"model-seed%s.sbm","train":"train-seed%s.json"}' "$seed" "$seed" "$seed"
|
|
done
|
|
printf ']}\n'
|
|
} >"$run/summary.json"
|
|
|
|
for f in "${publish[@]}"; do
|
|
(cd "$run" && sha256sum "$f" >"$f.sha256")
|
|
done
|
|
|
|
# Atomic publish: stage then rename, model and overlay each with its sidecar.
|
|
for f in "${publish[@]}"; do
|
|
install -m 0600 "$run/$f" "$dist_dir/.$f.new"
|
|
install -m 0600 "$run/$f.sha256" "$dist_dir/.$f.sha256.new"
|
|
mv -f "$dist_dir/.$f.new" "$dist_dir/$f"
|
|
mv -f "$dist_dir/.$f.sha256.new" "$dist_dir/$f.sha256"
|
|
done
|
|
|
|
# Gate candidates: all three seed models go to dist/gate/ so `orch.strawberry_gate` can have every
|
|
# sensor evaluate them on its own history (3 periods x 3 seeds). Evaluation only: sensors never
|
|
# load these as their shadow model, and nothing here activates a model.
|
|
if [ "$model" = trained ]; then
|
|
install -d -m 0700 "$dist_dir/gate"
|
|
for seed in 1 2 3; do
|
|
f=model-seed$seed.sbm
|
|
(cd "$run" && sha256sum "$f" >"$f.sha256")
|
|
install -m 0600 "$run/$f" "$dist_dir/gate/.$f.new"
|
|
install -m 0600 "$run/$f.sha256" "$dist_dir/gate/.$f.sha256.new"
|
|
mv -f "$dist_dir/gate/.$f.new" "$dist_dir/gate/$f"
|
|
mv -f "$dist_dir/gate/.$f.sha256.new" "$dist_dir/gate/$f.sha256"
|
|
done
|
|
fi
|
|
|
|
echo "cycle ok ($model): $run (published ${publish[*]} to $dist_dir; review seeds, then the operator decides)"
|