226 lines
9.9 KiB
Bash
Executable file
226 lines
9.9 KiB
Bash
Executable file
#!/bin/sh
|
|
# Quiet-host authentic 64K full-graph gate for a BK128 candidate advanced by the operator gate.
|
|
set -eu
|
|
|
|
[ "$#" -eq 3 ] || {
|
|
echo "usage: $0 OPERATOR_GATE_DIR MODEL.gguf OUTPUT_DIRECTORY" >&2
|
|
exit 2
|
|
}
|
|
operator_dir=$1
|
|
model=$2
|
|
output=$3
|
|
build_dir=${GEM_BQ8_GRAPH_GATE_BUILD:-build}
|
|
harness="$build_dir/gem-long-prefix-ab"
|
|
gem="$build_dir/gem"
|
|
max_load_per_cpu=${GEM_BQ8_GRAPH_GATE_MAX_LOAD_PER_CPU:-1.0}
|
|
max_short_busy=${GEM_BQ8_GRAPH_GATE_MAX_SHORT_WINDOW_BUSY_CPU_PERCENT:-10.0}
|
|
monitor_seconds=${GEM_BQ8_GRAPH_GATE_MONITOR_SECONDS:-30}
|
|
|
|
[ ! -e "$output" ] || { echo "gem: output already exists: $output" >&2; exit 2; }
|
|
[ -r "$model" ] || { echo "gem: model is not readable: $model" >&2; exit 2; }
|
|
[ -x "$harness" ] && [ -x "$gem" ] || {
|
|
echo "gem: required native binaries are missing from $build_dir" >&2
|
|
exit 2
|
|
}
|
|
for tool in jq shasum otool sw_vers sysctl pmset top caffeinate; do
|
|
command -v "$tool" >/dev/null 2>&1 || { echo "gem: required tool is missing: $tool" >&2; exit 2; }
|
|
done
|
|
if [ "$(uname -s)" != Darwin ]; then
|
|
echo "gem: BK128 evaluated-graph gate requires macOS" >&2
|
|
exit 2
|
|
fi
|
|
for value in "$max_load_per_cpu" "$max_short_busy"; do
|
|
awk -v value="$value" 'BEGIN {exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value > 0)}' || {
|
|
echo "gem: host thresholds must be positive numbers" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
case $monitor_seconds in ''|*[!0-9]*) echo "gem: monitor interval must be a positive integer" >&2; exit 2 ;; esac
|
|
[ "$monitor_seconds" -gt 0 ] || { echo "gem: monitor interval must be positive" >&2; exit 2; }
|
|
if command -v pgrep >/dev/null 2>&1 && pgrep -f 'build/gem serve' >/dev/null 2>&1; then
|
|
echo "gem: refusing to overlap a production server Metal workload" >&2
|
|
exit 1
|
|
fi
|
|
|
|
verified_operator=$(mktemp /private/tmp/gem-bk128-operator-decision.XXXXXX)
|
|
cleanup_temp() { rm -f "$verified_operator"; }
|
|
trap cleanup_temp EXIT INT TERM HUP
|
|
scripts/analyze-bq8-bk128-gate.sh "$operator_dir" "$verified_operator"
|
|
jq -e '.schema_version == 2 and
|
|
.screening.advance_to_evaluated_graph_gate == true and
|
|
.screening.promotion_eligible == false and
|
|
.screening.decision == "advance_to_evaluated_graph_gate"' \
|
|
"$verified_operator" >/dev/null || {
|
|
echo "gem: operator gate did not advance BK128" >&2
|
|
exit 1
|
|
}
|
|
|
|
mkdir -p "$output"
|
|
cp "$verified_operator" "$output/operator-decision.json"
|
|
date -u '+%Y-%m-%dT%H:%M:%SZ' > "$output/run-start-utc.txt"
|
|
sw_vers > "$output/macos-version.txt"
|
|
model_sha=${GEM_MODEL_SHA256:-}
|
|
if [ -n "$model_sha" ]; then
|
|
case $model_sha in *[!0-9a-fA-F]*) echo "gem: GEM_MODEL_SHA256 must be 64 hexadecimal digits" >&2; exit 2 ;; esac
|
|
[ "${#model_sha}" -eq 64 ] || {
|
|
echo "gem: GEM_MODEL_SHA256 must be 64 hexadecimal digits" >&2
|
|
exit 2
|
|
}
|
|
model_sha=$(printf '%s' "$model_sha" | tr 'A-F' 'a-f')
|
|
else
|
|
model_sha=$(shasum -a 256 "$model" | awk '{print $1}')
|
|
fi
|
|
printf '%s %s\n' "$model_sha" "$model" > "$output/model-sha256.txt"
|
|
shasum -a 256 "$harness" "$gem" src/long_prefix_ab.c src/metal.m src/metal_layer.c \
|
|
src/inference.c scripts/run-bq8-bk128-graph-gate.sh \
|
|
scripts/analyze-bq8-bk128-graph-gate.sh > "$output/sha256.txt"
|
|
otool -L "$harness" "$gem" > "$output/native-linkage.txt"
|
|
if grep -Eiq 'lib(llama|ggml)' "$output/native-linkage.txt"; then
|
|
echo "gem: native graph gate unexpectedly links llama/ggml" >&2
|
|
exit 1
|
|
fi
|
|
"$gem" print-config --model "$model" --context 65568 --kv-type f32 \
|
|
> "$output/print-config.json"
|
|
jq -e '.engine == "gem-native-c-metal" and .model_size == 31 and .layers == 60 and
|
|
.context == 65568 and .kv_type == "f32" and .matrix_type == "Q4_0" and
|
|
.prefill_batch == 32 and .attention_prefill_rows == 32 and
|
|
(.attention_kernel | contains("bq8-bk64"))' "$output/print-config.json" >/dev/null
|
|
jq -n --arg model "$model" --arg model_sha "$model_sha" \
|
|
--arg build_dir "$build_dir" --argjson max_load "$max_load_per_cpu" \
|
|
--argjson max_busy "$max_short_busy" --argjson monitor "$monitor_seconds" '
|
|
{schema_version: 1, engine: "gem-native-c-metal", model: $model,
|
|
model_sha256: $model_sha,
|
|
axis: {name: "global_prefill_cache_tile_width", baseline: 64, candidate: 128},
|
|
operator_gate_required: true, prefix_tokens: 65536, extension_tokens: 32,
|
|
repetitions: 3, kv_type: "f32", prefix_mode: "evaluated-prompt",
|
|
baseline_selector: 19, candidate_selector: 23,
|
|
minimum_improvement_pct: 3, minimum_logit_correlation: 0.9999,
|
|
maximum_logit_nmse: 0.0001, paired_confidence_level_pct: 95,
|
|
paired_t_critical_df2: 4.303,
|
|
host_policy: {max_load_per_logical_cpu: $max_load,
|
|
max_short_window_busy_cpu_percent: $max_busy,
|
|
requires_ac_power: true, requires_no_thermal_warning: true,
|
|
monitor_interval_seconds: $monitor, abort_after_consecutive_failures: 1,
|
|
sleep_assertion: "caffeinate -i -w benchmark_pid"},
|
|
build_directory: $build_dir}' > "$output/gate-config.json"
|
|
|
|
host_sample()
|
|
{
|
|
phase=$1
|
|
timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
load_average=$(sysctl -n vm.loadavg)
|
|
load_1m=$(printf '%s\n' "$load_average" | awk '{print $2}')
|
|
logical_cpus=$(sysctl -n hw.logicalcpu)
|
|
load_per_cpu=$(awk -v load="$load_1m" -v cpus="$logical_cpus" \
|
|
'BEGIN {printf "%.6f", load / cpus}')
|
|
recent_cpu=$(ps -Ao pcpu= | awk '{sum += $1} END {printf "%.3f", sum}')
|
|
short_busy=$(LC_ALL=C top -l 2 -s 1 -n 0 -stats pid,cpu,command |
|
|
awk -F'[:,%]' '/^CPU usage:/ {idle=$6} END {
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", idle)
|
|
if (idle !~ /^[0-9]+([.][0-9]+)?$/) exit 1
|
|
printf "%.3f", 100 - idle
|
|
}') || return 1
|
|
power=$(LC_ALL=C pmset -g batt 2>&1 || true)
|
|
thermal=$(LC_ALL=C pmset -g therm 2>&1 || true)
|
|
ac=false; case $power in *"Now drawing from 'AC Power'"*) ac=true ;; esac
|
|
no_warning=false
|
|
if printf '%s\n' "$thermal" | grep -Fq 'No thermal warning level has been recorded' &&
|
|
printf '%s\n' "$thermal" | grep -Fq 'No performance warning level has been recorded'; then
|
|
no_warning=true
|
|
elif printf '%s\n' "$thermal" | grep -Eq 'CPU_Scheduler_Limit[[:space:]]*=[[:space:]]*100' &&
|
|
printf '%s\n' "$thermal" | grep -Eq 'CPU_Speed_Limit[[:space:]]*=[[:space:]]*100'; then
|
|
no_warning=true
|
|
fi
|
|
status=pass
|
|
awk -v actual="$load_per_cpu" -v maximum="$max_load_per_cpu" \
|
|
'BEGIN {exit !(actual <= maximum)}' || status=fail
|
|
awk -v actual="$short_busy" -v maximum="$max_short_busy" \
|
|
'BEGIN {exit !(actual <= maximum)}' || status=fail
|
|
[ "$ac" = true ] || status=fail
|
|
[ "$no_warning" = true ] || status=fail
|
|
top_processes=$(ps -Ao pid,pcpu,pmem,etime,comm -r | sed -n '1,24p')
|
|
jq -cn --arg timestamp "$timestamp" --arg phase "$phase" --arg status "$status" \
|
|
--arg load_average "$load_average" --argjson load_1m "$load_1m" \
|
|
--argjson logical_cpus "$logical_cpus" --argjson load_per_cpu "$load_per_cpu" \
|
|
--argjson max_load "$max_load_per_cpu" --argjson recent_cpu "$recent_cpu" \
|
|
--argjson short_busy "$short_busy" --argjson max_busy "$max_short_busy" \
|
|
--arg power "$power" --argjson ac "$ac" --arg thermal "$thermal" \
|
|
--argjson no_warning "$no_warning" --arg top_processes "$top_processes" '
|
|
{schema_version: 2, timestamp_utc: $timestamp, phase: $phase, status: $status,
|
|
overloaded: ($status != "pass"), load_average: $load_average, load_1m: $load_1m,
|
|
logical_cpus: $logical_cpus, load_per_logical_cpu: $load_per_cpu,
|
|
max_load_per_logical_cpu: $max_load,
|
|
recent_aggregate_process_cpu_percent: $recent_cpu,
|
|
short_window_busy_cpu_percent: $short_busy,
|
|
max_short_window_busy_cpu_percent: $max_busy,
|
|
power_state: $power, ac_power: $ac, thermal_state: $thermal,
|
|
no_thermal_warning: $no_warning, top_processes: $top_processes}'
|
|
[ "$status" = pass ]
|
|
}
|
|
|
|
if ! host_sample preflight > "$output/host-preflight.json"; then
|
|
echo "gem: BK128 graph gate rejected the host before evaluated prefix build" >&2
|
|
exit 1
|
|
fi
|
|
|
|
benchmark_pid=
|
|
monitor_pid=
|
|
caffeinate_pid=
|
|
cleanup_run()
|
|
{
|
|
[ -z "$monitor_pid" ] || ! kill -0 "$monitor_pid" 2>/dev/null || kill "$monitor_pid"
|
|
[ -z "$benchmark_pid" ] || ! kill -0 "$benchmark_pid" 2>/dev/null || kill "$benchmark_pid"
|
|
[ -z "$caffeinate_pid" ] || ! kill -0 "$caffeinate_pid" 2>/dev/null || kill "$caffeinate_pid"
|
|
cleanup_temp
|
|
}
|
|
trap cleanup_run EXIT INT TERM HUP
|
|
|
|
monitor_host()
|
|
{
|
|
while kill -0 "$benchmark_pid" 2>/dev/null; do
|
|
if ! host_sample monitor >> "$output/host-monitor.ndjson"; then
|
|
printf '%s\n' "host policy violation during evaluated-prefix run" \
|
|
> "$output/host-monitor.failure.txt"
|
|
kill "$benchmark_pid" 2>/dev/null || true
|
|
return 1
|
|
fi
|
|
sleep "$monitor_seconds"
|
|
done
|
|
}
|
|
|
|
GEM_LONG_PREFIX_CANDIDATE=bk128 "$harness" "$model" 65536 3 \
|
|
> "$output/result.json.tmp" 2> "$output/run.log" &
|
|
benchmark_pid=$!
|
|
caffeinate -i -w "$benchmark_pid" &
|
|
caffeinate_pid=$!
|
|
monitor_host &
|
|
monitor_pid=$!
|
|
set +e
|
|
wait "$benchmark_pid"
|
|
benchmark_status=$?
|
|
kill "$monitor_pid" 2>/dev/null
|
|
wait "$monitor_pid" 2>/dev/null
|
|
kill "$caffeinate_pid" 2>/dev/null
|
|
wait "$caffeinate_pid" 2>/dev/null
|
|
set -e
|
|
benchmark_pid=
|
|
monitor_pid=
|
|
caffeinate_pid=
|
|
host_sample postflight > "$output/host-postflight.json" || true
|
|
[ "$benchmark_status" -eq 0 ] || {
|
|
echo "gem: BK128 evaluated-prefix harness failed; see $output/run.log" >&2
|
|
exit 1
|
|
}
|
|
[ ! -e "$output/host-monitor.failure.txt" ] || {
|
|
echo "gem: host policy changed during BK128 evaluated-prefix run" >&2
|
|
exit 1
|
|
}
|
|
jq -e '.schema_version == 1 and .engine == "gem-native-c-metal" and
|
|
.baseline_route == "selector19-bq8-bk64-matrix-global" and
|
|
.candidate_route == "selector23-bq8-bk128-matrix-global" and
|
|
.prefix_mode == "evaluated-prompt" and .prefix_builds == 1 and
|
|
.prefix_tokens == 65536 and .extension_tokens == 32 and .repetitions == 3' \
|
|
"$output/result.json.tmp" >/dev/null
|
|
mv "$output/result.json.tmp" "$output/result.json"
|
|
scripts/analyze-bq8-bk128-graph-gate.sh "$output" "$output/decision.json"
|
|
jq . "$output/decision.json"
|