555 lines
25 KiB
Bash
Executable file
555 lines
25 KiB
Bash
Executable file
#!/bin/sh
|
|
# Three uncached native-server repetitions for one exact long-prompt rung.
|
|
set -eu
|
|
|
|
if [ "$#" -ne 5 ]; then
|
|
echo "usage: $0 MODEL.gguf CONTEXT PROMPT_FILE OUTPUT_DIR PORT" >&2
|
|
exit 2
|
|
fi
|
|
|
|
model=$1
|
|
context=$2
|
|
prompt=$3
|
|
output_dir=$4
|
|
port=$5
|
|
max_tokens=${GEM_STREAM_MAX_TOKENS:-16}
|
|
kv_type=${GEM_KV_TYPE:-f32}
|
|
prefix_extension=${GEM_PREFIX_EXTENSION_TEXT:-}
|
|
resume=${GEM_GATE_RESUME:-0}
|
|
preflight_min_tokens=${GEM_GATE_PREFLIGHT_MIN_TOKENS:-65536}
|
|
max_load_per_cpu=${GEM_GATE_MAX_LOAD_PER_CPU:-1.0}
|
|
max_short_window_busy_cpu=${GEM_GATE_MAX_SHORT_WINDOW_BUSY_CPU_PERCENT:-10.0}
|
|
host_monitor_seconds=${GEM_GATE_HOST_MONITOR_SECONDS:-30}
|
|
host_monitor_failures=${GEM_GATE_HOST_MONITOR_FAILURES:-3}
|
|
case $context:$port:$max_tokens in
|
|
*[!0-9:]*|:*|*::*) echo "gem: context, port, and token limit must be positive integers" >&2; exit 2 ;;
|
|
esac
|
|
if [ "$context" -eq 0 ] || [ "$port" -eq 0 ] || [ "$max_tokens" -eq 0 ]; then
|
|
echo "gem: context, port, and token limit must be positive integers" >&2
|
|
exit 2
|
|
fi
|
|
case $kv_type in
|
|
f32|f16|f16-full) ;;
|
|
*) echo "gem: GEM_KV_TYPE must be f32, f16, or f16-full" >&2; exit 2 ;;
|
|
esac
|
|
case $resume in
|
|
0|1) ;;
|
|
*) echo "gem: GEM_GATE_RESUME must be 0 or 1" >&2; exit 2 ;;
|
|
esac
|
|
case $preflight_min_tokens in
|
|
''|*[!0-9]*)
|
|
echo "gem: GEM_GATE_PREFLIGHT_MIN_TOKENS must be a non-negative integer" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
case $host_monitor_seconds:$host_monitor_failures in
|
|
*[!0-9:]*|:*|*::*)
|
|
echo "gem: host monitor interval and failure count must be positive integers" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
if [ "$host_monitor_seconds" -eq 0 ] || [ "$host_monitor_failures" -eq 0 ]; then
|
|
echo "gem: host monitor interval and failure count must be positive integers" >&2
|
|
exit 2
|
|
fi
|
|
if ! awk -v value="$max_load_per_cpu" 'BEGIN {
|
|
exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value + 0 > 0)
|
|
}'; then
|
|
echo "gem: GEM_GATE_MAX_LOAD_PER_CPU must be a positive number" >&2
|
|
exit 2
|
|
fi
|
|
if ! awk -v value="$max_short_window_busy_cpu" 'BEGIN {
|
|
exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value + 0 > 0)
|
|
}'; then
|
|
echo "gem: GEM_GATE_MAX_SHORT_WINDOW_BUSY_CPU_PERCENT must be a positive number" >&2
|
|
exit 2
|
|
fi
|
|
if [ ! -r "$model" ] || [ ! -r "$prompt" ]; then
|
|
echo "gem: model and prompt must be readable" >&2
|
|
exit 2
|
|
fi
|
|
model_sha256=${GEM_MODEL_SHA256:-}
|
|
if ! command -v shasum >/dev/null 2>&1; then
|
|
echo "gem: shasum is required for benchmark provenance" >&2
|
|
exit 2
|
|
fi
|
|
if [ ! -x build/gem ]; then
|
|
echo "gem: native production binary is missing: build/gem" >&2
|
|
exit 2
|
|
fi
|
|
if [ -n "$model_sha256" ]; then
|
|
case $model_sha256 in
|
|
*[!0-9a-fA-F]*)
|
|
echo "gem: GEM_MODEL_SHA256 must contain exactly 64 hexadecimal digits" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
if [ "${#model_sha256}" -ne 64 ]; then
|
|
echo "gem: GEM_MODEL_SHA256 must contain exactly 64 hexadecimal digits" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
output_existed=0
|
|
if [ -e "$output_dir" ]; then
|
|
output_existed=1
|
|
if [ "$resume" -ne 1 ]; then
|
|
echo "gem: output directory already exists: $output_dir" >&2
|
|
exit 2
|
|
fi
|
|
if [ ! -d "$output_dir" ] || [ ! -r "$output_dir/print-config.json" ] ||
|
|
[ ! -r "$output_dir/gate-config.json" ]; then
|
|
echo "gem: resumable output must contain print-config.json and gate-config.json" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$model_sha256" ]; then
|
|
model_sha256=$(shasum -a 256 "$model" | awk '{print $1}')
|
|
fi
|
|
prompt_sha256=$(shasum -a 256 "$prompt" | awk '{print $1}')
|
|
binary_sha256=$(shasum -a 256 build/gem | awk '{print $1}')
|
|
|
|
prompt_count_json=$(build/gem-tokenize --count --file --chat-user "$model" "$prompt")
|
|
prompt_tokens=$(printf '%s\n' "$prompt_count_json" | jq -er '.token_count')
|
|
if [ "$context" -le "$max_tokens" ] ||
|
|
[ "$prompt_tokens" -gt $((context - max_tokens)) ]; then
|
|
echo "gem: $prompt_tokens prompt tokens plus $max_tokens output tokens exceed context $context" >&2
|
|
exit 2
|
|
fi
|
|
mkdir -p "$output_dir"
|
|
gate_config=$(jq -cn \
|
|
--arg engine "gem-native-c-metal" \
|
|
--arg model "$model" \
|
|
--arg model_sha256 "$model_sha256" \
|
|
--arg prompt "$prompt" \
|
|
--arg prompt_sha256 "$prompt_sha256" \
|
|
--arg kv_type "$kv_type" \
|
|
--arg prefix_extension "$prefix_extension" \
|
|
--arg sleep_assertion "caffeinate -i -w server_pid" \
|
|
--argjson context "$context" \
|
|
--argjson prompt_tokens "$prompt_tokens" \
|
|
--argjson max_tokens "$max_tokens" \
|
|
--argjson preflight_min_tokens "$preflight_min_tokens" \
|
|
--argjson max_load_per_cpu "$max_load_per_cpu" \
|
|
--argjson max_short_window_busy_cpu "$max_short_window_busy_cpu" \
|
|
--argjson host_monitor_seconds "$host_monitor_seconds" \
|
|
--argjson host_monitor_failures "$host_monitor_failures" \
|
|
'{schema_version: 4, engine: $engine, model: $model,
|
|
model_sha256: $model_sha256, prompt_file: $prompt,
|
|
prompt_sha256: $prompt_sha256, context: $context, kv_type: $kv_type,
|
|
prompt_tokens: $prompt_tokens, requested_max_tokens: $max_tokens,
|
|
prefix_extension_text: $prefix_extension, repetitions: 3,
|
|
host_policy: {preflight_min_tokens: $preflight_min_tokens,
|
|
max_load_per_logical_cpu: $max_load_per_cpu,
|
|
max_short_window_busy_cpu_percent: $max_short_window_busy_cpu,
|
|
requires_ac_power: true, requires_no_thermal_warning: true,
|
|
monitor_power_and_thermal: true,
|
|
monitor_interval_seconds: $host_monitor_seconds,
|
|
consecutive_failures_before_abort: $host_monitor_failures,
|
|
sleep_assertion: $sleep_assertion}}')
|
|
current_config=$(build/gem print-config --model "$model" --context "$context" --kv-type "$kv_type")
|
|
printf '%s\n' "$current_config" | jq -e --arg kv_type "$kv_type" --argjson context "$context" \
|
|
'.engine == "gem-native-c-metal" and .context == $context and .kv_type == $kv_type' \
|
|
>/dev/null
|
|
if [ "$output_existed" -eq 1 ]; then
|
|
saved_gate_config=$(jq -ceS . "$output_dir/gate-config.json")
|
|
canonical_gate_config=$(printf '%s\n' "$gate_config" | jq -ceS .)
|
|
if [ "$saved_gate_config" != "$canonical_gate_config" ]; then
|
|
echo "gem: refusing to resume with changed benchmark provenance" >&2
|
|
exit 2
|
|
fi
|
|
saved_config=$(jq -ceS . "$output_dir/print-config.json")
|
|
canonical_config=$(printf '%s\n' "$current_config" | jq -ceS .)
|
|
if [ "$saved_config" != "$canonical_config" ]; then
|
|
echo "gem: refusing to resume with a changed native configuration" >&2
|
|
exit 2
|
|
fi
|
|
if [ ! -s "$output_dir/live-binary-sha256.txt" ] ||
|
|
[ "$(awk 'NR == 1 {print $1}' "$output_dir/live-binary-sha256.txt")" != "$binary_sha256" ]; then
|
|
echo "gem: refusing to resume with a changed native production binary" >&2
|
|
exit 2
|
|
fi
|
|
else
|
|
printf '%s\n' "$gate_config" > "$output_dir/gate-config.json"
|
|
printf '%s\n' "$current_config" > "$output_dir/print-config.json"
|
|
printf '%s %s\n' "$binary_sha256" "build/gem" > "$output_dir/live-binary-sha256.txt"
|
|
fi
|
|
|
|
server_pid=
|
|
caffeinate_pid=
|
|
monitor_pid=
|
|
cleanup()
|
|
{
|
|
if [ -n "$server_pid" ] && kill -0 "$server_pid" 2>/dev/null; then
|
|
kill "$server_pid"
|
|
wait "$server_pid" 2>/dev/null || true
|
|
fi
|
|
server_pid=
|
|
if [ -n "$caffeinate_pid" ] && kill -0 "$caffeinate_pid" 2>/dev/null; then
|
|
kill "$caffeinate_pid"
|
|
wait "$caffeinate_pid" 2>/dev/null || true
|
|
fi
|
|
caffeinate_pid=
|
|
if [ -n "$monitor_pid" ] && kill -0 "$monitor_pid" 2>/dev/null; then
|
|
kill "$monitor_pid"
|
|
wait "$monitor_pid" 2>/dev/null || true
|
|
fi
|
|
monitor_pid=
|
|
}
|
|
trap cleanup EXIT INT TERM HUP
|
|
|
|
base_url="http://127.0.0.1:$port/v1"
|
|
resumed_samples=0
|
|
sample_complete()
|
|
{
|
|
sample=$1
|
|
log_file="$output_dir/server-$sample.log"
|
|
stream_file="$output_dir/stream-$sample.sse"
|
|
measurement_file="$output_dir/measurement-$sample.json"
|
|
[ -s "$log_file" ] && [ -s "$stream_file" ] && [ -s "$measurement_file" ] || return 1
|
|
jq -e --argjson prompt_tokens "$prompt_tokens" --argjson max_tokens "$max_tokens" '
|
|
.schema_version == 1 and .engine == "gem-native-c-metal" and
|
|
.model == "gemma4" and .requested_max_tokens == $max_tokens and
|
|
.http_code == 200 and .first_visible_event_seconds > .header_ttfb_seconds and
|
|
.completion_seconds + 0.001 >= .first_visible_event_seconds and
|
|
.visible_event_count > 0 and .finish_reason != null and
|
|
.usage.prompt_tokens == $prompt_tokens and .usage.completion_tokens > 0 and
|
|
.usage.total_tokens == (.usage.prompt_tokens + .usage.completion_tokens)
|
|
' "$measurement_file" >/dev/null || return 1
|
|
sed '/^$/d' "$stream_file" | tail -n 1 | grep -q '^data: \[DONE\]$' || return 1
|
|
if [ "$prompt_tokens" -ge 4096 ]; then
|
|
grep -Fq "gem: prompt prefill $prompt_tokens/$prompt_tokens tokens (100.00%)" \
|
|
"$log_file" || return 1
|
|
fi
|
|
if [ -n "$prefix_extension" ]; then
|
|
prefix_request_file="$output_dir/prefix-request-$sample.json"
|
|
prefix_stream_file="$output_dir/prefix-stream-$sample.sse"
|
|
prefix_measurement_file="$output_dir/prefix-measurement-$sample.json"
|
|
[ -s "$prefix_request_file" ] && [ -s "$prefix_stream_file" ] &&
|
|
[ -s "$prefix_measurement_file" ] || return 1
|
|
jq -e --arg extension "$prefix_extension" \
|
|
'.model == "gemma4" and .messages[-1].role == "user" and
|
|
.messages[-1].content == $extension' \
|
|
"$prefix_request_file" >/dev/null || return 1
|
|
jq -e --argjson prompt_tokens "$prompt_tokens" --argjson max_tokens "$max_tokens" '
|
|
.schema_version == 1 and .engine == "gem-native-c-metal" and
|
|
.model == "gemma4" and .requested_max_tokens == $max_tokens and
|
|
.http_code == 200 and .visible_event_count > 0 and .finish_reason != null and
|
|
.reused_prompt_tokens == $prompt_tokens and .appended_prompt_tokens > 0 and
|
|
.usage.prompt_tokens == (.reused_prompt_tokens + .appended_prompt_tokens)
|
|
' "$prefix_measurement_file" >/dev/null || return 1
|
|
extended_prompt_tokens=$(jq -er '.usage.prompt_tokens' "$prefix_measurement_file") ||
|
|
return 1
|
|
grep -Fq \
|
|
"gem: prompt prefill $extended_prompt_tokens/$extended_prompt_tokens tokens (100.00%)" \
|
|
"$log_file" || return 1
|
|
sed '/^$/d' "$prefix_stream_file" | tail -n 1 |
|
|
grep -q '^data: \[DONE\]$' || return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
host_preflight()
|
|
{
|
|
sample=$1
|
|
receipt="$output_dir/host-preflight-$sample.json"
|
|
timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
if [ "$prompt_tokens" -lt "$preflight_min_tokens" ]; then
|
|
jq -n --arg timestamp "$timestamp" --argjson prompt_tokens "$prompt_tokens" \
|
|
--argjson minimum "$preflight_min_tokens" \
|
|
'{schema_version: 2, timestamp_utc: $timestamp, status: "skipped",
|
|
reason: "prompt below configured long-context threshold",
|
|
prompt_tokens: $prompt_tokens, preflight_min_tokens: $minimum}' > "$receipt"
|
|
return 0
|
|
fi
|
|
if [ "$(uname -s)" != Darwin ] || ! command -v sysctl >/dev/null 2>&1 ||
|
|
! command -v pmset >/dev/null 2>&1 || ! command -v caffeinate >/dev/null 2>&1 ||
|
|
! command -v top >/dev/null 2>&1; then
|
|
jq -n --arg timestamp "$timestamp" \
|
|
'{schema_version: 2, timestamp_utc: $timestamp, status: "fail",
|
|
reason: "macOS host-state tools unavailable"}' > "$receipt"
|
|
echo "gem: long-context benchmark requires sysctl, pmset, caffeinate, and top on macOS" >&2
|
|
return 1
|
|
fi
|
|
load_average=$(sysctl -n vm.loadavg)
|
|
load_1m=$(printf '%s\n' "$load_average" | awk '{print $2}')
|
|
logical_cpus=$(sysctl -n hw.logicalcpu)
|
|
if ! awk -v load="$load_1m" -v cpus="$logical_cpus" 'BEGIN {
|
|
exit !(load ~ /^[0-9]+([.][0-9]+)?$/ && cpus ~ /^[1-9][0-9]*$/)
|
|
}'; then
|
|
echo "gem: could not parse macOS host load state" >&2
|
|
return 1
|
|
fi
|
|
load_per_cpu=$(awk -v load="$load_1m" -v cpus="$logical_cpus" \
|
|
'BEGIN {printf "%.6f", load / cpus}')
|
|
recent_process_cpu_percent=$(ps -Ao pcpu= |
|
|
awk '{sum += $1} END {printf "%.3f", sum}')
|
|
short_window_busy_cpu=$(LC_ALL=C top -l 2 -s 1 -n 0 -stats pid,cpu,command |
|
|
awk -F'[:,%]' '/^CPU usage:/ {idle=$6} END {
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", idle)
|
|
if (idle !~ /^[0-9]+([.][0-9]+)?$/) exit 1
|
|
printf "%.3f", 100 - idle
|
|
}') || {
|
|
echo "gem: could not sample short-window macOS CPU state" >&2
|
|
return 1
|
|
}
|
|
power_state=$(LC_ALL=C pmset -g batt 2>&1 || true)
|
|
thermal_state=$(LC_ALL=C pmset -g therm 2>&1 || true)
|
|
ac_power=false
|
|
case $power_state in
|
|
*"Now drawing from 'AC Power'"*) ac_power=true ;;
|
|
esac
|
|
no_thermal_warning=false
|
|
if printf '%s\n' "$thermal_state" |
|
|
grep -Fq 'No thermal warning level has been recorded' &&
|
|
printf '%s\n' "$thermal_state" |
|
|
grep -Fq 'No performance warning level has been recorded'; then
|
|
no_thermal_warning=true
|
|
elif printf '%s\n' "$thermal_state" |
|
|
grep -Eq 'CPU_Scheduler_Limit[[:space:]]*=[[:space:]]*100' &&
|
|
printf '%s\n' "$thermal_state" |
|
|
grep -Eq 'CPU_Speed_Limit[[:space:]]*=[[:space:]]*100'; then
|
|
no_thermal_warning=true
|
|
fi
|
|
assertions=$(pmset -g assertions 2>&1 || true)
|
|
top_processes=$(ps -Ao pid,pcpu,pmem,etime,comm -r | sed -n '1,24p')
|
|
status=pass
|
|
if ! awk -v actual="$load_per_cpu" -v maximum="$max_load_per_cpu" \
|
|
'BEGIN {exit !(actual <= maximum)}'; then
|
|
status=fail
|
|
fi
|
|
if ! awk -v actual="$short_window_busy_cpu" -v maximum="$max_short_window_busy_cpu" \
|
|
'BEGIN {exit !(actual <= maximum)}'; then
|
|
status=fail
|
|
fi
|
|
[ "$ac_power" = true ] || status=fail
|
|
[ "$no_thermal_warning" = true ] || status=fail
|
|
jq -n --arg timestamp "$timestamp" --arg status "$status" \
|
|
--arg load_average "$load_average" --argjson load_1m "$load_1m" \
|
|
--argjson logical_cpus "$logical_cpus" --argjson load_per_cpu "$load_per_cpu" \
|
|
--argjson maximum "$max_load_per_cpu" --arg power_state "$power_state" \
|
|
--argjson recent_process_cpu_percent "$recent_process_cpu_percent" \
|
|
--argjson short_window_busy_cpu "$short_window_busy_cpu" \
|
|
--argjson max_short_window_busy_cpu "$max_short_window_busy_cpu" \
|
|
--argjson ac_power "$ac_power" \
|
|
--argjson no_thermal_warning "$no_thermal_warning" \
|
|
--arg thermal_state "$thermal_state" --arg assertions "$assertions" \
|
|
--arg top_processes "$top_processes" \
|
|
'{schema_version: 2, timestamp_utc: $timestamp, status: $status,
|
|
load_average: $load_average, load_1m: $load_1m,
|
|
logical_cpus: $logical_cpus, load_per_logical_cpu: $load_per_cpu,
|
|
max_load_per_logical_cpu: $maximum, power_state: $power_state,
|
|
ac_power: $ac_power,
|
|
recent_aggregate_process_cpu_percent: $recent_process_cpu_percent,
|
|
short_window_busy_cpu_percent: $short_window_busy_cpu,
|
|
max_short_window_busy_cpu_percent: $max_short_window_busy_cpu,
|
|
thermal_state: $thermal_state, no_thermal_warning: $no_thermal_warning,
|
|
assertions_before_run: $assertions,
|
|
top_processes: $top_processes}' > "$receipt"
|
|
if [ "$status" != pass ]; then
|
|
echo "gem: host preflight rejected load, CPU, power, or thermal state; see $receipt" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
start_host_monitor()
|
|
{
|
|
sample=$1
|
|
[ "$prompt_tokens" -ge "$preflight_min_tokens" ] || return 0
|
|
monitor_file="$output_dir/host-monitor-$sample.ndjson"
|
|
: > "$monitor_file"
|
|
(
|
|
failures=0
|
|
while kill -0 "$server_pid" 2>/dev/null; do
|
|
sleep "$host_monitor_seconds"
|
|
kill -0 "$server_pid" 2>/dev/null || exit 0
|
|
timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
load_average=$(sysctl -n vm.loadavg)
|
|
load_1m=$(printf '%s\n' "$load_average" | awk '{print $2}')
|
|
logical_cpus=$(sysctl -n hw.logicalcpu)
|
|
load_per_cpu=$(awk -v load="$load_1m" -v cpus="$logical_cpus" \
|
|
'BEGIN {printf "%.6f", load / cpus}')
|
|
power_state=$(LC_ALL=C pmset -g batt 2>&1 || true)
|
|
thermal_state=$(LC_ALL=C pmset -g therm 2>&1 || true)
|
|
ac_power=false
|
|
case $power_state in
|
|
*"Now drawing from 'AC Power'"*) ac_power=true ;;
|
|
esac
|
|
no_thermal_warning=false
|
|
if printf '%s\n' "$thermal_state" |
|
|
grep -Fq 'No thermal warning level has been recorded' &&
|
|
printf '%s\n' "$thermal_state" |
|
|
grep -Fq 'No performance warning level has been recorded'; then
|
|
no_thermal_warning=true
|
|
elif printf '%s\n' "$thermal_state" |
|
|
grep -Eq 'CPU_Scheduler_Limit[[:space:]]*=[[:space:]]*100' &&
|
|
printf '%s\n' "$thermal_state" |
|
|
grep -Eq 'CPU_Speed_Limit[[:space:]]*=[[:space:]]*100'; then
|
|
no_thermal_warning=true
|
|
fi
|
|
overloaded=false
|
|
if ! awk -v actual="$load_per_cpu" -v maximum="$max_load_per_cpu" \
|
|
'BEGIN {exit !(actual <= maximum)}'; then
|
|
overloaded=true
|
|
failures=$((failures + 1))
|
|
elif [ "$ac_power" != true ] || [ "$no_thermal_warning" != true ]; then
|
|
overloaded=true
|
|
failures=$((failures + 1))
|
|
else
|
|
failures=0
|
|
fi
|
|
jq -cn --arg timestamp "$timestamp" --argjson load_1m "$load_1m" \
|
|
--argjson logical_cpus "$logical_cpus" \
|
|
--argjson load_per_cpu "$load_per_cpu" \
|
|
--argjson maximum "$max_load_per_cpu" --argjson overloaded "$overloaded" \
|
|
--argjson ac_power "$ac_power" \
|
|
--argjson no_thermal_warning "$no_thermal_warning" \
|
|
--arg power_state "$power_state" --arg thermal_state "$thermal_state" \
|
|
--argjson consecutive_failures "$failures" \
|
|
'{schema_version: 2, timestamp_utc: $timestamp, load_1m: $load_1m,
|
|
logical_cpus: $logical_cpus, load_per_logical_cpu: $load_per_cpu,
|
|
max_load_per_logical_cpu: $maximum, overloaded: $overloaded,
|
|
ac_power: $ac_power, no_thermal_warning: $no_thermal_warning,
|
|
power_state: $power_state, thermal_state: $thermal_state,
|
|
consecutive_failures: $consecutive_failures}' >> "$monitor_file"
|
|
if [ "$failures" -ge "$host_monitor_failures" ]; then
|
|
printf '%s\n' \
|
|
"gem: aborting sample $sample after $failures sustained host-load violations" \
|
|
>> "$output_dir/host-monitor-$sample.failure.txt"
|
|
kill "$server_pid" 2>/dev/null || true
|
|
exit 1
|
|
fi
|
|
done
|
|
) &
|
|
monitor_pid=$!
|
|
}
|
|
|
|
for repetition in 1 2 3; do
|
|
if [ "$resume" -eq 1 ] && sample_complete "$repetition"; then
|
|
resumed_samples=$((resumed_samples + 1))
|
|
continue
|
|
fi
|
|
if command -v lsof >/dev/null 2>&1 &&
|
|
lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
|
|
echo "gem: benchmark port $port is already occupied" >&2
|
|
exit 1
|
|
fi
|
|
host_preflight "$repetition"
|
|
log="$output_dir/server-$repetition.log"
|
|
build/gem serve --model "$model" --alias gemma4 --port "$port" \
|
|
--context "$context" --max-tokens "$max_tokens" --command-span token \
|
|
--kv-type "$kv_type" \
|
|
>"$log" 2>&1 &
|
|
server_pid=$!
|
|
caffeinate -i -w "$server_pid" >/dev/null 2>&1 &
|
|
caffeinate_pid=$!
|
|
start_host_monitor "$repetition"
|
|
ready=0
|
|
attempts=0
|
|
while [ "$attempts" -lt 1200 ]; do
|
|
if curl --fail --silent "http://127.0.0.1:$port/health" >/dev/null 2>&1; then
|
|
ready=1
|
|
break
|
|
fi
|
|
if ! kill -0 "$server_pid" 2>/dev/null; then
|
|
echo "gem: server exited before readiness; see $log" >&2
|
|
exit 1
|
|
fi
|
|
attempts=$((attempts + 1))
|
|
sleep 0.1
|
|
done
|
|
if [ "$ready" -ne 1 ]; then
|
|
echo "gem: server readiness timed out; see $log" >&2
|
|
exit 1
|
|
fi
|
|
GEM_STREAM_MAX_TOKENS=$max_tokens scripts/server-stream-probe.sh \
|
|
"$base_url" gemma4 "$prompt" "$output_dir/stream-$repetition.sse" \
|
|
"$output_dir/measurement-$repetition.json"
|
|
observed_prompt_tokens=$(jq -er '.usage.prompt_tokens' \
|
|
"$output_dir/measurement-$repetition.json")
|
|
if [ "$observed_prompt_tokens" -ne "$prompt_tokens" ]; then
|
|
echo "gem: server reported $observed_prompt_tokens prompt tokens; expected $prompt_tokens" >&2
|
|
exit 1
|
|
fi
|
|
if [ -n "$prefix_extension" ]; then
|
|
assistant_text=$(sed -n 's/^data: //p' "$output_dir/stream-$repetition.sse" |
|
|
sed '/^\[DONE\]$/d' |
|
|
jq -sr '[.[] | .choices[0].delta.content? // empty] | join("")')
|
|
if [ -z "$assistant_text" ]; then
|
|
assistant_text=OK
|
|
fi
|
|
request_file="$output_dir/prefix-request-$repetition.json"
|
|
jq -n --rawfile prompt "$prompt" --arg assistant "$assistant_text" \
|
|
--arg extension "$prefix_extension" --argjson max_tokens "$max_tokens" \
|
|
'{model: "gemma4", messages: [
|
|
{role: "user", content: $prompt},
|
|
{role: "assistant", content: $assistant},
|
|
{role: "user", content: $extension}],
|
|
max_tokens: $max_tokens}' > "$request_file"
|
|
GEM_STREAM_MAX_TOKENS=$max_tokens scripts/server-stream-probe.sh --request \
|
|
"$base_url" "$request_file" "$output_dir/prefix-stream-$repetition.sse" \
|
|
"$output_dir/prefix-measurement-$repetition.json"
|
|
first_prompt_tokens=$observed_prompt_tokens
|
|
extended_prompt_tokens=$(jq -er '.usage.prompt_tokens' \
|
|
"$output_dir/prefix-measurement-$repetition.json")
|
|
expected_appended=$((extended_prompt_tokens - first_prompt_tokens))
|
|
reuse_record=$(sed -n \
|
|
's/.*exact prefix cache hit (\([0-9][0-9]*\) reused, \([0-9][0-9]*\) appended).*/\1 \2/p' \
|
|
"$log" | tail -n 1)
|
|
if [ "$reuse_record" != "$first_prompt_tokens $expected_appended" ]; then
|
|
echo "gem: invalid prefix reuse '$reuse_record'; expected $first_prompt_tokens $expected_appended" >&2
|
|
exit 1
|
|
fi
|
|
jq --argjson reused "$first_prompt_tokens" --argjson appended "$expected_appended" \
|
|
'. + {reused_prompt_tokens: $reused, appended_prompt_tokens: $appended}' \
|
|
"$output_dir/prefix-measurement-$repetition.json" \
|
|
> "$output_dir/prefix-measurement-$repetition.tmp"
|
|
mv "$output_dir/prefix-measurement-$repetition.tmp" \
|
|
"$output_dir/prefix-measurement-$repetition.json"
|
|
fi
|
|
if ! sample_complete "$repetition"; then
|
|
echo "gem: completed sample $repetition failed retained-artifact validation" >&2
|
|
exit 1
|
|
fi
|
|
cleanup
|
|
done
|
|
|
|
jq -s \
|
|
--arg engine "gem-native-c-metal" \
|
|
--arg model "$model" \
|
|
--arg model_sha256 "$model_sha256" \
|
|
--arg prompt "$prompt" \
|
|
--arg prompt_sha256 "$prompt_sha256" \
|
|
--arg kv_type "$kv_type" \
|
|
--argjson context "$context" \
|
|
--argjson prompt_tokens "$prompt_tokens" \
|
|
--argjson max_tokens "$max_tokens" \
|
|
--argjson resumed_samples "$resumed_samples" \
|
|
'{schema_version: 1, engine: $engine, model: $model,
|
|
model_sha256: $model_sha256, prompt_file: $prompt,
|
|
prompt_sha256: $prompt_sha256,
|
|
context: $context, kv_type: $kv_type, prompt_tokens: $prompt_tokens,
|
|
requested_max_tokens: $max_tokens, repetitions: length,
|
|
resumed_samples: $resumed_samples, samples: .,
|
|
mean_first_visible_event_seconds:
|
|
(map(.first_visible_event_seconds) | add / length),
|
|
mean_completion_seconds: (map(.completion_seconds) | add / length)}' \
|
|
"$output_dir/measurement-1.json" "$output_dir/measurement-2.json" \
|
|
"$output_dir/measurement-3.json" > "$output_dir/summary.json"
|
|
if [ -n "$prefix_extension" ]; then
|
|
jq -s --arg extension "$prefix_extension" \
|
|
'{extension_user_text: $extension, repetitions: length, samples: .,
|
|
mean_first_visible_event_seconds:
|
|
(map(.first_visible_event_seconds) | add / length),
|
|
mean_completion_seconds: (map(.completion_seconds) | add / length),
|
|
reused_prompt_tokens: (map(.reused_prompt_tokens) | unique),
|
|
appended_prompt_tokens: (map(.appended_prompt_tokens) | unique)}' \
|
|
"$output_dir/prefix-measurement-1.json" \
|
|
"$output_dir/prefix-measurement-2.json" \
|
|
"$output_dir/prefix-measurement-3.json" > "$output_dir/prefix-summary.json"
|
|
summary_tmp="$output_dir/summary.tmp"
|
|
jq --slurpfile prefix "$output_dir/prefix-summary.json" \
|
|
'. + {prefix_extension: $prefix[0]}' "$output_dir/summary.json" > "$summary_tmp"
|
|
mv "$summary_tmp" "$output_dir/summary.json"
|
|
fi
|
|
jq . "$output_dir/summary.json"
|