#!/bin/sh # Three uncached native-server repetitions for one exact long-prompt rung. set -eu if [ "$#" -ne 5 ]; then echo "usage: $0 MODEL.gguf CONTEXT PROMPT_FILE OUTPUT_DIR PORT" >&2 exit 2 fi model=$1 context=$2 prompt=$3 output_dir=$4 port=$5 max_tokens=${GEM_STREAM_MAX_TOKENS:-16} kv_type=${GEM_KV_TYPE:-f32} prefix_extension=${GEM_PREFIX_EXTENSION_TEXT:-} resume=${GEM_GATE_RESUME:-0} preflight_min_tokens=${GEM_GATE_PREFLIGHT_MIN_TOKENS:-65536} max_load_per_cpu=${GEM_GATE_MAX_LOAD_PER_CPU:-1.0} max_short_window_busy_cpu=${GEM_GATE_MAX_SHORT_WINDOW_BUSY_CPU_PERCENT:-10.0} host_monitor_seconds=${GEM_GATE_HOST_MONITOR_SECONDS:-30} host_monitor_failures=${GEM_GATE_HOST_MONITOR_FAILURES:-3} case $context:$port:$max_tokens in *[!0-9:]*|:*|*::*) echo "gem: context, port, and token limit must be positive integers" >&2; exit 2 ;; esac if [ "$context" -eq 0 ] || [ "$port" -eq 0 ] || [ "$max_tokens" -eq 0 ]; then echo "gem: context, port, and token limit must be positive integers" >&2 exit 2 fi case $kv_type in f32|f16|f16-full) ;; *) echo "gem: GEM_KV_TYPE must be f32, f16, or f16-full" >&2; exit 2 ;; esac case $resume in 0|1) ;; *) echo "gem: GEM_GATE_RESUME must be 0 or 1" >&2; exit 2 ;; esac case $preflight_min_tokens in ''|*[!0-9]*) echo "gem: GEM_GATE_PREFLIGHT_MIN_TOKENS must be a non-negative integer" >&2 exit 2 ;; esac case $host_monitor_seconds:$host_monitor_failures in *[!0-9:]*|:*|*::*) echo "gem: host monitor interval and failure count must be positive integers" >&2 exit 2 ;; esac if [ "$host_monitor_seconds" -eq 0 ] || [ "$host_monitor_failures" -eq 0 ]; then echo "gem: host monitor interval and failure count must be positive integers" >&2 exit 2 fi if ! awk -v value="$max_load_per_cpu" 'BEGIN { exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value + 0 > 0) }'; then echo "gem: GEM_GATE_MAX_LOAD_PER_CPU must be a positive number" >&2 exit 2 fi if ! awk -v value="$max_short_window_busy_cpu" 'BEGIN { exit !(value ~ /^[0-9]+([.][0-9]+)?$/ && value + 0 > 0) }'; then echo "gem: GEM_GATE_MAX_SHORT_WINDOW_BUSY_CPU_PERCENT must be a positive number" >&2 exit 2 fi if [ ! -r "$model" ] || [ ! -r "$prompt" ]; then echo "gem: model and prompt must be readable" >&2 exit 2 fi model_sha256=${GEM_MODEL_SHA256:-} if ! command -v shasum >/dev/null 2>&1; then echo "gem: shasum is required for benchmark provenance" >&2 exit 2 fi if [ ! -x build/gem ]; then echo "gem: native production binary is missing: build/gem" >&2 exit 2 fi if [ -n "$model_sha256" ]; then case $model_sha256 in *[!0-9a-fA-F]*) echo "gem: GEM_MODEL_SHA256 must contain exactly 64 hexadecimal digits" >&2 exit 2 ;; esac if [ "${#model_sha256}" -ne 64 ]; then echo "gem: GEM_MODEL_SHA256 must contain exactly 64 hexadecimal digits" >&2 exit 2 fi fi output_existed=0 if [ -e "$output_dir" ]; then output_existed=1 if [ "$resume" -ne 1 ]; then echo "gem: output directory already exists: $output_dir" >&2 exit 2 fi if [ ! -d "$output_dir" ] || [ ! -r "$output_dir/print-config.json" ] || [ ! -r "$output_dir/gate-config.json" ]; then echo "gem: resumable output must contain print-config.json and gate-config.json" >&2 exit 2 fi fi if [ -z "$model_sha256" ]; then model_sha256=$(shasum -a 256 "$model" | awk '{print $1}') fi prompt_sha256=$(shasum -a 256 "$prompt" | awk '{print $1}') binary_sha256=$(shasum -a 256 build/gem | awk '{print $1}') prompt_count_json=$(build/gem-tokenize --count --file --chat-user "$model" "$prompt") prompt_tokens=$(printf '%s\n' "$prompt_count_json" | jq -er '.token_count') if [ "$context" -le "$max_tokens" ] || [ "$prompt_tokens" -gt $((context - max_tokens)) ]; then echo "gem: $prompt_tokens prompt tokens plus $max_tokens output tokens exceed context $context" >&2 exit 2 fi mkdir -p "$output_dir" gate_config=$(jq -cn \ --arg engine "gem-native-c-metal" \ --arg model "$model" \ --arg model_sha256 "$model_sha256" \ --arg prompt "$prompt" \ --arg prompt_sha256 "$prompt_sha256" \ --arg kv_type "$kv_type" \ --arg prefix_extension "$prefix_extension" \ --arg sleep_assertion "caffeinate -i -w server_pid" \ --argjson context "$context" \ --argjson prompt_tokens "$prompt_tokens" \ --argjson max_tokens "$max_tokens" \ --argjson preflight_min_tokens "$preflight_min_tokens" \ --argjson max_load_per_cpu "$max_load_per_cpu" \ --argjson max_short_window_busy_cpu "$max_short_window_busy_cpu" \ --argjson host_monitor_seconds "$host_monitor_seconds" \ --argjson host_monitor_failures "$host_monitor_failures" \ '{schema_version: 4, engine: $engine, model: $model, model_sha256: $model_sha256, prompt_file: $prompt, prompt_sha256: $prompt_sha256, context: $context, kv_type: $kv_type, prompt_tokens: $prompt_tokens, requested_max_tokens: $max_tokens, prefix_extension_text: $prefix_extension, repetitions: 3, host_policy: {preflight_min_tokens: $preflight_min_tokens, max_load_per_logical_cpu: $max_load_per_cpu, max_short_window_busy_cpu_percent: $max_short_window_busy_cpu, requires_ac_power: true, requires_no_thermal_warning: true, monitor_power_and_thermal: true, monitor_interval_seconds: $host_monitor_seconds, consecutive_failures_before_abort: $host_monitor_failures, sleep_assertion: $sleep_assertion}}') current_config=$(build/gem print-config --model "$model" --context "$context" --kv-type "$kv_type") printf '%s\n' "$current_config" | jq -e --arg kv_type "$kv_type" --argjson context "$context" \ '.engine == "gem-native-c-metal" and .context == $context and .kv_type == $kv_type' \ >/dev/null if [ "$output_existed" -eq 1 ]; then saved_gate_config=$(jq -ceS . "$output_dir/gate-config.json") canonical_gate_config=$(printf '%s\n' "$gate_config" | jq -ceS .) if [ "$saved_gate_config" != "$canonical_gate_config" ]; then echo "gem: refusing to resume with changed benchmark provenance" >&2 exit 2 fi saved_config=$(jq -ceS . "$output_dir/print-config.json") canonical_config=$(printf '%s\n' "$current_config" | jq -ceS .) if [ "$saved_config" != "$canonical_config" ]; then echo "gem: refusing to resume with a changed native configuration" >&2 exit 2 fi if [ ! -s "$output_dir/live-binary-sha256.txt" ] || [ "$(awk 'NR == 1 {print $1}' "$output_dir/live-binary-sha256.txt")" != "$binary_sha256" ]; then echo "gem: refusing to resume with a changed native production binary" >&2 exit 2 fi else printf '%s\n' "$gate_config" > "$output_dir/gate-config.json" printf '%s\n' "$current_config" > "$output_dir/print-config.json" printf '%s %s\n' "$binary_sha256" "build/gem" > "$output_dir/live-binary-sha256.txt" fi server_pid= caffeinate_pid= monitor_pid= cleanup() { if [ -n "$server_pid" ] && kill -0 "$server_pid" 2>/dev/null; then kill "$server_pid" wait "$server_pid" 2>/dev/null || true fi server_pid= if [ -n "$caffeinate_pid" ] && kill -0 "$caffeinate_pid" 2>/dev/null; then kill "$caffeinate_pid" wait "$caffeinate_pid" 2>/dev/null || true fi caffeinate_pid= if [ -n "$monitor_pid" ] && kill -0 "$monitor_pid" 2>/dev/null; then kill "$monitor_pid" wait "$monitor_pid" 2>/dev/null || true fi monitor_pid= } trap cleanup EXIT INT TERM HUP base_url="http://127.0.0.1:$port/v1" resumed_samples=0 sample_complete() { sample=$1 log_file="$output_dir/server-$sample.log" stream_file="$output_dir/stream-$sample.sse" measurement_file="$output_dir/measurement-$sample.json" [ -s "$log_file" ] && [ -s "$stream_file" ] && [ -s "$measurement_file" ] || return 1 jq -e --argjson prompt_tokens "$prompt_tokens" --argjson max_tokens "$max_tokens" ' .schema_version == 1 and .engine == "gem-native-c-metal" and .model == "gemma4" and .requested_max_tokens == $max_tokens and .http_code == 200 and .first_visible_event_seconds > .header_ttfb_seconds and .completion_seconds + 0.001 >= .first_visible_event_seconds and .visible_event_count > 0 and .finish_reason != null and .usage.prompt_tokens == $prompt_tokens and .usage.completion_tokens > 0 and .usage.total_tokens == (.usage.prompt_tokens + .usage.completion_tokens) ' "$measurement_file" >/dev/null || return 1 sed '/^$/d' "$stream_file" | tail -n 1 | grep -q '^data: \[DONE\]$' || return 1 if [ "$prompt_tokens" -ge 4096 ]; then grep -Fq "gem: prompt prefill $prompt_tokens/$prompt_tokens tokens (100.00%)" \ "$log_file" || return 1 fi if [ -n "$prefix_extension" ]; then prefix_request_file="$output_dir/prefix-request-$sample.json" prefix_stream_file="$output_dir/prefix-stream-$sample.sse" prefix_measurement_file="$output_dir/prefix-measurement-$sample.json" [ -s "$prefix_request_file" ] && [ -s "$prefix_stream_file" ] && [ -s "$prefix_measurement_file" ] || return 1 jq -e --arg extension "$prefix_extension" \ '.model == "gemma4" and .messages[-1].role == "user" and .messages[-1].content == $extension' \ "$prefix_request_file" >/dev/null || return 1 jq -e --argjson prompt_tokens "$prompt_tokens" --argjson max_tokens "$max_tokens" ' .schema_version == 1 and .engine == "gem-native-c-metal" and .model == "gemma4" and .requested_max_tokens == $max_tokens and .http_code == 200 and .visible_event_count > 0 and .finish_reason != null and .reused_prompt_tokens == $prompt_tokens and .appended_prompt_tokens > 0 and .usage.prompt_tokens == (.reused_prompt_tokens + .appended_prompt_tokens) ' "$prefix_measurement_file" >/dev/null || return 1 extended_prompt_tokens=$(jq -er '.usage.prompt_tokens' "$prefix_measurement_file") || return 1 grep -Fq \ "gem: prompt prefill $extended_prompt_tokens/$extended_prompt_tokens tokens (100.00%)" \ "$log_file" || return 1 sed '/^$/d' "$prefix_stream_file" | tail -n 1 | grep -q '^data: \[DONE\]$' || return 1 fi return 0 } host_preflight() { sample=$1 receipt="$output_dir/host-preflight-$sample.json" timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') if [ "$prompt_tokens" -lt "$preflight_min_tokens" ]; then jq -n --arg timestamp "$timestamp" --argjson prompt_tokens "$prompt_tokens" \ --argjson minimum "$preflight_min_tokens" \ '{schema_version: 2, timestamp_utc: $timestamp, status: "skipped", reason: "prompt below configured long-context threshold", prompt_tokens: $prompt_tokens, preflight_min_tokens: $minimum}' > "$receipt" return 0 fi if [ "$(uname -s)" != Darwin ] || ! command -v sysctl >/dev/null 2>&1 || ! command -v pmset >/dev/null 2>&1 || ! command -v caffeinate >/dev/null 2>&1 || ! command -v top >/dev/null 2>&1; then jq -n --arg timestamp "$timestamp" \ '{schema_version: 2, timestamp_utc: $timestamp, status: "fail", reason: "macOS host-state tools unavailable"}' > "$receipt" echo "gem: long-context benchmark requires sysctl, pmset, caffeinate, and top on macOS" >&2 return 1 fi load_average=$(sysctl -n vm.loadavg) load_1m=$(printf '%s\n' "$load_average" | awk '{print $2}') logical_cpus=$(sysctl -n hw.logicalcpu) if ! awk -v load="$load_1m" -v cpus="$logical_cpus" 'BEGIN { exit !(load ~ /^[0-9]+([.][0-9]+)?$/ && cpus ~ /^[1-9][0-9]*$/) }'; then echo "gem: could not parse macOS host load state" >&2 return 1 fi load_per_cpu=$(awk -v load="$load_1m" -v cpus="$logical_cpus" \ 'BEGIN {printf "%.6f", load / cpus}') recent_process_cpu_percent=$(ps -Ao pcpu= | awk '{sum += $1} END {printf "%.3f", sum}') short_window_busy_cpu=$(LC_ALL=C top -l 2 -s 1 -n 0 -stats pid,cpu,command | awk -F'[:,%]' '/^CPU usage:/ {idle=$6} END { gsub(/^[[:space:]]+|[[:space:]]+$/, "", idle) if (idle !~ /^[0-9]+([.][0-9]+)?$/) exit 1 printf "%.3f", 100 - idle }') || { echo "gem: could not sample short-window macOS CPU state" >&2 return 1 } power_state=$(LC_ALL=C pmset -g batt 2>&1 || true) thermal_state=$(LC_ALL=C pmset -g therm 2>&1 || true) ac_power=false case $power_state in *"Now drawing from 'AC Power'"*) ac_power=true ;; esac no_thermal_warning=false if printf '%s\n' "$thermal_state" | grep -Fq 'No thermal warning level has been recorded' && printf '%s\n' "$thermal_state" | grep -Fq 'No performance warning level has been recorded'; then no_thermal_warning=true elif printf '%s\n' "$thermal_state" | grep -Eq 'CPU_Scheduler_Limit[[:space:]]*=[[:space:]]*100' && printf '%s\n' "$thermal_state" | grep -Eq 'CPU_Speed_Limit[[:space:]]*=[[:space:]]*100'; then no_thermal_warning=true fi assertions=$(pmset -g assertions 2>&1 || true) top_processes=$(ps -Ao pid,pcpu,pmem,etime,comm -r | sed -n '1,24p') status=pass if ! awk -v actual="$load_per_cpu" -v maximum="$max_load_per_cpu" \ 'BEGIN {exit !(actual <= maximum)}'; then status=fail fi if ! awk -v actual="$short_window_busy_cpu" -v maximum="$max_short_window_busy_cpu" \ 'BEGIN {exit !(actual <= maximum)}'; then status=fail fi [ "$ac_power" = true ] || status=fail [ "$no_thermal_warning" = true ] || status=fail jq -n --arg timestamp "$timestamp" --arg status "$status" \ --arg load_average "$load_average" --argjson load_1m "$load_1m" \ --argjson logical_cpus "$logical_cpus" --argjson load_per_cpu "$load_per_cpu" \ --argjson maximum "$max_load_per_cpu" --arg power_state "$power_state" \ --argjson recent_process_cpu_percent "$recent_process_cpu_percent" \ --argjson short_window_busy_cpu "$short_window_busy_cpu" \ --argjson max_short_window_busy_cpu "$max_short_window_busy_cpu" \ --argjson ac_power "$ac_power" \ --argjson no_thermal_warning "$no_thermal_warning" \ --arg thermal_state "$thermal_state" --arg assertions "$assertions" \ --arg top_processes "$top_processes" \ '{schema_version: 2, timestamp_utc: $timestamp, status: $status, load_average: $load_average, load_1m: $load_1m, logical_cpus: $logical_cpus, load_per_logical_cpu: $load_per_cpu, max_load_per_logical_cpu: $maximum, power_state: $power_state, ac_power: $ac_power, recent_aggregate_process_cpu_percent: $recent_process_cpu_percent, short_window_busy_cpu_percent: $short_window_busy_cpu, max_short_window_busy_cpu_percent: $max_short_window_busy_cpu, thermal_state: $thermal_state, no_thermal_warning: $no_thermal_warning, assertions_before_run: $assertions, top_processes: $top_processes}' > "$receipt" if [ "$status" != pass ]; then echo "gem: host preflight rejected load, CPU, power, or thermal state; see $receipt" >&2 return 1 fi } start_host_monitor() { sample=$1 [ "$prompt_tokens" -ge "$preflight_min_tokens" ] || return 0 monitor_file="$output_dir/host-monitor-$sample.ndjson" : > "$monitor_file" ( failures=0 while kill -0 "$server_pid" 2>/dev/null; do sleep "$host_monitor_seconds" kill -0 "$server_pid" 2>/dev/null || exit 0 timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') load_average=$(sysctl -n vm.loadavg) load_1m=$(printf '%s\n' "$load_average" | awk '{print $2}') logical_cpus=$(sysctl -n hw.logicalcpu) load_per_cpu=$(awk -v load="$load_1m" -v cpus="$logical_cpus" \ 'BEGIN {printf "%.6f", load / cpus}') power_state=$(LC_ALL=C pmset -g batt 2>&1 || true) thermal_state=$(LC_ALL=C pmset -g therm 2>&1 || true) ac_power=false case $power_state in *"Now drawing from 'AC Power'"*) ac_power=true ;; esac no_thermal_warning=false if printf '%s\n' "$thermal_state" | grep -Fq 'No thermal warning level has been recorded' && printf '%s\n' "$thermal_state" | grep -Fq 'No performance warning level has been recorded'; then no_thermal_warning=true elif printf '%s\n' "$thermal_state" | grep -Eq 'CPU_Scheduler_Limit[[:space:]]*=[[:space:]]*100' && printf '%s\n' "$thermal_state" | grep -Eq 'CPU_Speed_Limit[[:space:]]*=[[:space:]]*100'; then no_thermal_warning=true fi overloaded=false if ! awk -v actual="$load_per_cpu" -v maximum="$max_load_per_cpu" \ 'BEGIN {exit !(actual <= maximum)}'; then overloaded=true failures=$((failures + 1)) elif [ "$ac_power" != true ] || [ "$no_thermal_warning" != true ]; then overloaded=true failures=$((failures + 1)) else failures=0 fi jq -cn --arg timestamp "$timestamp" --argjson load_1m "$load_1m" \ --argjson logical_cpus "$logical_cpus" \ --argjson load_per_cpu "$load_per_cpu" \ --argjson maximum "$max_load_per_cpu" --argjson overloaded "$overloaded" \ --argjson ac_power "$ac_power" \ --argjson no_thermal_warning "$no_thermal_warning" \ --arg power_state "$power_state" --arg thermal_state "$thermal_state" \ --argjson consecutive_failures "$failures" \ '{schema_version: 2, timestamp_utc: $timestamp, load_1m: $load_1m, logical_cpus: $logical_cpus, load_per_logical_cpu: $load_per_cpu, max_load_per_logical_cpu: $maximum, overloaded: $overloaded, ac_power: $ac_power, no_thermal_warning: $no_thermal_warning, power_state: $power_state, thermal_state: $thermal_state, consecutive_failures: $consecutive_failures}' >> "$monitor_file" if [ "$failures" -ge "$host_monitor_failures" ]; then printf '%s\n' \ "gem: aborting sample $sample after $failures sustained host-load violations" \ >> "$output_dir/host-monitor-$sample.failure.txt" kill "$server_pid" 2>/dev/null || true exit 1 fi done ) & monitor_pid=$! } for repetition in 1 2 3; do if [ "$resume" -eq 1 ] && sample_complete "$repetition"; then resumed_samples=$((resumed_samples + 1)) continue fi if command -v lsof >/dev/null 2>&1 && lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then echo "gem: benchmark port $port is already occupied" >&2 exit 1 fi host_preflight "$repetition" log="$output_dir/server-$repetition.log" build/gem serve --model "$model" --alias gemma4 --port "$port" \ --context "$context" --max-tokens "$max_tokens" --command-span token \ --kv-type "$kv_type" \ >"$log" 2>&1 & server_pid=$! caffeinate -i -w "$server_pid" >/dev/null 2>&1 & caffeinate_pid=$! start_host_monitor "$repetition" ready=0 attempts=0 while [ "$attempts" -lt 1200 ]; do if curl --fail --silent "http://127.0.0.1:$port/health" >/dev/null 2>&1; then ready=1 break fi if ! kill -0 "$server_pid" 2>/dev/null; then echo "gem: server exited before readiness; see $log" >&2 exit 1 fi attempts=$((attempts + 1)) sleep 0.1 done if [ "$ready" -ne 1 ]; then echo "gem: server readiness timed out; see $log" >&2 exit 1 fi GEM_STREAM_MAX_TOKENS=$max_tokens scripts/server-stream-probe.sh \ "$base_url" gemma4 "$prompt" "$output_dir/stream-$repetition.sse" \ "$output_dir/measurement-$repetition.json" observed_prompt_tokens=$(jq -er '.usage.prompt_tokens' \ "$output_dir/measurement-$repetition.json") if [ "$observed_prompt_tokens" -ne "$prompt_tokens" ]; then echo "gem: server reported $observed_prompt_tokens prompt tokens; expected $prompt_tokens" >&2 exit 1 fi if [ -n "$prefix_extension" ]; then assistant_text=$(sed -n 's/^data: //p' "$output_dir/stream-$repetition.sse" | sed '/^\[DONE\]$/d' | jq -sr '[.[] | .choices[0].delta.content? // empty] | join("")') if [ -z "$assistant_text" ]; then assistant_text=OK fi request_file="$output_dir/prefix-request-$repetition.json" jq -n --rawfile prompt "$prompt" --arg assistant "$assistant_text" \ --arg extension "$prefix_extension" --argjson max_tokens "$max_tokens" \ '{model: "gemma4", messages: [ {role: "user", content: $prompt}, {role: "assistant", content: $assistant}, {role: "user", content: $extension}], max_tokens: $max_tokens}' > "$request_file" GEM_STREAM_MAX_TOKENS=$max_tokens scripts/server-stream-probe.sh --request \ "$base_url" "$request_file" "$output_dir/prefix-stream-$repetition.sse" \ "$output_dir/prefix-measurement-$repetition.json" first_prompt_tokens=$observed_prompt_tokens extended_prompt_tokens=$(jq -er '.usage.prompt_tokens' \ "$output_dir/prefix-measurement-$repetition.json") expected_appended=$((extended_prompt_tokens - first_prompt_tokens)) reuse_record=$(sed -n \ 's/.*exact prefix cache hit (\([0-9][0-9]*\) reused, \([0-9][0-9]*\) appended).*/\1 \2/p' \ "$log" | tail -n 1) if [ "$reuse_record" != "$first_prompt_tokens $expected_appended" ]; then echo "gem: invalid prefix reuse '$reuse_record'; expected $first_prompt_tokens $expected_appended" >&2 exit 1 fi jq --argjson reused "$first_prompt_tokens" --argjson appended "$expected_appended" \ '. + {reused_prompt_tokens: $reused, appended_prompt_tokens: $appended}' \ "$output_dir/prefix-measurement-$repetition.json" \ > "$output_dir/prefix-measurement-$repetition.tmp" mv "$output_dir/prefix-measurement-$repetition.tmp" \ "$output_dir/prefix-measurement-$repetition.json" fi if ! sample_complete "$repetition"; then echo "gem: completed sample $repetition failed retained-artifact validation" >&2 exit 1 fi cleanup done jq -s \ --arg engine "gem-native-c-metal" \ --arg model "$model" \ --arg model_sha256 "$model_sha256" \ --arg prompt "$prompt" \ --arg prompt_sha256 "$prompt_sha256" \ --arg kv_type "$kv_type" \ --argjson context "$context" \ --argjson prompt_tokens "$prompt_tokens" \ --argjson max_tokens "$max_tokens" \ --argjson resumed_samples "$resumed_samples" \ '{schema_version: 1, engine: $engine, model: $model, model_sha256: $model_sha256, prompt_file: $prompt, prompt_sha256: $prompt_sha256, context: $context, kv_type: $kv_type, prompt_tokens: $prompt_tokens, requested_max_tokens: $max_tokens, repetitions: length, resumed_samples: $resumed_samples, samples: ., mean_first_visible_event_seconds: (map(.first_visible_event_seconds) | add / length), mean_completion_seconds: (map(.completion_seconds) | add / length)}' \ "$output_dir/measurement-1.json" "$output_dir/measurement-2.json" \ "$output_dir/measurement-3.json" > "$output_dir/summary.json" if [ -n "$prefix_extension" ]; then jq -s --arg extension "$prefix_extension" \ '{extension_user_text: $extension, repetitions: length, samples: ., mean_first_visible_event_seconds: (map(.first_visible_event_seconds) | add / length), mean_completion_seconds: (map(.completion_seconds) | add / length), reused_prompt_tokens: (map(.reused_prompt_tokens) | unique), appended_prompt_tokens: (map(.appended_prompt_tokens) | unique)}' \ "$output_dir/prefix-measurement-1.json" \ "$output_dir/prefix-measurement-2.json" \ "$output_dir/prefix-measurement-3.json" > "$output_dir/prefix-summary.json" summary_tmp="$output_dir/summary.tmp" jq --slurpfile prefix "$output_dir/prefix-summary.json" \ '. + {prefix_extension: $prefix[0]}' "$output_dir/summary.json" > "$summary_tmp" mv "$summary_tmp" "$output_dir/summary.json" fi jq . "$output_dir/summary.json"