195 lines
9.3 KiB
Bash
Executable file
195 lines
9.3 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 1 ]; then
|
|
echo "usage: $0 AUDITOR" >&2
|
|
exit 2
|
|
fi
|
|
|
|
auditor=$1
|
|
fixture=$(mktemp -d /private/tmp/gem-server-gate-audit.XXXXXX)
|
|
trap 'rm -rf "$fixture"' EXIT INT TERM HUP
|
|
printf 'fixture prompt\n' > "$fixture/prompt.txt"
|
|
prompt_sha=$(shasum -a 256 "$fixture/prompt.txt" | awk '{print $1}')
|
|
printf 'fixture model\n' > "$fixture/model.gguf"
|
|
model_sha=$(shasum -a 256 "$fixture/model.gguf" | awk '{print $1}')
|
|
printf '#!/bin/sh\nexit 0\n' > "$fixture/frozen-gem"
|
|
chmod +x "$fixture/frozen-gem"
|
|
shasum -a 256 "$fixture/frozen-gem" > "$fixture/live-binary-sha256.txt"
|
|
|
|
jq -n --arg model "$fixture/model.gguf" --arg model_sha "$model_sha" \
|
|
--arg prompt "$fixture/prompt.txt" --arg prompt_sha "$prompt_sha" '
|
|
{schema_version: 4, engine: "gem-native-c-metal", model: $model,
|
|
model_sha256: $model_sha,
|
|
prompt_file: $prompt, prompt_sha256: $prompt_sha, context: 4128, kv_type: "f32",
|
|
prompt_tokens: 4096, requested_max_tokens: 16,
|
|
prefix_extension_text: "Reply with DONE.", repetitions: 3,
|
|
host_policy: {preflight_min_tokens: 4096, max_load_per_logical_cpu: 1,
|
|
max_short_window_busy_cpu_percent: 10,
|
|
requires_ac_power: true, requires_no_thermal_warning: true,
|
|
monitor_power_and_thermal: true,
|
|
monitor_interval_seconds: 30, consecutive_failures_before_abort: 3,
|
|
sleep_assertion: "caffeinate -i -w server_pid"}}
|
|
' > "$fixture/gate-config.json"
|
|
jq -n --arg model "$fixture/model.gguf" '
|
|
{engine: "gem-native-c-metal", model: $model, model_size: 31, layers: 60,
|
|
context: 4128, kv_type: "f32", matrix_type: "Q4_0", kv_bytes: 2354053120,
|
|
prefill_batch: 32, attention_prefill_rows: 32,
|
|
prefill_kernel: "mma64x8-4sg-at8+mma64x32-swizzled",
|
|
attention_kernel: "rowwise-gqa4096+full-decode-value-dim32-split16@8192"}
|
|
' > "$fixture/print-config.json"
|
|
|
|
sample=1
|
|
while [ "$sample" -le 3 ]; do
|
|
jq -n '
|
|
{schema_version: 2, status: "pass", load_per_logical_cpu: 0.25,
|
|
max_load_per_logical_cpu: 1, recent_aggregate_process_cpu_percent: 250,
|
|
short_window_busy_cpu_percent: 5, max_short_window_busy_cpu_percent: 10,
|
|
power_state: "Now drawing from AC Power", ac_power: true,
|
|
thermal_state: "no warnings", no_thermal_warning: true}
|
|
' > "$fixture/host-preflight-$sample.json"
|
|
jq -cn '
|
|
{schema_version: 2, load_per_logical_cpu: 0.25,
|
|
max_load_per_logical_cpu: 1, overloaded: false,
|
|
power_state: "Now drawing from AC Power", ac_power: true,
|
|
thermal_state: "no warnings", no_thermal_warning: true}
|
|
' > "$fixture/host-monitor-$sample.ndjson"
|
|
cold=$((100 + sample))
|
|
jq -n --argjson cold "$cold" '
|
|
{schema_version: 1, engine: "gem-native-c-metal", model: "gemma4",
|
|
requested_max_tokens: 16, header_ttfb_seconds: 0.01,
|
|
first_visible_event_seconds: $cold, completion_seconds: ($cold + 1),
|
|
http_code: 200, visible_event_count: 1, finish_reason: "length",
|
|
usage: {prompt_tokens: 4096, completion_tokens: 16, total_tokens: 4112}}
|
|
' > "$fixture/measurement-$sample.json"
|
|
jq -n --argjson sample "$sample" '
|
|
{schema_version: 1, engine: "gem-native-c-metal", model: "gemma4",
|
|
requested_max_tokens: 16, header_ttfb_seconds: 0.01,
|
|
first_visible_event_seconds: (1 + $sample / 10),
|
|
completion_seconds: (1.5 + $sample / 10), http_code: 200,
|
|
visible_event_count: 1, finish_reason: "stop",
|
|
usage: {prompt_tokens: 4122, completion_tokens: 2, total_tokens: 4124},
|
|
reused_prompt_tokens: 4096, appended_prompt_tokens: 26}
|
|
' > "$fixture/prefix-measurement-$sample.json"
|
|
jq -n '
|
|
{model: "gemma4", messages: [{role: "user", content: "fixture"},
|
|
{role: "assistant", content: "OK"},
|
|
{role: "user", content: "Reply with DONE."}], max_tokens: 16}
|
|
' > "$fixture/prefix-request-$sample.json"
|
|
printf '%s\n\n' \
|
|
'data: {"choices":[{"delta":{"content":"x"},"finish_reason":null}]}' \
|
|
'data: {"choices":[{"delta":{},"finish_reason":"length"}]}' \
|
|
'data: {"choices":[],"usage":{"prompt_tokens":4096,"completion_tokens":16,"total_tokens":4112}}' \
|
|
'data: [DONE]' \
|
|
> "$fixture/stream-$sample.sse"
|
|
printf '%s\n\n' \
|
|
'data: {"choices":[{"delta":{"content":"DONE"},"finish_reason":null}]}' \
|
|
'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}' \
|
|
'data: {"choices":[],"usage":{"prompt_tokens":4122,"completion_tokens":2,"total_tokens":4124}}' \
|
|
'data: [DONE]' \
|
|
> "$fixture/prefix-stream-$sample.sse"
|
|
printf '%s\n' \
|
|
'gem: prompt prefill 4096/4096 tokens (100.00%), fixture' \
|
|
'gem: exact prefix cache hit (4096 reused, 26 appended)' \
|
|
'gem: prompt prefill 4122/4122 tokens (100.00%), fixture' \
|
|
> "$fixture/server-$sample.log"
|
|
sample=$((sample + 1))
|
|
done
|
|
|
|
jq -s '
|
|
{schema_version: 1, engine: "gem-native-c-metal", model: "fixture.gguf",
|
|
model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
|
prompt_file: "fixture", prompt_sha256: "fixture", context: 4128, kv_type: "f32",
|
|
prompt_tokens: 4096, requested_max_tokens: 16, repetitions: length, samples: .,
|
|
mean_first_visible_event_seconds: (map(.first_visible_event_seconds) | add / length),
|
|
mean_completion_seconds: (map(.completion_seconds) | add / length)}
|
|
' "$fixture/measurement-1.json" "$fixture/measurement-2.json" \
|
|
"$fixture/measurement-3.json" > "$fixture/summary.json"
|
|
jq -s '
|
|
{extension_user_text: "Reply with DONE.", repetitions: length, samples: .,
|
|
mean_first_visible_event_seconds: (map(.first_visible_event_seconds) | add / length),
|
|
mean_completion_seconds: (map(.completion_seconds) | add / length),
|
|
reused_prompt_tokens: (map(.reused_prompt_tokens) | unique),
|
|
appended_prompt_tokens: (map(.appended_prompt_tokens) | unique)}
|
|
' "$fixture/prefix-measurement-1.json" "$fixture/prefix-measurement-2.json" \
|
|
"$fixture/prefix-measurement-3.json" > "$fixture/prefix-summary.json"
|
|
jq --slurpfile prefix "$fixture/prefix-summary.json" '. + {prefix_extension: $prefix[0]}' \
|
|
"$fixture/summary.json" > "$fixture/summary.tmp"
|
|
mv "$fixture/summary.tmp" "$fixture/summary.json"
|
|
|
|
"$auditor" "$fixture" "$fixture/frozen-gem" > "$fixture/audit.json"
|
|
jq -e '
|
|
.result == "pass" and .context == 4128 and .prompt_tokens == 4096 and
|
|
.model_digest_verified == true and .prompt_digest_verified == true and
|
|
.binary_digest_verified == true and
|
|
.prefix_extension.reused_prompt_tokens == 4096 and
|
|
.prefix_extension.appended_prompt_tokens == 26 and
|
|
.prefix_extension.cold_to_prefix_visible_speedup > 70
|
|
' "$fixture/audit.json" >/dev/null
|
|
|
|
jq '.short_window_busy_cpu_percent = 12.5' "$fixture/host-preflight-2.json" \
|
|
> "$fixture/host-preflight-2.tmp"
|
|
mv "$fixture/host-preflight-2.tmp" "$fixture/host-preflight-2.json"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "overloaded short-window CPU preflight was accepted" >&2
|
|
exit 1
|
|
fi
|
|
jq '.short_window_busy_cpu_percent = 5' "$fixture/host-preflight-2.json" \
|
|
> "$fixture/host-preflight-2.tmp"
|
|
mv "$fixture/host-preflight-2.tmp" "$fixture/host-preflight-2.json"
|
|
|
|
jq '.ac_power = false | .power_state = "Now drawing from Battery Power"' \
|
|
"$fixture/host-preflight-2.json" > "$fixture/host-preflight-2.tmp"
|
|
mv "$fixture/host-preflight-2.tmp" "$fixture/host-preflight-2.json"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "battery-power server preflight was accepted" >&2
|
|
exit 1
|
|
fi
|
|
jq '.ac_power = true | .power_state = "Now drawing from AC Power"' \
|
|
"$fixture/host-preflight-2.json" > "$fixture/host-preflight-2.tmp"
|
|
mv "$fixture/host-preflight-2.tmp" "$fixture/host-preflight-2.json"
|
|
|
|
jq '.no_thermal_warning = false | .overloaded = true' \
|
|
"$fixture/host-monitor-2.ndjson" > "$fixture/host-monitor-2.tmp"
|
|
mv "$fixture/host-monitor-2.tmp" "$fixture/host-monitor-2.ndjson"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "thermal-warning server monitor receipt was accepted" >&2
|
|
exit 1
|
|
fi
|
|
jq -c '.no_thermal_warning = true | .overloaded = false' \
|
|
"$fixture/host-monitor-2.ndjson" > "$fixture/host-monitor-2.tmp"
|
|
mv "$fixture/host-monitor-2.tmp" "$fixture/host-monitor-2.ndjson"
|
|
|
|
sed 's/"total_tokens":4112/"total_tokens":4111/' "$fixture/stream-2.sse" \
|
|
> "$fixture/stream-2.tmp"
|
|
mv "$fixture/stream-2.tmp" "$fixture/stream-2.sse"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "tampered SSE usage receipt was accepted" >&2
|
|
exit 1
|
|
fi
|
|
sed 's/"total_tokens":4111/"total_tokens":4112/' "$fixture/stream-2.sse" \
|
|
> "$fixture/stream-2.tmp"
|
|
mv "$fixture/stream-2.tmp" "$fixture/stream-2.sse"
|
|
|
|
printf '#!/bin/sh\nexit 1\n' > "$fixture/frozen-gem"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "tampered frozen binary was accepted" >&2
|
|
exit 1
|
|
fi
|
|
printf '#!/bin/sh\nexit 0\n' > "$fixture/frozen-gem"
|
|
chmod +x "$fixture/frozen-gem"
|
|
|
|
printf 'tampered model\n' > "$fixture/model.gguf"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "tampered model was accepted" >&2
|
|
exit 1
|
|
fi
|
|
printf 'fixture model\n' > "$fixture/model.gguf"
|
|
|
|
jq '.reused_prompt_tokens = 4095' "$fixture/prefix-measurement-2.json" \
|
|
> "$fixture/prefix-measurement-2.tmp"
|
|
mv "$fixture/prefix-measurement-2.tmp" "$fixture/prefix-measurement-2.json"
|
|
if "$auditor" "$fixture" "$fixture/frozen-gem" >/dev/null 2>&1; then
|
|
echo "tampered prefix-reuse receipt was accepted" >&2
|
|
exit 1
|
|
fi
|