53 lines
1.6 KiB
Bash
Executable file
53 lines
1.6 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "usage: $0 GEM_PROBE FAKE_SERVER" >&2
|
|
exit 1
|
|
fi
|
|
|
|
probe=$1
|
|
server=$2
|
|
port=$((40000 + $$ % 15000))
|
|
prompt=$(mktemp /private/tmp/gem-probe-prompt.XXXXXX)
|
|
response=$(mktemp /private/tmp/gem-probe-response.XXXXXX)
|
|
summary=$(mktemp /private/tmp/gem-probe-summary.XXXXXX)
|
|
server_pid=
|
|
cleanup() {
|
|
if [ -n "$server_pid" ]; then
|
|
kill "$server_pid" 2>/dev/null || true
|
|
wait "$server_pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$prompt" "$response" "$summary"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
printf 'quote " slash \\\nline' > "$prompt"
|
|
"$server" --port "$port" &
|
|
server_pid=$!
|
|
ready=false
|
|
for _attempt in 1 2 3 4 5 6 7 8 9 10; do
|
|
if curl --fail --silent "http://127.0.0.1:$port/health" >/dev/null; then
|
|
ready=true
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
[ "$ready" = true ]
|
|
|
|
"$probe" --base-url "http://127.0.0.1:$port/v1" --model gemma-4-31b \
|
|
--prompt "$prompt" --max-tokens 8 --response "$response" > "$summary"
|
|
|
|
jq -e '.schema_version == 1 and .http_status == 200' "$summary" >/dev/null
|
|
jq -e '.first_delta_kind == "reasoning_content"' "$summary" >/dev/null
|
|
jq -e '(.ttft_ms - .ttfb_ms) >= 100' "$summary" >/dev/null
|
|
jq -e '(.total_ms - .ttft_ms) >= 50' "$summary" >/dev/null
|
|
jq -e '.response_bytes > 0 and .saw_done == true' "$summary" >/dev/null
|
|
grep -q 'reasoning_content.*think' "$response"
|
|
grep -q 'data: \[DONE\]' "$response"
|
|
|
|
if "$probe" --base-url "http://127.0.0.1:$port/v1" --model gemma-4-31b \
|
|
--prompt /does/not/exist --response "$response" >/dev/null 2>&1; then
|
|
echo "gem: probe accepted an unreadable prompt" >&2
|
|
exit 1
|
|
fi
|