50 lines
1.9 KiB
Bash
Executable file
50 lines
1.9 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
[ "$#" -eq 1 ] || { echo "usage: $0 ANALYZER" >&2; exit 2; }
|
|
analyzer=$1
|
|
fixture=$(mktemp -d /private/tmp/gem-decode-host-overhead.XXXXXX)
|
|
trap 'rm -rf "$fixture"' EXIT INT TERM HUP
|
|
|
|
jq -n '{
|
|
schema_version:1, engine:"gem-native-c-metal",
|
|
workload:"authentic-one-token-decode-stage-profile", model:"model.gguf",
|
|
attention_kernel:"split16", prefix_mode:"seeded-kv", context:65792,
|
|
position:65536, rows:1, repetitions:3, command_buffers:1, dispatches:1393,
|
|
production_execution:{
|
|
mean_wall_seconds:0.2, mean_gpu_seconds:0.198,
|
|
gpu_wall_fraction:0.99, mean_wall_minus_gpu_seconds:0.002,
|
|
mean_host_prepare_seconds:0.0002, mean_host_encode_seconds:0.0005,
|
|
mean_host_submit_wait_seconds:0.1992, mean_host_cleanup_seconds:0.00002,
|
|
mean_host_readback_seconds:0.00008,
|
|
wall_seconds_samples:[0.2,0.2,0.2], gpu_seconds_samples:[0.198,0.198,0.198]
|
|
}
|
|
}' > "$fixture/input.json"
|
|
|
|
"$analyzer" "$fixture/input.json" "$fixture/output.json"
|
|
jq -e '
|
|
.workload == "decode-host-overhead-ceiling-screen" and
|
|
.observed.wall_minus_gpu_ceiling_pct == 1 and
|
|
.screening.host_axis_admissible == false and
|
|
.screening.promotion_eligible == false and
|
|
(.input.sha256 | test("^[0-9a-f]{64}$"))
|
|
' "$fixture/output.json" >/dev/null
|
|
|
|
jq '.production_execution.mean_host_encode_seconds = 0.02' \
|
|
"$fixture/input.json" > "$fixture/bad-sum.json"
|
|
if "$analyzer" "$fixture/bad-sum.json" >/dev/null 2>&1; then
|
|
echo "decode host-overhead analyzer accepted inconsistent phase accounting" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq '.production_execution.mean_gpu_seconds = 0.201' \
|
|
"$fixture/input.json" > "$fixture/bad-gpu.json"
|
|
if "$analyzer" "$fixture/bad-gpu.json" >/dev/null 2>&1; then
|
|
echo "decode host-overhead analyzer accepted GPU time above wall time" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if "$analyzer" "$fixture/input.json" "$fixture/output.json" >/dev/null 2>&1; then
|
|
echo "decode host-overhead analyzer overwrote existing output" >&2
|
|
exit 1
|
|
fi
|