38 lines
1.1 KiB
Bash
Executable file
38 lines
1.1 KiB
Bash
Executable file
#!/bin/sh
|
|
# Run the first one-variable benchmark sweep and preserve its raw evidence.
|
|
set -eu
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "usage: $0 GEM_BINARY MODEL_GGUF" >&2
|
|
exit 1
|
|
fi
|
|
|
|
gem=$1
|
|
model=$2
|
|
root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
stamp=$(date +%Y%m%d-%H%M%S)
|
|
out=${GEM_BENCH_DIR:-"artifacts/$stamp"}
|
|
|
|
mkdir -p "$out"
|
|
"$gem" print-config --model "$model" > "$out/config.txt"
|
|
"$gem" bench --dry-run --model "$model" > "$out/baseline-command.txt"
|
|
sw_vers > "$out/macos.txt"
|
|
system_profiler SPHardwareDataType SPDisplaysDataType > "$out/hardware.txt"
|
|
|
|
llama_dir=${GEM_LLAMA_CPP_DIR:-"$root/../qwen38-llama-bench/llama.cpp"}
|
|
if [ -d "$llama_dir/.git" ]; then
|
|
git -C "$llama_dir" rev-parse HEAD > "$out/llama-commit.txt"
|
|
git -C "$llama_dir" status --short > "$out/llama-status.txt"
|
|
fi
|
|
|
|
for flash in auto on off; do
|
|
"$gem" bench --model "$model" --flash-attn "$flash" --repetitions 3 \
|
|
> "$out/flash-$flash.json"
|
|
done
|
|
|
|
for flash in on off; do
|
|
"$root/scripts/analyze-benchmarks.sh" "$out/flash-auto.json" \
|
|
"$out/flash-$flash.json" "$out/flash-$flash-comparison.json"
|
|
done
|
|
|
|
echo "gem: wrote benchmark evidence to $out"
|