45 lines
1.7 KiB
Bash
Executable file
45 lines
1.7 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "usage: $0 ANALYZER PROFILE_FIXTURE" >&2
|
|
exit 2
|
|
fi
|
|
analyzer=$1
|
|
fixture=$2
|
|
temporary=$(mktemp -d /private/tmp/gem-decode-attention-abba-test.XXXXXX)
|
|
trap 'rm -rf "$temporary"' EXIT INT TERM HUP
|
|
|
|
jq '.score_token_tile = 1' "$fixture" > "$temporary/a1.json"
|
|
jq '.score_token_tile = 8 |
|
|
.gpu_seconds_samples |= map(. * 0.90) |
|
|
.stages.score.samples_seconds |= map(. * 0.80)' \
|
|
"$fixture" > "$temporary/b1.json"
|
|
jq '.score_token_tile = 8 |
|
|
.gpu_seconds_samples |= map(. * 0.88) |
|
|
.stages.score.samples_seconds |= map(. * 0.78)' \
|
|
"$fixture" > "$temporary/b2.json"
|
|
jq '.score_token_tile = 1 |
|
|
.gpu_seconds_samples |= map(. * 1.01) |
|
|
.stages.score.samples_seconds |= map(. * 1.01)' \
|
|
"$fixture" > "$temporary/a2.json"
|
|
|
|
"$analyzer" "$temporary/a1.json" "$temporary/b1.json" \
|
|
"$temporary/b2.json" "$temporary/a2.json" "$temporary/report.json"
|
|
jq -e '
|
|
.axis.name == "score_token_tile" and .axis.baseline == 1 and .axis.candidate == 8 and
|
|
.controls.repetitions == 6 and .total_gpu.improvement_pct > 10 and
|
|
.bracket.order == ["baseline-a1", "candidate-b1", "candidate-b2", "baseline-a2"] and
|
|
.bracket.arm_repetitions == 3 and
|
|
.bracket.total_gpu.opening_improvement_pct > 9.9 and
|
|
.bracket.total_gpu.closing_improvement_pct > 12 and
|
|
.bracket.total_gpu.every_candidate_sample_faster == true and
|
|
.screening.promotion_eligible == false
|
|
' "$temporary/report.json" >/dev/null
|
|
|
|
jq '.score_token_tile = 1' "$temporary/b2.json" > "$temporary/invalid.json"
|
|
if "$analyzer" "$temporary/a1.json" "$temporary/b1.json" \
|
|
"$temporary/invalid.json" "$temporary/a2.json" >/dev/null 2>&1; then
|
|
echo "ABBA candidate-axis mismatch was accepted" >&2
|
|
exit 1
|
|
fi
|