49 lines
1.8 KiB
Bash
Executable file
49 lines
1.8 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
[ "$#" -eq 2 ] || [ "$#" -eq 3 ] || {
|
|
echo "usage: $0 BASELINE_JSON CANDIDATE_JSON [OUTPUT_JSON]" >&2
|
|
exit 2
|
|
}
|
|
baseline=$1
|
|
candidate=$2
|
|
output=${3:-}
|
|
|
|
analyze() {
|
|
jq -n --slurpfile baseline "$baseline" --slurpfile candidate "$candidate" '
|
|
$baseline[0] as $b | $candidate[0] as $c |
|
|
if ($b.schema_version != 1 or $c.schema_version != 1 or
|
|
$b.kernel != $c.kernel or $b.hardware != $c.hardware or
|
|
$b.input_size != $c.input_size or $b.output_size != $c.output_size or
|
|
$b.iterations != $c.iterations or
|
|
$b.warmup != $c.warmup or
|
|
($b.samples_gbps | length) < 3 or ($c.samples_gbps | length) < 3)
|
|
then error("kernel benchmark inputs are not comparable") else
|
|
(["buffer_mode", "command_span", "matrix_slots"] |
|
|
map(select($b[.] != $c[.]))) as $axes |
|
|
if ($axes | length) != 1 then error("exactly one tuning axis must change") else
|
|
(100 * ($c.mean_gbps / $b.mean_gbps - 1)) as $delta |
|
|
(100 * $b.stddev_gbps / $b.mean_gbps) as $bcv |
|
|
(100 * $c.stddev_gbps / $c.mean_gbps) as $ccv |
|
|
([ $bcv, $ccv ] | max) as $noise |
|
|
{
|
|
schema_version: 1,
|
|
axis: {name: $axes[0], baseline: $b[$axes[0]], candidate: $c[$axes[0]]},
|
|
metric: {
|
|
baseline_gbps: $b.mean_gbps,
|
|
candidate_gbps: $c.mean_gbps,
|
|
delta_pct: $delta,
|
|
baseline_cv_pct: $bcv,
|
|
candidate_cv_pct: $ccv,
|
|
observed_noise_floor_pct: $noise
|
|
},
|
|
promotion: {
|
|
threshold_passed: ($delta >= 3 and $delta > $noise),
|
|
eligible: false,
|
|
reason: "microbenchmark only; authentic full-model correctness and performance gates are required"
|
|
}
|
|
} end
|
|
end'
|
|
}
|
|
|
|
if [ -n "$output" ]; then analyze > "$output"; else analyze; fi
|