88 lines
4.3 KiB
Bash
Executable file
88 lines
4.3 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 11 ] && [ "$#" -ne 12 ]; then
|
|
echo "usage: $0 COMPARISON_64K COMPARISON_128K PREFILL_ANALYSIS ORACLE_SUMMARY MODEL MODEL_SHA256 GATE_CONFIG HOST_64K_PRE HOST_64K_POST HOST_128K_PRE HOST_128K_POST [OUTPUT_JSON]" >&2
|
|
exit 2
|
|
fi
|
|
c64=$1
|
|
c128=$2
|
|
prefill=$3
|
|
oracle=$4
|
|
model=$5
|
|
model_sha256=$6
|
|
gate=$7
|
|
h64pre=$8
|
|
h64post=$9
|
|
h128pre=${10}
|
|
h128post=${11}
|
|
output=${12:-}
|
|
|
|
analyze() {
|
|
jq -n --slurpfile c64 "$c64" --slurpfile c128 "$c128" \
|
|
--slurpfile prefill "$prefill" --slurpfile oracle "$oracle" \
|
|
--slurpfile gate "$gate" --slurpfile h64pre "$h64pre" \
|
|
--slurpfile h64post "$h64post" --slurpfile h128pre "$h128pre" \
|
|
--slurpfile h128post "$h128post" \
|
|
--arg model "$model" --arg model_sha256 "$model_sha256" '
|
|
def host_ok($h):
|
|
$h.schema_version == 2 and $h.status == "pass" and $h.logical_cpus > 0 and
|
|
$h.load_per_logical_cpu >= 0 and
|
|
$h.load_per_logical_cpu <= $h.max_load_per_logical_cpu and
|
|
$h.short_window_busy_cpu_percent >= 0 and
|
|
$h.short_window_busy_cpu_percent <= $h.max_short_window_busy_cpu_percent and
|
|
$h.ac_power == true and $h.no_thermal_warning == true and
|
|
($h.power_state | contains("AC Power"));
|
|
0.177716972 as $model64 |
|
|
0.304832097 as $model128 |
|
|
(100 * 10 * ($c64[0].score_stage.baseline_mean_seconds -
|
|
$c64[0].score_stage.candidate_mean_seconds) / $model64) as $projected64 |
|
|
(100 * 10 * ($c128[0].score_stage.baseline_mean_seconds -
|
|
$c128[0].score_stage.candidate_mean_seconds) / $model128) as $projected128 |
|
|
if ($model_sha256 | test("^[0-9a-fA-F]{64}$") | not) or
|
|
$c64[0].axis != {name: "score_token_tile", baseline: 1, candidate: 8} or
|
|
$c128[0].axis != {name: "score_token_tile", baseline: 1, candidate: 8} or
|
|
$c64[0].controls.tokens != 65536 or $c128[0].controls.tokens != 131072 or
|
|
$oracle[0].passed != $oracle[0].total or $oracle[0].passed <= 0 or
|
|
$prefill[0].screening.promotion_eligible != false or
|
|
$gate[0].schema_version != 2 or $gate[0].engine != "gem-native-c-metal" or
|
|
$gate[0].axis != {name: "score_token_tile", baseline: 1, candidate: 8} or
|
|
$gate[0].contexts != [65536, 131072] or
|
|
$gate[0].repetitions_per_arm != 3 or $gate[0].iterations != 5 or
|
|
$gate[0].warmup != 3 or
|
|
$gate[0].abba_order != ["baseline-a1", "candidate-b1", "candidate-b2", "baseline-a2"] or
|
|
$gate[0].requires_ac_power != true or
|
|
$gate[0].requires_no_thermal_warning != true or
|
|
$gate[0].max_load_per_logical_cpu <= 0 or
|
|
$gate[0].max_short_window_busy_cpu_percent <= 0 or
|
|
$gate[0].correctness_after_timing != true or
|
|
(host_ok($h64pre[0]) | not) or (host_ok($h64post[0]) | not) or
|
|
(host_ok($h128pre[0]) | not) or (host_ok($h128post[0]) | not)
|
|
then error("invalid score-tile8 decision inputs") else
|
|
{schema_version: 1, engine: "gem-native-c-metal", status: "diagnostic-screen-complete",
|
|
model: $model, model_sha256: $model_sha256,
|
|
axis: {name: "score_token_tile", baseline: 1, candidate: 8},
|
|
host: {at_65536: {preflight: $h64pre[0], postflight: $h64post[0]},
|
|
at_131072: {preflight: $h128pre[0], postflight: $h128post[0]}},
|
|
adjacent_oracle: $oracle[0], operator_64k: $c64[0], operator_128k: $c128[0],
|
|
prefill_attribution: $prefill[0],
|
|
projected_populated_model: {
|
|
full_attention_layers: 10,
|
|
baseline_token_seconds_64k: $model64,
|
|
baseline_token_seconds_128k: $model128,
|
|
projected_whole_token_improvement_pct_64k: $projected64,
|
|
projected_whole_token_improvement_pct_128k: $projected128,
|
|
admission_floor_pct: 3
|
|
},
|
|
operator_screen_passed:
|
|
($c64[0].screening.threshold_passed and $c128[0].screening.threshold_passed and
|
|
$c64[0].bracket.total_gpu.every_candidate_sample_faster and
|
|
$c128[0].bracket.total_gpu.every_candidate_sample_faster and
|
|
$projected64 >= 3 and $projected128 >= 3),
|
|
promotion_eligible: false,
|
|
reason: "diagnostic screen only; authentic logits and populated-model gates remain required"}
|
|
end
|
|
'
|
|
}
|
|
|
|
if [ -n "$output" ]; then analyze > "$output"; else analyze; fi
|