#!/bin/sh
set -eu

if [ "$#" -ne 3 ]; then
    echo "usage: $0 GEM_TOKENIZE LLAMA_TOKENIZE MODEL.gguf" >&2
    exit 2
fi

gem_tokenize=$1
llama_tokenize=$2
model=$3

compare() {
    prompt=$1
    native=$($gem_tokenize "$model" "$prompt")
    reference=$($llama_tokenize -m "$model" -p "$prompt" --ids --no-escape --log-disable)
    if [ "$native" != "$reference" ]; then
        echo "tokenizer mismatch" >&2
        echo "native:    $native" >&2
        echo "reference: $reference" >&2
        exit 1
    fi
}

compare 'Hello world! 123'
compare 'naïve café 日本語 🚀'
compare 'punctuation: a+b=c?!'
compare "<|turn>user
Hello<turn|>
<|turn>model
<|channel>thought
<channel|>"

echo "Gemma 4 tokenizer parity passed"