34 lines
727 B
Bash
Executable file
34 lines
727 B
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
if [ "$#" -ne 3 ]; then
|
|
echo "usage: $0 GEM_TOKENIZE LLAMA_TOKENIZE MODEL.gguf" >&2
|
|
exit 2
|
|
fi
|
|
|
|
gem_tokenize=$1
|
|
llama_tokenize=$2
|
|
model=$3
|
|
|
|
compare() {
|
|
prompt=$1
|
|
native=$($gem_tokenize "$model" "$prompt")
|
|
reference=$($llama_tokenize -m "$model" -p "$prompt" --ids --no-escape --log-disable)
|
|
if [ "$native" != "$reference" ]; then
|
|
echo "tokenizer mismatch" >&2
|
|
echo "native: $native" >&2
|
|
echo "reference: $reference" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
compare 'Hello world! 123'
|
|
compare 'naïve café 日本語 🚀'
|
|
compare 'punctuation: a+b=c?!'
|
|
compare "<|turn>user
|
|
Hello<turn|>
|
|
<|turn>model
|
|
<|channel>thought
|
|
<channel|>"
|
|
|
|
echo "Gemma 4 tokenizer parity passed"
|