40 lines
1.3 KiB
Bash
Executable file
40 lines
1.3 KiB
Bash
Executable file
#!/bin/sh
|
|
# Convert licensed Gemma 4 31B Hugging Face weights and quantize the text model.
|
|
set -eu
|
|
|
|
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
|
|
echo "usage: $0 HF_MODEL_DIR OUTPUT_DIR [QUANT_TYPE]" >&2
|
|
exit 1
|
|
fi
|
|
|
|
source_dir=$1
|
|
output_dir=$2
|
|
quant=${3:-Q4_K_M}
|
|
root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
llama_dir=${GEM_LLAMA_CPP_DIR:-"$root/../qwen38-llama-bench/llama.cpp"}
|
|
python=${GEM_CONVERT_PYTHON:-"$root/../qwen38-llama-bench/.venv/bin/python"}
|
|
quantize=${GEM_LLAMA_QUANTIZE:-"$llama_dir/build-gem-metal/bin/llama-quantize"}
|
|
|
|
if [ ! -d "$source_dir" ]; then
|
|
echo "gem: model directory does not exist: $source_dir" >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -x "$python" ]; then
|
|
echo "gem: conversion Python with torch is not executable: $python" >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -x "$quantize" ]; then
|
|
echo "gem: llama-quantize is not executable: $quantize" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$output_dir"
|
|
f16="$output_dir/gemma-4-31b-it-f16.gguf"
|
|
mmproj="$output_dir/mmproj-gemma-4-31b-it-f16.gguf"
|
|
quantized="$output_dir/gemma-4-31b-it-${quant}.gguf"
|
|
|
|
"$python" "$llama_dir/convert_hf_to_gguf.py" "$source_dir" --outtype f16 --outfile "$f16"
|
|
"$python" "$llama_dir/convert_hf_to_gguf.py" "$source_dir" --outtype f16 --outfile "$mmproj" --mmproj
|
|
"$quantize" "$f16" "$quantized" "$quant"
|
|
|
|
printf 'text_model=%s\nmmproj=%s\n' "$quantized" "$mmproj"
|