43 lines
1.6 KiB
CMake
43 lines
1.6 KiB
CMake
if(NOT DEFINED NATIVE_BINARY OR NOT DEFINED MODEL OR NOT DEFINED INPUT_FILE)
|
|
message(FATAL_ERROR "tokenize CLI test requires NATIVE_BINARY, MODEL, and INPUT_FILE")
|
|
endif()
|
|
|
|
execute_process(
|
|
COMMAND "${NATIVE_BINARY}" --count "${MODEL}" hello
|
|
RESULT_VARIABLE raw_status
|
|
OUTPUT_VARIABLE raw_json
|
|
ERROR_VARIABLE raw_error)
|
|
if(NOT raw_status EQUAL 0)
|
|
message(FATAL_ERROR "direct count failed: ${raw_error}")
|
|
endif()
|
|
string(JSON raw_count ERROR_VARIABLE raw_json_error GET "${raw_json}" token_count)
|
|
if(raw_json_error OR NOT raw_count EQUAL 2)
|
|
message(FATAL_ERROR "unexpected direct count: ${raw_json}")
|
|
endif()
|
|
|
|
execute_process(
|
|
COMMAND "${NATIVE_BINARY}" --count --file "${MODEL}" "${INPUT_FILE}"
|
|
RESULT_VARIABLE file_status
|
|
OUTPUT_VARIABLE file_json
|
|
ERROR_VARIABLE file_error)
|
|
if(NOT file_status EQUAL 0)
|
|
message(FATAL_ERROR "file count failed: ${file_error}")
|
|
endif()
|
|
string(JSON file_count ERROR_VARIABLE file_json_error GET "${file_json}" token_count)
|
|
if(file_json_error OR NOT file_count EQUAL 3)
|
|
message(FATAL_ERROR "unexpected newline-terminated file count: ${file_json}")
|
|
endif()
|
|
|
|
execute_process(
|
|
COMMAND "${NATIVE_BINARY}" --count --file --chat-user
|
|
"${MODEL}" "${INPUT_FILE}"
|
|
RESULT_VARIABLE chat_status
|
|
OUTPUT_VARIABLE chat_json
|
|
ERROR_VARIABLE chat_error)
|
|
if(NOT chat_status EQUAL 0)
|
|
message(FATAL_ERROR "chat count failed: ${chat_error}")
|
|
endif()
|
|
string(JSON chat_count ERROR_VARIABLE chat_json_error GET "${chat_json}" token_count)
|
|
if(chat_json_error OR NOT chat_count EQUAL 10)
|
|
message(FATAL_ERROR "unexpected canonical chat count: ${chat_json}")
|
|
endif()
|