if(NOT DEFINED NATIVE_BINARY OR NOT DEFINED MODEL OR NOT DEFINED INPUT_FILE)
    message(FATAL_ERROR "tokenize CLI test requires NATIVE_BINARY, MODEL, and INPUT_FILE")
endif()

execute_process(
    COMMAND "${NATIVE_BINARY}" --count "${MODEL}" hello
    RESULT_VARIABLE raw_status
    OUTPUT_VARIABLE raw_json
    ERROR_VARIABLE raw_error)
if(NOT raw_status EQUAL 0)
    message(FATAL_ERROR "direct count failed: ${raw_error}")
endif()
string(JSON raw_count ERROR_VARIABLE raw_json_error GET "${raw_json}" token_count)
if(raw_json_error OR NOT raw_count EQUAL 2)
    message(FATAL_ERROR "unexpected direct count: ${raw_json}")
endif()

execute_process(
    COMMAND "${NATIVE_BINARY}" --count --file "${MODEL}" "${INPUT_FILE}"
    RESULT_VARIABLE file_status
    OUTPUT_VARIABLE file_json
    ERROR_VARIABLE file_error)
if(NOT file_status EQUAL 0)
    message(FATAL_ERROR "file count failed: ${file_error}")
endif()
string(JSON file_count ERROR_VARIABLE file_json_error GET "${file_json}" token_count)
if(file_json_error OR NOT file_count EQUAL 3)
    message(FATAL_ERROR "unexpected newline-terminated file count: ${file_json}")
endif()

execute_process(
    COMMAND "${NATIVE_BINARY}" --count --file --chat-user
            "${MODEL}" "${INPUT_FILE}"
    RESULT_VARIABLE chat_status
    OUTPUT_VARIABLE chat_json
    ERROR_VARIABLE chat_error)
if(NOT chat_status EQUAL 0)
    message(FATAL_ERROR "chat count failed: ${chat_error}")
endif()
string(JSON chat_count ERROR_VARIABLE chat_json_error GET "${chat_json}" token_count)
if(chat_json_error OR NOT chat_count EQUAL 10)
    message(FATAL_ERROR "unexpected canonical chat count: ${chat_json}")
endif()