yams/tests/unit/common/utf8_utils_catch2_test.cpp

// Tests for UTF-8 sanitization helper.

#include <catch2/catch_test_macros.hpp>

#include <string>

#include <yams/common/utf8_utils.h>

using yams::common::ensureValidUtf8;
using yams::common::sanitizeUtf8;
using yams::common::sanitizeUtf8Strict;

TEST_CASE("common::sanitizeUtf8 preserves ASCII", "[common][utf8][catch2]") {
    CHECK(sanitizeUtf8("hello") == "hello");
    CHECK(sanitizeUtf8("") == "");
}

TEST_CASE("common::sanitizeUtf8 preserves valid multibyte sequences", "[common][utf8][catch2]") {
    // 2-byte: C2 A2
    CHECK(sanitizeUtf8(std::string("\xC2\xA2")) == std::string("\xC2\xA2"));
    // 3-byte: E2 82 AC
    CHECK(sanitizeUtf8(std::string("\xE2\x82\xAC")) == std::string("\xE2\x82\xAC"));
    // 4-byte: F0 9F 98 80
    CHECK(sanitizeUtf8(std::string("\xF0\x9F\x98\x80")) == std::string("\xF0\x9F\x98\x80"));
}

TEST_CASE("common::sanitizeUtf8 replaces invalid sequences with '?'", "[common][utf8][catch2]") {
    // Lone continuation byte
    CHECK(sanitizeUtf8(std::string("\x80")) == "?");

    // 2-byte lead followed by invalid continuation: consumes lead, keeps next byte
    CHECK(sanitizeUtf8(std::string("\xC2"
                                   "x")) == "?x");

    // Truncated 3-byte sequence at end: each invalid byte becomes '?'
    CHECK(sanitizeUtf8(std::string("\xE2\x82")) == "??");

    // Overlong starter (0xC0) is not treated as a valid lead
    CHECK(sanitizeUtf8(std::string("\xC0\x80")) == "??");
}

TEST_CASE("common::sanitizeUtf8Strict preserves valid UTF-8", "[common][utf8][strict]") {
    const std::string valid = "ASCII \xC2\xA2 \xE2\x82\xAC \xF0\x9F\x98\x80";
    CHECK(sanitizeUtf8Strict(valid) == valid);
}

TEST_CASE("common::sanitizeUtf8Strict replaces each invalid byte with U+FFFD",
          "[common][utf8][strict]") {
    const std::string replacement = "\xEF\xBF\xBD";

    CHECK(sanitizeUtf8Strict(std::string("\x80")) == replacement);
    CHECK(sanitizeUtf8Strict(std::string("\xE0\x80\x80")) ==
          replacement + replacement + replacement);
    CHECK(sanitizeUtf8Strict(std::string("\xED\xA0\x80")) ==
          replacement + replacement + replacement);
    CHECK(sanitizeUtf8Strict(std::string("\xF4\x90\x80\x80")) ==
          replacement + replacement + replacement + replacement);
    CHECK(sanitizeUtf8Strict(std::string("\xE2\x82")) == replacement + replacement);
}

// --- ensureValidUtf8 tests ---

TEST_CASE("ensureValidUtf8 returns input view for valid ASCII", "[common][utf8][lazy]") {
    std::string storage;
    std::string_view input = "hello world";
    auto result = ensureValidUtf8(input, storage);
    CHECK(result == "hello world");
    CHECK(result.data() == input.data()); // Points to original, not storage
    CHECK(storage.empty());
}

TEST_CASE("ensureValidUtf8 returns input view for valid multibyte", "[common][utf8][lazy]") {
    std::string storage;
    // 2-byte
    std::string two("\xC2\xA2");
    auto r2 = ensureValidUtf8(two, storage);
    CHECK(r2 == two);
    CHECK(r2.data() == two.data());
    CHECK(storage.empty());

    // 3-byte
    std::string three("\xE2\x82\xAC");
    auto r3 = ensureValidUtf8(three, storage);
    CHECK(r3 == three);
    CHECK(r3.data() == three.data());

    // 4-byte
    std::string four("\xF0\x9F\x98\x80");
    auto r4 = ensureValidUtf8(four, storage);
    CHECK(r4 == four);
    CHECK(r4.data() == four.data());
}

TEST_CASE("ensureValidUtf8 sanitizes invalid input into storage", "[common][utf8][lazy]") {
    std::string storage;
    std::string invalid("\x80invalid");
    auto result = ensureValidUtf8(invalid, storage);
    CHECK(result == sanitizeUtf8(invalid));
    CHECK(result.data() == storage.data()); // Points into storage
    CHECK(!storage.empty());
}

TEST_CASE("ensureValidUtf8 storage reuse across calls", "[common][utf8][lazy]") {
    std::string storage;

    // First call: valid input, storage untouched
    std::string valid = "abc";
    auto r1 = ensureValidUtf8(valid, storage);
    CHECK(r1.data() == valid.data());
    CHECK(storage.empty());

    // Second call: invalid input, storage written
    std::string invalid("\xFE\xFF");
    auto r2 = ensureValidUtf8(invalid, storage);
    CHECK(r2 == sanitizeUtf8(invalid));
    CHECK(!storage.empty());
}

TEST_CASE("ensureValidUtf8 handles empty input", "[common][utf8][lazy]") {
    std::string storage;
    std::string_view empty;
    auto result = ensureValidUtf8(empty, storage);
    CHECK(result.empty());
    CHECK(storage.empty());
}

// sanitizeUtf8 feeds proto3 string fields, which reject overlong encodings, UTF-16 surrogates
// and code points above U+10FFFF. Its output (and ensureValidUtf8's) must be well-formed UTF-8,
// i.e. unchanged by the strict sanitizer.
TEST_CASE("common::sanitizeUtf8 output is well-formed UTF-8", "[common][utf8][catch2]") {
    using yams::common::ensureValidUtf8;
    using yams::common::sanitizeUtf8;
    using yams::common::sanitizeUtf8Strict;

    const std::string cases[] = {
        std::string("\xE0\x80\x80", 3),     // overlong 3-byte (U+0000)
        std::string("\xE0\x9F\xBF", 3),     // overlong 3-byte (U+07FF)
        std::string("\xED\xA0\x80", 3),     // UTF-16 high surrogate
        std::string("\xED\xBF\xBF", 3),     // UTF-16 low surrogate
        std::string("\xF0\x80\x80\x80", 4), // overlong 4-byte
        std::string("\xF4\x90\x80\x80", 4), // above U+10FFFF
        std::string("ok \xED\xA0\x80 tail", 10),
    };
    for (const auto& input : cases) {
        const auto lenient = sanitizeUtf8(input);
        INFO("input bytes: " << input.size());
        CHECK(sanitizeUtf8Strict(lenient) == lenient);

        std::string storage;
        const std::string ensured(ensureValidUtf8(input, storage));
        CHECK(sanitizeUtf8Strict(ensured) == ensured);
    }
    // Valid boundary code points are preserved.
    const std::string valid =
        "\xE0\xA0\x80\xED\x9F\xBF\xEE\x80\x80\xF0\x90\x80\x80\xF4\x8F\xBF\xBF";
    CHECK(sanitizeUtf8(valid) == valid);
}