9e44b172ae
- fuzz_fts5_query: sanitized MCP/CLI search text must execute as MATCH on an FTS5 table shaped like documents_fts, also after re-sanitizing. - fuzz_utf8: sanitizer output is well-formed and preserves valid input; a UTF-8 verdict implies well-formed bytes; UTF-16/Latin-1 conversions produce well-formed UTF-8. - fuzz_text_extractors: HTML/plain-text extraction, title and section detection on arbitrary bytes, with an output-size bound; run with -timeout to catch super-linear scans. - fuzz_compressed_block: stored blocks decode within the header's size and -rss_limit_mb; zstd/LZMA round-trip exactly. Seeds in generate_corpus.sh; the Forgejo fuzz-smoke lane builds and runs them and triggers on the parsers' sources. Five minutes per harness on a Linux x86 host after the fixes in this series: no findings.
63 lines
2.2 KiB
C++
63 lines
2.2 KiB
C++
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
// Copyright 2026 YAMS Contributors
|
|
|
|
// UTF-8 sanitizers and text encoding detection/conversion on ingested file bytes. Oracles:
|
|
// sanitizer output is well-formed UTF-8 (unchanged by the strict sanitizer) and leaves valid
|
|
// input intact; a "UTF-8" detection means the bytes really are well-formed; UTF-16 and
|
|
// Latin-1 conversions always produce well-formed UTF-8.
|
|
|
|
#include <yams/common/utf8_utils.h>
|
|
#include <yams/extraction/text_extractor.h>
|
|
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <span>
|
|
#include <string>
|
|
#include <string_view>
|
|
|
|
namespace {
|
|
|
|
void fuzzRequire(bool condition) {
|
|
if (!condition) {
|
|
__builtin_trap();
|
|
}
|
|
}
|
|
|
|
bool wellFormed(std::string_view text) {
|
|
return yams::common::sanitizeUtf8Strict(text) == text;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) {
|
|
if (data == nullptr || size > std::size_t{256} * 1024) {
|
|
return 0;
|
|
}
|
|
const std::string input(reinterpret_cast<const char*>(data), size);
|
|
const bool inputValid = wellFormed(input);
|
|
|
|
const std::string lenient = yams::common::sanitizeUtf8(input);
|
|
fuzzRequire(wellFormed(lenient));
|
|
fuzzRequire(!inputValid || lenient == input);
|
|
|
|
const std::string strict = yams::common::sanitizeUtf8Strict(input);
|
|
fuzzRequire(wellFormed(strict));
|
|
|
|
std::string storage;
|
|
const std::string_view ensured = yams::common::ensureValidUtf8(input, storage);
|
|
fuzzRequire(wellFormed(ensured));
|
|
fuzzRequire(!inputValid || ensured == input);
|
|
|
|
using yams::extraction::EncodingDetector;
|
|
const auto bytes = std::span<const std::byte>(reinterpret_cast<const std::byte*>(data), size);
|
|
const std::string detected = EncodingDetector::detectEncoding(bytes, nullptr);
|
|
const bool bom = size >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF;
|
|
fuzzRequire(detected != "UTF-8" || bom || inputValid);
|
|
|
|
for (const char* encoding : {"UTF-16LE", "UTF-16BE", "ISO-8859-1"}) {
|
|
auto converted = EncodingDetector::convertToUtf8(input, encoding);
|
|
fuzzRequire(converted.has_value());
|
|
fuzzRequire(wellFormed(converted.value()));
|
|
}
|
|
return 0;
|
|
}
|