4228a0c5e9* implemented stubbed IGraphContextService methods * feat: wire graph communication updates in ipc, mcp and cli * fix: fix call graph recall and add profiling coverage to graph calls sites * fix: fixes for work where we added agent navigation to graph tooling * fix(corpus): clean FTS and KG orphans on document delete * fix(repair): clean orphans, honor cancellation, surface stats * fix(rm): order content removal before metadata/KG delete. keep manifest while content is still referenced * fix(daemon): seed built-in content extractor for readiness * chore(cli): remove dead stubs and unwired helpers * feat(search): named-collection scoping + provable relevance gate * fix(refactor): rm dead stubs, inline simd, add proto/metrics tests * chore: rm dead interfaces/enums, consolidate guards, add tests * refactor: rm dr vaporware, collapse download handlers, clean list flags * docs: rm stale testing investigation logs * fix: address Copilot review findings from PR #50 - relationship_ops: return existing id when INSERT OR IGNORE skips - document_crud_ops: propagate FTS5 cleanup error (was silent skip) - builtin_text_content_extractor: unsigned char cast for tolower, add cctype - mcp_server download_jobs: validate job_id for status/cancel actions - list_command tests: unsigned char cast for isspace, add cctype include
75 lines
2.6 KiB
C++
75 lines
2.6 KiB
C++
#include <catch2/catch_test_macros.hpp>
|
|
|
|
#include <cstddef>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <yams/extraction/builtin_text_content_extractor.h>
|
|
#include <yams/extraction/content_extractor.h>
|
|
|
|
using namespace yams::extraction;
|
|
|
|
namespace {
|
|
|
|
std::vector<std::byte> toBytes(const std::string& s) {
|
|
std::vector<std::byte> bytes;
|
|
bytes.reserve(s.size());
|
|
for (char c : s) {
|
|
bytes.push_back(static_cast<std::byte>(static_cast<unsigned char>(c)));
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// The daemon gates content_extractors_ready on the count of IContentExtractor
|
|
// instances. Plugin-sourced extractors (e.g. zyp/PDF) are optional and
|
|
// toolchain-gated (require Zig), so on platforms without that toolchain the count
|
|
// would be zero and readiness would never flip. The built-in text content extractor
|
|
// must therefore be constructible and functional unconditionally, on every platform.
|
|
TEST_CASE("BuiltinTextContentExtractor is always constructible as IContentExtractor",
|
|
"[extraction][content_extractor][platform]") {
|
|
std::shared_ptr<IContentExtractor> extractor = std::make_shared<BuiltinTextContentExtractor>();
|
|
REQUIRE(extractor != nullptr);
|
|
|
|
ContentExtractorList list{extractor};
|
|
REQUIRE(list.size() == 1);
|
|
}
|
|
|
|
TEST_CASE("BuiltinTextContentExtractor supports text by extension and mime",
|
|
"[extraction][content_extractor]") {
|
|
BuiltinTextContentExtractor extractor;
|
|
|
|
CHECK(extractor.supports("text/plain", ".txt"));
|
|
CHECK(extractor.supports("text/markdown", ".md"));
|
|
CHECK(extractor.supports("text/plain", ""));
|
|
|
|
// Plain extension with no leading dot must normalize.
|
|
CHECK(extractor.supports("", "txt"));
|
|
}
|
|
|
|
TEST_CASE("BuiltinTextContentExtractor extracts plain text bytes",
|
|
"[extraction][content_extractor]") {
|
|
BuiltinTextContentExtractor extractor;
|
|
|
|
const std::string content = "hello world\nsecond line\n";
|
|
auto bytes = toBytes(content);
|
|
|
|
auto text = extractor.extractText(bytes, "text/plain", ".txt");
|
|
REQUIRE(text.has_value());
|
|
CHECK(text->find("hello world") != std::string::npos);
|
|
CHECK(text->find("second line") != std::string::npos);
|
|
}
|
|
|
|
TEST_CASE("BuiltinTextContentExtractor falls back to raw bytes for text mime",
|
|
"[extraction][content_extractor]") {
|
|
BuiltinTextContentExtractor extractor;
|
|
|
|
const std::string content = "raw text payload";
|
|
auto bytes = toBytes(content);
|
|
|
|
// Unknown extension but text mime: should still yield content.
|
|
auto text = extractor.extractText(bytes, "text/plain", ".unknownext");
|
|
REQUIRE(text.has_value());
|
|
CHECK(text->find("raw text payload") != std::string::npos);
|
|
}
|