* refactor(tuning): prune dormant runtime knobs Remove unused tuning controls and adaptive search budget state. Extract testable ONNX slot reconciliation and search telemetry hydration seams, and correct the Apple Silicon sysctl value width. * refactor(vector): retire dormant backends Remove unused FAISS, model-infrastructure, VectorIndexManager, and TurboQuant paths while keeping Simeon PQ and float-vector storage. Migrate unused V2.2 quantized schemas transactionally to V2.3. Databases containing non-default quantized formats or packed-code blobs fail closed with recovery guidance and no mutation. * refactor(plugins): remove legacy SDK tooling Remove the Ghidra and SDK submodules, their bespoke tests, and stale scan exclusions. Retain generic PluginProcess coverage against the repository mock fixture. * build(vector): remove stale FAISS overrides * refactor(ipc): remove dormant FSM snapshots * refactor(ingest): unify bounded KG enqueue * fix(plugins): harden allocation contracts Keep ABI outputs null or fully initialized when allocations fail, and represent empty object payloads without relying on malloc(0). * fix(storage): define empty object reads Return a null buffer with zero length for successful empty-object reads instead of treating malloc(0) as an allocation contract. * refactor(storage): trim through string views * test: prune dormant harness helpers Remove unused readiness and fixture helpers, and keep benchmark client-count doubling in a wide integer domain. * chore(cpp): initialize analyzer-facing locals * build(simeon): advance analyzer cleanup * refactor(config): drop dead daemon client keys * refactor(ipc): inject daemon client transport Socket and in-process execution now share DaemonClient request dispatch, so CLI fallbacks do not need alternate service paths. * refactor(retrieval): route CLI through daemon Search and grep now use the same daemon handlers over socket or in-process transport. Live grep remains an explicit freshness inspection. * refactor(graph): move CLI scope into daemon Graph CLI requests now share daemon handlers across socket and in-process transports. CWD filtering occurs before pagination so scoped totals and pages stay correct. * build: isolate test-only runtime controls * refactor(metadata): decouple lock telemetry * fix(services): return on request timeout * fix(retrieval): enforce workspace scope * fix(graph): scope nodes before pagination * fix(grep): drop binary result lines * fix(daemon): bound checkpoint wait * fix(repair): use KG health checks * fix(storage): always verify compressed CRC * perf(vector): skip unchanged PQ snapshots * build(zyp): exclude bench from default build * build: protect Meson files from formatter * refactor: remove unreachable production APIs * refactor(daemon): remove dormant runtime state * fix(vector): harden daemon provider calls Timed-out coroutines must own their client and requests. Provider discovery must also fail instead of returning synthetic embeddings in production. * refactor(storage): remove unusable adapters * fix(cli): unify in-process add requests * fix(tsan): preserve async worker progress * test(cli): isolate daemon-dependent suites * fix(repair): expose queue backoff progress
365 lines
10 KiB
C++
365 lines
10 KiB
C++
/**
|
|
* @file pdf_metadata.cpp
|
|
* @brief Minimal PDF metadata extraction implementation
|
|
*/
|
|
|
|
#include "pdf_metadata.h"
|
|
|
|
#include <algorithm>
|
|
#include <charconv>
|
|
#include <cstring>
|
|
#include <string_view>
|
|
|
|
namespace yams::zyp {
|
|
|
|
namespace detail {
|
|
|
|
const uint8_t* findBackwards(std::span<const uint8_t> data, std::string_view needle) {
|
|
if (needle.empty() || data.size() < needle.size()) {
|
|
return nullptr;
|
|
}
|
|
|
|
const auto* needleData = reinterpret_cast<const uint8_t*>(needle.data());
|
|
const size_t candidateCount = data.size() - needle.size() + 1;
|
|
for (size_t candidate = candidateCount; candidate > 0; --candidate) {
|
|
const size_t offset = candidate - 1;
|
|
const auto* match = data.data() + offset;
|
|
if (std::memcmp(match, needleData, needle.size()) == 0) {
|
|
return match;
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
} // namespace detail
|
|
|
|
namespace {
|
|
|
|
const uint8_t* findBytes(const uint8_t* haystack, size_t haystackLen, const uint8_t* needle,
|
|
size_t needleLen) {
|
|
if (needleLen == 0 || haystackLen < needleLen) {
|
|
return nullptr;
|
|
}
|
|
|
|
const auto* haystackEnd = haystack + haystackLen;
|
|
auto it = std::search(haystack, haystackEnd, needle, needle + needleLen);
|
|
if (it == haystackEnd) {
|
|
return nullptr;
|
|
}
|
|
return it;
|
|
}
|
|
|
|
/**
|
|
* Skip whitespace characters.
|
|
*/
|
|
const uint8_t* skipWhitespace(const uint8_t* p, const uint8_t* end) {
|
|
while (p < end && (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')) {
|
|
++p;
|
|
}
|
|
return p;
|
|
}
|
|
|
|
/**
|
|
* Parse a PDF string (parentheses or hex).
|
|
* Returns the string content without delimiters.
|
|
*/
|
|
std::optional<std::string> parsePdfString(const uint8_t*& p, const uint8_t* end) {
|
|
p = skipWhitespace(p, end);
|
|
if (p >= end) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
if (*p == '(') {
|
|
// Literal string
|
|
++p;
|
|
std::string result;
|
|
int parenDepth = 1;
|
|
|
|
while (p < end && parenDepth > 0) {
|
|
if (*p == '\\' && p + 1 < end) {
|
|
// Escape sequence
|
|
++p;
|
|
switch (*p) {
|
|
case 'n':
|
|
result += '\n';
|
|
break;
|
|
case 'r':
|
|
result += '\r';
|
|
break;
|
|
case 't':
|
|
result += '\t';
|
|
break;
|
|
case 'b':
|
|
result += '\b';
|
|
break;
|
|
case 'f':
|
|
result += '\f';
|
|
break;
|
|
case '(':
|
|
result += '(';
|
|
break;
|
|
case ')':
|
|
result += ')';
|
|
break;
|
|
case '\\':
|
|
result += '\\';
|
|
break;
|
|
default:
|
|
// Octal or just the character
|
|
if (*p >= '0' && *p <= '7') {
|
|
int octal = *p - '0';
|
|
if (p + 1 < end && p[1] >= '0' && p[1] <= '7') {
|
|
++p;
|
|
octal = octal * 8 + (*p - '0');
|
|
if (p + 1 < end && p[1] >= '0' && p[1] <= '7') {
|
|
++p;
|
|
octal = octal * 8 + (*p - '0');
|
|
}
|
|
}
|
|
result += static_cast<char>(octal);
|
|
} else {
|
|
result += static_cast<char>(*p);
|
|
}
|
|
break;
|
|
}
|
|
} else if (*p == '(') {
|
|
++parenDepth;
|
|
result += '(';
|
|
} else if (*p == ')') {
|
|
--parenDepth;
|
|
if (parenDepth > 0) {
|
|
result += ')';
|
|
}
|
|
} else {
|
|
result += static_cast<char>(*p);
|
|
}
|
|
++p;
|
|
}
|
|
return result;
|
|
|
|
} else if (*p == '<' && p + 1 < end && p[1] != '<') {
|
|
// Hex string
|
|
++p;
|
|
std::string result;
|
|
|
|
while (p < end && *p != '>') {
|
|
if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') {
|
|
++p;
|
|
continue;
|
|
}
|
|
|
|
auto hexVal = [](uint8_t c) -> int {
|
|
if (c >= '0' && c <= '9')
|
|
return c - '0';
|
|
if (c >= 'A' && c <= 'F')
|
|
return c - 'A' + 10;
|
|
if (c >= 'a' && c <= 'f')
|
|
return c - 'a' + 10;
|
|
return -1;
|
|
};
|
|
|
|
int high = hexVal(*p);
|
|
if (high < 0) {
|
|
++p;
|
|
continue;
|
|
}
|
|
++p;
|
|
|
|
int low = 0;
|
|
if (p < end && *p != '>') {
|
|
low = hexVal(*p);
|
|
if (low >= 0) {
|
|
++p;
|
|
}
|
|
}
|
|
|
|
result += static_cast<char>((high << 4) | (low >= 0 ? low : 0));
|
|
}
|
|
|
|
if (p < end && *p == '>') {
|
|
++p;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
/**
|
|
* Parse an indirect object reference (e.g., "5 0 R").
|
|
* Returns the object number, or -1 on failure.
|
|
*/
|
|
int parseIndirectRef(const uint8_t*& p, const uint8_t* end) {
|
|
p = skipWhitespace(p, end);
|
|
|
|
// Parse object number
|
|
const auto* numStart = p;
|
|
while (p < end && *p >= '0' && *p <= '9') {
|
|
++p;
|
|
}
|
|
if (p == numStart) {
|
|
return -1;
|
|
}
|
|
|
|
int objNum = 0;
|
|
std::from_chars(reinterpret_cast<const char*>(numStart), reinterpret_cast<const char*>(p),
|
|
objNum);
|
|
|
|
// Skip generation number and 'R'
|
|
p = skipWhitespace(p, end);
|
|
while (p < end && *p >= '0' && *p <= '9') {
|
|
++p;
|
|
}
|
|
p = skipWhitespace(p, end);
|
|
if (p < end && *p == 'R') {
|
|
++p;
|
|
}
|
|
|
|
return objNum;
|
|
}
|
|
|
|
/**
|
|
* Find an object by number in the PDF.
|
|
* Returns pointer to start of object content (after "obj" keyword).
|
|
*/
|
|
const uint8_t* findObject(std::span<const uint8_t> data, int objNum) {
|
|
char pattern[32]{};
|
|
const int len = std::snprintf(pattern, sizeof(pattern), "%d 0 obj", objNum);
|
|
if (len <= 0 || static_cast<std::size_t>(len) >= sizeof(pattern)) {
|
|
return nullptr;
|
|
}
|
|
const std::string_view needle(pattern, static_cast<std::size_t>(len));
|
|
|
|
// Search forward (objects are typically near the start)
|
|
auto it = std::search(data.begin(), data.end(), needle.begin(), needle.end());
|
|
if (it == data.end()) {
|
|
it = std::search(data.begin(), data.end(), needle.begin(), needle.end());
|
|
}
|
|
|
|
if (it == data.end()) {
|
|
return nullptr;
|
|
}
|
|
|
|
// Skip past "obj"
|
|
const auto* p = &*it + needle.size();
|
|
return skipWhitespace(p, data.data() + data.size());
|
|
}
|
|
|
|
/**
|
|
* Extract a named entry from a dictionary.
|
|
*/
|
|
std::optional<std::string> extractDictEntry(const uint8_t* dictStart, const uint8_t* end,
|
|
std::string_view name) {
|
|
// Look for /Name pattern
|
|
std::string pattern = "/" + std::string(name);
|
|
const auto* p = dictStart;
|
|
|
|
while (p < end - pattern.size()) {
|
|
if (std::memcmp(p, pattern.data(), pattern.size()) == 0) {
|
|
p += pattern.size();
|
|
p = skipWhitespace(p, end);
|
|
return parsePdfString(p, end);
|
|
}
|
|
++p;
|
|
}
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
std::unordered_map<std::string, std::string> PdfMetadata::toMap() const {
|
|
std::unordered_map<std::string, std::string> result;
|
|
|
|
if (title)
|
|
result["pdf_title"] = *title;
|
|
if (author)
|
|
result["pdf_author"] = *author;
|
|
if (subject)
|
|
result["pdf_subject"] = *subject;
|
|
if (keywords)
|
|
result["pdf_keywords"] = *keywords;
|
|
if (creator)
|
|
result["pdf_creator"] = *creator;
|
|
if (producer)
|
|
result["pdf_producer"] = *producer;
|
|
if (creationDate)
|
|
result["creation_date"] = *creationDate;
|
|
if (modDate)
|
|
result["modification_date"] = *modDate;
|
|
|
|
return result;
|
|
}
|
|
|
|
bool PdfMetadata::empty() const {
|
|
return !title && !author && !subject && !keywords && !creator && !producer && !creationDate &&
|
|
!modDate;
|
|
}
|
|
|
|
std::optional<PdfMetadata> extractMetadata(std::span<const uint8_t> data) {
|
|
if (data.size() < 100) {
|
|
return std::nullopt; // Too small to be a valid PDF
|
|
}
|
|
|
|
// Find trailer
|
|
const auto* trailer = detail::findBackwards(data, "trailer");
|
|
if (!trailer) {
|
|
// Try startxref for linearized PDFs
|
|
return std::nullopt;
|
|
}
|
|
|
|
const auto* end = data.data() + data.size();
|
|
|
|
// Find /Info reference in trailer
|
|
const auto* p = trailer;
|
|
const auto* infoRef =
|
|
findBytes(p, static_cast<size_t>(end - p), reinterpret_cast<const uint8_t*>("/Info"), 5);
|
|
|
|
if (!infoRef) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Parse the /Info reference
|
|
p = infoRef + 5;
|
|
p = skipWhitespace(p, end);
|
|
|
|
int infoObjNum = parseIndirectRef(p, end);
|
|
if (infoObjNum < 0) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Find the info object
|
|
const auto* infoObj = findObject(data, infoObjNum);
|
|
if (!infoObj) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Find the dictionary start
|
|
p = skipWhitespace(infoObj, end);
|
|
if (p >= end || *p != '<' || p + 1 >= end || p[1] != '<') {
|
|
return std::nullopt;
|
|
}
|
|
p += 2; // Skip "<<"
|
|
|
|
// Find dictionary end
|
|
const auto* dictEnd =
|
|
findBytes(p, static_cast<size_t>(end - p), reinterpret_cast<const uint8_t*>(">>"), 2);
|
|
if (!dictEnd) {
|
|
dictEnd = end;
|
|
}
|
|
|
|
// Extract metadata fields
|
|
PdfMetadata metadata;
|
|
metadata.title = extractDictEntry(p, dictEnd, "Title");
|
|
metadata.author = extractDictEntry(p, dictEnd, "Author");
|
|
metadata.subject = extractDictEntry(p, dictEnd, "Subject");
|
|
metadata.keywords = extractDictEntry(p, dictEnd, "Keywords");
|
|
metadata.creator = extractDictEntry(p, dictEnd, "Creator");
|
|
metadata.producer = extractDictEntry(p, dictEnd, "Producer");
|
|
metadata.creationDate = extractDictEntry(p, dictEnd, "CreationDate");
|
|
metadata.modDate = extractDictEntry(p, dictEnd, "ModDate");
|
|
|
|
return metadata;
|
|
}
|
|
|
|
} // namespace yams::zyp
|