0dd3985db4* refactor(tuning): prune dormant runtime knobs Remove unused tuning controls and adaptive search budget state. Extract testable ONNX slot reconciliation and search telemetry hydration seams, and correct the Apple Silicon sysctl value width. * refactor(vector): retire dormant backends Remove unused FAISS, model-infrastructure, VectorIndexManager, and TurboQuant paths while keeping Simeon PQ and float-vector storage. Migrate unused V2.2 quantized schemas transactionally to V2.3. Databases containing non-default quantized formats or packed-code blobs fail closed with recovery guidance and no mutation. * refactor(plugins): remove legacy SDK tooling Remove the Ghidra and SDK submodules, their bespoke tests, and stale scan exclusions. Retain generic PluginProcess coverage against the repository mock fixture. * build(vector): remove stale FAISS overrides * refactor(ipc): remove dormant FSM snapshots * refactor(ingest): unify bounded KG enqueue * fix(plugins): harden allocation contracts Keep ABI outputs null or fully initialized when allocations fail, and represent empty object payloads without relying on malloc(0). * fix(storage): define empty object reads Return a null buffer with zero length for successful empty-object reads instead of treating malloc(0) as an allocation contract. * refactor(storage): trim through string views * test: prune dormant harness helpers Remove unused readiness and fixture helpers, and keep benchmark client-count doubling in a wide integer domain. * chore(cpp): initialize analyzer-facing locals * build(simeon): advance analyzer cleanup * refactor(config): drop dead daemon client keys * refactor(ipc): inject daemon client transport Socket and in-process execution now share DaemonClient request dispatch, so CLI fallbacks do not need alternate service paths. * refactor(retrieval): route CLI through daemon Search and grep now use the same daemon handlers over socket or in-process transport. Live grep remains an explicit freshness inspection. * refactor(graph): move CLI scope into daemon Graph CLI requests now share daemon handlers across socket and in-process transports. CWD filtering occurs before pagination so scoped totals and pages stay correct. * build: isolate test-only runtime controls * refactor(metadata): decouple lock telemetry * fix(services): return on request timeout * fix(retrieval): enforce workspace scope * fix(graph): scope nodes before pagination * fix(grep): drop binary result lines * fix(daemon): bound checkpoint wait * fix(repair): use KG health checks * fix(storage): always verify compressed CRC * perf(vector): skip unchanged PQ snapshots * build(zyp): exclude bench from default build * build: protect Meson files from formatter * refactor: remove unreachable production APIs * refactor(daemon): remove dormant runtime state * fix(vector): harden daemon provider calls Timed-out coroutines must own their client and requests. Provider discovery must also fail instead of returning synthetic embeddings in production. * refactor(storage): remove unusable adapters * fix(cli): unify in-process add requests * fix(tsan): preserve async worker progress * test(cli): isolate daemon-dependent suites * fix(repair): expose queue backoff progress
8025 lines
359 KiB
C++
8025 lines
359 KiB
C++
// Multi-client ingestion and concurrent access benchmark
|
|
// Benchmarks the daemon's ability to handle:
|
|
// 1. Concurrent ingestion from multiple clients
|
|
// 2. Mixed read/write workloads (ingest + search + list)
|
|
// 3. Connection fairness and latency under contention
|
|
// 4. Per-client throughput scaling vs total throughput
|
|
// 5. Backpressure and queue depth behavior under multi-client load
|
|
//
|
|
// Designed for recursive tuning: run repeatedly, adjusting TuneAdvisor knobs,
|
|
// connection pool sizes, and I/O thread counts to optimize multi-client performance.
|
|
//
|
|
// Environment variables:
|
|
// YAMS_BENCH_NUM_CLIENTS - Number of concurrent clients (default: 4)
|
|
// YAMS_BENCH_DOCS_PER_CLIENT - Documents each client ingests (default: 100)
|
|
// YAMS_BENCH_DOC_SIZE_BYTES - Average document size in bytes (default: 2048)
|
|
// YAMS_BENCH_SEARCH_RATIO - Fraction of ops that are search (default: 0.2)
|
|
// YAMS_BENCH_MIXED_OPS_PER_CLIENT - Fixed operation budget per mixed client (default: derived)
|
|
// YAMS_BENCH_WARMUP_DOCS - Docs to pre-ingest for search queries (default: 50)
|
|
// YAMS_BENCH_OUTPUT - JSONL output path (default: bench_results/multi_client.jsonl)
|
|
// YAMS_BENCH_DRAIN_TIMEOUT_S - Seconds to wait for queue drain (default: 120)
|
|
// YAMS_BENCH_SCALING_MAX_CLIENTS - Max clients in scaling sweep (default: 32)
|
|
// YAMS_TUNING_PROFILE - TuneAdvisor profile (Quiet/Balanced/Performance)
|
|
// YAMS_BENCH_DATA_DIR - Existing YAMS data dir (for large-corpus bench)
|
|
// YAMS_BENCH_ENABLE_PLUGINS - Enable plugin loading: 1/true/yes (default: off)
|
|
// YAMS_BENCH_PLUGIN_DIR - Directory containing plugin .dylib/.so files
|
|
// YAMS_BENCH_TRUSTED_PLUGIN_PATHS - Colon-separated trusted plugin search paths
|
|
// YAMS_BENCH_REAL_MODEL_PROVIDER - Use real ONNX model provider: 1/true/yes
|
|
// YAMS_BENCH_SEARCH_QUERY - Daemon search query for mixed-load phases
|
|
// YAMS_BENCH_SEARCH_TYPE - Daemon search type: keyword|semantic|hybrid
|
|
// YAMS_BENCH_SEARCH_LIMIT - Daemon search result limit
|
|
// YAMS_BENCH_SEARCH_PATHS_ONLY - Skip result hydration and return paths only
|
|
// YAMS_BENCH_SNIPPET_HYDRATION_TIMEOUT_MS - Search snippet hydration budget
|
|
// YAMS_BENCH_PROFILE_HYDRATION_SURFACES - Add gated grep/graph render probes
|
|
// YAMS_BENCH_CLI_SEARCH_QUERY - CLI search query for full subprocess probes
|
|
// YAMS_BENCH_CLI_SEARCH_TYPE - CLI search type: keyword|semantic|hybrid
|
|
// YAMS_BENCH_CLI_SEARCH_LIMIT - CLI search result limit
|
|
// YAMS_BENCH_MCP_SESSION_CHURN_EVERY_N - MCP only: every Nth request uses a fresh
|
|
// MCP session (default: 0; profile may override)
|
|
// YAMS_BENCH_IDLE_PROBE - Enable post-run idle transition probe (default: on)
|
|
// YAMS_BENCH_IDLE_PROBE_BASELINE_MS - Idle baseline sampling window (default: 1500)
|
|
// YAMS_BENCH_IDLE_PROBE_SAMPLE_MS - Idle probe sampling interval (default: 50)
|
|
// YAMS_BENCH_IDLE_PROBE_DETECT_TIMEOUT_MS - Max wake detection window (default: 5000)
|
|
// YAMS_BENCH_MIXED_WRITERS - Mixed-ops writer client count (default: 4)
|
|
// YAMS_BENCH_MIXED_SEARCHERS - Mixed-ops search client count (default: 4)
|
|
// YAMS_BENCH_MIXED_LISTERS - Mixed-ops list client count (default: 4)
|
|
// YAMS_BENCH_MIXED_STATUS - Mixed-ops status/get client count (default: 4)
|
|
// YAMS_BENCH_MIXED_READER_OPS - Mixed-ops ops per reader client (default: 200)
|
|
// YAMS_BENCH_MIXED_MAX_FAIL_RATE - Mixed-ops maximum tolerated fail rate (default: 0.0)
|
|
|
|
#define CATCH_CONFIG_MAIN
|
|
#include <catch2/catch_session.hpp>
|
|
#include <catch2/catch_test_macros.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <atomic>
|
|
#include <cerrno>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <future>
|
|
#include <iomanip>
|
|
#include <iostream>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <mutex>
|
|
#include <numeric>
|
|
#include <optional>
|
|
#include <random>
|
|
#include <sstream>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <unordered_map>
|
|
#include <unordered_set>
|
|
#include <vector>
|
|
|
|
#ifdef _WIN32
|
|
#include <process.h>
|
|
#define getpid _getpid
|
|
#else
|
|
#include <dirent.h>
|
|
#include <fcntl.h>
|
|
#include <limits.h>
|
|
#include <spawn.h>
|
|
#include <unistd.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/types.h>
|
|
#include <sys/wait.h>
|
|
|
|
extern char** environ;
|
|
#endif
|
|
|
|
#include <nlohmann/json.hpp>
|
|
#include <spdlog/spdlog.h>
|
|
#include "../common/test_helpers_catch2.h"
|
|
#include "../integration/daemon/test_async_helpers.h"
|
|
#include "../integration/daemon/test_daemon_harness.h"
|
|
#include "bench_utils.h"
|
|
#include <boost/asio/connect.hpp>
|
|
#include <boost/asio/io_context.hpp>
|
|
#include <boost/asio/local/stream_protocol.hpp>
|
|
#include <yams/daemon/client/asio_connection_pool.h>
|
|
#include <yams/daemon/client/daemon_client.h>
|
|
#include <yams/daemon/components/ServiceManager.h>
|
|
#include <yams/daemon/components/TuneAdvisor.h>
|
|
#include <yams/daemon/metric_keys.h>
|
|
#include <yams/mcp/mcp_server.h>
|
|
#include <yams/metadata/knowledge_graph_store.h>
|
|
|
|
namespace fs = std::filesystem;
|
|
using json = nlohmann::json;
|
|
using namespace std::chrono_literals;
|
|
using namespace yams::daemon;
|
|
using namespace yams::test;
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Configuration
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
namespace {
|
|
|
|
constexpr int kMaxBenchClients = 512;
|
|
constexpr int kMaxBenchDocsPerClient = 1'000'000;
|
|
constexpr int kMaxBenchWarmupDocs = 1'000'000;
|
|
constexpr int kMaxBenchSeconds = 24 * 60 * 60;
|
|
constexpr int kMaxBenchMilliseconds = 60 * 60 * 1000;
|
|
constexpr int kMaxBenchStressRounds = 1000;
|
|
constexpr int kMaxBenchOpsPerClient = 1'000'000;
|
|
constexpr int kMaxBenchPageSize = 1'000'000;
|
|
constexpr int kMaxBenchHashLimit = 10'000'000;
|
|
constexpr std::size_t kMaxBenchDocSizeBytes = 1024ULL * 1024ULL * 1024ULL;
|
|
constexpr std::size_t kMaxBenchSearchLimit = 1'000'000;
|
|
constexpr std::int64_t kMaxBenchmarkLatencyUs = 30LL * 24 * 60 * 60 * 1'000'000;
|
|
constexpr std::int64_t kMaxBenchmarkWallMs = 30LL * 24 * 60 * 60 * 1000;
|
|
|
|
int clampInt64ToInt(std::int64_t value, int minValue, int maxValue) noexcept {
|
|
if (value < static_cast<std::int64_t>(minValue)) {
|
|
return minValue;
|
|
}
|
|
if (value > static_cast<std::int64_t>(maxValue)) {
|
|
return maxValue;
|
|
}
|
|
return static_cast<int>(value);
|
|
}
|
|
|
|
int parseEnvIntClamped(const char* name, int defaultValue, int minValue, int maxValue) noexcept {
|
|
const char* raw = std::getenv(name);
|
|
if (raw == nullptr || *raw == '\0') {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
char* end = nullptr;
|
|
errno = 0;
|
|
const long long parsed = std::strtoll(raw, &end, 10);
|
|
if (end == raw || (end != nullptr && *end != '\0')) {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
if (errno == ERANGE) {
|
|
return parsed < 0 ? minValue : maxValue;
|
|
}
|
|
return clampInt64ToInt(static_cast<std::int64_t>(parsed), minValue, maxValue);
|
|
}
|
|
|
|
bool parseEnvBool(const char* name, bool defaultValue = false) {
|
|
const char* raw = std::getenv(name);
|
|
if (raw == nullptr) {
|
|
return defaultValue;
|
|
}
|
|
const std::string value(raw);
|
|
return value == "1" || value == "true" || value == "yes" || value == "on";
|
|
}
|
|
|
|
std::size_t parseEnvSizeClamped(const char* name, std::size_t defaultValue, std::size_t minValue,
|
|
std::size_t maxValue) noexcept {
|
|
const char* raw = std::getenv(name);
|
|
if (raw == nullptr || *raw == '\0' || *raw == '-') {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
char* end = nullptr;
|
|
errno = 0;
|
|
const unsigned long long parsed = std::strtoull(raw, &end, 10);
|
|
if (end == raw || (end != nullptr && *end != '\0')) {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
if (errno == ERANGE) {
|
|
return maxValue;
|
|
}
|
|
return std::clamp(static_cast<std::size_t>(parsed), minValue, maxValue);
|
|
}
|
|
|
|
double parseEnvDoubleClamped(const char* name, double defaultValue, double minValue,
|
|
double maxValue) noexcept {
|
|
const char* raw = std::getenv(name);
|
|
if (raw == nullptr || *raw == '\0') {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
char* end = nullptr;
|
|
errno = 0;
|
|
const double parsed = std::strtod(raw, &end);
|
|
if (end == raw || (end != nullptr && *end != '\0') || errno == ERANGE ||
|
|
!std::isfinite(parsed)) {
|
|
return std::clamp(defaultValue, minValue, maxValue);
|
|
}
|
|
return std::clamp(parsed, minValue, maxValue);
|
|
}
|
|
|
|
int checkedMulToInt(int lhs, int rhs, int maxValue = std::numeric_limits<int>::max()) noexcept {
|
|
const auto product = static_cast<std::int64_t>(lhs) * static_cast<std::int64_t>(rhs);
|
|
return clampInt64ToInt(product, 0, maxValue);
|
|
}
|
|
|
|
int checkedAddToInt(std::initializer_list<int> values,
|
|
int maxValue = std::numeric_limits<int>::max()) noexcept {
|
|
std::int64_t total = 0;
|
|
for (int value : values) {
|
|
total += static_cast<std::int64_t>(value);
|
|
if (total > static_cast<std::int64_t>(maxValue)) {
|
|
return maxValue;
|
|
}
|
|
}
|
|
return clampInt64ToInt(total, 0, maxValue);
|
|
}
|
|
|
|
std::int64_t clampMetricSample(std::int64_t value,
|
|
std::int64_t maxValue = kMaxBenchmarkLatencyUs) noexcept {
|
|
if (value <= 0) {
|
|
return 0;
|
|
}
|
|
return std::min(value, maxValue);
|
|
}
|
|
|
|
std::int64_t clampUnsignedMetricSample(std::uint64_t value,
|
|
std::int64_t maxValue = kMaxBenchmarkLatencyUs) noexcept {
|
|
const auto unsignedMax = static_cast<std::uint64_t>(maxValue);
|
|
return value > unsignedMax ? maxValue : static_cast<std::int64_t>(value);
|
|
}
|
|
|
|
void pushMetricSample(std::vector<std::int64_t>& values, std::int64_t value,
|
|
std::int64_t maxValue = kMaxBenchmarkLatencyUs) {
|
|
if (value >= 0) {
|
|
values.push_back(clampMetricSample(value, maxValue));
|
|
}
|
|
}
|
|
|
|
std::int64_t checkedMetricAdd(std::int64_t lhs, std::int64_t rhs,
|
|
std::int64_t maxValue = std::numeric_limits<std::int64_t>::max()) {
|
|
lhs = clampMetricSample(lhs, maxValue);
|
|
rhs = clampMetricSample(rhs, maxValue);
|
|
if (lhs > maxValue - rhs) {
|
|
return maxValue;
|
|
}
|
|
return lhs + rhs;
|
|
}
|
|
|
|
std::int64_t checkedMetricMul(std::int64_t lhs, std::int64_t rhs,
|
|
std::int64_t maxValue = std::numeric_limits<std::int64_t>::max()) {
|
|
lhs = clampMetricSample(lhs, maxValue);
|
|
rhs = clampMetricSample(rhs, maxValue);
|
|
if (lhs != 0 && rhs > maxValue / lhs) {
|
|
return maxValue;
|
|
}
|
|
return lhs * rhs;
|
|
}
|
|
|
|
struct BenchConfig {
|
|
int numClients{4};
|
|
int docsPerClient{100};
|
|
size_t docSizeBytes{2048};
|
|
double searchRatio{0.2};
|
|
int warmupDocs{50};
|
|
fs::path outputPath{"bench_results/multi_client.jsonl"};
|
|
int drainTimeoutSecs{120};
|
|
int scalingMaxClients{32};
|
|
int stressRounds{1};
|
|
int cliProbeOpsPerCommand{3};
|
|
int cliProbeTimeoutSecs{10};
|
|
std::string tuningProfile;
|
|
std::optional<fs::path> cliBinary;
|
|
std::string searchQuery{"architecture performance tuning daemon"};
|
|
std::string searchType{"hybrid"};
|
|
size_t searchLimit{5};
|
|
bool searchPathsOnly{false};
|
|
int snippetHydrationTimeoutMs{5000};
|
|
bool profileHydrationSurfaces{false};
|
|
int resourceSampleMs{500};
|
|
std::string cliSearchQuery{"architecture performance tuning daemon"};
|
|
std::string cliSearchType{"hybrid"};
|
|
size_t cliSearchLimit{5};
|
|
bool idleProbeEnabled{true};
|
|
int idleProbeBaselineMs{1500};
|
|
int idleProbeSampleMs{50};
|
|
int idleProbeDetectTimeoutMs{5000};
|
|
int mixedWriterClients{4};
|
|
int mixedSearchClients{4};
|
|
int mixedListClients{4};
|
|
int mixedStatusClients{4};
|
|
int mixedOpsPerReader{200};
|
|
int mixedOpsPerClient{0};
|
|
double mixedMaxFailRate{0.0};
|
|
|
|
// --- Storage / corpus overrides ---
|
|
// Point at an existing YAMS data directory (e.g. /Volumes/picaso/yams)
|
|
// When set, the daemon serves this corpus instead of creating an empty temp db.
|
|
std::optional<fs::path> dataDir;
|
|
// Optional explicit config path used for daemon startup/config lookup.
|
|
std::optional<fs::path> configPath;
|
|
|
|
// --- Plugin / embedding config ---
|
|
// Enable real plugin loading (ONNX embeddings, PDF extraction, etc.)
|
|
bool enablePlugins{false};
|
|
// Directory containing built plugin .dylib/.so files
|
|
std::optional<fs::path> pluginDir;
|
|
// Trusted plugin search paths (from plugins.trust)
|
|
std::vector<fs::path> trustedPluginPaths;
|
|
// Use real model provider instead of mock (requires ONNX runtime + models)
|
|
bool useRealModelProvider{false};
|
|
bool useMcpPath{false};
|
|
std::string usageProfile{"mixed"};
|
|
std::string backend{"unknown"};
|
|
std::string authMode{""};
|
|
|
|
static BenchConfig fromEnv() {
|
|
BenchConfig cfg;
|
|
cfg.numClients =
|
|
parseEnvIntClamped("YAMS_BENCH_NUM_CLIENTS", cfg.numClients, 1, kMaxBenchClients);
|
|
cfg.docsPerClient = parseEnvIntClamped("YAMS_BENCH_DOCS_PER_CLIENT", cfg.docsPerClient, 1,
|
|
kMaxBenchDocsPerClient);
|
|
cfg.docSizeBytes = parseEnvSizeClamped("YAMS_BENCH_DOC_SIZE_BYTES", cfg.docSizeBytes, 64,
|
|
kMaxBenchDocSizeBytes);
|
|
cfg.searchRatio =
|
|
parseEnvDoubleClamped("YAMS_BENCH_SEARCH_RATIO", cfg.searchRatio, 0.0, 1.0);
|
|
cfg.warmupDocs =
|
|
parseEnvIntClamped("YAMS_BENCH_WARMUP_DOCS", cfg.warmupDocs, 0, kMaxBenchWarmupDocs);
|
|
if (auto* v = std::getenv("YAMS_BENCH_OUTPUT"))
|
|
cfg.outputPath = v;
|
|
cfg.drainTimeoutSecs = parseEnvIntClamped("YAMS_BENCH_DRAIN_TIMEOUT_S",
|
|
cfg.drainTimeoutSecs, 10, kMaxBenchSeconds);
|
|
cfg.scalingMaxClients = parseEnvIntClamped("YAMS_BENCH_SCALING_MAX_CLIENTS",
|
|
cfg.scalingMaxClients, 1, kMaxBenchClients);
|
|
cfg.stressRounds = parseEnvIntClamped("YAMS_BENCH_STRESS_ROUNDS", cfg.stressRounds, 1,
|
|
kMaxBenchStressRounds);
|
|
cfg.cliProbeOpsPerCommand = parseEnvIntClamped(
|
|
"YAMS_BENCH_CLI_PROBE_OPS", cfg.cliProbeOpsPerCommand, 0, kMaxBenchOpsPerClient);
|
|
cfg.cliProbeTimeoutSecs = parseEnvIntClamped("YAMS_BENCH_CLI_PROBE_TIMEOUT_S",
|
|
cfg.cliProbeTimeoutSecs, 1, kMaxBenchSeconds);
|
|
if (auto* v = std::getenv("YAMS_BENCH_CLI_BIN"))
|
|
cfg.cliBinary = fs::path(v);
|
|
if (auto* v = std::getenv("YAMS_BENCH_SEARCH_QUERY"))
|
|
cfg.searchQuery = v;
|
|
if (auto* v = std::getenv("YAMS_BENCH_SEARCH_TYPE"))
|
|
cfg.searchType = v;
|
|
cfg.searchLimit = parseEnvSizeClamped("YAMS_BENCH_SEARCH_LIMIT", cfg.searchLimit, 1,
|
|
kMaxBenchSearchLimit);
|
|
cfg.searchPathsOnly = parseEnvBool("YAMS_BENCH_SEARCH_PATHS_ONLY");
|
|
cfg.snippetHydrationTimeoutMs =
|
|
parseEnvIntClamped("YAMS_BENCH_SNIPPET_HYDRATION_TIMEOUT_MS",
|
|
cfg.snippetHydrationTimeoutMs, 0, kMaxBenchMilliseconds);
|
|
cfg.profileHydrationSurfaces = parseEnvBool("YAMS_BENCH_PROFILE_HYDRATION_SURFACES");
|
|
cfg.resourceSampleMs = parseEnvIntClamped("YAMS_BENCH_RESOURCE_SAMPLE_MS",
|
|
cfg.resourceSampleMs, 10, kMaxBenchMilliseconds);
|
|
if (auto* v = std::getenv("YAMS_BENCH_CLI_SEARCH_QUERY"))
|
|
cfg.cliSearchQuery = v;
|
|
if (auto* v = std::getenv("YAMS_BENCH_CLI_SEARCH_TYPE"))
|
|
cfg.cliSearchType = v;
|
|
cfg.cliSearchLimit = parseEnvSizeClamped("YAMS_BENCH_CLI_SEARCH_LIMIT", cfg.cliSearchLimit,
|
|
1, kMaxBenchSearchLimit);
|
|
if (auto* v = std::getenv("YAMS_BENCH_IDLE_PROBE")) {
|
|
std::string s(v);
|
|
cfg.idleProbeEnabled = !(s == "0" || s == "false" || s == "no" || s == "off");
|
|
}
|
|
cfg.idleProbeBaselineMs = parseEnvIntClamped(
|
|
"YAMS_BENCH_IDLE_PROBE_BASELINE_MS", cfg.idleProbeBaselineMs, 0, kMaxBenchMilliseconds);
|
|
cfg.idleProbeSampleMs = parseEnvIntClamped(
|
|
"YAMS_BENCH_IDLE_PROBE_SAMPLE_MS", cfg.idleProbeSampleMs, 10, kMaxBenchMilliseconds);
|
|
cfg.idleProbeDetectTimeoutMs =
|
|
parseEnvIntClamped("YAMS_BENCH_IDLE_PROBE_DETECT_TIMEOUT_MS",
|
|
cfg.idleProbeDetectTimeoutMs, 250, kMaxBenchMilliseconds);
|
|
cfg.mixedWriterClients = parseEnvIntClamped("YAMS_BENCH_MIXED_WRITERS",
|
|
cfg.mixedWriterClients, 1, kMaxBenchClients);
|
|
cfg.mixedSearchClients = parseEnvIntClamped("YAMS_BENCH_MIXED_SEARCHERS",
|
|
cfg.mixedSearchClients, 1, kMaxBenchClients);
|
|
cfg.mixedListClients = parseEnvIntClamped("YAMS_BENCH_MIXED_LISTERS", cfg.mixedListClients,
|
|
1, kMaxBenchClients);
|
|
cfg.mixedStatusClients = parseEnvIntClamped("YAMS_BENCH_MIXED_STATUS",
|
|
cfg.mixedStatusClients, 1, kMaxBenchClients);
|
|
cfg.mixedOpsPerReader = parseEnvIntClamped("YAMS_BENCH_MIXED_READER_OPS",
|
|
cfg.mixedOpsPerReader, 1, kMaxBenchOpsPerClient);
|
|
cfg.mixedOpsPerClient = parseEnvIntClamped("YAMS_BENCH_MIXED_OPS_PER_CLIENT",
|
|
cfg.mixedOpsPerClient, 0, kMaxBenchOpsPerClient);
|
|
cfg.mixedMaxFailRate =
|
|
parseEnvDoubleClamped("YAMS_BENCH_MIXED_MAX_FAIL_RATE", cfg.mixedMaxFailRate, 0.0, 1.0);
|
|
cfg.scalingMaxClients =
|
|
std::max(cfg.scalingMaxClients, checkedMulToInt(cfg.numClients, 2, kMaxBenchClients));
|
|
cfg.cliProbeOpsPerCommand = std::max(cfg.cliProbeOpsPerCommand, cfg.stressRounds);
|
|
if (cfg.cliSearchQuery.empty())
|
|
cfg.cliSearchQuery = cfg.searchQuery;
|
|
if (cfg.cliSearchType.empty())
|
|
cfg.cliSearchType = cfg.searchType;
|
|
if (cfg.cliSearchLimit == 0)
|
|
cfg.cliSearchLimit = cfg.searchLimit;
|
|
if (auto* v = std::getenv("YAMS_TUNING_PROFILE"))
|
|
cfg.tuningProfile = v;
|
|
|
|
// Storage path override: point at an existing corpus
|
|
if (auto* v = std::getenv("YAMS_BENCH_DATA_DIR"))
|
|
cfg.dataDir = fs::path(v);
|
|
if (auto* v = std::getenv("YAMS_BENCH_CONFIG"))
|
|
cfg.configPath = fs::path(v);
|
|
|
|
// Plugin / embedding configuration
|
|
if (auto* v = std::getenv("YAMS_BENCH_ENABLE_PLUGINS")) {
|
|
std::string s(v);
|
|
cfg.enablePlugins = (s == "1" || s == "true" || s == "yes");
|
|
}
|
|
if (auto* v = std::getenv("YAMS_BENCH_PLUGIN_DIR"))
|
|
cfg.pluginDir = fs::path(v);
|
|
if (auto* v = std::getenv("YAMS_BENCH_TRUSTED_PLUGIN_PATHS")) {
|
|
// Colon-separated list of paths
|
|
std::string paths(v);
|
|
std::istringstream iss(paths);
|
|
std::string path;
|
|
while (std::getline(iss, path, ':')) {
|
|
if (!path.empty())
|
|
cfg.trustedPluginPaths.push_back(fs::path(path));
|
|
}
|
|
}
|
|
if (auto* v = std::getenv("YAMS_BENCH_REAL_MODEL_PROVIDER")) {
|
|
std::string s(v);
|
|
cfg.useRealModelProvider = (s == "1" || s == "true" || s == "yes");
|
|
}
|
|
|
|
if (auto* v = std::getenv("YAMS_BENCH_USE_MCP")) {
|
|
std::string s(v);
|
|
cfg.useMcpPath = (s == "1" || s == "true" || s == "yes");
|
|
}
|
|
if (auto* v = std::getenv("YAMS_BENCH_USAGE_PROFILE")) {
|
|
cfg.usageProfile = v;
|
|
}
|
|
if (auto* v = std::getenv("YAMS_BENCH_BACKEND")) {
|
|
cfg.backend = v;
|
|
}
|
|
if (auto* v = std::getenv("YAMS_BENCH_AUTH_MODE")) {
|
|
cfg.authMode = v;
|
|
}
|
|
|
|
return cfg;
|
|
}
|
|
};
|
|
|
|
struct OutputLivenessCounters {
|
|
std::atomic<int64_t> attempts{0};
|
|
std::atomic<int64_t> nonemptySuccesses{0};
|
|
std::atomic<int64_t> emptySuccesses{0};
|
|
std::atomic<int64_t> errors{0};
|
|
|
|
void recordSuccess(bool nonempty) {
|
|
attempts.fetch_add(1, std::memory_order_relaxed);
|
|
(nonempty ? nonemptySuccesses : emptySuccesses).fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
|
|
void recordError() {
|
|
attempts.fetch_add(1, std::memory_order_relaxed);
|
|
errors.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
|
|
[[nodiscard]] json toJson() const {
|
|
return {
|
|
{"attempts", attempts.load(std::memory_order_relaxed)},
|
|
{"nonempty_successes", nonemptySuccesses.load(std::memory_order_relaxed)},
|
|
{"empty_successes", emptySuccesses.load(std::memory_order_relaxed)},
|
|
{"errors", errors.load(std::memory_order_relaxed)},
|
|
};
|
|
}
|
|
};
|
|
|
|
TEST_CASE("Multi-client benchmark env parsing clamps extreme numeric values",
|
|
"[multi-client][config]") {
|
|
ScopedEnvVar clients("YAMS_BENCH_NUM_CLIENTS", std::string("999999999999999999999"));
|
|
ScopedEnvVar docs("YAMS_BENCH_DOCS_PER_CLIENT", std::string("999999999999999999999"));
|
|
ScopedEnvVar docSize("YAMS_BENCH_DOC_SIZE_BYTES", std::string("999999999999999999999"));
|
|
ScopedEnvVar ratio("YAMS_BENCH_SEARCH_RATIO", std::string("inf"));
|
|
ScopedEnvVar scaling("YAMS_BENCH_SCALING_MAX_CLIENTS", std::string("999999999999999999999"));
|
|
ScopedEnvVar mixedReaders("YAMS_BENCH_MIXED_READER_OPS", std::string("999999999999999999999"));
|
|
|
|
const auto cfg = BenchConfig::fromEnv();
|
|
|
|
CHECK(cfg.numClients == kMaxBenchClients);
|
|
CHECK(cfg.docsPerClient == kMaxBenchDocsPerClient);
|
|
CHECK(cfg.docSizeBytes == kMaxBenchDocSizeBytes);
|
|
CHECK(cfg.searchRatio == 0.2);
|
|
CHECK(cfg.scalingMaxClients == kMaxBenchClients);
|
|
CHECK(cfg.mixedOpsPerReader == kMaxBenchOpsPerClient);
|
|
}
|
|
|
|
std::vector<std::string> buildCliSearchArgs(const BenchConfig& cfg) {
|
|
std::vector<std::string> args{
|
|
"search",
|
|
cfg.cliSearchQuery,
|
|
"--limit",
|
|
std::to_string(cfg.cliSearchLimit),
|
|
};
|
|
if (!cfg.cliSearchType.empty()) {
|
|
args.push_back("--type");
|
|
args.push_back(cfg.cliSearchType);
|
|
}
|
|
return args;
|
|
}
|
|
|
|
class NullTransport : public yams::mcp::ITransport {
|
|
public:
|
|
void send(const nlohmann::json&) override {}
|
|
yams::mcp::MessageResult receive() override {
|
|
return yams::Error{yams::ErrorCode::NotImplemented, "Null transport"};
|
|
}
|
|
bool isConnected() const override { return false; }
|
|
void close() override {}
|
|
yams::mcp::TransportState getState() const override {
|
|
return yams::mcp::TransportState::Disconnected;
|
|
}
|
|
};
|
|
|
|
class MCPPipelineClient {
|
|
public:
|
|
explicit MCPPipelineClient(const fs::path& socketPath) {
|
|
::setenv("YAMS_DAEMON_SOCKET_PATH", socketPath.string().c_str(), 1);
|
|
::setenv("YAMS_CLI_DISABLE_DAEMON_AUTOSTART", "1", 1);
|
|
server_ = std::make_shared<yams::mcp::MCPServer>(std::make_unique<NullTransport>());
|
|
initialize();
|
|
}
|
|
|
|
yams::Result<nlohmann::json> queryStep(const std::string& op, const nlohmann::json& params) {
|
|
std::lock_guard<std::mutex> lock(mu_);
|
|
return queryStepUnlocked(op, params);
|
|
}
|
|
|
|
private:
|
|
static bool isQueryCompositeReadOp(const std::string& op) {
|
|
return op == "search" || op == "grep" || op == "list" || op == "list_collections" ||
|
|
op == "list_snapshots" || op == "graph" || op == "get" || op == "status";
|
|
}
|
|
|
|
yams::Result<nlohmann::json> queryStepUnlocked(const std::string& op,
|
|
const nlohmann::json& params) {
|
|
nlohmann::json request = nlohmann::json::object();
|
|
request["jsonrpc"] = "2.0";
|
|
request["id"] = requestId_++;
|
|
request["method"] = "tools/call";
|
|
request["params"] = nlohmann::json::object();
|
|
const bool useQueryTool = hasQueryTool_ && isQueryCompositeReadOp(op);
|
|
request["params"]["name"] = useQueryTool ? "query" : op;
|
|
if (useQueryTool) {
|
|
request["params"]["arguments"] = nlohmann::json::object();
|
|
request["params"]["arguments"]["steps"] =
|
|
nlohmann::json::array({{{"op", op}, {"params", params}}});
|
|
} else {
|
|
request["params"]["arguments"] = params.is_object() ? params : nlohmann::json::object();
|
|
}
|
|
|
|
auto responseResult = server_->processMessage(request);
|
|
if (!responseResult) {
|
|
return yams::Error{responseResult.error()};
|
|
}
|
|
|
|
const auto& response = responseResult.value();
|
|
if (response.contains("error")) {
|
|
const std::string msg = response["error"].value("message", "MCP tools/call error");
|
|
if (hasQueryTool_ && msg.rfind("Unknown tool: query", 0) == 0) {
|
|
hasQueryTool_ = false;
|
|
return queryStepUnlocked(op, params);
|
|
}
|
|
return yams::Error{yams::ErrorCode::InvalidState, msg};
|
|
}
|
|
if (!response.contains("result") || !response["result"].is_object()) {
|
|
return yams::Error{yams::ErrorCode::InvalidState,
|
|
"MCP tools/call missing result object"};
|
|
}
|
|
|
|
const auto& result = response["result"];
|
|
if (result.value("isError", false)) {
|
|
std::string msg{"MCP tool returned error"};
|
|
if (result.contains("content") && result["content"].is_array() &&
|
|
!result["content"].empty() && result["content"][0].contains("text")) {
|
|
msg = result["content"][0]["text"].get<std::string>();
|
|
}
|
|
return yams::Error{yams::ErrorCode::InvalidState, msg};
|
|
}
|
|
|
|
if (result.contains("structuredContent") && result["structuredContent"].is_object()) {
|
|
const auto& sc = result["structuredContent"];
|
|
if (sc.contains("data")) {
|
|
return sc["data"];
|
|
}
|
|
}
|
|
|
|
if (result.contains("content") && result["content"].is_array() &&
|
|
!result["content"].empty() && result["content"][0].contains("text")) {
|
|
try {
|
|
return nlohmann::json::parse(result["content"][0]["text"].get<std::string>());
|
|
} catch (...) {
|
|
return nlohmann::json{{"text", result["content"][0]["text"]}};
|
|
}
|
|
}
|
|
|
|
return yams::Error{yams::ErrorCode::InvalidState, "MCP tool result had no data"};
|
|
}
|
|
|
|
void initialize() {
|
|
nlohmann::json initReq = nlohmann::json::object();
|
|
initReq["jsonrpc"] = "2.0";
|
|
initReq["id"] = requestId_++;
|
|
initReq["method"] = "initialize";
|
|
initReq["params"] = nlohmann::json::object();
|
|
initReq["params"]["protocolVersion"] = "2025-06-18";
|
|
initReq["params"]["capabilities"] = nlohmann::json::object();
|
|
initReq["params"]["clientInfo"] =
|
|
nlohmann::json{{"name", "multi_client_ingestion_bench"}, {"version", "1.0"}};
|
|
auto initRes = server_->processMessage(initReq);
|
|
if (!initRes) {
|
|
throw std::runtime_error("MCP initialize failed: " + initRes.error().message);
|
|
}
|
|
|
|
nlohmann::json initializedNotif = {{"jsonrpc", "2.0"},
|
|
{"method", "notifications/initialized"},
|
|
{"params", nlohmann::json::object()}};
|
|
(void)server_->processMessage(initializedNotif);
|
|
|
|
nlohmann::json toolsReq = nlohmann::json::object();
|
|
toolsReq["jsonrpc"] = "2.0";
|
|
toolsReq["id"] = requestId_++;
|
|
toolsReq["method"] = "tools/list";
|
|
toolsReq["params"] = nlohmann::json::object();
|
|
auto toolsRes = server_->processMessage(toolsReq);
|
|
if (!toolsRes) {
|
|
throw std::runtime_error("MCP tools/list failed: " + toolsRes.error().message);
|
|
}
|
|
const auto& toolsResp = toolsRes.value();
|
|
if (!toolsResp.contains("result") || !toolsResp["result"].is_object() ||
|
|
!toolsResp["result"].contains("tools") || !toolsResp["result"]["tools"].is_array()) {
|
|
throw std::runtime_error("MCP tools/list returned invalid shape");
|
|
}
|
|
hasQueryTool_ = false;
|
|
for (const auto& tool : toolsResp["result"]["tools"]) {
|
|
if (!tool.is_object() || !tool.contains("name") || !tool["name"].is_string()) {
|
|
continue;
|
|
}
|
|
if (tool["name"].get<std::string>() == "query") {
|
|
hasQueryTool_ = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::shared_ptr<yams::mcp::MCPServer> server_;
|
|
std::mutex mu_;
|
|
bool hasQueryTool_{false};
|
|
std::uint64_t requestId_{1};
|
|
};
|
|
|
|
class MCPPipelineClientPool {
|
|
public:
|
|
MCPPipelineClientPool(const fs::path& socketPath, size_t poolSize) {
|
|
const size_t size = std::max<size_t>(1, poolSize);
|
|
clients_.reserve(size);
|
|
for (size_t i = 0; i < size; ++i) {
|
|
clients_.push_back(std::make_unique<MCPPipelineClient>(socketPath));
|
|
}
|
|
}
|
|
|
|
yams::Result<nlohmann::json> queryStepForSlot(size_t slot, const std::string& op,
|
|
const nlohmann::json& params) {
|
|
const size_t idx = slot % clients_.size();
|
|
return clients_[idx]->queryStep(op, params);
|
|
}
|
|
|
|
private:
|
|
std::vector<std::unique_ptr<MCPPipelineClient>> clients_;
|
|
};
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Latency & throughput tracking
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
struct OpLatency {
|
|
std::string opType; // "add", "search", "list", "status"
|
|
int clientId{0};
|
|
int64_t latencyUs{0};
|
|
bool success{false};
|
|
std::string errorMsg;
|
|
};
|
|
|
|
struct ClientResult {
|
|
int clientId{0};
|
|
int docsIngested{0};
|
|
int searchesExecuted{0};
|
|
int listsExecuted{0};
|
|
int statusChecks{0};
|
|
int failures{0};
|
|
double elapsedSeconds{0.0};
|
|
double addThroughputDocsPerSec{0.0};
|
|
std::vector<OpLatency> latencies;
|
|
std::vector<int64_t> snippetHydrationUs;
|
|
std::vector<int64_t> snippetLookupUs;
|
|
std::vector<int64_t> snippetContentFetchUs;
|
|
std::vector<int64_t> snippetRenderUs;
|
|
int64_t snippetTimeoutHits{0};
|
|
int64_t snippetHydrationCandidates{0};
|
|
int64_t snippetHydrationMissing{0};
|
|
int64_t snippetHydratedResults{0};
|
|
};
|
|
|
|
double safeRate(double numerator, double elapsedSeconds) {
|
|
return (numerator > 0.0 && elapsedSeconds > 0.0) ? numerator / elapsedSeconds : 0.0;
|
|
}
|
|
|
|
struct PercentileStats {
|
|
int64_t min{0};
|
|
int64_t p50{0};
|
|
int64_t p90{0};
|
|
int64_t p95{0};
|
|
int64_t p99{0};
|
|
int64_t max{0};
|
|
double mean{0.0};
|
|
size_t count{0};
|
|
|
|
static PercentileStats compute(std::vector<int64_t>& values) {
|
|
values.erase(
|
|
std::remove_if(values.begin(), values.end(), [](int64_t value) { return value < 0; }),
|
|
values.end());
|
|
for (auto& value : values) {
|
|
value = clampMetricSample(value);
|
|
}
|
|
|
|
PercentileStats stats;
|
|
stats.count = values.size();
|
|
if (values.empty())
|
|
return stats;
|
|
|
|
std::sort(values.begin(), values.end());
|
|
const auto percentileIndex = [&](double percentile) {
|
|
return std::min(values.size() - 1,
|
|
static_cast<size_t>(percentile * static_cast<double>(values.size())));
|
|
};
|
|
stats.min = values.front();
|
|
stats.max = values.back();
|
|
stats.p50 = values[percentileIndex(0.50)];
|
|
stats.p90 = values[percentileIndex(0.90)];
|
|
stats.p95 = values[percentileIndex(0.95)];
|
|
stats.p99 = values[percentileIndex(0.99)];
|
|
const long double total =
|
|
std::accumulate(values.begin(), values.end(), 0.0L, [](long double sum, int64_t value) {
|
|
return sum + static_cast<long double>(value);
|
|
});
|
|
stats.mean = static_cast<double>(total / static_cast<long double>(values.size()));
|
|
return stats;
|
|
}
|
|
|
|
json toJson() const {
|
|
return json{{"count", count}, {"min_us", min}, {"p50_us", p50}, {"p90_us", p90},
|
|
{"p95_us", p95}, {"p99_us", p99}, {"max_us", max}, {"mean_us", mean}};
|
|
}
|
|
};
|
|
|
|
std::optional<fs::path> findExecutableInPath(const std::string& name) {
|
|
if (const char* pathEnv = std::getenv("PATH")) {
|
|
std::stringstream ss(pathEnv);
|
|
std::string entry;
|
|
while (std::getline(ss, entry, ':')) {
|
|
if (entry.empty()) {
|
|
continue;
|
|
}
|
|
fs::path candidate = fs::path(entry) / name;
|
|
std::error_code ec;
|
|
if (fs::exists(candidate, ec) && !ec) {
|
|
return candidate;
|
|
}
|
|
}
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
struct CliBinarySelection {
|
|
std::optional<fs::path> path;
|
|
std::string source{"missing"};
|
|
std::string note;
|
|
bool explicitSelection{false};
|
|
bool pathFallback{false};
|
|
bool usableForStress{false};
|
|
|
|
json toJson() const {
|
|
return json{{"path", path ? path->string() : std::string()},
|
|
{"source", source},
|
|
{"note", note},
|
|
{"explicit_selection", explicitSelection},
|
|
{"path_fallback", pathFallback},
|
|
{"usable_for_stress", usableForStress}};
|
|
}
|
|
};
|
|
|
|
bool isExecutableBinary(const fs::path& candidate) {
|
|
std::error_code ec;
|
|
if (!fs::exists(candidate, ec) || ec) {
|
|
return false;
|
|
}
|
|
#ifdef _WIN32
|
|
return true;
|
|
#else
|
|
return ::access(candidate.c_str(), X_OK) == 0;
|
|
#endif
|
|
}
|
|
|
|
CliBinarySelection findYamsBinary(const BenchConfig& cfg) {
|
|
auto resolveCandidate = [](const fs::path& candidate) -> std::optional<fs::path> {
|
|
if (!isExecutableBinary(candidate)) {
|
|
return std::nullopt;
|
|
}
|
|
return fs::absolute(candidate);
|
|
};
|
|
|
|
if (cfg.cliBinary && fs::exists(cfg.cliBinary.value())) {
|
|
CliBinarySelection selection;
|
|
selection.path = fs::absolute(cfg.cliBinary.value());
|
|
selection.source = "explicit";
|
|
selection.explicitSelection = true;
|
|
selection.usableForStress = true;
|
|
return selection;
|
|
}
|
|
|
|
if (const char* buildRoot = std::getenv("MESON_BUILD_ROOT")) {
|
|
for (const auto& relative : {fs::path("tools/yams-cli/yams-cli"),
|
|
fs::path("tools/yams-cli/yams"), fs::path("src/cli/yams")}) {
|
|
#ifdef _WIN32
|
|
const fs::path candidate = fs::path(buildRoot) / relative.replace_extension(".exe");
|
|
#else
|
|
const fs::path candidate = fs::path(buildRoot) / relative;
|
|
#endif
|
|
if (auto resolved = resolveCandidate(candidate)) {
|
|
CliBinarySelection selection;
|
|
selection.path = resolved.value();
|
|
selection.source = "meson_build_root";
|
|
selection.usableForStress = true;
|
|
return selection;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const auto& relative :
|
|
{fs::path("builddir/tools/yams-cli/yams-cli"), fs::path("build/tools/yams-cli/yams-cli"),
|
|
fs::path("build/debug/tools/yams-cli/yams-cli"),
|
|
fs::path("build/release/tools/yams-cli/yams-cli"),
|
|
fs::path("build/asan/tools/yams-cli/yams-cli"),
|
|
fs::path("build/opencode-tests/tools/yams-cli/yams-cli"),
|
|
fs::path("builddir/src/cli/yams"), fs::path("build/src/cli/yams"),
|
|
fs::path("build/debug/src/cli/yams"), fs::path("build/release/src/cli/yams"),
|
|
fs::path("build/asan/src/cli/yams"), fs::path("build/opencode-tests/src/cli/yams")}) {
|
|
#ifdef _WIN32
|
|
const fs::path candidate = relative.replace_extension(".exe");
|
|
#else
|
|
const fs::path candidate = relative;
|
|
#endif
|
|
if (auto resolved = resolveCandidate(candidate)) {
|
|
CliBinarySelection selection;
|
|
selection.path = resolved.value();
|
|
selection.source = "local_build_tree";
|
|
selection.usableForStress = true;
|
|
return selection;
|
|
}
|
|
}
|
|
|
|
#ifdef _WIN32
|
|
if (auto candidate = findExecutableInPath("yams.exe")) {
|
|
CliBinarySelection selection;
|
|
selection.path = candidate.value();
|
|
selection.source = "path_fallback";
|
|
selection.note =
|
|
"PATH fallback may not match the build under test; set YAMS_BENCH_CLI_BIN for "
|
|
"deterministic CLI stress results";
|
|
selection.pathFallback = true;
|
|
selection.usableForStress = false;
|
|
return selection;
|
|
}
|
|
#else
|
|
if (auto candidate = findExecutableInPath("yams")) {
|
|
CliBinarySelection selection;
|
|
selection.path = candidate.value();
|
|
selection.source = "path_fallback";
|
|
selection.note =
|
|
"PATH fallback may not match the build under test; set YAMS_BENCH_CLI_BIN for "
|
|
"deterministic CLI stress results";
|
|
selection.pathFallback = true;
|
|
selection.usableForStress = false;
|
|
return selection;
|
|
}
|
|
#endif
|
|
|
|
return CliBinarySelection{.source = "missing",
|
|
.note =
|
|
"No in-tree CLI binary found; set YAMS_BENCH_CLI_BIN or build "
|
|
"tools/yams-cli/yams-cli to enable CLI stress probes",
|
|
.usableForStress = false};
|
|
}
|
|
|
|
struct ProbeDaemonState {
|
|
bool captured{false};
|
|
size_t activeConnections{0};
|
|
size_t maxConnections{0};
|
|
size_t connectionSlotsFree{0};
|
|
int pressureLevel{0};
|
|
uint32_t retryAfterMs{0};
|
|
std::string error;
|
|
|
|
bool underStress() const {
|
|
if (captured) {
|
|
return (maxConnections > 0 && activeConnections >= maxConnections) ||
|
|
connectionSlotsFree == 0 || retryAfterMs > 0 || pressureLevel >= 1;
|
|
}
|
|
if (error.empty()) {
|
|
return false;
|
|
}
|
|
std::string lower;
|
|
lower.reserve(error.size());
|
|
for (unsigned char ch : error) {
|
|
lower.push_back(static_cast<char>(std::tolower(ch)));
|
|
}
|
|
auto has = [&lower](std::string_view needle) {
|
|
return lower.find(needle) != std::string::npos;
|
|
};
|
|
return has("ipc:eof") || has("eof") || has("end of file") || has("timed out") ||
|
|
has("timeout") || has("connection reset") || has("broken pipe") ||
|
|
has("resource exhausted") || has("retry after") || has("server busy") ||
|
|
has("bad file descriptor");
|
|
}
|
|
|
|
json toJson() const {
|
|
return json{{"captured", captured},
|
|
{"active_connections", activeConnections},
|
|
{"max_connections", maxConnections},
|
|
{"connection_slots_free", connectionSlotsFree},
|
|
{"pressure_level", pressureLevel},
|
|
{"retry_after_ms", retryAfterMs},
|
|
{"error", error}};
|
|
}
|
|
};
|
|
|
|
struct CliProbeResult {
|
|
bool attempted{false};
|
|
bool success{false};
|
|
int exitCode{-1};
|
|
int signal{-1};
|
|
int64_t latencyUs{0};
|
|
bool timedOut{false};
|
|
std::string opType;
|
|
std::string error;
|
|
std::string stderrText;
|
|
std::string failureBucket;
|
|
std::string failureMode;
|
|
ProbeDaemonState beforeState;
|
|
ProbeDaemonState afterState;
|
|
std::unordered_map<std::string, int64_t> stageElapsedUs;
|
|
|
|
json toJson() const {
|
|
return json{{"attempted", attempted},
|
|
{"success", success},
|
|
{"op_type", opType},
|
|
{"exit_code", exitCode},
|
|
{"signal", signal},
|
|
{"latency_us", latencyUs},
|
|
{"timed_out", timedOut},
|
|
{"error", error},
|
|
{"stderr", stderrText},
|
|
{"failure_bucket", failureBucket},
|
|
{"failure_mode", failureMode},
|
|
{"before_state", beforeState.toJson()},
|
|
{"after_state", afterState.toJson()},
|
|
{"stage_elapsed_us", stageElapsedUs}};
|
|
}
|
|
};
|
|
|
|
#ifndef _WIN32
|
|
namespace {} // namespace
|
|
|
|
CliProbeResult runCliProbe(const fs::path& yamsBinary, const fs::path& socketPath,
|
|
const std::vector<std::string>& args,
|
|
std::chrono::milliseconds timeout = 10s) {
|
|
CliProbeResult result;
|
|
result.attempted = true;
|
|
|
|
int sinkFd = ::open("/dev/null", O_WRONLY);
|
|
if (sinkFd < 0) {
|
|
result.error = "open(/dev/null) failed";
|
|
return result;
|
|
}
|
|
int tracePipe[2] = {-1, -1};
|
|
if (::pipe(tracePipe) != 0) {
|
|
::close(sinkFd);
|
|
result.error = "pipe() failed";
|
|
return result;
|
|
}
|
|
|
|
std::vector<std::string> owned;
|
|
owned.reserve(args.size() + 1);
|
|
owned.push_back(yamsBinary.string());
|
|
owned.insert(owned.end(), args.begin(), args.end());
|
|
|
|
std::vector<char*> argv;
|
|
argv.reserve(owned.size() + 1);
|
|
for (auto& arg : owned) {
|
|
argv.push_back(arg.data());
|
|
}
|
|
argv.push_back(nullptr);
|
|
|
|
std::vector<std::string> envOwned;
|
|
if (environ != nullptr) {
|
|
for (char** env = environ; *env != nullptr; ++env) {
|
|
envOwned.emplace_back(*env);
|
|
}
|
|
}
|
|
auto setOrReplaceEnv = [&](std::string key, std::string value) {
|
|
const std::string prefix = key + "=";
|
|
for (auto& entry : envOwned) {
|
|
if (entry.rfind(prefix, 0) == 0) {
|
|
entry = prefix + value;
|
|
return;
|
|
}
|
|
}
|
|
envOwned.push_back(prefix + value);
|
|
};
|
|
setOrReplaceEnv("YAMS_DAEMON_SOCKET_PATH", socketPath.string());
|
|
setOrReplaceEnv("YAMS_CLI_DISABLE_DAEMON_AUTOSTART", "1");
|
|
setOrReplaceEnv("YAMS_CLI_ONE_SHOT", "1");
|
|
setOrReplaceEnv("YAMS_CLI_PERF_TRACE", "1");
|
|
setOrReplaceEnv("YAMS_LOG_LEVEL", "error");
|
|
|
|
std::vector<char*> envp;
|
|
envp.reserve(envOwned.size() + 1);
|
|
for (auto& entry : envOwned) {
|
|
envp.push_back(entry.data());
|
|
}
|
|
envp.push_back(nullptr);
|
|
|
|
posix_spawn_file_actions_t actions;
|
|
if (::posix_spawn_file_actions_init(&actions) != 0) {
|
|
::close(sinkFd);
|
|
::close(tracePipe[0]);
|
|
::close(tracePipe[1]);
|
|
result.error = "posix_spawn file actions init failed";
|
|
return result;
|
|
}
|
|
(void)::posix_spawn_file_actions_adddup2(&actions, sinkFd, STDOUT_FILENO);
|
|
(void)::posix_spawn_file_actions_adddup2(&actions, tracePipe[1], STDERR_FILENO);
|
|
(void)::posix_spawn_file_actions_addclose(&actions, sinkFd);
|
|
(void)::posix_spawn_file_actions_addclose(&actions, tracePipe[0]);
|
|
(void)::posix_spawn_file_actions_addclose(&actions, tracePipe[1]);
|
|
|
|
posix_spawnattr_t attr;
|
|
(void)::posix_spawnattr_init(&attr);
|
|
#ifdef POSIX_SPAWN_CLOEXEC_DEFAULT
|
|
short spawnFlags = 0;
|
|
spawnFlags |= POSIX_SPAWN_CLOEXEC_DEFAULT;
|
|
(void)::posix_spawnattr_setflags(&attr, spawnFlags);
|
|
#endif
|
|
|
|
const auto start = std::chrono::steady_clock::now();
|
|
pid_t pid = -1;
|
|
const int spawnErr =
|
|
::posix_spawn(&pid, yamsBinary.c_str(), &actions, &attr, argv.data(), envp.data());
|
|
(void)::posix_spawn_file_actions_destroy(&actions);
|
|
(void)::posix_spawnattr_destroy(&attr);
|
|
if (spawnErr != 0) {
|
|
::close(sinkFd);
|
|
::close(tracePipe[0]);
|
|
::close(tracePipe[1]);
|
|
result.error = "posix_spawn() failed: " + std::string(std::strerror(spawnErr));
|
|
return result;
|
|
}
|
|
|
|
::close(sinkFd);
|
|
::close(tracePipe[1]);
|
|
int status = 0;
|
|
const auto deadline = std::chrono::steady_clock::now() + timeout;
|
|
for (;;) {
|
|
pid_t waited = ::waitpid(pid, &status, WNOHANG);
|
|
if (waited == pid) {
|
|
break;
|
|
}
|
|
if (waited < 0) {
|
|
if (errno == EINTR) {
|
|
continue;
|
|
}
|
|
result.error = "waitpid() failed";
|
|
break;
|
|
}
|
|
if (std::chrono::steady_clock::now() >= deadline) {
|
|
::kill(pid, SIGKILL);
|
|
while (::waitpid(pid, &status, 0) < 0 && errno == EINTR) {
|
|
}
|
|
result.timedOut = true;
|
|
result.error = "cli probe timed out";
|
|
result.exitCode = 124;
|
|
result.latencyUs = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
|
|
result.latencyUs = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
|
|
if (WIFSIGNALED(status)) {
|
|
result.signal = WTERMSIG(status);
|
|
result.error = "cli probe terminated by signal " + std::to_string(result.signal);
|
|
} else if (WIFEXITED(status)) {
|
|
result.exitCode = WEXITSTATUS(status);
|
|
result.success = (result.exitCode == 0);
|
|
} else {
|
|
result.error = "child terminated abnormally";
|
|
}
|
|
|
|
std::string trace;
|
|
std::array<char, 1024> buffer{};
|
|
for (;;) {
|
|
ssize_t n = ::read(tracePipe[0], buffer.data(), buffer.size());
|
|
if (n > 0) {
|
|
trace.append(buffer.data(), static_cast<size_t>(n));
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
::close(tracePipe[0]);
|
|
|
|
std::stringstream traceStream(trace);
|
|
std::string line;
|
|
std::vector<std::string> stderrLines;
|
|
while (std::getline(traceStream, line)) {
|
|
constexpr std::string_view kPrefix = "[yams-cli-perf] stage=";
|
|
auto prefixPos = line.find(kPrefix);
|
|
if (prefixPos == std::string::npos) {
|
|
if (!line.empty()) {
|
|
stderrLines.push_back(line);
|
|
}
|
|
continue;
|
|
}
|
|
auto stageStart = prefixPos + kPrefix.size();
|
|
auto elapsedPos = line.find(" elapsed_us=", stageStart);
|
|
if (elapsedPos == std::string::npos) {
|
|
continue;
|
|
}
|
|
auto stage = line.substr(stageStart, elapsedPos - stageStart);
|
|
auto valueStart = elapsedPos + std::string_view(" elapsed_us=").size();
|
|
auto valueEnd = line.find(' ', valueStart);
|
|
auto valueText = line.substr(
|
|
valueStart, valueEnd == std::string::npos ? std::string::npos : valueEnd - valueStart);
|
|
try {
|
|
result.stageElapsedUs[stage] = std::stoll(valueText);
|
|
} catch (...) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (!stderrLines.empty()) {
|
|
std::ostringstream stderrBuf;
|
|
for (size_t i = 0; i < stderrLines.size(); ++i) {
|
|
if (i > 0) {
|
|
stderrBuf << '\n';
|
|
}
|
|
stderrBuf << stderrLines[i];
|
|
}
|
|
result.stderrText = stderrBuf.str();
|
|
}
|
|
|
|
if (!result.success && result.error.empty()) {
|
|
if (!result.stderrText.empty()) {
|
|
result.error = result.stderrText;
|
|
} else if (result.exitCode >= 0) {
|
|
result.error = "cli probe exited with code " + std::to_string(result.exitCode);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
#endif
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Daemon status snapshot helper
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
struct FdCategoryCounts {
|
|
uint64_t total{0};
|
|
uint64_t socket{0};
|
|
uint64_t pipe{0};
|
|
uint64_t file{0};
|
|
uint64_t device{0};
|
|
uint64_t kqueue{0};
|
|
uint64_t other{0};
|
|
};
|
|
|
|
FdCategoryCounts countOpenFileDescriptorCategories() {
|
|
FdCategoryCounts counts;
|
|
#ifdef _WIN32
|
|
return counts;
|
|
#else
|
|
DIR* dir = opendir("/dev/fd");
|
|
if (!dir) {
|
|
return counts;
|
|
}
|
|
|
|
while (auto* entry = readdir(dir)) {
|
|
const std::string_view name(entry->d_name);
|
|
if (name != "." && name != "..") {
|
|
++counts.total;
|
|
char* end = nullptr;
|
|
const auto fdNumber = std::strtol(entry->d_name, &end, 10);
|
|
if (end && *end == '\0' && fdNumber >= 0) {
|
|
struct stat st{};
|
|
if (::fstat(static_cast<int>(fdNumber), &st) == 0) {
|
|
if (S_ISSOCK(st.st_mode)) {
|
|
++counts.socket;
|
|
} else if (S_ISFIFO(st.st_mode)) {
|
|
++counts.pipe;
|
|
} else if (S_ISREG(st.st_mode) || S_ISDIR(st.st_mode)) {
|
|
++counts.file;
|
|
} else if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode)) {
|
|
++counts.device;
|
|
} else {
|
|
++counts.other;
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
const std::string fdPath = std::string{"/dev/fd/"} + std::string{name};
|
|
std::array<char, PATH_MAX + 1> target{};
|
|
const auto len = ::readlink(fdPath.c_str(), target.data(), PATH_MAX);
|
|
if (len < 0) {
|
|
++counts.other;
|
|
continue;
|
|
}
|
|
target[static_cast<std::size_t>(len)] = '\0';
|
|
const std::string_view t(target.data(), static_cast<std::size_t>(len));
|
|
if (t.find("socket") != std::string_view::npos) {
|
|
++counts.socket;
|
|
} else if (t.find("pipe") != std::string_view::npos) {
|
|
++counts.pipe;
|
|
} else if (t.find("kqueue") != std::string_view::npos) {
|
|
++counts.kqueue;
|
|
} else if (t.starts_with("/dev/")) {
|
|
++counts.device;
|
|
} else if (t.starts_with("/") || t.starts_with(".")) {
|
|
++counts.file;
|
|
} else {
|
|
++counts.other;
|
|
}
|
|
}
|
|
}
|
|
closedir(dir);
|
|
return counts;
|
|
#endif
|
|
}
|
|
|
|
constexpr uint64_t backgroundWorkerActive(uint64_t observedWorkerActive) {
|
|
// StatusRequest is itself tracked as an active worker before its snapshot is captured.
|
|
return observedWorkerActive > 0 ? observedWorkerActive - 1 : 0;
|
|
}
|
|
|
|
struct DaemonSnapshot {
|
|
size_t activeConnections{0};
|
|
size_t maxConnections{0};
|
|
size_t connectionSlotsFree{0};
|
|
uint64_t oldestConnectionAge{0};
|
|
uint64_t forcedCloseCount{0};
|
|
uint64_t requestsProcessed{0};
|
|
double memoryUsageMb{0.0};
|
|
double cpuUsagePercent{0.0};
|
|
uint64_t rssBytes{0};
|
|
uint64_t postIngestQueued{0};
|
|
uint64_t postIngestInflight{0};
|
|
uint64_t workerActive{0};
|
|
uint64_t workerQueued{0};
|
|
uint64_t deferredQueueDepth{0};
|
|
int pressureLevel{0};
|
|
uint64_t documentsTotal{0};
|
|
uint64_t searchActive{0};
|
|
uint64_t searchQueued{0};
|
|
uint64_t dbWritePoolWaiting{0};
|
|
uint64_t dbReadPoolWaiting{0};
|
|
uint64_t dbWritePoolMaxObservedWaiting{0};
|
|
uint64_t dbReadPoolMaxObservedWaiting{0};
|
|
uint64_t dbWritePoolTotalWaitMicros{0};
|
|
uint64_t dbReadPoolTotalWaitMicros{0};
|
|
uint64_t dbWritePoolTimeoutCount{0};
|
|
uint64_t dbReadPoolTimeoutCount{0};
|
|
uint64_t dbWritePoolFailedAcquisitions{0};
|
|
uint64_t dbReadPoolFailedAcquisitions{0};
|
|
uint64_t dbWritePoolSlowHolders{0};
|
|
uint64_t dbReadPoolSlowHolders{0};
|
|
uint64_t dbWritePoolMaxHolderMicros{0};
|
|
uint64_t dbReadPoolMaxHolderMicros{0};
|
|
uint64_t muxWriterBudgetBytes{0};
|
|
uint64_t writeQueueDepth{0};
|
|
uint64_t writeQueueCapacity{0};
|
|
uint64_t writeQueueDepthMax{0};
|
|
uint64_t writeQueueCapacityRejections{0};
|
|
uint64_t writeQueueForcedOverCapacity{0};
|
|
uint64_t writeInFlight{0};
|
|
uint64_t writeMaxBatchApplyMs{0};
|
|
uint64_t writeMaxBatchQueueWaitMs{0};
|
|
uint64_t writeMaxBatchExcessQueueWaitMs{0};
|
|
uint64_t processFdCount{0};
|
|
uint64_t processFdSockets{0};
|
|
uint64_t processFdPipes{0};
|
|
uint64_t processFdFiles{0};
|
|
uint64_t processFdDevices{0};
|
|
uint64_t processFdKqueues{0};
|
|
uint64_t processFdOther{0};
|
|
uint64_t postCommitContentIndexCalls{0};
|
|
uint64_t postCommitContentIndexTotalMs{0};
|
|
uint64_t postCommitContentIndexMaxMs{0};
|
|
uint64_t addDispatchSamples{0};
|
|
uint64_t addDispatchTotalUs{0};
|
|
uint64_t addDispatchMaxUs{0};
|
|
uint64_t addFingerprintTotalUs{0};
|
|
uint64_t addFingerprintMaxUs{0};
|
|
uint64_t addEnqueueTotalUs{0};
|
|
uint64_t addEnqueueMaxUs{0};
|
|
uint64_t clientPoolOpenSockets{0};
|
|
uint64_t clientPoolAliveConnections{0};
|
|
uint64_t clientPoolInUseConnections{0};
|
|
std::string writeHotSources;
|
|
uint32_t ipcPoolSize{0};
|
|
uint32_t ioPoolSize{0};
|
|
uint32_t retryAfterMs{0};
|
|
uint64_t metadataWalBytes{0};
|
|
|
|
static DaemonSnapshot capture(DaemonClient& client, const fs::path& dataDir = {}) {
|
|
DaemonSnapshot snap;
|
|
if (!dataDir.empty()) {
|
|
std::error_code ec;
|
|
const auto walPath = dataDir / "yams.db-wal";
|
|
if (fs::exists(walPath, ec) && !ec) {
|
|
snap.metadataWalBytes = static_cast<uint64_t>(fs::file_size(walPath, ec));
|
|
}
|
|
}
|
|
auto st = yams::cli::run_sync(client.status(), 5s);
|
|
if (!st)
|
|
return snap;
|
|
|
|
const auto fdCounts = countOpenFileDescriptorCategories();
|
|
snap.processFdCount = fdCounts.total;
|
|
snap.processFdSockets = fdCounts.socket;
|
|
snap.processFdPipes = fdCounts.pipe;
|
|
snap.processFdFiles = fdCounts.file;
|
|
snap.processFdDevices = fdCounts.device;
|
|
snap.processFdKqueues = fdCounts.kqueue;
|
|
snap.processFdOther = fdCounts.other;
|
|
const auto poolStats = AsioConnectionPool::registry_stats();
|
|
snap.clientPoolOpenSockets = poolStats.openSockets;
|
|
snap.clientPoolAliveConnections = poolStats.aliveConnections;
|
|
snap.clientPoolInUseConnections = poolStats.inUseConnections;
|
|
|
|
const auto& s = st.value();
|
|
snap.activeConnections = s.activeConnections;
|
|
snap.maxConnections = s.maxConnections;
|
|
snap.connectionSlotsFree = s.connectionSlotsFree;
|
|
snap.oldestConnectionAge = s.oldestConnectionAge;
|
|
snap.forcedCloseCount = s.forcedCloseCount;
|
|
snap.requestsProcessed = s.requestsProcessed;
|
|
snap.memoryUsageMb = s.memoryUsageMb;
|
|
snap.cpuUsagePercent = s.cpuUsagePercent;
|
|
snap.muxWriterBudgetBytes = s.muxWriterBudgetBytes;
|
|
snap.ipcPoolSize = s.ipcPoolSize;
|
|
snap.ioPoolSize = s.ioPoolSize;
|
|
snap.retryAfterMs = s.retryAfterMs;
|
|
|
|
auto getCount = [&](std::string_view key) -> uint64_t {
|
|
auto it = s.requestCounts.find(std::string(key));
|
|
return (it != s.requestCounts.end()) ? it->second : 0;
|
|
};
|
|
|
|
if (snap.maxConnections == 0) {
|
|
snap.maxConnections = static_cast<size_t>(getCount("status_max_connections"));
|
|
}
|
|
if (snap.connectionSlotsFree == 0) {
|
|
snap.connectionSlotsFree =
|
|
static_cast<size_t>(getCount("status_connection_slots_free"));
|
|
}
|
|
if (snap.oldestConnectionAge == 0) {
|
|
snap.oldestConnectionAge = getCount("status_oldest_connection_age_s");
|
|
}
|
|
if (snap.forcedCloseCount == 0) {
|
|
snap.forcedCloseCount = getCount("status_forced_close_count");
|
|
}
|
|
if (snap.muxWriterBudgetBytes == 0) {
|
|
snap.muxWriterBudgetBytes = getCount("status_mux_writer_budget_bytes");
|
|
}
|
|
if (snap.ipcPoolSize == 0) {
|
|
snap.ipcPoolSize = static_cast<uint32_t>(getCount("status_ipc_pool_size"));
|
|
}
|
|
if (snap.ioPoolSize == 0) {
|
|
snap.ioPoolSize = static_cast<uint32_t>(getCount("status_io_pool_size"));
|
|
}
|
|
if (snap.retryAfterMs == 0) {
|
|
snap.retryAfterMs = static_cast<uint32_t>(getCount("status_retry_after_ms"));
|
|
}
|
|
snap.rssBytes = getCount("status_rss_bytes");
|
|
|
|
snap.postIngestQueued = getCount(metrics::kPostIngestQueued);
|
|
snap.postIngestInflight = getCount(metrics::kPostIngestInflight);
|
|
snap.workerActive = backgroundWorkerActive(getCount(metrics::kWorkerActive));
|
|
snap.workerQueued = getCount(metrics::kWorkerQueued);
|
|
snap.deferredQueueDepth = getCount(metrics::kDeferredQueueDepth);
|
|
snap.pressureLevel = static_cast<int>(getCount(metrics::kPressureLevel));
|
|
snap.documentsTotal = getCount(metrics::kDocumentsTotal);
|
|
snap.searchActive = getCount(metrics::kSearchActive);
|
|
snap.searchQueued = getCount(metrics::kSearchQueued);
|
|
snap.dbWritePoolWaiting = getCount(metrics::kDbWritePoolWaitingRequests);
|
|
snap.dbReadPoolWaiting = getCount(metrics::kDbReadPoolWaitingRequests);
|
|
snap.addDispatchSamples = getCount("add_phase_dispatch_samples");
|
|
snap.addDispatchTotalUs = getCount("add_phase_dispatch_total_us");
|
|
snap.addDispatchMaxUs = getCount("add_phase_dispatch_max_us");
|
|
snap.addFingerprintTotalUs = getCount("add_phase_fingerprint_total_us");
|
|
snap.addFingerprintMaxUs = getCount("add_phase_fingerprint_max_us");
|
|
snap.addEnqueueTotalUs = getCount("add_phase_enqueue_total_us");
|
|
snap.addEnqueueMaxUs = getCount("add_phase_enqueue_max_us");
|
|
GetStatsRequest statsReq;
|
|
auto statsRes = yams::cli::run_sync(client.getStats(statsReq), 3s);
|
|
const auto* additionalStats = statsRes ? &statsRes.value().additionalStats : nullptr;
|
|
auto getAdditionalUint = [&](std::string_view key) -> uint64_t {
|
|
if (!additionalStats)
|
|
return 0;
|
|
auto it = additionalStats->find(std::string(key));
|
|
if (it == additionalStats->end() || it->second.empty())
|
|
return 0;
|
|
try {
|
|
return static_cast<uint64_t>(std::stoull(it->second));
|
|
} catch (...) {
|
|
return 0;
|
|
}
|
|
};
|
|
snap.writeMaxBatchApplyMs = getAdditionalUint("write_max_batch_apply_ms");
|
|
snap.writeQueueDepth = getAdditionalUint("write_queue_depth");
|
|
snap.writeInFlight = getAdditionalUint("write_in_flight");
|
|
snap.writeQueueCapacity = getAdditionalUint("write_queue_capacity");
|
|
snap.writeQueueDepthMax = getAdditionalUint("write_queue_depth_max");
|
|
snap.writeQueueCapacityRejections = getAdditionalUint("write_queue_capacity_rejections");
|
|
snap.writeQueueForcedOverCapacity = getAdditionalUint("write_queue_forced_over_capacity");
|
|
snap.dbWritePoolMaxObservedWaiting =
|
|
getAdditionalUint(metrics::kDbWritePoolMaxObservedWaiting);
|
|
snap.dbReadPoolMaxObservedWaiting =
|
|
getAdditionalUint(metrics::kDbReadPoolMaxObservedWaiting);
|
|
snap.dbWritePoolTotalWaitMicros = getAdditionalUint(metrics::kDbWritePoolTotalWaitMicros);
|
|
snap.dbReadPoolTotalWaitMicros = getAdditionalUint(metrics::kDbReadPoolTotalWaitMicros);
|
|
snap.dbWritePoolTimeoutCount = getAdditionalUint(metrics::kDbWritePoolTimeoutCount);
|
|
snap.dbReadPoolTimeoutCount = getAdditionalUint(metrics::kDbReadPoolTimeoutCount);
|
|
snap.dbWritePoolFailedAcquisitions =
|
|
getAdditionalUint(metrics::kDbWritePoolFailedAcquisitions);
|
|
snap.dbReadPoolFailedAcquisitions =
|
|
getAdditionalUint(metrics::kDbReadPoolFailedAcquisitions);
|
|
snap.dbWritePoolSlowHolders = getAdditionalUint(metrics::kDbWritePoolSlowHolders);
|
|
snap.dbReadPoolSlowHolders = getAdditionalUint(metrics::kDbReadPoolSlowHolders);
|
|
snap.dbWritePoolMaxHolderMicros = getAdditionalUint(metrics::kDbWritePoolMaxHolderMicros);
|
|
snap.dbReadPoolMaxHolderMicros = getAdditionalUint(metrics::kDbReadPoolMaxHolderMicros);
|
|
snap.writeMaxBatchQueueWaitMs = getAdditionalUint("write_max_batch_queue_wait_ms");
|
|
snap.writeMaxBatchExcessQueueWaitMs =
|
|
getAdditionalUint("write_max_batch_excess_queue_wait_ms");
|
|
snap.postCommitContentIndexCalls =
|
|
getAdditionalUint("post_timing_commit_content_index_calls");
|
|
snap.postCommitContentIndexTotalMs =
|
|
getAdditionalUint("post_timing_commit_content_index_total_ms");
|
|
snap.postCommitContentIndexMaxMs =
|
|
getAdditionalUint("post_timing_commit_content_index_max_ms");
|
|
if (additionalStats) {
|
|
if (auto it = additionalStats->find("write_hot_sources");
|
|
it != additionalStats->end()) {
|
|
snap.writeHotSources = it->second;
|
|
}
|
|
}
|
|
return snap;
|
|
}
|
|
|
|
json toJson() const {
|
|
return json{{"active_connections", activeConnections},
|
|
{"max_connections", maxConnections},
|
|
{"connection_slots_free", connectionSlotsFree},
|
|
{"oldest_connection_age_s", oldestConnectionAge},
|
|
{"forced_close_count", forcedCloseCount},
|
|
{"requests_processed", requestsProcessed},
|
|
{"memory_mb", memoryUsageMb},
|
|
{"cpu_pct", cpuUsagePercent},
|
|
{"rss_bytes", rssBytes},
|
|
{"post_ingest_queued", postIngestQueued},
|
|
{"post_ingest_inflight", postIngestInflight},
|
|
{"worker_active", workerActive},
|
|
{"worker_queued", workerQueued},
|
|
{"deferred_queue_depth", deferredQueueDepth},
|
|
{"pressure_level", pressureLevel},
|
|
{"documents_total", documentsTotal},
|
|
{"search_active", searchActive},
|
|
{"search_queued", searchQueued},
|
|
{"db_write_pool_waiting", dbWritePoolWaiting},
|
|
{"db_read_pool_waiting", dbReadPoolWaiting},
|
|
{"db_write_pool_max_observed_waiting", dbWritePoolMaxObservedWaiting},
|
|
{"db_read_pool_max_observed_waiting", dbReadPoolMaxObservedWaiting},
|
|
{"db_write_pool_total_wait_us", dbWritePoolTotalWaitMicros},
|
|
{"db_read_pool_total_wait_us", dbReadPoolTotalWaitMicros},
|
|
{"db_write_pool_timeouts", dbWritePoolTimeoutCount},
|
|
{"db_read_pool_timeouts", dbReadPoolTimeoutCount},
|
|
{"db_write_pool_failed_acquisitions", dbWritePoolFailedAcquisitions},
|
|
{"db_read_pool_failed_acquisitions", dbReadPoolFailedAcquisitions},
|
|
{"db_write_pool_slow_holders", dbWritePoolSlowHolders},
|
|
{"db_read_pool_slow_holders", dbReadPoolSlowHolders},
|
|
{"db_write_pool_max_holder_us", dbWritePoolMaxHolderMicros},
|
|
{"db_read_pool_max_holder_us", dbReadPoolMaxHolderMicros},
|
|
{"write_queue_depth", writeQueueDepth},
|
|
{"write_queue_capacity", writeQueueCapacity},
|
|
{"write_queue_depth_max", writeQueueDepthMax},
|
|
{"write_queue_capacity_rejections", writeQueueCapacityRejections},
|
|
{"write_queue_forced_over_capacity", writeQueueForcedOverCapacity},
|
|
{"write_in_flight", writeInFlight},
|
|
{"write_max_batch_apply_ms", writeMaxBatchApplyMs},
|
|
{"write_max_batch_queue_wait_ms", writeMaxBatchQueueWaitMs},
|
|
{"write_max_batch_excess_queue_wait_ms", writeMaxBatchExcessQueueWaitMs},
|
|
{"process_fd_count", processFdCount},
|
|
{"process_fd_sockets", processFdSockets},
|
|
{"process_fd_pipes", processFdPipes},
|
|
{"process_fd_files", processFdFiles},
|
|
{"process_fd_devices", processFdDevices},
|
|
{"process_fd_kqueues", processFdKqueues},
|
|
{"process_fd_other", processFdOther},
|
|
{"post_commit_content_index_calls", postCommitContentIndexCalls},
|
|
{"post_commit_content_index_total_ms", postCommitContentIndexTotalMs},
|
|
{"post_commit_content_index_max_ms", postCommitContentIndexMaxMs},
|
|
{"add_dispatch_samples", addDispatchSamples},
|
|
{"add_dispatch_total_us", addDispatchTotalUs},
|
|
{"add_dispatch_max_us", addDispatchMaxUs},
|
|
{"add_fingerprint_total_us", addFingerprintTotalUs},
|
|
{"add_fingerprint_max_us", addFingerprintMaxUs},
|
|
{"add_enqueue_total_us", addEnqueueTotalUs},
|
|
{"add_enqueue_max_us", addEnqueueMaxUs},
|
|
{"client_pool_open_sockets", clientPoolOpenSockets},
|
|
{"client_pool_alive_connections", clientPoolAliveConnections},
|
|
{"client_pool_in_use_connections", clientPoolInUseConnections},
|
|
{"write_hot_sources", writeHotSources},
|
|
{"mux_writer_budget_bytes", muxWriterBudgetBytes},
|
|
{"ipc_pool_size", ipcPoolSize},
|
|
{"io_pool_size", ioPoolSize},
|
|
{"retry_after_ms", retryAfterMs},
|
|
{"metadata_wal_bytes", metadataWalBytes}};
|
|
}
|
|
};
|
|
|
|
ProbeDaemonState captureProbeDaemonState(const fs::path& socketPath,
|
|
std::chrono::milliseconds timeout = 3s) {
|
|
ProbeDaemonState state;
|
|
ClientConfig cfg;
|
|
cfg.socketPath = socketPath;
|
|
cfg.autoStart = false;
|
|
cfg.singleUseConnections = true;
|
|
cfg.requestTimeout = timeout;
|
|
DaemonClient client(cfg);
|
|
|
|
auto st = yams::cli::run_sync(client.status(), timeout);
|
|
if (!st) {
|
|
state.error = st.error().message;
|
|
return state;
|
|
}
|
|
|
|
state.captured = true;
|
|
state.activeConnections = st.value().activeConnections;
|
|
state.maxConnections = st.value().maxConnections;
|
|
state.connectionSlotsFree = st.value().connectionSlotsFree;
|
|
state.pressureLevel = 0;
|
|
auto getCount = [&](std::string_view key) -> uint64_t {
|
|
auto it = st.value().requestCounts.find(std::string(key));
|
|
return (it != st.value().requestCounts.end()) ? it->second : 0ULL;
|
|
};
|
|
if (state.maxConnections == 0) {
|
|
state.maxConnections = static_cast<size_t>(getCount("status_max_connections"));
|
|
}
|
|
if (state.connectionSlotsFree == 0) {
|
|
state.connectionSlotsFree = static_cast<size_t>(getCount("status_connection_slots_free"));
|
|
}
|
|
auto it = st.value().requestCounts.find(std::string(metrics::kPressureLevel));
|
|
if (it != st.value().requestCounts.end()) {
|
|
state.pressureLevel = static_cast<int>(it->second);
|
|
}
|
|
state.retryAfterMs = st.value().retryAfterMs;
|
|
if (state.retryAfterMs == 0) {
|
|
state.retryAfterMs = static_cast<uint32_t>(getCount("status_retry_after_ms"));
|
|
}
|
|
return state;
|
|
}
|
|
|
|
std::string classifyFailureMessage(const std::string& errorMsg);
|
|
|
|
std::string classifyCliProbeBucket(const CliProbeResult& probe) {
|
|
if (probe.success) {
|
|
return "success";
|
|
}
|
|
if (probe.timedOut) {
|
|
return "timeout";
|
|
}
|
|
if (probe.signal > 0) {
|
|
return "signal";
|
|
}
|
|
if (probe.exitCode == 127) {
|
|
return "exec_failed";
|
|
}
|
|
return classifyFailureMessage(!probe.stderrText.empty() ? probe.stderrText : probe.error);
|
|
}
|
|
|
|
std::string classifyCliProbeMode(const CliProbeResult& probe) {
|
|
if (probe.success) {
|
|
return "success";
|
|
}
|
|
|
|
const bool underStress = probe.beforeState.underStress() || probe.afterState.underStress();
|
|
const std::string bucket =
|
|
probe.failureBucket.empty() ? classifyCliProbeBucket(probe) : probe.failureBucket;
|
|
|
|
if (bucket == "exec_failed") {
|
|
return "benchmark_wiring";
|
|
}
|
|
if (bucket == "signal") {
|
|
return "unexpected_signal";
|
|
}
|
|
if (bucket == "protocol") {
|
|
return "protocol_or_cli_bug";
|
|
}
|
|
if (bucket == "connect_refused") {
|
|
return underStress ? "stress_expected" : "ambient_daemon_down";
|
|
}
|
|
if (bucket == "backpressure") {
|
|
return underStress ? "overload_under_load" : "ambient_failure";
|
|
}
|
|
if (bucket == "connection_drop") {
|
|
return underStress ? "stress_expected" : "ambient_failure";
|
|
}
|
|
if (bucket == "timeout") {
|
|
return underStress ? "timeout_under_load" : "ambient_failure";
|
|
}
|
|
return underStress ? "stress_expected" : "unexpected_cli_failure";
|
|
}
|
|
|
|
struct RecoveryCheck {
|
|
bool statusOk{false};
|
|
bool listOk{false};
|
|
std::string statusError;
|
|
std::string listError;
|
|
DaemonSnapshot snapshot;
|
|
|
|
bool ok() const { return statusOk && listOk; }
|
|
|
|
json toJson() const {
|
|
return json{{"status_ok", statusOk},
|
|
{"list_ok", listOk},
|
|
{"status_error", statusError},
|
|
{"list_error", listError},
|
|
{"snapshot", snapshot.toJson()}};
|
|
}
|
|
};
|
|
|
|
RecoveryCheck runRecoveryCheck(const fs::path& socketPath) {
|
|
RecoveryCheck check;
|
|
ClientConfig cfg;
|
|
cfg.socketPath = socketPath;
|
|
cfg.autoStart = false;
|
|
cfg.singleUseConnections = true;
|
|
cfg.requestTimeout = 10s;
|
|
DaemonClient client(cfg);
|
|
|
|
auto statusRes = yams::cli::run_sync(client.status(), 10s);
|
|
if (statusRes) {
|
|
check.statusOk = true;
|
|
check.snapshot = DaemonSnapshot::capture(client);
|
|
} else {
|
|
check.statusError = statusRes.error().message;
|
|
}
|
|
|
|
ListRequest listReq;
|
|
listReq.limit = 10;
|
|
auto listRes = yams::cli::run_sync(client.list(listReq), 10s);
|
|
if (listRes) {
|
|
check.listOk = true;
|
|
} else {
|
|
check.listError = listRes.error().message;
|
|
}
|
|
|
|
return check;
|
|
}
|
|
|
|
void printRecoverySummary(const RecoveryCheck& recovery) {
|
|
std::cout << " Recovery: status=" << (recovery.statusOk ? "ok" : "FAIL")
|
|
<< " list=" << (recovery.listOk ? "ok" : "FAIL") << "\n";
|
|
if (!recovery.statusOk && !recovery.statusError.empty()) {
|
|
std::cout << " status error: " << recovery.statusError << "\n";
|
|
}
|
|
if (!recovery.listOk && !recovery.listError.empty()) {
|
|
std::cout << " list error: " << recovery.listError << "\n";
|
|
}
|
|
}
|
|
|
|
CliProbeResult runObservedCliProbe(const std::string& opType, const fs::path& cliBinary,
|
|
const fs::path& socketPath, const std::vector<std::string>& args,
|
|
std::chrono::milliseconds timeout = 10s) {
|
|
auto beforeState = captureProbeDaemonState(socketPath, std::min(timeout, 1500ms));
|
|
auto probe = runCliProbe(cliBinary, socketPath, args, timeout);
|
|
probe.opType = opType;
|
|
probe.beforeState = beforeState;
|
|
probe.afterState = captureProbeDaemonState(socketPath, std::min(timeout, 1500ms));
|
|
probe.failureBucket = classifyCliProbeBucket(probe);
|
|
probe.failureMode = classifyCliProbeMode(probe);
|
|
return probe;
|
|
}
|
|
|
|
void printCliProbeFailureSummary(const std::map<std::string, int>& failureModes,
|
|
const std::map<std::string, std::string>& failureSamples) {
|
|
if (failureModes.empty()) {
|
|
return;
|
|
}
|
|
std::cout << " CLI failure modes:";
|
|
for (const auto& [mode, count] : failureModes) {
|
|
std::cout << " " << mode << "=" << count;
|
|
}
|
|
std::cout << "\n";
|
|
for (const auto& [mode, sample] : failureSamples) {
|
|
std::cout << " sample[" << mode << "]: " << sample << "\n";
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Time-series sampler (runs in background during benchmark)
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
class TimeSeriesSampler {
|
|
public:
|
|
struct Sample {
|
|
uint64_t elapsedMs{0};
|
|
DaemonSnapshot snap;
|
|
};
|
|
|
|
~TimeSeriesSampler() { stop(); }
|
|
|
|
void start(DaemonClient& client, std::chrono::milliseconds interval = 500ms,
|
|
fs::path dataDir = {}) {
|
|
stop();
|
|
stop_ = false;
|
|
startTime_ = std::chrono::steady_clock::now();
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
samples_.clear();
|
|
}
|
|
thread_ = std::thread([this, &client, interval, dataDir = std::move(dataDir)]() {
|
|
while (!stop_) {
|
|
Sample sample;
|
|
sample.elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - startTime_)
|
|
.count();
|
|
sample.snap = DaemonSnapshot::capture(client, dataDir);
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
samples_.push_back(sample);
|
|
}
|
|
std::this_thread::sleep_for(interval);
|
|
}
|
|
});
|
|
}
|
|
|
|
void stop() {
|
|
stop_ = true;
|
|
if (thread_.joinable())
|
|
thread_.join();
|
|
}
|
|
|
|
std::vector<Sample> getSamples() const {
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
return samples_;
|
|
}
|
|
|
|
private:
|
|
std::atomic<bool> stop_{false};
|
|
std::chrono::steady_clock::time_point startTime_;
|
|
std::thread thread_;
|
|
mutable std::mutex mutex_;
|
|
std::vector<Sample> samples_;
|
|
};
|
|
|
|
struct SlotScalingSummary {
|
|
size_t initialLimit{0};
|
|
size_t peakLimit{0};
|
|
size_t finalLimit{0};
|
|
size_t minFreeSlots{0};
|
|
size_t peakActiveConnections{0};
|
|
double peakUtilization{0.0};
|
|
int growEvents{0};
|
|
int shrinkEvents{0};
|
|
int retryAfterSamples{0};
|
|
};
|
|
|
|
SlotScalingSummary summarizeSlotScaling(const std::vector<TimeSeriesSampler::Sample>& samples) {
|
|
SlotScalingSummary summary;
|
|
if (samples.empty()) {
|
|
return summary;
|
|
}
|
|
|
|
summary.initialLimit = samples.front().snap.maxConnections;
|
|
summary.peakLimit = samples.front().snap.maxConnections;
|
|
summary.finalLimit = samples.back().snap.maxConnections;
|
|
summary.minFreeSlots = samples.front().snap.connectionSlotsFree;
|
|
|
|
size_t previousLimit = samples.front().snap.maxConnections;
|
|
for (const auto& sample : samples) {
|
|
const auto& snap = sample.snap;
|
|
summary.peakLimit = std::max(summary.peakLimit, snap.maxConnections);
|
|
summary.minFreeSlots = std::min(summary.minFreeSlots, snap.connectionSlotsFree);
|
|
summary.peakActiveConnections =
|
|
std::max(summary.peakActiveConnections, snap.activeConnections);
|
|
if (snap.maxConnections > 0) {
|
|
summary.peakUtilization =
|
|
std::max(summary.peakUtilization, static_cast<double>(snap.activeConnections) /
|
|
static_cast<double>(snap.maxConnections));
|
|
}
|
|
if (snap.retryAfterMs > 0) {
|
|
summary.retryAfterSamples++;
|
|
}
|
|
if (snap.maxConnections > previousLimit) {
|
|
summary.growEvents++;
|
|
} else if (snap.maxConnections < previousLimit) {
|
|
summary.shrinkEvents++;
|
|
}
|
|
previousLimit = snap.maxConnections;
|
|
}
|
|
|
|
return summary;
|
|
}
|
|
|
|
struct TimedLatencySample {
|
|
std::string opType;
|
|
uint64_t wallClockMs{0};
|
|
int64_t latencyUs{0};
|
|
bool success{false};
|
|
};
|
|
|
|
std::optional<int64_t> parseInt64(std::string_view value) {
|
|
try {
|
|
return static_cast<int64_t>(std::stoll(std::string(value)));
|
|
} catch (...) {
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
|
|
std::optional<int64_t> metricInt(const std::map<std::string, std::string>& stats,
|
|
std::string_view key) {
|
|
const auto it = stats.find(std::string(key));
|
|
if (it == stats.end()) {
|
|
return std::nullopt;
|
|
}
|
|
return parseInt64(it->second);
|
|
}
|
|
|
|
void collectSearchHydrationTelemetry(const SearchResponse& response, ClientResult& result) {
|
|
const auto appendTiming = [&](std::string_view key, std::vector<int64_t>& samples) {
|
|
if (auto value = metricInt(response.searchStats, key)) {
|
|
samples.push_back(*value);
|
|
}
|
|
};
|
|
const auto addCounter = [&](std::string_view key, int64_t& counter) {
|
|
if (auto value = metricInt(response.searchStats, key)) {
|
|
counter = checkedMetricAdd(counter, *value);
|
|
}
|
|
};
|
|
|
|
appendTiming("timing_snippet_hydration_us", result.snippetHydrationUs);
|
|
appendTiming("timing_snippet_lookup_us", result.snippetLookupUs);
|
|
appendTiming("timing_snippet_content_fetch_us", result.snippetContentFetchUs);
|
|
appendTiming("timing_snippet_render_us", result.snippetRenderUs);
|
|
addCounter("snippet_hydration_candidates", result.snippetHydrationCandidates);
|
|
addCounter("snippet_hydration_missing", result.snippetHydrationMissing);
|
|
addCounter("snippet_hydrated_results", result.snippetHydratedResults);
|
|
const auto timeout = response.searchStats.find("snippet_timeout_hit");
|
|
if (timeout != response.searchStats.end() && timeout->second == "true") {
|
|
++result.snippetTimeoutHits;
|
|
}
|
|
}
|
|
|
|
void appendStringTimingStats(std::map<std::string, std::vector<int64_t>>& dest, std::string_view op,
|
|
const std::map<std::string, std::string>& stats, int64_t wallUs) {
|
|
std::optional<int64_t> dispatchTotalUs;
|
|
for (const auto& [key, value] : stats) {
|
|
if (!key.ends_with("_us")) {
|
|
continue;
|
|
}
|
|
auto parsed = parseInt64(value);
|
|
if (!parsed) {
|
|
continue;
|
|
}
|
|
dest[std::string(op) + ":" + key].push_back(*parsed);
|
|
if (key == "phase_dispatch_total_us") {
|
|
dispatchTotalUs = *parsed;
|
|
}
|
|
}
|
|
if (dispatchTotalUs) {
|
|
dest[std::string(op) + ":client_transport_overhead_us"].push_back(
|
|
std::max<int64_t>(0, wallUs - *dispatchTotalUs));
|
|
}
|
|
}
|
|
|
|
void appendStatusTimingStats(std::map<std::string, std::vector<int64_t>>& dest,
|
|
const std::map<std::string, size_t>& stats, int64_t wallUs) {
|
|
auto it = stats.find("phase_dispatch_total_us");
|
|
if (it == stats.end()) {
|
|
return;
|
|
}
|
|
const auto dispatchTotalUs = static_cast<int64_t>(it->second);
|
|
dest["status:phase_dispatch_total_us"].push_back(dispatchTotalUs);
|
|
dest["status:client_transport_overhead_us"].push_back(
|
|
std::max<int64_t>(0, wallUs - dispatchTotalUs));
|
|
}
|
|
|
|
json summarizeTimingStats(std::map<std::string, std::vector<int64_t>> stats) {
|
|
json out = json::object();
|
|
for (auto& [key, values] : stats) {
|
|
out[key] = PercentileStats::compute(values).toJson();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
struct ResourcePeaks {
|
|
double peakMemoryMb{0.0};
|
|
double peakCpuPercent{0.0};
|
|
uint64_t peakRssBytes{0};
|
|
size_t peakActiveConnections{0};
|
|
size_t peakMaxConnections{0};
|
|
uint64_t peakPostIngestQueued{0};
|
|
uint64_t peakPostIngestInflight{0};
|
|
uint64_t peakSearchActive{0};
|
|
uint64_t peakSearchQueued{0};
|
|
uint64_t peakDbWritePoolWaiting{0};
|
|
uint64_t peakDbReadPoolWaiting{0};
|
|
uint64_t peakDbWritePoolMaxObservedWaiting{0};
|
|
uint64_t peakDbReadPoolMaxObservedWaiting{0};
|
|
uint64_t peakDbWritePoolTotalWaitMicros{0};
|
|
uint64_t peakDbReadPoolTotalWaitMicros{0};
|
|
uint64_t peakDbWritePoolTimeoutCount{0};
|
|
uint64_t peakDbReadPoolTimeoutCount{0};
|
|
uint64_t peakDbWritePoolFailedAcquisitions{0};
|
|
uint64_t peakDbReadPoolFailedAcquisitions{0};
|
|
uint64_t peakDbWritePoolSlowHolders{0};
|
|
uint64_t peakDbReadPoolSlowHolders{0};
|
|
uint64_t peakDbWritePoolMaxHolderMicros{0};
|
|
uint64_t peakDbReadPoolMaxHolderMicros{0};
|
|
uint64_t peakMuxWriterBudgetBytes{0};
|
|
uint64_t peakWriteQueueDepth{0};
|
|
uint64_t writeQueueCapacity{0};
|
|
uint64_t peakWriteQueueDepthMax{0};
|
|
uint64_t peakWriteQueueCapacityRejections{0};
|
|
uint64_t peakWriteQueueForcedOverCapacity{0};
|
|
uint64_t peakWriteInFlight{0};
|
|
uint64_t peakWriteMaxBatchApplyMs{0};
|
|
uint64_t peakWriteMaxBatchQueueWaitMs{0};
|
|
uint64_t peakWriteMaxBatchExcessQueueWaitMs{0};
|
|
uint64_t peakProcessFdCount{0};
|
|
uint64_t peakProcessFdSockets{0};
|
|
uint64_t peakProcessFdPipes{0};
|
|
uint64_t peakProcessFdFiles{0};
|
|
uint64_t peakProcessFdDevices{0};
|
|
uint64_t peakProcessFdKqueues{0};
|
|
uint64_t peakProcessFdOther{0};
|
|
uint64_t peakPostCommitContentIndexCalls{0};
|
|
uint64_t peakPostCommitContentIndexTotalMs{0};
|
|
uint64_t peakPostCommitContentIndexMaxMs{0};
|
|
uint64_t peakClientPoolOpenSockets{0};
|
|
uint64_t peakClientPoolAliveConnections{0};
|
|
uint64_t peakClientPoolInUseConnections{0};
|
|
uint64_t peakMetadataWalBytes{0};
|
|
std::string lastWriteHotSources;
|
|
uint32_t maxPressureLevel{0};
|
|
|
|
json toJson() const {
|
|
return json{
|
|
{"peak_memory_mb", peakMemoryMb},
|
|
{"peak_cpu_pct", peakCpuPercent},
|
|
{"peak_rss_bytes", peakRssBytes},
|
|
{"peak_active_connections", peakActiveConnections},
|
|
{"peak_max_connections", peakMaxConnections},
|
|
{"peak_post_ingest_queued", peakPostIngestQueued},
|
|
{"peak_post_ingest_inflight", peakPostIngestInflight},
|
|
{"peak_search_active", peakSearchActive},
|
|
{"peak_search_queued", peakSearchQueued},
|
|
{"peak_db_write_pool_waiting", peakDbWritePoolWaiting},
|
|
{"peak_db_read_pool_waiting", peakDbReadPoolWaiting},
|
|
{"peak_db_write_pool_max_observed_waiting", peakDbWritePoolMaxObservedWaiting},
|
|
{"peak_db_read_pool_max_observed_waiting", peakDbReadPoolMaxObservedWaiting},
|
|
{"peak_db_write_pool_total_wait_us", peakDbWritePoolTotalWaitMicros},
|
|
{"peak_db_read_pool_total_wait_us", peakDbReadPoolTotalWaitMicros},
|
|
{"peak_db_write_pool_timeouts", peakDbWritePoolTimeoutCount},
|
|
{"peak_db_read_pool_timeouts", peakDbReadPoolTimeoutCount},
|
|
{"peak_db_write_pool_failed_acquisitions", peakDbWritePoolFailedAcquisitions},
|
|
{"peak_db_read_pool_failed_acquisitions", peakDbReadPoolFailedAcquisitions},
|
|
{"peak_db_write_pool_slow_holders", peakDbWritePoolSlowHolders},
|
|
{"peak_db_read_pool_slow_holders", peakDbReadPoolSlowHolders},
|
|
{"peak_db_write_pool_max_holder_us", peakDbWritePoolMaxHolderMicros},
|
|
{"peak_db_read_pool_max_holder_us", peakDbReadPoolMaxHolderMicros},
|
|
{"peak_mux_writer_budget_bytes", peakMuxWriterBudgetBytes},
|
|
{"peak_write_queue_depth", peakWriteQueueDepth},
|
|
{"write_queue_capacity", writeQueueCapacity},
|
|
{"peak_write_queue_depth_max", peakWriteQueueDepthMax},
|
|
{"peak_write_queue_capacity_rejections", peakWriteQueueCapacityRejections},
|
|
{"peak_write_queue_forced_over_capacity", peakWriteQueueForcedOverCapacity},
|
|
{"peak_write_in_flight", peakWriteInFlight},
|
|
{"peak_write_max_batch_apply_ms", peakWriteMaxBatchApplyMs},
|
|
{"peak_write_max_batch_queue_wait_ms", peakWriteMaxBatchQueueWaitMs},
|
|
{"peak_write_max_batch_excess_queue_wait_ms", peakWriteMaxBatchExcessQueueWaitMs},
|
|
{"peak_process_fd_count", peakProcessFdCount},
|
|
{"peak_process_fd_sockets", peakProcessFdSockets},
|
|
{"peak_process_fd_pipes", peakProcessFdPipes},
|
|
{"peak_process_fd_files", peakProcessFdFiles},
|
|
{"peak_process_fd_devices", peakProcessFdDevices},
|
|
{"peak_process_fd_kqueues", peakProcessFdKqueues},
|
|
{"peak_process_fd_other", peakProcessFdOther},
|
|
{"peak_post_commit_content_index_calls", peakPostCommitContentIndexCalls},
|
|
{"peak_post_commit_content_index_total_ms", peakPostCommitContentIndexTotalMs},
|
|
{"peak_post_commit_content_index_max_ms", peakPostCommitContentIndexMaxMs},
|
|
{"peak_client_pool_open_sockets", peakClientPoolOpenSockets},
|
|
{"peak_client_pool_alive_connections", peakClientPoolAliveConnections},
|
|
{"peak_client_pool_in_use_connections", peakClientPoolInUseConnections},
|
|
{"peak_metadata_wal_bytes", peakMetadataWalBytes},
|
|
{"last_write_hot_sources", lastWriteHotSources},
|
|
{"max_pressure_level", maxPressureLevel}};
|
|
}
|
|
};
|
|
|
|
ResourcePeaks summarizeResourcePeaks(const std::vector<TimeSeriesSampler::Sample>& samples) {
|
|
ResourcePeaks peaks;
|
|
for (const auto& sample : samples) {
|
|
const auto& snap = sample.snap;
|
|
peaks.peakMemoryMb = std::max(peaks.peakMemoryMb, snap.memoryUsageMb);
|
|
peaks.peakCpuPercent = std::max(peaks.peakCpuPercent, snap.cpuUsagePercent);
|
|
peaks.peakRssBytes = std::max(peaks.peakRssBytes, snap.rssBytes);
|
|
peaks.peakActiveConnections = std::max(peaks.peakActiveConnections, snap.activeConnections);
|
|
peaks.peakMaxConnections = std::max(peaks.peakMaxConnections, snap.maxConnections);
|
|
peaks.peakPostIngestQueued = std::max(peaks.peakPostIngestQueued, snap.postIngestQueued);
|
|
peaks.peakPostIngestInflight =
|
|
std::max(peaks.peakPostIngestInflight, snap.postIngestInflight);
|
|
peaks.peakSearchActive = std::max(peaks.peakSearchActive, snap.searchActive);
|
|
peaks.peakSearchQueued = std::max(peaks.peakSearchQueued, snap.searchQueued);
|
|
peaks.peakDbWritePoolWaiting =
|
|
std::max(peaks.peakDbWritePoolWaiting, snap.dbWritePoolWaiting);
|
|
peaks.peakDbReadPoolWaiting = std::max(peaks.peakDbReadPoolWaiting, snap.dbReadPoolWaiting);
|
|
peaks.peakDbWritePoolMaxObservedWaiting =
|
|
std::max(peaks.peakDbWritePoolMaxObservedWaiting, snap.dbWritePoolMaxObservedWaiting);
|
|
peaks.peakDbReadPoolMaxObservedWaiting =
|
|
std::max(peaks.peakDbReadPoolMaxObservedWaiting, snap.dbReadPoolMaxObservedWaiting);
|
|
peaks.peakDbWritePoolTotalWaitMicros =
|
|
std::max(peaks.peakDbWritePoolTotalWaitMicros, snap.dbWritePoolTotalWaitMicros);
|
|
peaks.peakDbReadPoolTotalWaitMicros =
|
|
std::max(peaks.peakDbReadPoolTotalWaitMicros, snap.dbReadPoolTotalWaitMicros);
|
|
peaks.peakDbWritePoolTimeoutCount =
|
|
std::max(peaks.peakDbWritePoolTimeoutCount, snap.dbWritePoolTimeoutCount);
|
|
peaks.peakDbReadPoolTimeoutCount =
|
|
std::max(peaks.peakDbReadPoolTimeoutCount, snap.dbReadPoolTimeoutCount);
|
|
peaks.peakDbWritePoolFailedAcquisitions =
|
|
std::max(peaks.peakDbWritePoolFailedAcquisitions, snap.dbWritePoolFailedAcquisitions);
|
|
peaks.peakDbReadPoolFailedAcquisitions =
|
|
std::max(peaks.peakDbReadPoolFailedAcquisitions, snap.dbReadPoolFailedAcquisitions);
|
|
peaks.peakDbWritePoolSlowHolders =
|
|
std::max(peaks.peakDbWritePoolSlowHolders, snap.dbWritePoolSlowHolders);
|
|
peaks.peakDbReadPoolSlowHolders =
|
|
std::max(peaks.peakDbReadPoolSlowHolders, snap.dbReadPoolSlowHolders);
|
|
peaks.peakDbWritePoolMaxHolderMicros =
|
|
std::max(peaks.peakDbWritePoolMaxHolderMicros, snap.dbWritePoolMaxHolderMicros);
|
|
peaks.peakDbReadPoolMaxHolderMicros =
|
|
std::max(peaks.peakDbReadPoolMaxHolderMicros, snap.dbReadPoolMaxHolderMicros);
|
|
peaks.peakMuxWriterBudgetBytes =
|
|
std::max(peaks.peakMuxWriterBudgetBytes, snap.muxWriterBudgetBytes);
|
|
peaks.peakWriteQueueDepth = std::max(peaks.peakWriteQueueDepth, snap.writeQueueDepth);
|
|
peaks.writeQueueCapacity = std::max(peaks.writeQueueCapacity, snap.writeQueueCapacity);
|
|
peaks.peakWriteQueueDepthMax =
|
|
std::max(peaks.peakWriteQueueDepthMax, snap.writeQueueDepthMax);
|
|
peaks.peakWriteQueueCapacityRejections =
|
|
std::max(peaks.peakWriteQueueCapacityRejections, snap.writeQueueCapacityRejections);
|
|
peaks.peakWriteQueueForcedOverCapacity =
|
|
std::max(peaks.peakWriteQueueForcedOverCapacity, snap.writeQueueForcedOverCapacity);
|
|
peaks.peakWriteInFlight = std::max(peaks.peakWriteInFlight, snap.writeInFlight);
|
|
peaks.peakWriteMaxBatchApplyMs =
|
|
std::max(peaks.peakWriteMaxBatchApplyMs, snap.writeMaxBatchApplyMs);
|
|
peaks.peakWriteMaxBatchQueueWaitMs =
|
|
std::max(peaks.peakWriteMaxBatchQueueWaitMs, snap.writeMaxBatchQueueWaitMs);
|
|
peaks.peakWriteMaxBatchExcessQueueWaitMs =
|
|
std::max(peaks.peakWriteMaxBatchExcessQueueWaitMs, snap.writeMaxBatchExcessQueueWaitMs);
|
|
peaks.peakProcessFdCount = std::max(peaks.peakProcessFdCount, snap.processFdCount);
|
|
peaks.peakProcessFdSockets = std::max(peaks.peakProcessFdSockets, snap.processFdSockets);
|
|
peaks.peakProcessFdPipes = std::max(peaks.peakProcessFdPipes, snap.processFdPipes);
|
|
peaks.peakProcessFdFiles = std::max(peaks.peakProcessFdFiles, snap.processFdFiles);
|
|
peaks.peakProcessFdDevices = std::max(peaks.peakProcessFdDevices, snap.processFdDevices);
|
|
peaks.peakProcessFdKqueues = std::max(peaks.peakProcessFdKqueues, snap.processFdKqueues);
|
|
peaks.peakProcessFdOther = std::max(peaks.peakProcessFdOther, snap.processFdOther);
|
|
peaks.peakPostCommitContentIndexCalls =
|
|
std::max(peaks.peakPostCommitContentIndexCalls, snap.postCommitContentIndexCalls);
|
|
peaks.peakPostCommitContentIndexTotalMs =
|
|
std::max(peaks.peakPostCommitContentIndexTotalMs, snap.postCommitContentIndexTotalMs);
|
|
peaks.peakPostCommitContentIndexMaxMs =
|
|
std::max(peaks.peakPostCommitContentIndexMaxMs, snap.postCommitContentIndexMaxMs);
|
|
peaks.peakClientPoolOpenSockets =
|
|
std::max(peaks.peakClientPoolOpenSockets, snap.clientPoolOpenSockets);
|
|
peaks.peakClientPoolAliveConnections =
|
|
std::max(peaks.peakClientPoolAliveConnections, snap.clientPoolAliveConnections);
|
|
peaks.peakClientPoolInUseConnections =
|
|
std::max(peaks.peakClientPoolInUseConnections, snap.clientPoolInUseConnections);
|
|
peaks.peakMetadataWalBytes = std::max(peaks.peakMetadataWalBytes, snap.metadataWalBytes);
|
|
if (!snap.writeHotSources.empty()) {
|
|
peaks.lastWriteHotSources = snap.writeHotSources;
|
|
}
|
|
peaks.maxPressureLevel = std::max(peaks.maxPressureLevel,
|
|
static_cast<uint32_t>(std::max(0, snap.pressureLevel)));
|
|
}
|
|
return peaks;
|
|
}
|
|
|
|
json summarizeLatencyHotspots(const std::vector<TimedLatencySample>& samples,
|
|
uint64_t windowMs = 500) {
|
|
json out = json{{"window_ms", windowMs},
|
|
{"slowest_ops", json::array()},
|
|
{"worst_window", json::object()},
|
|
{"per_op_p95_rank", json::array()}};
|
|
if (samples.empty()) {
|
|
return out;
|
|
}
|
|
|
|
auto sorted = samples;
|
|
std::sort(sorted.begin(), sorted.end(),
|
|
[](const auto& a, const auto& b) { return a.latencyUs > b.latencyUs; });
|
|
for (size_t i = 0; i < std::min<size_t>(5, sorted.size()); ++i) {
|
|
out["slowest_ops"].push_back(json{{"op_type", sorted[i].opType},
|
|
{"wall_clock_ms", sorted[i].wallClockMs},
|
|
{"latency_us", sorted[i].latencyUs},
|
|
{"success", sorted[i].success}});
|
|
}
|
|
|
|
std::unordered_map<std::string, std::vector<int64_t>> byOp;
|
|
for (const auto& sample : samples) {
|
|
if (sample.success) {
|
|
pushMetricSample(byOp[sample.opType], sample.latencyUs);
|
|
}
|
|
}
|
|
std::vector<std::pair<std::string, PercentileStats>> ranked;
|
|
ranked.reserve(byOp.size());
|
|
for (auto& [op, values] : byOp) {
|
|
ranked.emplace_back(op, PercentileStats::compute(values));
|
|
}
|
|
std::sort(ranked.begin(), ranked.end(),
|
|
[](const auto& a, const auto& b) { return a.second.p95 > b.second.p95; });
|
|
for (const auto& [op, stats] : ranked) {
|
|
out["per_op_p95_rank"].push_back(json{{"op_type", op},
|
|
{"p95_us", stats.p95},
|
|
{"mean_us", stats.mean},
|
|
{"count", stats.count}});
|
|
}
|
|
|
|
size_t left = 0;
|
|
int64_t currentSumUs = 0;
|
|
size_t bestCount = 0;
|
|
int64_t bestSumUs = 0;
|
|
uint64_t bestStartMs = static_cast<uint64_t>(
|
|
clampUnsignedMetricSample(samples.front().wallClockMs, kMaxBenchmarkWallMs));
|
|
uint64_t bestEndMs = bestStartMs;
|
|
auto byTime = samples;
|
|
std::sort(byTime.begin(), byTime.end(),
|
|
[](const auto& a, const auto& b) { return a.wallClockMs < b.wallClockMs; });
|
|
const uint64_t boundedWindowMs =
|
|
std::max<uint64_t>(1, std::min<uint64_t>(windowMs, kMaxBenchmarkWallMs));
|
|
for (size_t right = 0; right < byTime.size(); ++right) {
|
|
const auto rightLatency = clampMetricSample(byTime[right].latencyUs);
|
|
currentSumUs = checkedMetricAdd(currentSumUs, rightLatency);
|
|
auto rightWallMs = static_cast<uint64_t>(
|
|
clampUnsignedMetricSample(byTime[right].wallClockMs, kMaxBenchmarkWallMs));
|
|
auto leftWallMs = static_cast<uint64_t>(
|
|
clampUnsignedMetricSample(byTime[left].wallClockMs, kMaxBenchmarkWallMs));
|
|
while (rightWallMs >= leftWallMs && rightWallMs - leftWallMs > boundedWindowMs) {
|
|
currentSumUs =
|
|
std::max<int64_t>(0, currentSumUs - clampMetricSample(byTime[left].latencyUs));
|
|
++left;
|
|
leftWallMs = static_cast<uint64_t>(
|
|
clampUnsignedMetricSample(byTime[left].wallClockMs, kMaxBenchmarkWallMs));
|
|
}
|
|
const size_t count = right - left + 1;
|
|
if (count > 0 &&
|
|
(currentSumUs > bestSumUs || (currentSumUs == bestSumUs && count > bestCount))) {
|
|
bestSumUs = currentSumUs;
|
|
bestCount = count;
|
|
bestStartMs = leftWallMs;
|
|
bestEndMs = rightWallMs;
|
|
}
|
|
}
|
|
out["worst_window"] = json{
|
|
{"start_ms", bestStartMs},
|
|
{"end_ms", bestEndMs},
|
|
{"sample_count", bestCount},
|
|
{"latency_sum_us", bestSumUs},
|
|
{"avg_latency_us",
|
|
bestCount > 0 ? static_cast<double>(bestSumUs) / static_cast<double>(bestCount) : 0.0}};
|
|
return out;
|
|
}
|
|
|
|
class HeldSocketConnections {
|
|
public:
|
|
bool open(const fs::path& socketPath, int count) {
|
|
lastError_.clear();
|
|
lastFailedIndex_ = -1;
|
|
openedCount_ = 0;
|
|
sockets_.clear();
|
|
sockets_.reserve(static_cast<size_t>(std::max(0, count)));
|
|
|
|
boost::system::error_code ec;
|
|
for (int i = 0; i < count; ++i) {
|
|
auto sock = std::make_unique<boost::asio::local::stream_protocol::socket>(io_);
|
|
sock->connect(boost::asio::local::stream_protocol::endpoint(socketPath.string()), ec);
|
|
if (ec) {
|
|
lastFailedIndex_ = i;
|
|
openedCount_ = static_cast<int>(sockets_.size());
|
|
lastError_ = ec.message();
|
|
closeAll();
|
|
return false;
|
|
}
|
|
sockets_.push_back(std::move(sock));
|
|
openedCount_ = static_cast<int>(sockets_.size());
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void closeAll() {
|
|
for (auto& sock : sockets_) {
|
|
if (!sock) {
|
|
continue;
|
|
}
|
|
boost::system::error_code ec;
|
|
sock->close(ec);
|
|
}
|
|
sockets_.clear();
|
|
}
|
|
|
|
~HeldSocketConnections() { closeAll(); }
|
|
|
|
[[nodiscard]] const std::string& lastError() const noexcept { return lastError_; }
|
|
|
|
[[nodiscard]] int lastFailedIndex() const noexcept { return lastFailedIndex_; }
|
|
|
|
[[nodiscard]] int openedCount() const noexcept { return openedCount_; }
|
|
|
|
private:
|
|
boost::asio::io_context io_;
|
|
std::vector<std::unique_ptr<boost::asio::local::stream_protocol::socket>> sockets_;
|
|
std::string lastError_;
|
|
int lastFailedIndex_{-1};
|
|
int openedCount_{0};
|
|
};
|
|
|
|
class ScopedConnectionSlotOverrides {
|
|
public:
|
|
ScopedConnectionSlotOverrides(uint32_t minSlots, uint32_t maxSlots, uint32_t scaleStep) {
|
|
TuneAdvisor::setConnectionSlotsMin(minSlots);
|
|
TuneAdvisor::setConnectionSlotsMax(maxSlots);
|
|
TuneAdvisor::setConnectionSlotsScaleStep(scaleStep);
|
|
}
|
|
|
|
~ScopedConnectionSlotOverrides() {
|
|
TuneAdvisor::setConnectionSlotsMin(0);
|
|
TuneAdvisor::setConnectionSlotsMax(0);
|
|
TuneAdvisor::setConnectionSlotsScaleStep(0);
|
|
}
|
|
|
|
ScopedConnectionSlotOverrides(const ScopedConnectionSlotOverrides&) = delete;
|
|
ScopedConnectionSlotOverrides& operator=(const ScopedConnectionSlotOverrides&) = delete;
|
|
};
|
|
|
|
bool waitForSlotLimit(DaemonClient& client, size_t targetLimit, std::chrono::milliseconds timeout,
|
|
bool requireAtLeast) {
|
|
const auto deadline = std::chrono::steady_clock::now() + timeout;
|
|
while (std::chrono::steady_clock::now() < deadline) {
|
|
const auto snap = DaemonSnapshot::capture(client);
|
|
if (requireAtLeast) {
|
|
if (snap.maxConnections >= targetLimit) {
|
|
return true;
|
|
}
|
|
} else if (snap.maxConnections <= targetLimit) {
|
|
return true;
|
|
}
|
|
std::this_thread::sleep_for(250ms);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool waitForActiveConnectionsAtMost(DaemonClient& client, size_t targetActive,
|
|
std::chrono::milliseconds timeout) {
|
|
const auto deadline = std::chrono::steady_clock::now() + timeout;
|
|
while (std::chrono::steady_clock::now() < deadline) {
|
|
const auto snap = DaemonSnapshot::capture(client);
|
|
if (snap.activeConnections <= targetActive) {
|
|
return true;
|
|
}
|
|
std::this_thread::sleep_for(250ms);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Document generator
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
std::string generateDocument(int clientId, int docIndex, size_t targetBytes) {
|
|
std::ostringstream oss;
|
|
oss << "# Client " << clientId << " Document " << docIndex << "\n\n";
|
|
oss << "Created at: " << std::chrono::system_clock::now().time_since_epoch().count() << "\n\n";
|
|
|
|
// Fill with pseudo-realistic text to reach target size
|
|
static const char* paragraphs[] = {
|
|
"The system architecture follows a layered design pattern with clear separation "
|
|
"of concerns between the daemon core, service layer, and client interfaces.\n\n",
|
|
"Performance tuning parameters are exposed through TuneAdvisor knobs allowing "
|
|
"runtime adjustment of thread pool sizes, queue capacities, and concurrency limits.\n\n",
|
|
"The ingestion pipeline processes documents through extraction, indexing, knowledge "
|
|
"graph construction, symbol analysis, entity recognition, and embedding stages.\n\n",
|
|
"Resource governance monitors memory pressure, CPU utilization, and I/O throughput "
|
|
"to dynamically scale worker thread counts and admission control thresholds.\n\n",
|
|
"The IPC transport uses Unix domain sockets with a binary framing protocol supporting "
|
|
"multiplexed request/response streams with backpressure signaling.\n\n",
|
|
};
|
|
constexpr int numParagraphs = 5;
|
|
|
|
int paraIdx = 0;
|
|
while (oss.tellp() < static_cast<std::streamoff>(targetBytes)) {
|
|
oss << paragraphs[paraIdx % numParagraphs];
|
|
paraIdx++;
|
|
}
|
|
|
|
auto result = oss.str();
|
|
if (result.size() > targetBytes)
|
|
result.resize(targetBytes);
|
|
return result;
|
|
}
|
|
|
|
using yams::bench::isoTimestamp;
|
|
|
|
std::string toLowerCopy(std::string value) {
|
|
std::transform(value.begin(), value.end(), value.begin(),
|
|
[](unsigned char ch) { return static_cast<char>(std::tolower(ch)); });
|
|
return value;
|
|
}
|
|
|
|
bool containsInsensitive(const std::string& haystackLower, std::string_view needle) {
|
|
return haystackLower.find(std::string(needle)) != std::string::npos;
|
|
}
|
|
|
|
std::string classifyFailureMessage(const std::string& errorMsg) {
|
|
if (errorMsg.empty()) {
|
|
return "unknown";
|
|
}
|
|
|
|
const std::string lower = toLowerCopy(errorMsg);
|
|
|
|
if (containsInsensitive(lower, "document content not found")) {
|
|
return "content_not_found";
|
|
}
|
|
if (containsInsensitive(lower, "document not found") ||
|
|
containsInsensitive(lower, "no documents found matching hash prefix")) {
|
|
return "document_not_found";
|
|
}
|
|
|
|
if (containsInsensitive(lower, "timed out") || containsInsensitive(lower, "timeout")) {
|
|
return "timeout";
|
|
}
|
|
if (containsInsensitive(lower, "retry after") || containsInsensitive(lower, "retry in ") ||
|
|
containsInsensitive(lower, "server busy") ||
|
|
containsInsensitive(lower, "resource exhausted") ||
|
|
containsInsensitive(lower, "backpressure")) {
|
|
return "backpressure";
|
|
}
|
|
if (containsInsensitive(lower, "connection refused") ||
|
|
containsInsensitive(lower, "no such file or directory")) {
|
|
return "connect_refused";
|
|
}
|
|
if (containsInsensitive(lower, "broken pipe") || containsInsensitive(lower, "eof") ||
|
|
containsInsensitive(lower, "connection reset")) {
|
|
return "connection_drop";
|
|
}
|
|
if (containsInsensitive(lower, "cancel")) {
|
|
return "cancelled";
|
|
}
|
|
if (containsInsensitive(lower, "database is locked") ||
|
|
containsInsensitive(lower, "sql_busy")) {
|
|
return "db_locked";
|
|
}
|
|
if (containsInsensitive(lower, "queue") &&
|
|
(containsInsensitive(lower, "full") || containsInsensitive(lower, "overflow"))) {
|
|
return "queue_full";
|
|
}
|
|
if (containsInsensitive(lower, "parse") || containsInsensitive(lower, "protocol") ||
|
|
containsInsensitive(lower, "invalid response")) {
|
|
return "protocol";
|
|
}
|
|
|
|
return "other";
|
|
}
|
|
|
|
bool isDocumentMissingFailure(const std::string& errorMsg) {
|
|
if (errorMsg.empty()) {
|
|
return false;
|
|
}
|
|
const std::string lower = toLowerCopy(errorMsg);
|
|
return containsInsensitive(lower, "document not found") ||
|
|
containsInsensitive(lower, "document content not found") ||
|
|
containsInsensitive(lower, "no documents found matching hash prefix");
|
|
}
|
|
|
|
bool isHotspotWindowEligible(int ops, int totalClients) {
|
|
return ops >= std::max(10, totalClients);
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Wait for queue drain
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
struct DrainObservation {
|
|
bool drained{false};
|
|
uint64_t elapsedMs{0};
|
|
int polls{0};
|
|
int stablePolls{0};
|
|
uint64_t minimumDocuments{0};
|
|
uint64_t documentsBelowMinimumPolls{0};
|
|
uint64_t postIngestQueuedPolls{0};
|
|
uint64_t postIngestInflightPolls{0};
|
|
uint64_t workerActivePolls{0};
|
|
uint64_t workerQueuedPolls{0};
|
|
uint64_t deferredQueuePolls{0};
|
|
uint64_t coordinatedWritePolls{0};
|
|
uint64_t peakPostIngestQueued{0};
|
|
uint64_t peakPostIngestInflight{0};
|
|
uint64_t peakWorkerActive{0};
|
|
uint64_t peakWorkerQueued{0};
|
|
uint64_t peakDeferredQueueDepth{0};
|
|
double peakCpuPercent{0.0};
|
|
double peakMemoryMb{0.0};
|
|
uint32_t maxPressureLevel{0};
|
|
DaemonSnapshot finalSnapshot;
|
|
|
|
json toJson() const {
|
|
return json{{"drained", drained},
|
|
{"elapsed_ms", elapsedMs},
|
|
{"polls", polls},
|
|
{"stable_polls", stablePolls},
|
|
{"minimum_documents", minimumDocuments},
|
|
{"blocker_polls",
|
|
{{"documents_below_minimum", documentsBelowMinimumPolls},
|
|
{"post_ingest_queued", postIngestQueuedPolls},
|
|
{"post_ingest_inflight", postIngestInflightPolls},
|
|
{"worker_active", workerActivePolls},
|
|
{"worker_queued", workerQueuedPolls},
|
|
{"deferred_queue", deferredQueuePolls},
|
|
{"coordinated_write", coordinatedWritePolls}}},
|
|
{"peak_post_ingest_queued", peakPostIngestQueued},
|
|
{"peak_post_ingest_inflight", peakPostIngestInflight},
|
|
{"peak_worker_active", peakWorkerActive},
|
|
{"peak_worker_queued", peakWorkerQueued},
|
|
{"peak_deferred_queue_depth", peakDeferredQueueDepth},
|
|
{"peak_cpu_pct", peakCpuPercent},
|
|
{"peak_memory_mb", peakMemoryMb},
|
|
{"max_pressure_level", maxPressureLevel},
|
|
{"final_snapshot", finalSnapshot.toJson()}};
|
|
}
|
|
};
|
|
|
|
struct IdleBaselineObservation {
|
|
uint64_t durationMs{0};
|
|
int samples{0};
|
|
double meanCpuPercent{0.0};
|
|
double peakCpuPercent{0.0};
|
|
double meanMemoryMb{0.0};
|
|
double peakMemoryMb{0.0};
|
|
uint64_t maxPostIngestQueued{0};
|
|
uint64_t maxPostIngestInflight{0};
|
|
size_t maxActiveConnections{0};
|
|
uint32_t maxPressureLevel{0};
|
|
|
|
json toJson() const {
|
|
return json{{"duration_ms", durationMs},
|
|
{"samples", samples},
|
|
{"mean_cpu_pct", meanCpuPercent},
|
|
{"peak_cpu_pct", peakCpuPercent},
|
|
{"mean_memory_mb", meanMemoryMb},
|
|
{"peak_memory_mb", peakMemoryMb},
|
|
{"max_post_ingest_queued", maxPostIngestQueued},
|
|
{"max_post_ingest_inflight", maxPostIngestInflight},
|
|
{"max_active_connections", maxActiveConnections},
|
|
{"max_pressure_level", maxPressureLevel}};
|
|
}
|
|
};
|
|
|
|
struct IdleWakeProbeObservation {
|
|
bool enabled{false};
|
|
bool attempted{false};
|
|
bool addCompleted{false};
|
|
bool addSucceeded{false};
|
|
bool workVisible{false};
|
|
uint64_t addLatencyUs{0};
|
|
uint64_t workVisibleUs{0};
|
|
std::string addError;
|
|
DaemonSnapshot beforeSnapshot;
|
|
DaemonSnapshot visibleSnapshot;
|
|
DrainObservation preProbeDrain;
|
|
IdleBaselineObservation idleBaseline;
|
|
DrainObservation postProbeDrain;
|
|
|
|
json toJson() const {
|
|
return json{{"enabled", enabled},
|
|
{"attempted", attempted},
|
|
{"add_completed", addCompleted},
|
|
{"add_succeeded", addSucceeded},
|
|
{"work_visible", workVisible},
|
|
{"add_latency_us", addLatencyUs},
|
|
{"work_visible_us", workVisibleUs},
|
|
{"add_error", addError},
|
|
{"before_snapshot", beforeSnapshot.toJson()},
|
|
{"visible_snapshot", visibleSnapshot.toJson()},
|
|
{"pre_probe_drain", preProbeDrain.toJson()},
|
|
{"idle_baseline", idleBaseline.toJson()},
|
|
{"post_probe_drain", postProbeDrain.toJson()}};
|
|
}
|
|
};
|
|
|
|
bool writePipelineDrained(const DaemonSnapshot& snapshot, uint64_t minimumDocuments = 0) {
|
|
return snapshot.postIngestQueued == 0 && snapshot.postIngestInflight == 0 &&
|
|
snapshot.workerActive == 0 && snapshot.workerQueued == 0 &&
|
|
snapshot.deferredQueueDepth == 0 && snapshot.writeQueueDepth == 0 &&
|
|
snapshot.writeInFlight == 0 && snapshot.documentsTotal >= minimumDocuments;
|
|
}
|
|
|
|
TEST_CASE("Drain contract includes post-ingest and coordinated writes",
|
|
"[benchmark-contract][drain]") {
|
|
CHECK(backgroundWorkerActive(0) == 0);
|
|
CHECK(backgroundWorkerActive(1) == 0);
|
|
CHECK(backgroundWorkerActive(3) == 2);
|
|
|
|
DaemonSnapshot snapshot;
|
|
CHECK(writePipelineDrained(snapshot));
|
|
CHECK(writePipelineDrained(snapshot, 0));
|
|
|
|
snapshot.postIngestQueued = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.postIngestQueued = 0;
|
|
|
|
snapshot.postIngestInflight = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.postIngestInflight = 0;
|
|
|
|
snapshot.writeQueueDepth = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.writeQueueDepth = 0;
|
|
|
|
snapshot.writeInFlight = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.writeInFlight = 0;
|
|
|
|
snapshot.workerActive = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.workerActive = 0;
|
|
|
|
snapshot.workerQueued = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.workerQueued = 0;
|
|
|
|
snapshot.deferredQueueDepth = 1;
|
|
CHECK_FALSE(writePipelineDrained(snapshot));
|
|
snapshot.deferredQueueDepth = 0;
|
|
|
|
snapshot.documentsTotal = 41;
|
|
CHECK_FALSE(writePipelineDrained(snapshot, 42));
|
|
snapshot.documentsTotal = 42;
|
|
CHECK(writePipelineDrained(snapshot, 42));
|
|
}
|
|
|
|
DrainObservation waitForDrainObservation(DaemonClient& client, std::chrono::milliseconds timeout,
|
|
std::chrono::milliseconds sampleInterval = 500ms,
|
|
uint64_t minimumDocuments = 0) {
|
|
DrainObservation observation;
|
|
observation.minimumDocuments = minimumDocuments;
|
|
const auto start = std::chrono::steady_clock::now();
|
|
const auto deadline = start + timeout;
|
|
constexpr int stableRequired = 5;
|
|
int stableCount = 0;
|
|
|
|
while (std::chrono::steady_clock::now() < deadline) {
|
|
auto snap = DaemonSnapshot::capture(client);
|
|
observation.polls++;
|
|
observation.peakPostIngestQueued =
|
|
std::max(observation.peakPostIngestQueued, snap.postIngestQueued);
|
|
observation.peakPostIngestInflight =
|
|
std::max(observation.peakPostIngestInflight, snap.postIngestInflight);
|
|
observation.peakWorkerActive = std::max(observation.peakWorkerActive, snap.workerActive);
|
|
observation.peakWorkerQueued = std::max(observation.peakWorkerQueued, snap.workerQueued);
|
|
observation.peakDeferredQueueDepth =
|
|
std::max(observation.peakDeferredQueueDepth, snap.deferredQueueDepth);
|
|
observation.peakCpuPercent = std::max(observation.peakCpuPercent, snap.cpuUsagePercent);
|
|
observation.peakMemoryMb = std::max(observation.peakMemoryMb, snap.memoryUsageMb);
|
|
observation.maxPressureLevel = std::max(
|
|
observation.maxPressureLevel, static_cast<uint32_t>(std::max(0, snap.pressureLevel)));
|
|
observation.documentsBelowMinimumPolls += snap.documentsTotal < minimumDocuments ? 1u : 0u;
|
|
observation.postIngestQueuedPolls += snap.postIngestQueued > 0 ? 1u : 0u;
|
|
observation.postIngestInflightPolls += snap.postIngestInflight > 0 ? 1u : 0u;
|
|
observation.workerActivePolls += snap.workerActive > 0 ? 1u : 0u;
|
|
observation.workerQueuedPolls += snap.workerQueued > 0 ? 1u : 0u;
|
|
observation.deferredQueuePolls += snap.deferredQueueDepth > 0 ? 1u : 0u;
|
|
observation.coordinatedWritePolls +=
|
|
(snap.writeQueueDepth > 0 || snap.writeInFlight > 0) ? 1u : 0u;
|
|
|
|
const bool drained = writePipelineDrained(snap, minimumDocuments);
|
|
if (drained) {
|
|
++stableCount;
|
|
if (stableCount >= stableRequired) {
|
|
observation.drained = true;
|
|
observation.stablePolls = stableCount;
|
|
observation.finalSnapshot = snap;
|
|
observation.elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
return observation;
|
|
}
|
|
} else {
|
|
stableCount = 0;
|
|
}
|
|
|
|
std::this_thread::sleep_for(sampleInterval);
|
|
}
|
|
|
|
observation.stablePolls = stableCount;
|
|
observation.elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
observation.finalSnapshot = DaemonSnapshot::capture(client);
|
|
return observation;
|
|
}
|
|
|
|
IdleBaselineObservation sampleIdleBaseline(DaemonClient& client, std::chrono::milliseconds duration,
|
|
std::chrono::milliseconds sampleInterval) {
|
|
IdleBaselineObservation observation;
|
|
observation.durationMs = static_cast<uint64_t>(std::max<int64_t>(0, duration.count()));
|
|
if (duration <= 0ms) {
|
|
return observation;
|
|
}
|
|
|
|
const auto start = std::chrono::steady_clock::now();
|
|
const auto deadline = start + duration;
|
|
double cpuSum = 0.0;
|
|
double memorySum = 0.0;
|
|
|
|
while (true) {
|
|
auto snap = DaemonSnapshot::capture(client);
|
|
observation.samples++;
|
|
cpuSum += snap.cpuUsagePercent;
|
|
memorySum += snap.memoryUsageMb;
|
|
observation.peakCpuPercent = std::max(observation.peakCpuPercent, snap.cpuUsagePercent);
|
|
observation.peakMemoryMb = std::max(observation.peakMemoryMb, snap.memoryUsageMb);
|
|
observation.maxPostIngestQueued =
|
|
std::max(observation.maxPostIngestQueued, snap.postIngestQueued);
|
|
observation.maxPostIngestInflight =
|
|
std::max(observation.maxPostIngestInflight, snap.postIngestInflight);
|
|
observation.maxActiveConnections =
|
|
std::max(observation.maxActiveConnections, snap.activeConnections);
|
|
observation.maxPressureLevel = std::max(
|
|
observation.maxPressureLevel, static_cast<uint32_t>(std::max(0, snap.pressureLevel)));
|
|
|
|
if (std::chrono::steady_clock::now() >= deadline) {
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(sampleInterval);
|
|
}
|
|
|
|
if (observation.samples > 0) {
|
|
observation.meanCpuPercent = cpuSum / static_cast<double>(observation.samples);
|
|
observation.meanMemoryMb = memorySum / static_cast<double>(observation.samples);
|
|
}
|
|
observation.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
return observation;
|
|
}
|
|
|
|
bool waitForDrain(DaemonClient& client, std::chrono::seconds timeout) {
|
|
return waitForDrainObservation(client, timeout).drained;
|
|
}
|
|
|
|
IdleWakeProbeObservation runIdleWakeProbe(const BenchConfig& cfg, const fs::path& socketPath,
|
|
DaemonClient& monitorClient) {
|
|
IdleWakeProbeObservation observation;
|
|
observation.enabled = cfg.idleProbeEnabled;
|
|
if (!cfg.idleProbeEnabled) {
|
|
return observation;
|
|
}
|
|
|
|
observation.attempted = true;
|
|
const auto sampleInterval = std::chrono::milliseconds(cfg.idleProbeSampleMs);
|
|
observation.preProbeDrain = waitForDrainObservation(
|
|
monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs), sampleInterval);
|
|
if (!observation.preProbeDrain.drained) {
|
|
return observation;
|
|
}
|
|
|
|
observation.idleBaseline = sampleIdleBaseline(
|
|
monitorClient, std::chrono::milliseconds(cfg.idleProbeBaselineMs), sampleInterval);
|
|
observation.beforeSnapshot = DaemonSnapshot::capture(monitorClient);
|
|
|
|
struct ProbeAddResult {
|
|
bool success{false};
|
|
std::string error;
|
|
int64_t latencyUs{0};
|
|
};
|
|
|
|
auto addFuture = std::async(std::launch::async, [socketPath, &cfg]() {
|
|
ProbeAddResult result;
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = socketPath;
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = std::chrono::seconds(15);
|
|
ccfg.singleUseConnections = true;
|
|
DaemonClient client(ccfg);
|
|
|
|
AddDocumentRequest req;
|
|
req.name = "idle_probe_" + std::to_string(::getpid()) + ".md";
|
|
req.content = generateDocument(
|
|
777, 0, std::max<size_t>(256, std::min<size_t>(1024, cfg.docSizeBytes)));
|
|
req.tags = {"bench", "idle-probe"};
|
|
req.collection = "bench_idle_probe";
|
|
req.noEmbeddings = true;
|
|
|
|
const auto start = std::chrono::steady_clock::now();
|
|
auto addResult = yams::cli::run_sync(client.streamingAddDocument(req), 15s);
|
|
result.latencyUs = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - start)
|
|
.count();
|
|
result.success = static_cast<bool>(addResult);
|
|
if (!addResult) {
|
|
result.error = addResult.error().message;
|
|
}
|
|
return result;
|
|
});
|
|
|
|
const auto probeStart = std::chrono::steady_clock::now();
|
|
const auto detectDeadline =
|
|
probeStart + std::chrono::milliseconds(cfg.idleProbeDetectTimeoutMs);
|
|
while (std::chrono::steady_clock::now() < detectDeadline) {
|
|
if (!observation.addCompleted && addFuture.wait_for(0ms) == std::future_status::ready) {
|
|
const auto addResult = addFuture.get();
|
|
observation.addCompleted = true;
|
|
observation.addSucceeded = addResult.success;
|
|
observation.addLatencyUs =
|
|
static_cast<uint64_t>(std::max<int64_t>(0, addResult.latencyUs));
|
|
observation.addError = addResult.error;
|
|
}
|
|
|
|
auto snap = DaemonSnapshot::capture(monitorClient);
|
|
if (!observation.workVisible &&
|
|
(snap.documentsTotal > observation.beforeSnapshot.documentsTotal ||
|
|
snap.postIngestQueued > 0 || snap.postIngestInflight > 0 || snap.searchQueued > 0 ||
|
|
snap.searchActive > 0)) {
|
|
observation.workVisible = true;
|
|
observation.workVisibleUs = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - probeStart)
|
|
.count();
|
|
observation.visibleSnapshot = snap;
|
|
}
|
|
|
|
if (observation.workVisible && observation.addCompleted) {
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(sampleInterval);
|
|
}
|
|
|
|
if (!observation.addCompleted &&
|
|
addFuture.wait_for(std::chrono::seconds(20)) == std::future_status::ready) {
|
|
const auto addResult = addFuture.get();
|
|
observation.addCompleted = true;
|
|
observation.addSucceeded = addResult.success;
|
|
observation.addLatencyUs = static_cast<uint64_t>(std::max<int64_t>(0, addResult.latencyUs));
|
|
observation.addError = addResult.error;
|
|
}
|
|
|
|
if (observation.visibleSnapshot.maxConnections == 0) {
|
|
observation.visibleSnapshot = DaemonSnapshot::capture(monitorClient);
|
|
}
|
|
|
|
if (observation.addSucceeded) {
|
|
observation.postProbeDrain = waitForDrainObservation(
|
|
monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs), sampleInterval);
|
|
}
|
|
|
|
return observation;
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Output helpers
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
void emitJsonl(const fs::path& outputPath, const json& record) {
|
|
json enriched = record;
|
|
enriched["schema_version"] = "multi_client_record_v4";
|
|
if (const auto* v = std::getenv("YAMS_BENCH_RUN_ID"); v && *v) {
|
|
enriched["run_id"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_PHASE"); v && *v) {
|
|
enriched["run_phase"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_CASE"); v && *v) {
|
|
enriched["run_case"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_TRANSPORT"); v && *v) {
|
|
enriched["run_transport"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_GIT_SHA"); v && *v) {
|
|
enriched["git_sha"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_BACKEND"); v && *v) {
|
|
enriched["run_backend"] = v;
|
|
}
|
|
if (const auto* v = std::getenv("YAMS_BENCH_AUTH_MODE"); v && *v) {
|
|
enriched["run_auth_mode"] = v;
|
|
}
|
|
|
|
fs::create_directories(outputPath.parent_path());
|
|
std::ofstream out(outputPath, std::ios::app);
|
|
if (out) {
|
|
out << enriched.dump() << '\n';
|
|
}
|
|
}
|
|
|
|
std::optional<int64_t> jsonToInt64(const nlohmann::json& value) {
|
|
try {
|
|
if (value.is_number_integer()) {
|
|
return value.get<int64_t>();
|
|
}
|
|
if (value.is_number_unsigned()) {
|
|
return static_cast<int64_t>(value.get<uint64_t>());
|
|
}
|
|
if (value.is_number_float()) {
|
|
return static_cast<int64_t>(value.get<double>());
|
|
}
|
|
if (value.is_string()) {
|
|
const auto& s = value.get_ref<const std::string&>();
|
|
if (!s.empty()) {
|
|
return std::stoll(s);
|
|
}
|
|
}
|
|
} catch (const std::exception&) {
|
|
return std::nullopt;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
std::optional<int64_t> jsonGetInt64AnyKey(const nlohmann::json& object,
|
|
std::initializer_list<const char*> keys) {
|
|
if (!object.is_object()) {
|
|
return std::nullopt;
|
|
}
|
|
for (const auto* key : keys) {
|
|
if (!key) {
|
|
continue;
|
|
}
|
|
auto it = object.find(key);
|
|
if (it == object.end()) {
|
|
continue;
|
|
}
|
|
if (auto parsed = jsonToInt64(*it); parsed.has_value()) {
|
|
return parsed;
|
|
}
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
void printPercentiles(const std::string& label, const PercentileStats& stats) {
|
|
std::cout << " " << label << ": n=" << stats.count << " min=" << stats.min << "us"
|
|
<< " p50=" << stats.p50 << "us"
|
|
<< " p95=" << stats.p95 << "us"
|
|
<< " p99=" << stats.p99 << "us"
|
|
<< " max=" << stats.max << "us"
|
|
<< " mean=" << std::fixed << std::setprecision(0) << stats.mean << "us" << std::endl;
|
|
}
|
|
|
|
void printDrainAndIdleProbeSummary(const DrainObservation& drainObservation,
|
|
const IdleWakeProbeObservation& idleProbe) {
|
|
std::cout << " Drain time: " << drainObservation.elapsedMs << " ms\n";
|
|
if (idleProbe.attempted) {
|
|
std::cout << " Idle probe: baseline_cpu=" << std::fixed << std::setprecision(2)
|
|
<< idleProbe.idleBaseline.meanCpuPercent
|
|
<< "% peak=" << idleProbe.idleBaseline.peakCpuPercent
|
|
<< "% work_visible=" << idleProbe.workVisibleUs
|
|
<< "us add_latency=" << idleProbe.addLatencyUs
|
|
<< "us post_drain=" << idleProbe.postProbeDrain.elapsedMs << "ms\n";
|
|
}
|
|
}
|
|
|
|
// Create harness options tuned for benchmarking:
|
|
// - Disable model provider (we don't need embeddings for ingestion benchmarks)
|
|
// - Use mock model provider as fallback
|
|
// - No auto-loaded plugins (reduce startup variance)
|
|
// Overloads accept a BenchConfig to enable plugins, real models, and custom data dirs.
|
|
DaemonHarnessOptions benchHarnessOptions() {
|
|
return DaemonHarnessOptions{
|
|
.enableModelProvider = false,
|
|
.useMockModelProvider = true,
|
|
.autoLoadPlugins = false,
|
|
.enableAutoRepair = true,
|
|
.isolateState = true,
|
|
};
|
|
}
|
|
|
|
DaemonHarnessOptions benchHarnessOptions(const BenchConfig& cfg) {
|
|
DaemonHarnessOptions opts{
|
|
.enableModelProvider = cfg.enablePlugins || cfg.useRealModelProvider,
|
|
.useMockModelProvider = !cfg.useRealModelProvider,
|
|
.autoLoadPlugins = cfg.enablePlugins,
|
|
.enableAutoRepair = true,
|
|
.isolateState = true,
|
|
.pluginDir = cfg.pluginDir,
|
|
.dataDir = cfg.dataDir,
|
|
.configPath = cfg.configPath,
|
|
.trustedPluginPaths = cfg.trustedPluginPaths,
|
|
};
|
|
return opts;
|
|
}
|
|
|
|
// Startup timeout: 30s to handle TSan/debug builds where init is ~5-10x slower
|
|
constexpr auto kStartTimeout = std::chrono::seconds(30);
|
|
constexpr auto kCorpusStartTimeout = std::chrono::seconds(180);
|
|
|
|
std::chrono::seconds benchStartTimeout(const BenchConfig& cfg) {
|
|
return cfg.dataDir.has_value() ? kCorpusStartTimeout : kStartTimeout;
|
|
}
|
|
|
|
TEST_CASE("Multi-client benchmark allows explicit corpora a longer startup window",
|
|
"[multi-client][config]") {
|
|
BenchConfig synthetic;
|
|
CHECK(benchStartTimeout(synthetic) == kStartTimeout);
|
|
|
|
BenchConfig corpus;
|
|
corpus.dataDir = fs::path("symbol-rich-corpus");
|
|
CHECK(benchStartTimeout(corpus) == kCorpusStartTimeout);
|
|
}
|
|
|
|
bool startHarnessWithRetry(DaemonHarness& harness, std::chrono::milliseconds timeout,
|
|
int maxAttempts = 3) {
|
|
for (int attempt = 0; attempt < maxAttempts; ++attempt) {
|
|
if (harness.start(timeout)) {
|
|
return true;
|
|
}
|
|
harness.stop();
|
|
std::this_thread::sleep_for(500ms);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// =============================================================================
|
|
// TEST CASES
|
|
// =============================================================================
|
|
|
|
TEST_CASE("Multi-client ingestion: baseline single client",
|
|
"[!benchmark][multi-client][ingestion]") {
|
|
auto cfg = BenchConfig::fromEnv();
|
|
cfg.numClients = 1; // Force single client for baseline
|
|
|
|
DaemonHarness harness(benchHarnessOptions(cfg));
|
|
REQUIRE(harness.start(benchStartTimeout(cfg)));
|
|
|
|
// Create client
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
// Ingest documents
|
|
std::vector<int64_t> addLatencies;
|
|
auto startTime = std::chrono::steady_clock::now();
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
auto doc = generateDocument(0, i, cfg.docSizeBytes);
|
|
AddDocumentRequest req;
|
|
req.name = "baseline_doc_" + std::to_string(i) + ".md";
|
|
req.content = std::move(doc);
|
|
req.tags = {"bench", "baseline"};
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto result = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (result) {
|
|
addLatencies.push_back(latUs);
|
|
} else {
|
|
WARN("Add failed at doc " << i << ": " << result.error().message);
|
|
}
|
|
}
|
|
|
|
auto elapsed = std::chrono::steady_clock::now() - startTime;
|
|
double elapsedSec = std::chrono::duration<double>(elapsed).count();
|
|
double throughput = static_cast<double>(addLatencies.size()) / elapsedSec;
|
|
|
|
// Wait for pipeline drain and then probe idle->work transition once.
|
|
const auto drainObservation =
|
|
waitForDrainObservation(client, std::chrono::seconds(cfg.drainTimeoutSecs));
|
|
REQUIRE(drainObservation.drained);
|
|
const auto idleProbe = runIdleWakeProbe(cfg, harness.socketPath(), client);
|
|
|
|
auto snap = DaemonSnapshot::capture(client);
|
|
const auto recovery = runRecoveryCheck(harness.socketPath());
|
|
auto stats = PercentileStats::compute(addLatencies);
|
|
|
|
std::cout << "\n=== Single Client Baseline ===\n";
|
|
std::cout << " Docs ingested: " << addLatencies.size() << "/" << cfg.docsPerClient << "\n";
|
|
std::cout << " Elapsed: " << std::fixed << std::setprecision(2) << elapsedSec << "s\n";
|
|
std::cout << " Throughput: " << throughput << " docs/s\n";
|
|
printPercentiles("Add latency", stats);
|
|
printDrainAndIdleProbeSummary(drainObservation, idleProbe);
|
|
printRecoverySummary(recovery);
|
|
std::cout << " Final docs_total: " << snap.documentsTotal << "\n";
|
|
std::cout << " Memory: " << snap.memoryUsageMb << " MB\n\n";
|
|
|
|
CHECK(static_cast<int>(addLatencies.size()) == cfg.docsPerClient);
|
|
CHECK(throughput > 0.0);
|
|
CHECK(recovery.ok());
|
|
|
|
// Emit JSONL record for regression tracking
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "baseline_single_client"},
|
|
{"num_clients", 1},
|
|
{"docs_per_client", cfg.docsPerClient},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"elapsed_seconds", elapsedSec},
|
|
{"throughput_docs_per_sec", throughput},
|
|
{"add_latency", stats.toJson()},
|
|
{"drain_metrics", drainObservation.toJson()},
|
|
{"idle_probe", idleProbe.toJson()},
|
|
{"recovery", recovery.toJson()},
|
|
{"daemon_snapshot", snap.toJson()},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: concurrent pure ingest",
|
|
"[!benchmark][multi-client][ingestion]") {
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
DaemonHarness harness(benchHarnessOptions(cfg));
|
|
REQUIRE(harness.start(kStartTimeout));
|
|
|
|
// Status-monitoring client (separate from worker clients)
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
monCfg.singleUseConnections = true;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
// Start time-series sampler
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, std::chrono::milliseconds(cfg.resourceSampleMs),
|
|
harness.dataDir());
|
|
|
|
// Shared counters
|
|
std::atomic<int> totalDocsIngested{0};
|
|
std::atomic<int> totalFailures{0};
|
|
std::vector<ClientResult> clientResults(cfg.numClients);
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
// Launch concurrent client threads
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(cfg.numClients);
|
|
|
|
for (int c = 0; c < cfg.numClients; ++c) {
|
|
threads.emplace_back([&, c]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
ClientResult& result = clientResults[c];
|
|
result.clientId = c;
|
|
|
|
auto clientStart = std::chrono::steady_clock::now();
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
auto doc = generateDocument(c, i, cfg.docSizeBytes);
|
|
AddDocumentRequest req;
|
|
req.name = "client" + std::to_string(c) + "_doc_" + std::to_string(i) + ".md";
|
|
req.content = std::move(doc);
|
|
req.tags = {"bench", "multi-client", "client-" + std::to_string(c)};
|
|
req.collection = "bench_client_" + std::to_string(c);
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
OpLatency op;
|
|
op.opType = "add";
|
|
op.clientId = c;
|
|
op.latencyUs = latUs;
|
|
op.success = static_cast<bool>(res);
|
|
if (!res)
|
|
op.errorMsg = res.error().message;
|
|
|
|
result.latencies.push_back(op);
|
|
|
|
if (res) {
|
|
result.docsIngested++;
|
|
totalDocsIngested.fetch_add(1);
|
|
} else {
|
|
result.failures++;
|
|
totalFailures.fetch_add(1);
|
|
}
|
|
}
|
|
|
|
auto clientEnd = std::chrono::steady_clock::now();
|
|
result.elapsedSeconds = std::chrono::duration<double>(clientEnd - clientStart).count();
|
|
result.addThroughputDocsPerSec =
|
|
(result.docsIngested > 0)
|
|
? static_cast<double>(result.docsIngested) / result.elapsedSeconds
|
|
: 0.0;
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
sampler.stop();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
double aggregateThroughput = static_cast<double>(totalDocsIngested.load()) / globalElapsed;
|
|
|
|
// Wait for pipeline drain and then probe idle->work transition once.
|
|
const auto drainObservation =
|
|
waitForDrainObservation(monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs));
|
|
const bool drained = drainObservation.drained;
|
|
const auto idleProbe = runIdleWakeProbe(cfg, harness.socketPath(), monitorClient);
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
const auto recovery = runRecoveryCheck(harness.socketPath());
|
|
|
|
// Compute aggregate latency stats
|
|
std::vector<int64_t> allAddLatencies;
|
|
for (auto& cr : clientResults) {
|
|
for (auto& op : cr.latencies) {
|
|
if (op.opType == "add" && op.success)
|
|
allAddLatencies.push_back(op.latencyUs);
|
|
}
|
|
}
|
|
auto aggStats = PercentileStats::compute(allAddLatencies);
|
|
|
|
std::map<std::string, int> failureBreakdown;
|
|
std::map<std::string, std::string> failureSamples;
|
|
for (const auto& cr : clientResults) {
|
|
for (const auto& op : cr.latencies) {
|
|
if (op.opType != "add" || op.success) {
|
|
continue;
|
|
}
|
|
const auto bucket = classifyFailureMessage(op.errorMsg);
|
|
failureBreakdown[bucket] += 1;
|
|
if (!op.errorMsg.empty() && !failureSamples.contains(bucket)) {
|
|
failureSamples[bucket] = op.errorMsg;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Print summary
|
|
std::cout << "\n=== Concurrent Pure Ingest (N=" << cfg.numClients << ") ===\n";
|
|
const int expectedDocs = checkedMulToInt(cfg.numClients, cfg.docsPerClient);
|
|
std::cout << " Total docs: " << totalDocsIngested.load() << "/" << expectedDocs << "\n";
|
|
std::cout << " Total failures: " << totalFailures.load() << "\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
std::cout << " Aggregate throughput: " << aggregateThroughput << " docs/s\n";
|
|
printPercentiles("Add latency (all)", aggStats);
|
|
|
|
if (!failureBreakdown.empty()) {
|
|
std::cout << "\n Failure breakdown (add requests):\n";
|
|
for (const auto& [bucket, count] : failureBreakdown) {
|
|
std::cout << " " << bucket << ": " << count;
|
|
auto sampleIt = failureSamples.find(bucket);
|
|
if (sampleIt != failureSamples.end()) {
|
|
std::cout << " (sample: " << sampleIt->second << ")";
|
|
}
|
|
std::cout << "\n";
|
|
}
|
|
}
|
|
|
|
std::cout << "\n Per-client breakdown:\n";
|
|
for (auto& cr : clientResults) {
|
|
std::cout << " Client " << cr.clientId << ": " << cr.docsIngested << " docs, "
|
|
<< std::fixed << std::setprecision(1) << cr.addThroughputDocsPerSec
|
|
<< " docs/s, failures=" << cr.failures << "\n";
|
|
}
|
|
|
|
// Fairness metric: coefficient of variation of per-client throughput
|
|
{
|
|
std::vector<double> perClientThroughput;
|
|
for (auto& cr : clientResults) {
|
|
if (cr.addThroughputDocsPerSec > 0)
|
|
perClientThroughput.push_back(cr.addThroughputDocsPerSec);
|
|
}
|
|
if (perClientThroughput.size() > 1) {
|
|
double mean =
|
|
std::accumulate(perClientThroughput.begin(), perClientThroughput.end(), 0.0) /
|
|
static_cast<double>(perClientThroughput.size());
|
|
double variance = 0.0;
|
|
for (auto v : perClientThroughput)
|
|
variance += (v - mean) * (v - mean);
|
|
variance /= static_cast<double>(perClientThroughput.size());
|
|
double cv = (mean > 0) ? std::sqrt(variance) / mean : 0.0;
|
|
std::cout << " Fairness (CV of throughput): " << std::fixed << std::setprecision(3)
|
|
<< cv << " (lower = more fair)\n";
|
|
}
|
|
}
|
|
|
|
std::cout << " Queue drained: " << (drained ? "yes" : "NO") << "\n";
|
|
printDrainAndIdleProbeSummary(drainObservation, idleProbe);
|
|
printRecoverySummary(recovery);
|
|
std::cout << " Final docs_total: " << finalSnap.documentsTotal << "\n";
|
|
std::cout << " Final memory: " << finalSnap.memoryUsageMb << " MB\n\n";
|
|
|
|
CHECK(totalDocsIngested.load() == expectedDocs);
|
|
CHECK(drained);
|
|
CHECK(recovery.ok());
|
|
|
|
// Emit JSONL
|
|
json perClient = json::array();
|
|
for (auto& cr : clientResults) {
|
|
perClient.push_back(json{
|
|
{"client_id", cr.clientId},
|
|
{"docs_ingested", cr.docsIngested},
|
|
{"throughput_docs_per_sec", cr.addThroughputDocsPerSec},
|
|
{"elapsed_seconds", cr.elapsedSeconds},
|
|
{"failures", cr.failures},
|
|
});
|
|
}
|
|
|
|
// Time-series data
|
|
json timeSeries = json::array();
|
|
for (auto& s : sampler.getSamples()) {
|
|
timeSeries.push_back(json{{"elapsed_ms", s.elapsedMs}, {"snapshot", s.snap.toJson()}});
|
|
}
|
|
|
|
json failureBreakdownJson = json::object();
|
|
for (const auto& [bucket, count] : failureBreakdown) {
|
|
failureBreakdownJson[bucket] = count;
|
|
}
|
|
|
|
json failureSamplesJson = json::object();
|
|
for (const auto& [bucket, sample] : failureSamples) {
|
|
failureSamplesJson[bucket] = sample;
|
|
}
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "concurrent_pure_ingest"},
|
|
{"num_clients", cfg.numClients},
|
|
{"docs_per_client", cfg.docsPerClient},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"total_docs_ingested", totalDocsIngested.load()},
|
|
{"total_failures", totalFailures.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"aggregate_throughput_docs_per_sec", aggregateThroughput},
|
|
{"add_latency", aggStats.toJson()},
|
|
{"failure_breakdown", failureBreakdownJson},
|
|
{"failure_samples", failureSamplesJson},
|
|
{"per_client", perClient},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"drain_metrics", drainObservation.toJson()},
|
|
{"idle_probe", idleProbe.toJson()},
|
|
{"recovery", recovery.toJson()},
|
|
{"time_series", timeSeries},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
{"drained", drained},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: mixed read/write workload",
|
|
"[!benchmark][multi-client][mixed]") {
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
DaemonHarness harness(benchHarnessOptions(cfg));
|
|
REQUIRE(harness.start(benchStartTimeout(cfg)));
|
|
|
|
// Monitor client
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
// Pre-seed corpus so search queries return results
|
|
if (cfg.warmupDocs > 0) {
|
|
ClientConfig warmCfg;
|
|
warmCfg.socketPath = harness.socketPath();
|
|
warmCfg.autoStart = false;
|
|
warmCfg.requestTimeout = 30s;
|
|
warmCfg.singleUseConnections = true;
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
std::cout << " Pre-seeding " << cfg.warmupDocs << " warmup docs...\n";
|
|
for (int i = 0; i < cfg.warmupDocs; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "warmup_doc_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(99, i, cfg.docSizeBytes);
|
|
req.tags = {"warmup"};
|
|
auto res = yams::cli::run_sync(warmClient.streamingAddDocument(req), 30s);
|
|
if (!res) {
|
|
WARN("Warmup doc " << i << " failed: " << res.error().message);
|
|
}
|
|
}
|
|
// Wait for warmup to settle
|
|
(void)waitForDrain(warmClient, 30s);
|
|
std::this_thread::sleep_for(1s);
|
|
}
|
|
|
|
constexpr std::string_view kHydrationGraphSymbol = "benchHydratedSymbol";
|
|
if (cfg.profileHydrationSurfaces) {
|
|
ClientConfig setupCfg;
|
|
setupCfg.socketPath = harness.socketPath();
|
|
setupCfg.autoStart = false;
|
|
setupCfg.requestTimeout = 30s;
|
|
setupCfg.singleUseConnections = true;
|
|
DaemonClient setupClient(setupCfg);
|
|
|
|
const auto sourcePath = harness.dataDir() / "bench_hydration_source.cpp";
|
|
const std::string source = "int benchHydratedSymbol() {\n return 42;\n}\n";
|
|
{
|
|
std::ofstream out(sourcePath);
|
|
REQUIRE(out.good());
|
|
out << source;
|
|
}
|
|
|
|
AddDocumentRequest addReq;
|
|
addReq.name = sourcePath.string();
|
|
addReq.content = source;
|
|
addReq.tags = {"bench", "hydration"};
|
|
auto addResult = yams::cli::run_sync(setupClient.streamingAddDocument(addReq), 30s);
|
|
REQUIRE(addResult);
|
|
|
|
(void)waitForDrain(setupClient, 30s);
|
|
|
|
auto* serviceManager = harness.daemon()->getServiceManager();
|
|
REQUIRE(serviceManager != nullptr);
|
|
auto metadataRepo = serviceManager->getMetadataRepo();
|
|
REQUIRE(metadataRepo != nullptr);
|
|
auto kgStore = metadataRepo->getKnowledgeGraphStore();
|
|
REQUIRE(kgStore != nullptr);
|
|
yams::metadata::KGNode node;
|
|
node.nodeKey = "function:bench::benchHydratedSymbol@" + sourcePath.string();
|
|
node.label = std::string(kHydrationGraphSymbol);
|
|
node.type = "function";
|
|
node.properties = nlohmann::json{{"file_path", sourcePath.string()},
|
|
{"qualified_name", "bench::benchHydratedSymbol"},
|
|
{"start_line", 1},
|
|
{"end_line", 3}}
|
|
.dump();
|
|
REQUIRE(kgStore->upsertNode(node));
|
|
}
|
|
|
|
const auto resourceBaseline = DaemonSnapshot::capture(monitorClient, harness.dataDir());
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, std::chrono::milliseconds(cfg.resourceSampleMs),
|
|
harness.dataDir());
|
|
|
|
std::atomic<int> totalAdds{0};
|
|
std::atomic<int> totalSearches{0};
|
|
std::atomic<int> totalLists{0};
|
|
std::atomic<int> totalFailures{0};
|
|
OutputLivenessCounters searchOutputLiveness;
|
|
OutputLivenessCounters grepOutputLiveness;
|
|
OutputLivenessCounters graphOutputLiveness;
|
|
std::vector<ClientResult> clientResults(cfg.numClients);
|
|
std::mutex hydrationSurfaceMutex;
|
|
std::vector<int64_t> grepContentRetrievalMs;
|
|
std::vector<int64_t> grepMatchRenderUs;
|
|
std::vector<int64_t> grepWorkerCriticalTotalMs;
|
|
std::vector<int64_t> graphSnippetRenderUs;
|
|
std::vector<int64_t> graphSnippetsRendered;
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(cfg.numClients + (cfg.profileHydrationSurfaces ? 1 : 0));
|
|
|
|
for (int c = 0; c < cfg.numClients; ++c) {
|
|
threads.emplace_back([&, c]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
ccfg.singleUseConnections = true;
|
|
DaemonClient client(ccfg);
|
|
|
|
ClientResult& result = clientResults[c];
|
|
result.clientId = c;
|
|
|
|
std::mt19937 rng(static_cast<unsigned>(c * 1000 + 42));
|
|
std::uniform_real_distribution<double> opDist(0.0, 1.0);
|
|
|
|
auto clientStart = std::chrono::steady_clock::now();
|
|
int docIdx = 0;
|
|
|
|
// Total operations = docsPerClient ingestions + proportional reads
|
|
int totalOps = cfg.mixedOpsPerClient;
|
|
if (totalOps <= 0 && cfg.searchRatio >= 1.0) {
|
|
totalOps = cfg.docsPerClient;
|
|
} else if (totalOps <= 0 && cfg.searchRatio > 0.0) {
|
|
totalOps = static_cast<int>(static_cast<double>(cfg.docsPerClient) /
|
|
(1.0 - cfg.searchRatio));
|
|
} else if (totalOps <= 0) {
|
|
totalOps = cfg.docsPerClient;
|
|
}
|
|
|
|
for (int op = 0; op < totalOps; ++op) {
|
|
double roll = opDist(rng);
|
|
bool doSearch = (roll < cfg.searchRatio) && (docIdx > 0 || cfg.warmupDocs > 0);
|
|
bool doList = (roll >= cfg.searchRatio && roll < cfg.searchRatio + 0.05);
|
|
|
|
if (doSearch) {
|
|
// Search operation
|
|
SearchRequest req;
|
|
req.query = cfg.searchQuery;
|
|
req.searchType = cfg.searchType;
|
|
req.limit = cfg.searchLimit;
|
|
req.pathsOnly = cfg.searchPathsOnly;
|
|
req.snippetHydrationTimeoutMs = cfg.snippetHydrationTimeoutMs;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.search(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
OpLatency lat;
|
|
lat.opType = "search";
|
|
lat.clientId = c;
|
|
lat.latencyUs = latUs;
|
|
lat.success = static_cast<bool>(res);
|
|
if (!res)
|
|
lat.errorMsg = res.error().message;
|
|
result.latencies.push_back(lat);
|
|
|
|
if (res) {
|
|
result.searchesExecuted++;
|
|
totalSearches.fetch_add(1);
|
|
searchOutputLiveness.recordSuccess(!res.value().results.empty());
|
|
collectSearchHydrationTelemetry(res.value(), result);
|
|
} else {
|
|
searchOutputLiveness.recordError();
|
|
result.failures++;
|
|
totalFailures.fetch_add(1);
|
|
}
|
|
} else if (doList) {
|
|
// List operation
|
|
ListRequest req;
|
|
req.limit = 20;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.list(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
OpLatency lat;
|
|
lat.opType = "list";
|
|
lat.clientId = c;
|
|
lat.latencyUs = latUs;
|
|
lat.success = static_cast<bool>(res);
|
|
if (!res)
|
|
lat.errorMsg = res.error().message;
|
|
result.latencies.push_back(lat);
|
|
|
|
if (res) {
|
|
result.listsExecuted++;
|
|
totalLists.fetch_add(1);
|
|
} else {
|
|
result.failures++;
|
|
totalFailures.fetch_add(1);
|
|
}
|
|
} else {
|
|
// Add operation
|
|
if (docIdx >= cfg.docsPerClient)
|
|
continue; // Already ingested all docs for this client
|
|
|
|
auto doc = generateDocument(c, docIdx, cfg.docSizeBytes);
|
|
AddDocumentRequest req;
|
|
req.name =
|
|
"mixed_c" + std::to_string(c) + "_d" + std::to_string(docIdx) + ".md";
|
|
req.content = std::move(doc);
|
|
req.tags = {"bench", "mixed", "client-" + std::to_string(c)};
|
|
req.collection = "bench_mixed_" + std::to_string(c);
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
OpLatency lat;
|
|
lat.opType = "add";
|
|
lat.clientId = c;
|
|
lat.latencyUs = latUs;
|
|
lat.success = static_cast<bool>(res);
|
|
if (!res)
|
|
lat.errorMsg = res.error().message;
|
|
result.latencies.push_back(lat);
|
|
|
|
if (res) {
|
|
result.docsIngested++;
|
|
totalAdds.fetch_add(1);
|
|
} else {
|
|
result.failures++;
|
|
totalFailures.fetch_add(1);
|
|
}
|
|
docIdx++;
|
|
}
|
|
}
|
|
|
|
auto clientEnd = std::chrono::steady_clock::now();
|
|
result.elapsedSeconds = std::chrono::duration<double>(clientEnd - clientStart).count();
|
|
result.addThroughputDocsPerSec =
|
|
(result.docsIngested > 0)
|
|
? static_cast<double>(result.docsIngested) / result.elapsedSeconds
|
|
: 0.0;
|
|
});
|
|
}
|
|
|
|
if (cfg.profileHydrationSurfaces) {
|
|
threads.emplace_back([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
ccfg.singleUseConnections = true;
|
|
DaemonClient client(ccfg);
|
|
const int probeOps = std::clamp(cfg.mixedOpsPerClient / 4, 4, 25);
|
|
const auto recordGrepRender = [&](const yams::Result<GrepResponse>& result) {
|
|
if (!result) {
|
|
grepOutputLiveness.recordError();
|
|
return;
|
|
}
|
|
grepOutputLiveness.recordSuccess(!result.value().matches.empty());
|
|
std::lock_guard<std::mutex> lock(hydrationSurfaceMutex);
|
|
if (const auto value =
|
|
metricInt(result.value().searchStats, "content_retrieval_ms")) {
|
|
grepContentRetrievalMs.push_back(*value);
|
|
}
|
|
if (const auto value = metricInt(result.value().searchStats, "match_render_us")) {
|
|
grepMatchRenderUs.push_back(*value);
|
|
}
|
|
if (const auto value =
|
|
metricInt(result.value().searchStats, "worker_critical_total_ms")) {
|
|
grepWorkerCriticalTotalMs.push_back(*value);
|
|
}
|
|
};
|
|
const auto recordGraphRender = [&](const yams::Result<GraphExploreResponse>& result) {
|
|
if (!result) {
|
|
graphOutputLiveness.recordError();
|
|
return;
|
|
}
|
|
const auto& response = result.value();
|
|
graphOutputLiveness.recordSuccess(!response.entrySymbols.empty() ||
|
|
!response.files.empty() ||
|
|
!response.relationships.empty());
|
|
std::lock_guard<std::mutex> lock(hydrationSurfaceMutex);
|
|
graphSnippetRenderUs.push_back(static_cast<int64_t>(response.snippetRenderMicros));
|
|
graphSnippetsRendered.push_back(static_cast<int64_t>(response.snippetsRendered));
|
|
};
|
|
|
|
for (int i = 0; i < probeOps; ++i) {
|
|
GrepRequest grepReq;
|
|
grepReq.pattern = std::string(kHydrationGraphSymbol);
|
|
grepReq.literalText = true;
|
|
grepReq.maxMatches = 20;
|
|
auto grepResult = yams::cli::run_sync(client.grep(grepReq), 30s);
|
|
recordGrepRender(grepResult);
|
|
|
|
GraphExploreRequest graphReq;
|
|
graphReq.query = std::string(kHydrationGraphSymbol);
|
|
graphReq.maxFiles = 4;
|
|
auto graphResult = yams::cli::run_sync(client.call(graphReq), 30s);
|
|
recordGraphRender(graphResult);
|
|
}
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
const uint64_t minimumDocuments =
|
|
resourceBaseline.documentsTotal + static_cast<uint64_t>(totalAdds.load());
|
|
const auto drainObservation = waitForDrainObservation(
|
|
monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs), 500ms, minimumDocuments);
|
|
sampler.stop();
|
|
const auto resourcePeaks = summarizeResourcePeaks(sampler.getSamples());
|
|
const bool drained = drainObservation.drained;
|
|
const auto idleProbe = runIdleWakeProbe(cfg, harness.socketPath(), monitorClient);
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient, harness.dataDir());
|
|
const auto recovery = runRecoveryCheck(harness.socketPath());
|
|
|
|
// Compute per-op-type latency stats
|
|
std::vector<int64_t> addLats, searchLats, listLats;
|
|
std::vector<int64_t> snippetHydrationUs, snippetLookupUs, snippetContentFetchUs,
|
|
snippetRenderUs;
|
|
int64_t snippetTimeoutHits = 0;
|
|
int64_t snippetHydrationCandidates = 0;
|
|
int64_t snippetHydrationMissing = 0;
|
|
int64_t snippetHydratedResults = 0;
|
|
for (auto& cr : clientResults) {
|
|
for (auto& op : cr.latencies) {
|
|
if (!op.success)
|
|
continue;
|
|
if (op.opType == "add")
|
|
addLats.push_back(op.latencyUs);
|
|
else if (op.opType == "search")
|
|
searchLats.push_back(op.latencyUs);
|
|
else if (op.opType == "list")
|
|
listLats.push_back(op.latencyUs);
|
|
}
|
|
snippetHydrationUs.insert(snippetHydrationUs.end(), cr.snippetHydrationUs.begin(),
|
|
cr.snippetHydrationUs.end());
|
|
snippetLookupUs.insert(snippetLookupUs.end(), cr.snippetLookupUs.begin(),
|
|
cr.snippetLookupUs.end());
|
|
snippetContentFetchUs.insert(snippetContentFetchUs.end(), cr.snippetContentFetchUs.begin(),
|
|
cr.snippetContentFetchUs.end());
|
|
snippetRenderUs.insert(snippetRenderUs.end(), cr.snippetRenderUs.begin(),
|
|
cr.snippetRenderUs.end());
|
|
snippetTimeoutHits = checkedMetricAdd(snippetTimeoutHits, cr.snippetTimeoutHits);
|
|
snippetHydrationCandidates =
|
|
checkedMetricAdd(snippetHydrationCandidates, cr.snippetHydrationCandidates);
|
|
snippetHydrationMissing =
|
|
checkedMetricAdd(snippetHydrationMissing, cr.snippetHydrationMissing);
|
|
snippetHydratedResults =
|
|
checkedMetricAdd(snippetHydratedResults, cr.snippetHydratedResults);
|
|
}
|
|
auto addStats = PercentileStats::compute(addLats);
|
|
auto searchStats = PercentileStats::compute(searchLats);
|
|
auto listStats = PercentileStats::compute(listLats);
|
|
auto snippetHydrationStats = PercentileStats::compute(snippetHydrationUs);
|
|
auto snippetLookupStats = PercentileStats::compute(snippetLookupUs);
|
|
auto snippetContentFetchStats = PercentileStats::compute(snippetContentFetchUs);
|
|
auto snippetRenderStats = PercentileStats::compute(snippetRenderUs);
|
|
auto grepContentRetrievalStats = PercentileStats::compute(grepContentRetrievalMs);
|
|
auto grepMatchRenderStats = PercentileStats::compute(grepMatchRenderUs);
|
|
auto grepWorkerCriticalTotalStats = PercentileStats::compute(grepWorkerCriticalTotalMs);
|
|
auto graphSnippetRenderStats = PercentileStats::compute(graphSnippetRenderUs);
|
|
const auto graphSnippetsRenderedTotal =
|
|
std::accumulate(graphSnippetsRendered.begin(), graphSnippetsRendered.end(), int64_t{0});
|
|
|
|
std::cout << "\n=== Mixed Read/Write Workload (N=" << cfg.numClients
|
|
<< ", search_ratio=" << cfg.searchRatio << ") ===\n";
|
|
std::cout << " Adds: " << totalAdds.load() << ", Searches: " << totalSearches.load()
|
|
<< ", Lists: " << totalLists.load() << ", Failures: " << totalFailures.load() << "\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
printPercentiles("Add latency", addStats);
|
|
printPercentiles("Search latency", searchStats);
|
|
printPercentiles("List latency", listStats);
|
|
std::cout << " Queue drained: " << (drained ? "yes" : "NO") << "\n";
|
|
printDrainAndIdleProbeSummary(drainObservation, idleProbe);
|
|
printRecoverySummary(recovery);
|
|
std::cout << " Final docs_total: " << finalSnap.documentsTotal << "\n\n";
|
|
|
|
if (cfg.searchRatio < 1.0) {
|
|
CHECK(totalAdds.load() > 0);
|
|
}
|
|
CHECK((totalAdds.load() + totalSearches.load() + totalLists.load()) > 0);
|
|
if (cfg.profileHydrationSurfaces) {
|
|
CHECK_FALSE(grepMatchRenderUs.empty());
|
|
CHECK_FALSE(graphSnippetRenderUs.empty());
|
|
CHECK(grepOutputLiveness.attempts.load(std::memory_order_relaxed) > 0);
|
|
CHECK(graphOutputLiveness.attempts.load(std::memory_order_relaxed) > 0);
|
|
}
|
|
CHECK(drained);
|
|
CHECK(recovery.ok());
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "mixed_read_write"},
|
|
{"num_clients", cfg.numClients},
|
|
{"docs_per_client", cfg.docsPerClient},
|
|
{"mixed_ops_per_client", cfg.mixedOpsPerClient},
|
|
{"search_ratio", cfg.searchRatio},
|
|
{"search_paths_only", cfg.searchPathsOnly},
|
|
{"snippet_hydration_timeout_ms", cfg.snippetHydrationTimeoutMs},
|
|
{"warmup_docs", cfg.warmupDocs},
|
|
{"total_adds", totalAdds.load()},
|
|
{"total_searches", totalSearches.load()},
|
|
{"total_lists", totalLists.load()},
|
|
{"total_failures", totalFailures.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"add_latency", addStats.toJson()},
|
|
{"search_latency", searchStats.toJson()},
|
|
{"list_latency", listStats.toJson()},
|
|
{"search_server",
|
|
{{"timing_snippet_hydration_us", snippetHydrationStats.toJson()},
|
|
{"timing_snippet_lookup_us", snippetLookupStats.toJson()},
|
|
{"timing_snippet_content_fetch_us", snippetContentFetchStats.toJson()},
|
|
{"timing_snippet_render_us", snippetRenderStats.toJson()},
|
|
{"snippet_timeout_hits", snippetTimeoutHits},
|
|
{"snippet_hydration_candidates", snippetHydrationCandidates},
|
|
{"snippet_hydration_missing", snippetHydrationMissing},
|
|
{"snippet_hydrated_results", snippetHydratedResults}}},
|
|
{"grep_server",
|
|
{{"content_retrieval_ms", grepContentRetrievalStats.toJson()},
|
|
{"match_render_us", grepMatchRenderStats.toJson()},
|
|
{"worker_critical_total_ms", grepWorkerCriticalTotalStats.toJson()}}},
|
|
{"graph_server",
|
|
{{"snippet_render_us", graphSnippetRenderStats.toJson()},
|
|
{"snippets_rendered", graphSnippetsRenderedTotal}}},
|
|
{"output_liveness",
|
|
{{"search", searchOutputLiveness.toJson()},
|
|
{"grep", grepOutputLiveness.toJson()},
|
|
{"graph", graphOutputLiveness.toJson()}}},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"drain_metrics", drainObservation.toJson()},
|
|
{"idle_probe", idleProbe.toJson()},
|
|
{"recovery", recovery.toJson()},
|
|
{"resource_baseline", resourceBaseline.toJson()},
|
|
{"resource_peaks", resourcePeaks.toJson()},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
{"drained", drained},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: socket scaling vs ux latency",
|
|
"[!benchmark][multi-client][ux-scaling]") {
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
// Three phases:
|
|
// 1. Warmup: seed a small searchable corpus.
|
|
// 2. Realistic mixed load: measure ingest + UX latency under pressure.
|
|
// 3. Deterministic control: force slot growth, then force shrink, and assert only on
|
|
// sampled slot metrics rather than transient active-connection counts.
|
|
constexpr size_t kInitialSlotLimit = 12;
|
|
constexpr size_t kSlotMax = 16;
|
|
constexpr size_t kSlotStep = 4;
|
|
ScopedConnectionSlotOverrides slotOverrides(static_cast<uint32_t>(kInitialSlotLimit),
|
|
static_cast<uint32_t>(kSlotMax),
|
|
static_cast<uint32_t>(kSlotStep));
|
|
yams::test::ScopedEnvVar maxActiveConn("YAMS_MAX_ACTIVE_CONN",
|
|
std::to_string(kInitialSlotLimit));
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(startHarnessWithRetry(harness, kStartTimeout));
|
|
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
// Pre-seed a small searchable corpus so UX requests hit real work.
|
|
{
|
|
ClientConfig warmCfg;
|
|
warmCfg.socketPath = harness.socketPath();
|
|
warmCfg.autoStart = false;
|
|
warmCfg.requestTimeout = 30s;
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
const int warmDocs = std::max(16, cfg.warmupDocs);
|
|
for (int i = 0; i < warmDocs; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "ux_scale_warmup_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(77, i, cfg.docSizeBytes);
|
|
req.tags = {"warmup", "ux-scaling"};
|
|
auto res = yams::cli::run_sync(warmClient.streamingAddDocument(req), 30s);
|
|
REQUIRE(res);
|
|
}
|
|
REQUIRE(waitForDrain(warmClient, 30s));
|
|
}
|
|
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, 100ms);
|
|
|
|
const int writerClients = std::max(2, cfg.numClients);
|
|
const int uxClients = std::max(4, checkedMulToInt(cfg.numClients, 3, kMaxBenchClients * 3));
|
|
const int writerOpsPerClient =
|
|
checkedMulToInt(cfg.docsPerClient, cfg.stressRounds, kMaxBenchOpsPerClient);
|
|
const int uxOpsPerClient =
|
|
checkedMulToInt(std::max(24, cfg.docsPerClient), cfg.stressRounds, kMaxBenchOpsPerClient);
|
|
const int cliProbeOpsPerCommand = cfg.cliProbeOpsPerCommand;
|
|
const size_t grownSlotTarget = std::min(kSlotMax, kInitialSlotLimit + kSlotStep);
|
|
// Allow one lingering request/observer connection beyond the steady monitor connection.
|
|
const size_t controlLightLoadTarget =
|
|
std::max<size_t>(3, std::min<size_t>(4, static_cast<size_t>(cfg.stressRounds + 2)));
|
|
const auto yamsBinary = findYamsBinary(cfg);
|
|
const bool cliStressProbesEnabled =
|
|
yamsBinary.path.has_value() && yamsBinary.usableForStress && cliProbeOpsPerCommand > 0;
|
|
|
|
std::atomic<int> addOps{0};
|
|
std::atomic<int> addFails{0};
|
|
std::atomic<int> uxOps{0};
|
|
std::atomic<int> uxFails{0};
|
|
std::atomic<int> retryAfterCount{0};
|
|
std::atomic<int> cliOps{0};
|
|
std::atomic<int> cliFails{0};
|
|
std::mutex latMutex;
|
|
std::vector<int64_t> addLatencies;
|
|
std::vector<int64_t> searchLatencies;
|
|
std::vector<int64_t> listLatencies;
|
|
std::vector<int64_t> statusLatencies;
|
|
std::vector<int64_t> cliListLatencies;
|
|
std::vector<int64_t> cliSearchLatencies;
|
|
std::vector<int64_t> cliStatusLatencies;
|
|
std::map<std::string, std::vector<int64_t>> cliStageLatencies;
|
|
std::vector<CliProbeResult> cliProbeResults;
|
|
std::vector<TimedLatencySample> hotspotSamples;
|
|
std::map<std::string, std::vector<int64_t>> opStageLatencies;
|
|
std::map<std::string, int> uxErrorCategories;
|
|
std::map<std::string, std::string> uxFailureSamples;
|
|
|
|
HeldSocketConnections heldConnections;
|
|
const bool heldConnectionsOpened =
|
|
heldConnections.open(harness.socketPath(), static_cast<int>(kInitialSlotLimit));
|
|
INFO("HeldSocketConnections.open initial ramp socket="
|
|
<< harness.socketPath().string() << " requested=" << kInitialSlotLimit << " opened="
|
|
<< heldConnections.openedCount() << " failed_index=" << heldConnections.lastFailedIndex()
|
|
<< " error='" << heldConnections.lastError() << "'");
|
|
REQUIRE(heldConnectionsOpened);
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(writerClients + uxClients);
|
|
|
|
for (int writer = 0; writer < writerClients; ++writer) {
|
|
threads.emplace_back([&, writer]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
std::vector<int64_t> localAdds;
|
|
for (int i = 0; i < writerOpsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name =
|
|
"ux_scale_writer_" + std::to_string(writer) + "_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(writer, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "ux-scaling", "writer-" + std::to_string(writer)};
|
|
req.collection = "ux_scaling_writer_" + std::to_string(writer);
|
|
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
const auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
addOps.fetch_add(1);
|
|
localAdds.push_back(latUs);
|
|
} else {
|
|
addFails.fetch_add(1);
|
|
}
|
|
|
|
{
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
hotspotSamples.push_back(TimedLatencySample{
|
|
.opType = "add",
|
|
.wallClockMs = static_cast<uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count()),
|
|
.latencyUs = latUs,
|
|
.success = static_cast<bool>(res),
|
|
});
|
|
}
|
|
}
|
|
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
addLatencies.insert(addLatencies.end(), localAdds.begin(), localAdds.end());
|
|
});
|
|
}
|
|
|
|
for (int ux = 0; ux < uxClients; ++ux) {
|
|
threads.emplace_back([&]() {
|
|
std::vector<int64_t> localSearch;
|
|
std::vector<int64_t> localList;
|
|
std::vector<int64_t> localStatus;
|
|
std::map<std::string, int> localErrorCategories;
|
|
std::map<std::string, std::string> localFailureSamples;
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
|
|
for (int i = 0; i < uxOpsPerClient; ++i) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
bool success = false;
|
|
std::string errorMsg;
|
|
std::string opType;
|
|
|
|
switch (i % 3) {
|
|
case 0: {
|
|
opType = "status";
|
|
auto res = yams::cli::run_sync(client.status(), 10s);
|
|
success = static_cast<bool>(res);
|
|
if (!res) {
|
|
errorMsg = res.error().message;
|
|
}
|
|
if (res && res.value().retryAfterMs > 0) {
|
|
retryAfterCount.fetch_add(1);
|
|
}
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
const auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
if (success) {
|
|
localStatus.push_back(latUs);
|
|
}
|
|
{
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
if (res) {
|
|
appendStatusTimingStats(opStageLatencies, res.value().requestCounts,
|
|
latUs);
|
|
}
|
|
hotspotSamples.push_back(TimedLatencySample{
|
|
.opType = "status",
|
|
.wallClockMs = static_cast<uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
t1 - globalStart)
|
|
.count()),
|
|
.latencyUs = latUs,
|
|
.success = success,
|
|
});
|
|
}
|
|
break;
|
|
}
|
|
case 1: {
|
|
opType = "list";
|
|
ListRequest req;
|
|
req.limit = 10;
|
|
auto res = yams::cli::run_sync(client.list(req), 10s);
|
|
success = static_cast<bool>(res);
|
|
if (!res) {
|
|
errorMsg = res.error().message;
|
|
}
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
const auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
if (success) {
|
|
localList.push_back(latUs);
|
|
}
|
|
{
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
if (res) {
|
|
appendStringTimingStats(opStageLatencies, "list",
|
|
res.value().listStats, latUs);
|
|
}
|
|
hotspotSamples.push_back(TimedLatencySample{
|
|
.opType = "list",
|
|
.wallClockMs = static_cast<uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
t1 - globalStart)
|
|
.count()),
|
|
.latencyUs = latUs,
|
|
.success = success,
|
|
});
|
|
}
|
|
break;
|
|
}
|
|
default: {
|
|
opType = "search";
|
|
SearchRequest req;
|
|
req.query = cfg.searchQuery;
|
|
req.searchType = cfg.searchType;
|
|
req.limit = cfg.searchLimit;
|
|
auto res = yams::cli::run_sync(client.search(req), 10s);
|
|
success = static_cast<bool>(res);
|
|
if (!res) {
|
|
errorMsg = res.error().message;
|
|
}
|
|
const auto t1 = std::chrono::steady_clock::now();
|
|
if (success) {
|
|
localSearch.push_back(
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0)
|
|
.count());
|
|
}
|
|
{
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
hotspotSamples.push_back(TimedLatencySample{
|
|
.opType = "search",
|
|
.wallClockMs = static_cast<uint64_t>(
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
t1 - globalStart)
|
|
.count()),
|
|
.latencyUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0)
|
|
.count(),
|
|
.success = success,
|
|
});
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
uxOps.fetch_add(1);
|
|
if (!success) {
|
|
uxFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(errorMsg);
|
|
const auto key = opType + ":" + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!errorMsg.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = errorMsg;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
searchLatencies.insert(searchLatencies.end(), localSearch.begin(), localSearch.end());
|
|
listLatencies.insert(listLatencies.end(), localList.begin(), localList.end());
|
|
statusLatencies.insert(statusLatencies.end(), localStatus.begin(), localStatus.end());
|
|
for (const auto& [key, count] : localErrorCategories) {
|
|
uxErrorCategories[key] += count;
|
|
}
|
|
for (const auto& [key, sample] : localFailureSamples) {
|
|
if (!uxFailureSamples.contains(key)) {
|
|
uxFailureSamples[key] = sample;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
#ifndef _WIN32
|
|
if (cliStressProbesEnabled) {
|
|
threads.emplace_back([&, cliBin = yamsBinary.path.value()]() {
|
|
std::vector<int64_t> localCliList;
|
|
std::vector<int64_t> localCliSearch;
|
|
std::vector<int64_t> localCliStatus;
|
|
std::map<std::string, std::vector<int64_t>> localCliStageLatencies;
|
|
std::vector<CliProbeResult> localCliResults;
|
|
const auto cliProbeTimeout = std::chrono::seconds(cfg.cliProbeTimeoutSecs);
|
|
|
|
auto recordProbe = [&](const std::string& opType, const CliProbeResult& probe,
|
|
std::vector<int64_t>& storage) {
|
|
cliOps.fetch_add(1);
|
|
if (probe.success) {
|
|
storage.push_back(probe.latencyUs);
|
|
} else {
|
|
cliFails.fetch_add(1);
|
|
}
|
|
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
hotspotSamples.push_back(TimedLatencySample{
|
|
.opType = opType,
|
|
.wallClockMs =
|
|
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - globalStart)
|
|
.count()),
|
|
.latencyUs = probe.latencyUs,
|
|
.success = probe.success,
|
|
});
|
|
for (const auto& [stage, elapsedUs] : probe.stageElapsedUs) {
|
|
localCliStageLatencies[opType + ":" + stage].push_back(elapsedUs);
|
|
}
|
|
localCliResults.push_back(probe);
|
|
};
|
|
|
|
for (int i = 0; i < cliProbeOpsPerCommand; ++i) {
|
|
recordProbe("cli_list",
|
|
runObservedCliProbe("cli_list", cliBin, harness.socketPath(),
|
|
{"list", "--limit", "10"}, cliProbeTimeout),
|
|
localCliList);
|
|
recordProbe("cli_status",
|
|
runObservedCliProbe("cli_status", cliBin, harness.socketPath(),
|
|
{"status"}, cliProbeTimeout),
|
|
localCliStatus);
|
|
recordProbe("cli_search",
|
|
runObservedCliProbe("cli_search", cliBin, harness.socketPath(),
|
|
buildCliSearchArgs(cfg), cliProbeTimeout),
|
|
localCliSearch);
|
|
}
|
|
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
cliListLatencies.insert(cliListLatencies.end(), localCliList.begin(),
|
|
localCliList.end());
|
|
cliSearchLatencies.insert(cliSearchLatencies.end(), localCliSearch.begin(),
|
|
localCliSearch.end());
|
|
cliStatusLatencies.insert(cliStatusLatencies.end(), localCliStatus.begin(),
|
|
localCliStatus.end());
|
|
for (auto& [stage, values] : localCliStageLatencies) {
|
|
auto& dest = cliStageLatencies[stage];
|
|
dest.insert(dest.end(), values.begin(), values.end());
|
|
}
|
|
cliProbeResults.insert(cliProbeResults.end(), localCliResults.begin(),
|
|
localCliResults.end());
|
|
});
|
|
}
|
|
#endif
|
|
|
|
for (auto& thread : threads) {
|
|
thread.join();
|
|
}
|
|
|
|
sampler.stop();
|
|
heldConnections.closeAll();
|
|
|
|
std::this_thread::sleep_for(500ms);
|
|
|
|
const auto globalEnd = std::chrono::steady_clock::now();
|
|
const double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
const auto drainObservation =
|
|
waitForDrainObservation(monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs));
|
|
const bool drained = drainObservation.drained;
|
|
const auto idleProbe = runIdleWakeProbe(cfg, harness.socketPath(), monitorClient);
|
|
const bool recoveryQuiesced =
|
|
waitForActiveConnectionsAtMost(monitorClient, controlLightLoadTarget, 20s);
|
|
const auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
const auto recovery = runRecoveryCheck(harness.socketPath());
|
|
const bool cliRecoveryAttempted = yamsBinary.path.has_value() && yamsBinary.usableForStress;
|
|
const auto cliRecoveryStatus =
|
|
cliRecoveryAttempted ? runObservedCliProbe("cli_status_recovery", yamsBinary.path.value(),
|
|
harness.socketPath(), {"status"}, 10s)
|
|
: CliProbeResult{};
|
|
const auto cliRecoveryList =
|
|
cliRecoveryAttempted
|
|
? runObservedCliProbe("cli_list_recovery", yamsBinary.path.value(),
|
|
harness.socketPath(), {"list", "--limit", "10"}, 10s)
|
|
: CliProbeResult{};
|
|
const bool cliRecoveryOk =
|
|
!cliRecoveryAttempted || (cliRecoveryStatus.success && cliRecoveryList.success);
|
|
const auto samples = sampler.getSamples();
|
|
const auto slotSummary = summarizeSlotScaling(samples);
|
|
const auto resourcePeaks = summarizeResourcePeaks(samples);
|
|
const auto hotspotSummary = summarizeLatencyHotspots(hotspotSamples);
|
|
const auto opStageSummary = summarizeTimingStats(opStageLatencies);
|
|
const auto addStats = PercentileStats::compute(addLatencies);
|
|
const auto searchStats = PercentileStats::compute(searchLatencies);
|
|
const auto listStats = PercentileStats::compute(listLatencies);
|
|
const auto statusStats = PercentileStats::compute(statusLatencies);
|
|
const auto cliListStats = PercentileStats::compute(cliListLatencies);
|
|
const auto cliSearchStats = PercentileStats::compute(cliSearchLatencies);
|
|
const auto cliStatusStats = PercentileStats::compute(cliStatusLatencies);
|
|
json cliProbeResultsJson = json::array();
|
|
for (const auto& probe : cliProbeResults) {
|
|
cliProbeResultsJson.push_back(probe.toJson());
|
|
}
|
|
std::map<std::string, int> cliFailureBuckets;
|
|
std::map<std::string, int> cliFailureModes;
|
|
std::map<std::string, std::string> cliFailureSamples;
|
|
for (const auto& probe : cliProbeResults) {
|
|
if (probe.success) {
|
|
continue;
|
|
}
|
|
cliFailureBuckets[probe.failureBucket] += 1;
|
|
cliFailureModes[probe.failureMode] += 1;
|
|
if (!probe.error.empty() && !cliFailureSamples.contains(probe.failureMode)) {
|
|
cliFailureSamples[probe.failureMode] = probe.error;
|
|
}
|
|
}
|
|
int unacceptableCliFailures = 0;
|
|
for (const auto& [mode, count] : cliFailureModes) {
|
|
if (mode != "timeout_under_load" && mode != "overload_under_load" &&
|
|
mode != "stress_expected") {
|
|
unacceptableCliFailures += count;
|
|
}
|
|
}
|
|
const bool observedGrowth =
|
|
slotSummary.peakLimit >= grownSlotTarget || slotSummary.growEvents > 0;
|
|
const bool observedShrink =
|
|
slotSummary.finalLimit <= kInitialSlotLimit || slotSummary.shrinkEvents > 0;
|
|
|
|
const double addThroughput =
|
|
(globalElapsed > 0.0) ? static_cast<double>(addOps.load()) / globalElapsed : 0.0;
|
|
const double uxThroughput =
|
|
(globalElapsed > 0.0) ? static_cast<double>(uxOps.load()) / globalElapsed : 0.0;
|
|
const double uxFailureRate =
|
|
(uxOps.load() > 0) ? static_cast<double>(uxFails.load()) / static_cast<double>(uxOps.load())
|
|
: 0.0;
|
|
|
|
std::cout << "\n=== Socket Scaling vs UX Latency ===\n";
|
|
std::cout << " Adds: " << addOps.load() << " (fails=" << addFails.load() << ")"
|
|
<< " UX ops: " << uxOps.load() << " (fails=" << uxFails.load() << ")\n";
|
|
std::cout << " Throughput: add=" << std::fixed << std::setprecision(1) << addThroughput
|
|
<< " docs/s, ux=" << uxThroughput << " ops/s\n";
|
|
std::cout << " Search config: daemon(type=" << cfg.searchType << ", limit=" << cfg.searchLimit
|
|
<< ", query=\"" << cfg.searchQuery << "\")"
|
|
<< " cli(type=" << cfg.cliSearchType << ", limit=" << cfg.cliSearchLimit
|
|
<< ", query=\"" << cfg.cliSearchQuery << "\")\n";
|
|
if (cliStressProbesEnabled) {
|
|
std::cout << " CLI probes: " << cliOps.load() << " (fails=" << cliFails.load()
|
|
<< ") binary=" << yamsBinary.path.value().string() << "\n";
|
|
} else {
|
|
std::cout << " CLI probes: skipped";
|
|
if (!yamsBinary.note.empty()) {
|
|
std::cout << " (" << yamsBinary.note << ")";
|
|
}
|
|
std::cout << "\n";
|
|
}
|
|
if (!cliFailureModes.empty()) {
|
|
printCliProbeFailureSummary(cliFailureModes, cliFailureSamples);
|
|
}
|
|
if (!uxErrorCategories.empty()) {
|
|
std::cout << " UX failure categories:";
|
|
for (const auto& [key, count] : uxErrorCategories) {
|
|
std::cout << " " << key << "=" << count;
|
|
}
|
|
std::cout << "\n";
|
|
}
|
|
std::cout << " Slot scaling: initial=" << slotSummary.initialLimit
|
|
<< " peak=" << slotSummary.peakLimit << " final=" << slotSummary.finalLimit
|
|
<< " grow_events=" << slotSummary.growEvents
|
|
<< " shrink_events=" << slotSummary.shrinkEvents
|
|
<< " min_free=" << slotSummary.minFreeSlots
|
|
<< " peak_active=" << slotSummary.peakActiveConnections
|
|
<< " peak_util=" << std::setprecision(2) << (slotSummary.peakUtilization * 100.0)
|
|
<< "% retry_after_samples=" << slotSummary.retryAfterSamples
|
|
<< " observed_growth=" << (observedGrowth ? "yes" : "NO")
|
|
<< " observed_shrink=" << (observedShrink ? "yes" : "NO") << "\n";
|
|
printPercentiles("Add latency", addStats);
|
|
printPercentiles("Search latency", searchStats);
|
|
printPercentiles("List latency", listStats);
|
|
printPercentiles("Status latency", statusStats);
|
|
if (!opStageSummary.empty()) {
|
|
auto printStageP95 = [&](std::string_view key) {
|
|
auto it = opStageSummary.find(std::string(key));
|
|
if (it != opStageSummary.end() && it->is_object()) {
|
|
std::cout << " " << key << " p95=" << it->value("p95_us", 0)
|
|
<< "us mean=" << std::fixed << std::setprecision(1)
|
|
<< it->value("mean_us", 0.0) << "us\n";
|
|
}
|
|
};
|
|
std::cout << " Direct op overhead trace:\n";
|
|
printStageP95("list:phase_dispatch_total_us");
|
|
printStageP95("list:phase_dispatch_service_us");
|
|
printStageP95("list:phase_dispatch_map_us");
|
|
printStageP95("list:client_transport_overhead_us");
|
|
printStageP95("status:phase_dispatch_total_us");
|
|
printStageP95("status:client_transport_overhead_us");
|
|
if (finalSnap.addDispatchSamples > 0) {
|
|
const auto meanDispatch = static_cast<double>(finalSnap.addDispatchTotalUs) /
|
|
static_cast<double>(finalSnap.addDispatchSamples);
|
|
const auto meanFingerprint = static_cast<double>(finalSnap.addFingerprintTotalUs) /
|
|
static_cast<double>(finalSnap.addDispatchSamples);
|
|
const auto meanEnqueue = static_cast<double>(finalSnap.addEnqueueTotalUs) /
|
|
static_cast<double>(finalSnap.addDispatchSamples);
|
|
std::cout << " add:server_aggregate samples=" << finalSnap.addDispatchSamples
|
|
<< " mean_dispatch=" << std::fixed << std::setprecision(1) << meanDispatch
|
|
<< "us max_dispatch=" << finalSnap.addDispatchMaxUs
|
|
<< "us mean_fingerprint=" << meanFingerprint
|
|
<< "us max_fingerprint=" << finalSnap.addFingerprintMaxUs
|
|
<< "us mean_enqueue=" << meanEnqueue
|
|
<< "us max_enqueue=" << finalSnap.addEnqueueMaxUs << "us\n";
|
|
}
|
|
}
|
|
if (cliListStats.count + cliSearchStats.count + cliStatusStats.count > 0) {
|
|
printPercentiles("CLI list latency", cliListStats);
|
|
printPercentiles("CLI search latency", cliSearchStats);
|
|
printPercentiles("CLI status latency", cliStatusStats);
|
|
for (auto& [stage, values] : cliStageLatencies) {
|
|
auto stageStats = PercentileStats::compute(values);
|
|
printPercentiles("CLI stage " + stage, stageStats);
|
|
}
|
|
}
|
|
std::cout << " Resource peaks: memory=" << std::fixed << std::setprecision(1)
|
|
<< resourcePeaks.peakMemoryMb << " MB cpu=" << resourcePeaks.peakCpuPercent
|
|
<< "% rss=" << resourcePeaks.peakRssBytes
|
|
<< "B post_q=" << resourcePeaks.peakPostIngestQueued
|
|
<< " post_inflight=" << resourcePeaks.peakPostIngestInflight
|
|
<< " search_active=" << resourcePeaks.peakSearchActive
|
|
<< " search_q=" << resourcePeaks.peakSearchQueued
|
|
<< " db_write_wait=" << resourcePeaks.peakDbWritePoolWaiting
|
|
<< " db_read_wait=" << resourcePeaks.peakDbReadPoolWaiting
|
|
<< " write_apply_ms=" << resourcePeaks.peakWriteMaxBatchApplyMs
|
|
<< " write_wait_ms=" << resourcePeaks.peakWriteMaxBatchQueueWaitMs
|
|
<< " write_excess_wait_ms=" << resourcePeaks.peakWriteMaxBatchExcessQueueWaitMs
|
|
<< " fds=" << resourcePeaks.peakProcessFdCount
|
|
<< " (socket=" << resourcePeaks.peakProcessFdSockets
|
|
<< " pipe=" << resourcePeaks.peakProcessFdPipes
|
|
<< " file=" << resourcePeaks.peakProcessFdFiles
|
|
<< " dev=" << resourcePeaks.peakProcessFdDevices
|
|
<< " kqueue=" << resourcePeaks.peakProcessFdKqueues
|
|
<< " other=" << resourcePeaks.peakProcessFdOther << ")"
|
|
<< " post_content_index_calls=" << resourcePeaks.peakPostCommitContentIndexCalls
|
|
<< " post_content_index_total_ms=" << resourcePeaks.peakPostCommitContentIndexTotalMs
|
|
<< " post_content_index_max_ms=" << resourcePeaks.peakPostCommitContentIndexMaxMs
|
|
<< " client_pool_open=" << resourcePeaks.peakClientPoolOpenSockets
|
|
<< " client_pool_alive=" << resourcePeaks.peakClientPoolAliveConnections
|
|
<< " client_pool_in_use=" << resourcePeaks.peakClientPoolInUseConnections << "\n";
|
|
if (!resourcePeaks.lastWriteHotSources.empty()) {
|
|
std::cout << " Write hot sources: " << resourcePeaks.lastWriteHotSources << "\n";
|
|
}
|
|
if (hotspotSummary.contains("worst_window")) {
|
|
const auto& window = hotspotSummary["worst_window"];
|
|
std::cout << " Worst latency window: start=" << window.value("start_ms", 0)
|
|
<< "ms end=" << window.value("end_ms", 0)
|
|
<< "ms avg=" << window.value("avg_latency_us", 0.0)
|
|
<< "us samples=" << window.value("sample_count", 0) << "\n";
|
|
}
|
|
std::cout << " Queue drained: " << (drained ? "yes" : "NO")
|
|
<< " UX failure rate: " << std::setprecision(3) << uxFailureRate
|
|
<< " Final active/max: " << finalSnap.activeConnections << "/"
|
|
<< finalSnap.maxConnections << "\n\n";
|
|
printDrainAndIdleProbeSummary(drainObservation, idleProbe);
|
|
std::cout << " Recovery quiesced: " << (recoveryQuiesced ? "yes" : "NO") << "\n";
|
|
printRecoverySummary(recovery);
|
|
if (cliRecoveryAttempted) {
|
|
std::cout << " CLI post-stress recovery: status="
|
|
<< (cliRecoveryStatus.success ? "ok" : "FAIL")
|
|
<< " list=" << (cliRecoveryList.success ? "ok" : "FAIL") << "\n";
|
|
}
|
|
std::cout << "\n";
|
|
|
|
// Phase 3: deterministic control cycle for growth/shrink validation on a fresh daemon
|
|
// instance so realistic phase-2 idle sessions do not distort shrink behavior.
|
|
harness.stop();
|
|
DaemonHarness controlHarness(benchHarnessOptions(cfg));
|
|
REQUIRE(startHarnessWithRetry(controlHarness, kStartTimeout));
|
|
|
|
ClientConfig controlMonCfg;
|
|
controlMonCfg.socketPath = controlHarness.socketPath();
|
|
controlMonCfg.autoStart = false;
|
|
controlMonCfg.requestTimeout = 10s;
|
|
DaemonClient controlMonitorClient(controlMonCfg);
|
|
|
|
TimeSeriesSampler controlSampler;
|
|
controlSampler.start(controlMonitorClient, 100ms);
|
|
|
|
std::atomic<int> controlAddOps{0};
|
|
std::atomic<int> controlUxOps{0};
|
|
HeldSocketConnections controlHeldConnections;
|
|
const size_t controlHeldCount = kInitialSlotLimit - 2;
|
|
const bool controlHeldConnectionsOpened = controlHeldConnections.open(
|
|
controlHarness.socketPath(), static_cast<int>(controlHeldCount));
|
|
INFO("HeldSocketConnections.open control ramp socket="
|
|
<< controlHarness.socketPath().string() << " requested=" << controlHeldCount
|
|
<< " opened=" << controlHeldConnections.openedCount()
|
|
<< " failed_index=" << controlHeldConnections.lastFailedIndex() << " error='"
|
|
<< controlHeldConnections.lastError() << "'");
|
|
REQUIRE(controlHeldConnectionsOpened);
|
|
|
|
const int controlAddsPerWriter =
|
|
checkedMulToInt(std::max(8, checkedMulToInt(cfg.docsPerClient, 2, kMaxBenchOpsPerClient)),
|
|
cfg.stressRounds, kMaxBenchOpsPerClient);
|
|
const int controlUxOpsPerClient = checkedMulToInt(12, cfg.stressRounds, kMaxBenchOpsPerClient);
|
|
|
|
std::thread controlWriter([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = controlHarness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
for (int i = 0; i < controlAddsPerWriter; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "ux_scale_control_add_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(901, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "ux-scaling", "control"};
|
|
req.collection = "ux_scaling_control";
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
if (res) {
|
|
controlAddOps.fetch_add(1);
|
|
}
|
|
}
|
|
});
|
|
|
|
std::thread controlUx([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = controlHarness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
|
|
for (int i = 0; i < controlUxOpsPerClient; ++i) {
|
|
switch (i % 3) {
|
|
case 0: {
|
|
(void)yams::cli::run_sync(client.status(), 10s);
|
|
break;
|
|
}
|
|
case 1: {
|
|
ListRequest req;
|
|
req.limit = 10;
|
|
(void)yams::cli::run_sync(client.list(req), 10s);
|
|
break;
|
|
}
|
|
default: {
|
|
SearchRequest req;
|
|
req.query = cfg.searchQuery;
|
|
req.searchType = cfg.searchType;
|
|
req.limit = cfg.searchLimit;
|
|
(void)yams::cli::run_sync(client.search(req), 10s);
|
|
break;
|
|
}
|
|
}
|
|
controlUxOps.fetch_add(1);
|
|
}
|
|
});
|
|
|
|
controlWriter.join();
|
|
controlUx.join();
|
|
|
|
const bool controlGrew = waitForSlotLimit(controlMonitorClient, grownSlotTarget, 6s, true);
|
|
controlHeldConnections.closeAll();
|
|
const bool controlReachedLightLoad =
|
|
waitForActiveConnectionsAtMost(controlMonitorClient, controlLightLoadTarget, 12s);
|
|
const bool controlShrank =
|
|
waitForSlotLimit(controlMonitorClient, kInitialSlotLimit, 10s, false);
|
|
controlSampler.stop();
|
|
|
|
const auto controlSamples = controlSampler.getSamples();
|
|
const auto controlSlotSummary = summarizeSlotScaling(controlSamples);
|
|
const auto controlFinalSnap = DaemonSnapshot::capture(controlMonitorClient);
|
|
|
|
std::cout << "=== Deterministic Control Phase ===\n";
|
|
std::cout << " Adds: " << controlAddOps.load() << " UX ops: " << controlUxOps.load() << "\n";
|
|
std::cout << " Slot scaling: initial=" << controlSlotSummary.initialLimit
|
|
<< " peak=" << controlSlotSummary.peakLimit
|
|
<< " final=" << controlSlotSummary.finalLimit
|
|
<< " grow_events=" << controlSlotSummary.growEvents
|
|
<< " shrink_events=" << controlSlotSummary.shrinkEvents
|
|
<< " peak_active=" << controlSlotSummary.peakActiveConnections
|
|
<< " final_active=" << controlFinalSnap.activeConnections
|
|
<< " final_fds=" << controlFinalSnap.processFdCount
|
|
<< " grew=" << (controlGrew ? "yes" : "NO")
|
|
<< " light_load=" << (controlReachedLightLoad ? "yes" : "NO")
|
|
<< " shrank=" << (controlShrank ? "yes" : "NO") << "\n\n";
|
|
|
|
CHECK((addOps.load() + addFails.load()) > 0);
|
|
CHECK(uxOps.load() > 0);
|
|
CHECK((searchStats.count + listStats.count + statusStats.count) > 0);
|
|
CHECK(drained);
|
|
INFO("recoveryQuiesced=" << recoveryQuiesced);
|
|
CHECK(recovery.ok());
|
|
CHECK(cliRecoveryOk);
|
|
CHECK(unacceptableCliFailures == 0);
|
|
CHECK(controlAddOps.load() > 0);
|
|
CHECK(controlUxOps.load() > 0);
|
|
CHECK(controlGrew);
|
|
CHECK(controlSlotSummary.peakLimit >= grownSlotTarget);
|
|
CHECK(controlSlotSummary.growEvents > 0);
|
|
if (controlReachedLightLoad) {
|
|
CHECK(controlShrank);
|
|
CHECK(controlSlotSummary.finalLimit <= kInitialSlotLimit);
|
|
CHECK(controlSlotSummary.shrinkEvents > 0);
|
|
} else {
|
|
WARN("Skipping shrink assertions because control active connections did not reach "
|
|
"the light-load target; final_active="
|
|
<< controlFinalSnap.activeConnections << " target=" << controlLightLoadTarget
|
|
<< " final_limit=" << controlSlotSummary.finalLimit
|
|
<< " final_fd_count=" << controlFinalSnap.processFdCount);
|
|
}
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "socket_scaling_vs_ux_latency"},
|
|
{"num_writer_clients", writerClients},
|
|
{"num_ux_clients", uxClients},
|
|
{"docs_per_writer", writerOpsPerClient},
|
|
{"ux_ops_per_client", uxOpsPerClient},
|
|
{"stress_rounds", cfg.stressRounds},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"search_config", json{{"daemon", json{{"query", cfg.searchQuery},
|
|
{"type", cfg.searchType},
|
|
{"limit", cfg.searchLimit}}},
|
|
{"cli", json{{"query", cfg.cliSearchQuery},
|
|
{"type", cfg.cliSearchType},
|
|
{"limit", cfg.cliSearchLimit}}}}},
|
|
{"add", json{{"ops", addOps.load()},
|
|
{"fails", addFails.load()},
|
|
{"throughput_per_sec", addThroughput},
|
|
{"latency", addStats.toJson()}}},
|
|
{"search", json{{"latency", searchStats.toJson()}}},
|
|
{"list", json{{"latency", listStats.toJson()}}},
|
|
{"status", json{{"latency", statusStats.toJson()}}},
|
|
{"cli", json{{"enabled", cliStressProbesEnabled},
|
|
{"binary", yamsBinary.path ? yamsBinary.path.value().string() : std::string()},
|
|
{"selection", yamsBinary.toJson()},
|
|
{"ops", cliOps.load()},
|
|
{"fails", cliFails.load()},
|
|
{"ops_per_command", cliProbeOpsPerCommand},
|
|
{"failure_buckets", cliFailureBuckets},
|
|
{"failure_modes", cliFailureModes},
|
|
{"unacceptable_failures", unacceptableCliFailures},
|
|
{"failure_samples", cliFailureSamples},
|
|
{"probe_results", cliProbeResultsJson},
|
|
{"post_stress_recovery", json{{"attempted", cliRecoveryAttempted},
|
|
{"ok", cliRecoveryOk},
|
|
{"status_probe", cliRecoveryStatus.toJson()},
|
|
{"list_probe", cliRecoveryList.toJson()}}},
|
|
{"list_latency", cliListStats.toJson()},
|
|
{"search_latency", cliSearchStats.toJson()},
|
|
{"status_latency", cliStatusStats.toJson()}}},
|
|
{"ux", json{{"ops", uxOps.load()},
|
|
{"fails", uxFails.load()},
|
|
{"throughput_per_sec", uxThroughput},
|
|
{"failure_rate", uxFailureRate},
|
|
{"retry_after_count", retryAfterCount.load()},
|
|
{"error_categories", uxErrorCategories},
|
|
{"failure_samples", uxFailureSamples}}},
|
|
{"slot_scaling", json{{"initial_limit", slotSummary.initialLimit},
|
|
{"peak_limit", slotSummary.peakLimit},
|
|
{"final_limit", slotSummary.finalLimit},
|
|
{"observed_growth", observedGrowth},
|
|
{"observed_shrink", observedShrink},
|
|
{"growth_target", grownSlotTarget},
|
|
{"final_floor_target", kInitialSlotLimit},
|
|
{"min_free_slots", slotSummary.minFreeSlots},
|
|
{"peak_active_connections", slotSummary.peakActiveConnections},
|
|
{"peak_utilization", slotSummary.peakUtilization},
|
|
{"grow_events", slotSummary.growEvents},
|
|
{"shrink_events", slotSummary.shrinkEvents},
|
|
{"retry_after_samples", slotSummary.retryAfterSamples}}},
|
|
{"deterministic_control",
|
|
json{{"add_ops", controlAddOps.load()},
|
|
{"ux_ops", controlUxOps.load()},
|
|
{"grew_to_target", controlGrew},
|
|
{"reached_light_load", controlReachedLightLoad},
|
|
{"shrank_to_floor", controlShrank},
|
|
{"slot_scaling",
|
|
json{{"initial_limit", controlSlotSummary.initialLimit},
|
|
{"peak_limit", controlSlotSummary.peakLimit},
|
|
{"final_limit", controlSlotSummary.finalLimit},
|
|
{"min_free_slots", controlSlotSummary.minFreeSlots},
|
|
{"peak_active_connections", controlSlotSummary.peakActiveConnections},
|
|
{"peak_utilization", controlSlotSummary.peakUtilization},
|
|
{"grow_events", controlSlotSummary.growEvents},
|
|
{"shrink_events", controlSlotSummary.shrinkEvents},
|
|
{"retry_after_samples", controlSlotSummary.retryAfterSamples},
|
|
{"final_active_connections", controlFinalSnap.activeConnections}}}}},
|
|
{"resource_peaks", resourcePeaks.toJson()},
|
|
{"hotspots", hotspotSummary},
|
|
{"op_stage_latency", opStageSummary},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"drained", drained},
|
|
{"recovery_quiesced", recoveryQuiesced},
|
|
{"drain_metrics", drainObservation.toJson()},
|
|
{"idle_probe", idleProbe.toJson()},
|
|
{"recovery", recovery.toJson()},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"time_series_samples", samples.size()},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: scaling curve", "[!benchmark][multi-client][scaling]") {
|
|
// Run the same workload at increasing powers-of-two clients to find scaling limits.
|
|
auto baseCfg = BenchConfig::fromEnv();
|
|
std::vector<int> clientCounts;
|
|
for (std::int64_t n = 1; n <= baseCfg.scalingMaxClients;) {
|
|
clientCounts.push_back(static_cast<int>(n));
|
|
n *= 2;
|
|
}
|
|
if (!clientCounts.empty() && clientCounts.back() != baseCfg.scalingMaxClients) {
|
|
clientCounts.push_back(baseCfg.scalingMaxClients);
|
|
}
|
|
|
|
json scalingResults = json::array();
|
|
|
|
for (int numClients : clientCounts) {
|
|
std::cout << "\n--- Scaling test: " << numClients << " client(s) ---\n";
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(startHarnessWithRetry(harness, kStartTimeout));
|
|
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
std::atomic<int> totalDocs{0};
|
|
std::atomic<int> totalFail{0};
|
|
std::vector<ClientResult> results(numClients);
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
std::vector<std::thread> threads;
|
|
for (int c = 0; c < numClients; ++c) {
|
|
threads.emplace_back([&, c]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
auto start = std::chrono::steady_clock::now();
|
|
for (int i = 0; i < baseCfg.docsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "scale_c" + std::to_string(c) + "_d" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(c, i, baseCfg.docSizeBytes);
|
|
req.tags = {"bench", "scaling"};
|
|
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
if (res) {
|
|
results[c].docsIngested++;
|
|
totalDocs.fetch_add(1);
|
|
} else {
|
|
results[c].failures++;
|
|
totalFail.fetch_add(1);
|
|
}
|
|
}
|
|
auto end = std::chrono::steady_clock::now();
|
|
results[c].elapsedSeconds = std::chrono::duration<double>(end - start).count();
|
|
results[c].addThroughputDocsPerSec =
|
|
results[c].docsIngested / results[c].elapsedSeconds;
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
double aggThroughput = static_cast<double>(totalDocs.load()) / globalElapsed;
|
|
|
|
bool drained = waitForDrain(monitorClient, std::chrono::seconds(baseCfg.drainTimeoutSecs));
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
|
|
// Per-client throughput stats
|
|
std::vector<double> perClient;
|
|
for (auto& r : results)
|
|
perClient.push_back(r.addThroughputDocsPerSec);
|
|
|
|
double meanPerClient = std::accumulate(perClient.begin(), perClient.end(), 0.0) /
|
|
static_cast<double>(perClient.size());
|
|
|
|
std::cout << " Clients: " << numClients << " Aggregate: " << std::fixed
|
|
<< std::setprecision(1) << aggThroughput << " docs/s"
|
|
<< " Per-client mean: " << meanPerClient << " docs/s"
|
|
<< " Failures: " << totalFail.load() << " Memory: " << finalSnap.memoryUsageMb
|
|
<< " MB"
|
|
<< " Drained: " << (drained ? "yes" : "NO") << "\n";
|
|
|
|
CHECK(totalDocs.load() == checkedMulToInt(numClients, baseCfg.docsPerClient));
|
|
|
|
scalingResults.push_back(json{
|
|
{"num_clients", numClients},
|
|
{"aggregate_throughput", aggThroughput},
|
|
{"per_client_mean_throughput", meanPerClient},
|
|
{"total_docs", totalDocs.load()},
|
|
{"total_failures", totalFail.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"memory_mb", finalSnap.memoryUsageMb},
|
|
{"drained", drained},
|
|
});
|
|
|
|
// Harness destructor will stop the daemon
|
|
}
|
|
|
|
// Print scaling summary
|
|
std::cout << "\n=== Scaling Curve Summary ===\n";
|
|
std::cout << std::setw(10) << "Clients" << std::setw(18) << "Agg docs/s" << std::setw(18)
|
|
<< "Per-client docs/s" << std::setw(12) << "Efficiency"
|
|
<< "\n";
|
|
double baselineThroughput = 0;
|
|
for (auto& r : scalingResults) {
|
|
double agg = r["aggregate_throughput"];
|
|
double perClient = r["per_client_mean_throughput"];
|
|
int n = r["num_clients"];
|
|
if (n == 1)
|
|
baselineThroughput = agg;
|
|
double efficiency =
|
|
(baselineThroughput > 0) ? (agg / (baselineThroughput * n)) * 100.0 : 0.0;
|
|
std::cout << std::setw(10) << n << std::setw(18) << std::fixed << std::setprecision(1)
|
|
<< agg << std::setw(18) << perClient << std::setw(11) << std::setprecision(1)
|
|
<< efficiency << "%\n";
|
|
}
|
|
std::cout << std::endl;
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "scaling_curve"},
|
|
{"docs_per_client", baseCfg.docsPerClient},
|
|
{"doc_size_bytes", baseCfg.docSizeBytes},
|
|
{"scaling_results", scalingResults},
|
|
{"tuning_profile", baseCfg.tuningProfile},
|
|
};
|
|
emitJsonl(baseCfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: 16-client concurrent mixed ops",
|
|
"[!benchmark][multi-client][mixed-16]") {
|
|
// 16 clients performing list, search, status, and get simultaneously while
|
|
// background writers are ingesting. Measures per-operation-type latency and
|
|
// throughput under realistic concurrent access patterns.
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
const int kWriterClients = cfg.mixedWriterClients;
|
|
const int kSearchClients = cfg.mixedSearchClients;
|
|
const int kListClients = cfg.mixedListClients;
|
|
const int kStatusClients = cfg.mixedStatusClients;
|
|
const int kTotalClients = checkedAddToInt(
|
|
{kWriterClients, kSearchClients, kListClients, kStatusClients}, kMaxBenchClients * 4);
|
|
|
|
// Each reader client runs for a fixed number of operations
|
|
const int kOpsPerReader = cfg.mixedOpsPerReader;
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(harness.start(kStartTimeout));
|
|
|
|
// Pre-seed 100 docs so search/list have data from the start
|
|
std::vector<std::string> knownHashes;
|
|
{
|
|
ClientConfig warmCfg;
|
|
warmCfg.socketPath = harness.socketPath();
|
|
warmCfg.autoStart = false;
|
|
warmCfg.requestTimeout = 30s;
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
std::cout << " Pre-seeding 100 warmup docs...\n";
|
|
for (int i = 0; i < 100; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "mixed16_warmup_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(99, i, cfg.docSizeBytes);
|
|
req.tags = {"warmup", "bench"};
|
|
auto res = yams::cli::run_sync(warmClient.streamingAddDocument(req), 30s);
|
|
if (res && !res.value().hash.empty()) {
|
|
knownHashes.push_back(res.value().hash);
|
|
}
|
|
}
|
|
(void)waitForDrain(warmClient, 60s);
|
|
std::this_thread::sleep_for(1s);
|
|
std::cout << " Warmup complete (" << knownHashes.size() << " hashes)\n";
|
|
}
|
|
|
|
// Monitor client
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, 500ms);
|
|
|
|
// Per-thread latency collectors
|
|
std::mutex searchLatMutex, listLatMutex, statusLatMutex, getLatMutex, addLatMutex;
|
|
std::vector<int64_t> searchLatencies, listLatencies, statusLatencies, getLatencies,
|
|
addLatencies;
|
|
std::map<std::string, int> mixedErrorCategories;
|
|
std::map<std::string, std::string> mixedFailureSamples;
|
|
|
|
std::atomic<int> searchOps{0}, listOps{0}, statusOps{0}, getOps{0}, addOps{0};
|
|
std::atomic<int> searchFails{0}, listFails{0}, statusFails{0}, getFails{0}, addFails{0};
|
|
|
|
// Barrier: all threads start at the same instant
|
|
std::atomic<bool> go{false};
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(kTotalClients);
|
|
|
|
// --- Writer threads (4): continuous ingestion ---
|
|
for (int w = 0; w < kWriterClients; ++w) {
|
|
threads.emplace_back([&, w]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
std::map<std::string, int> localErrorCategories;
|
|
std::map<std::string, std::string> localFailureSamples;
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "mixed16_w" + std::to_string(w) + "_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(w, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "mixed16", "writer-" + std::to_string(w)};
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
addOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(addLatMutex);
|
|
addLatencies.push_back(latUs);
|
|
} else {
|
|
addFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(res.error().message);
|
|
const auto key = std::string("add:") + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!res.error().message.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = res.error().message;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!localErrorCategories.empty()) {
|
|
std::lock_guard<std::mutex> lk(addLatMutex);
|
|
for (const auto& [key, count] : localErrorCategories) {
|
|
mixedErrorCategories[key] += count;
|
|
}
|
|
for (const auto& [key, sample] : localFailureSamples) {
|
|
if (!mixedFailureSamples.contains(key)) {
|
|
mixedFailureSamples[key] = sample;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- Search threads (4): continuous search queries ---
|
|
for (int s = 0; s < kSearchClients; ++s) {
|
|
threads.emplace_back([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
std::map<std::string, int> localErrorCategories;
|
|
std::map<std::string, std::string> localFailureSamples;
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
static const char* queries[] = {
|
|
"architecture performance tuning",
|
|
"ingestion pipeline extraction",
|
|
"resource governance memory pressure",
|
|
"thread pool concurrency limits",
|
|
"IPC transport protocol",
|
|
};
|
|
|
|
for (int i = 0; i < kOpsPerReader; ++i) {
|
|
SearchRequest req;
|
|
req.query = queries[i % 5];
|
|
req.limit = 10;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.search(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
searchOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(searchLatMutex);
|
|
searchLatencies.push_back(latUs);
|
|
} else {
|
|
searchFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(res.error().message);
|
|
const auto key = std::string("search:") + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!res.error().message.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = res.error().message;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!localErrorCategories.empty()) {
|
|
std::lock_guard<std::mutex> lk(searchLatMutex);
|
|
for (const auto& [key, count] : localErrorCategories) {
|
|
mixedErrorCategories[key] += count;
|
|
}
|
|
for (const auto& [key, sample] : localFailureSamples) {
|
|
if (!mixedFailureSamples.contains(key)) {
|
|
mixedFailureSamples[key] = sample;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- List threads (4): continuous list queries with varying filters ---
|
|
for (int l = 0; l < kListClients; ++l) {
|
|
threads.emplace_back([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
std::map<std::string, int> localErrorCategories;
|
|
std::map<std::string, std::string> localFailureSamples;
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerReader; ++i) {
|
|
ListRequest req;
|
|
req.limit = (i % 3 == 0) ? 5 : (i % 3 == 1) ? 20 : 50;
|
|
// Alternate between filtered and unfiltered
|
|
if (i % 4 == 0) {
|
|
req.tags = {"warmup"};
|
|
}
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.list(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
listOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(listLatMutex);
|
|
listLatencies.push_back(latUs);
|
|
} else {
|
|
listFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(res.error().message);
|
|
const auto key = std::string("list:") + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!res.error().message.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = res.error().message;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!localErrorCategories.empty()) {
|
|
std::lock_guard<std::mutex> lk(listLatMutex);
|
|
for (const auto& [key, count] : localErrorCategories) {
|
|
mixedErrorCategories[key] += count;
|
|
}
|
|
for (const auto& [key, sample] : localFailureSamples) {
|
|
if (!mixedFailureSamples.contains(key)) {
|
|
mixedFailureSamples[key] = sample;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- Status threads (4): continuous status + get queries ---
|
|
for (int t = 0; t < kStatusClients; ++t) {
|
|
threads.emplace_back([&, t]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
std::mt19937 rng(static_cast<unsigned>(t * 997 + 13));
|
|
std::map<std::string, int> localErrorCategories;
|
|
std::map<std::string, std::string> localFailureSamples;
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerReader; ++i) {
|
|
if (i % 3 == 0) {
|
|
// Status query
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.status(), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
statusOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(statusLatMutex);
|
|
statusLatencies.push_back(latUs);
|
|
} else {
|
|
statusFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(res.error().message);
|
|
const auto key = std::string("status:") + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!res.error().message.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = res.error().message;
|
|
}
|
|
}
|
|
} else {
|
|
// Get (metadata) by hash
|
|
std::string hash;
|
|
{
|
|
if (!knownHashes.empty()) {
|
|
std::uniform_int_distribution<std::size_t> dist(0,
|
|
knownHashes.size() - 1);
|
|
hash = knownHashes[dist(rng)];
|
|
}
|
|
}
|
|
if (hash.empty()) {
|
|
statusOps.fetch_add(1); // count as no-op
|
|
continue;
|
|
}
|
|
|
|
GetRequest req;
|
|
req.hash = hash;
|
|
req.metadataOnly = true;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.get(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
getOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(getLatMutex);
|
|
getLatencies.push_back(latUs);
|
|
} else {
|
|
getFails.fetch_add(1);
|
|
const auto bucket = classifyFailureMessage(res.error().message);
|
|
const auto key = std::string("get:") + bucket;
|
|
localErrorCategories[key] += 1;
|
|
if (!res.error().message.empty() && !localFailureSamples.contains(key)) {
|
|
localFailureSamples[key] = res.error().message;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!localErrorCategories.empty()) {
|
|
std::lock_guard<std::mutex> lk(statusLatMutex);
|
|
for (const auto& [key, count] : localErrorCategories) {
|
|
mixedErrorCategories[key] += count;
|
|
}
|
|
for (const auto& [key, sample] : localFailureSamples) {
|
|
if (!mixedFailureSamples.contains(key)) {
|
|
mixedFailureSamples[key] = sample;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// Release all threads simultaneously
|
|
go.store(true);
|
|
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
sampler.stop();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
// Drain
|
|
bool drained = waitForDrain(monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs));
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
auto resourcePeaks = summarizeResourcePeaks(sampler.getSamples());
|
|
|
|
// Compute stats
|
|
auto searchStats = PercentileStats::compute(searchLatencies);
|
|
auto listStats = PercentileStats::compute(listLatencies);
|
|
auto statusStats = PercentileStats::compute(statusLatencies);
|
|
auto getStats = PercentileStats::compute(getLatencies);
|
|
auto addStats = PercentileStats::compute(addLatencies);
|
|
|
|
int totalOps =
|
|
searchOps.load() + listOps.load() + statusOps.load() + getOps.load() + addOps.load();
|
|
int totalFails = searchFails.load() + listFails.load() + statusFails.load() + getFails.load() +
|
|
addFails.load();
|
|
double opsPerSec = static_cast<double>(totalOps) / globalElapsed;
|
|
double failRate = (totalOps > 0) ? (static_cast<double>(totalFails) / totalOps) : 0.0;
|
|
const std::string stressTier = !drained ? "failed" : (totalFails == 0 ? "clean" : "degraded");
|
|
|
|
// Print summary
|
|
std::cout << "\n=== " << kTotalClients << "-Client Concurrent Mixed Ops ===\n";
|
|
std::cout << " Layout: " << kWriterClients << " writers + " << kSearchClients << " searchers"
|
|
<< " + " << kListClients << " listers + " << kStatusClients << " status/get\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
std::cout << " Total ops: " << totalOps << " (" << std::setprecision(0) << opsPerSec
|
|
<< " ops/s)"
|
|
<< " Failures: " << totalFails << " (rate=" << std::setprecision(4) << failRate
|
|
<< ")\n\n";
|
|
|
|
std::cout << " Op breakdown:\n";
|
|
std::cout << " Add: " << addOps.load() << " ok, " << addFails.load() << " fail\n";
|
|
std::cout << " Search: " << searchOps.load() << " ok, " << searchFails.load() << " fail\n";
|
|
std::cout << " List: " << listOps.load() << " ok, " << listFails.load() << " fail\n";
|
|
std::cout << " Status: " << statusOps.load() << " ok, " << statusFails.load() << " fail\n";
|
|
std::cout << " Get: " << getOps.load() << " ok, " << getFails.load() << " fail\n\n";
|
|
if (!mixedErrorCategories.empty()) {
|
|
std::cout << " Failure categories:";
|
|
for (const auto& [key, count] : mixedErrorCategories) {
|
|
std::cout << " " << key << "=" << count;
|
|
}
|
|
std::cout << "\n\n";
|
|
}
|
|
|
|
printPercentiles("Add latency ", addStats);
|
|
printPercentiles("Search latency", searchStats);
|
|
printPercentiles("List latency ", listStats);
|
|
printPercentiles("Status latency", statusStats);
|
|
printPercentiles("Get latency ", getStats);
|
|
|
|
std::cout << "\n Queue drained: " << (drained ? "yes" : "NO") << "\n";
|
|
std::cout << " Peak DB read/write waiting: " << resourcePeaks.peakDbReadPoolWaiting << "/"
|
|
<< resourcePeaks.peakDbWritePoolWaiting << "\n";
|
|
std::cout << " Final docs_total: " << finalSnap.documentsTotal << "\n";
|
|
std::cout << " Final memory: " << finalSnap.memoryUsageMb << " MB\n\n";
|
|
|
|
// Key assertions: all op types must make progress
|
|
CHECK(addOps.load() > 0);
|
|
CHECK(searchOps.load() > 0);
|
|
CHECK(listOps.load() > 0);
|
|
CHECK(statusOps.load() > 0);
|
|
CHECK(getOps.load() > 0);
|
|
CHECK(drained);
|
|
CHECK(failRate <= cfg.mixedMaxFailRate);
|
|
|
|
// Emit JSONL
|
|
json timeSeries = json::array();
|
|
for (auto& s : sampler.getSamples()) {
|
|
timeSeries.push_back(json{{"elapsed_ms", s.elapsedMs}, {"snapshot", s.snap.toJson()}});
|
|
}
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "16_client_concurrent_mixed_ops"},
|
|
{"layout", json{{"writers", kWriterClients},
|
|
{"searchers", kSearchClients},
|
|
{"listers", kListClients},
|
|
{"status_get", kStatusClients}}},
|
|
{"ops_per_reader", kOpsPerReader},
|
|
{"docs_per_writer", cfg.docsPerClient},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"total_ops", totalOps},
|
|
{"total_failures", totalFails},
|
|
{"fail_rate", failRate},
|
|
{"stress_tier", stressTier},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"ops_per_sec", opsPerSec},
|
|
{"add",
|
|
json{{"ops", addOps.load()}, {"fails", addFails.load()}, {"latency", addStats.toJson()}}},
|
|
{"search", json{{"ops", searchOps.load()},
|
|
{"fails", searchFails.load()},
|
|
{"latency", searchStats.toJson()}}},
|
|
{"list", json{{"ops", listOps.load()},
|
|
{"fails", listFails.load()},
|
|
{"latency", listStats.toJson()}}},
|
|
{"status", json{{"ops", statusOps.load()},
|
|
{"fails", statusFails.load()},
|
|
{"latency", statusStats.toJson()}}},
|
|
{"get",
|
|
json{{"ops", getOps.load()}, {"fails", getFails.load()}, {"latency", getStats.toJson()}}},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"resource_peaks", resourcePeaks.toJson()},
|
|
{"error_categories", mixedErrorCategories},
|
|
{"failure_samples", mixedFailureSamples},
|
|
{"time_series", timeSeries},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
{"drained", drained},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
// =============================================================================
|
|
// EXISTING CORPUS BENCHMARK
|
|
// =============================================================================
|
|
|
|
TEST_CASE("Multi-client ingestion: large corpus reads",
|
|
"[!benchmark][multi-client][large-corpus]") {
|
|
// Instrumented benchmark against an existing YAMS corpus (e.g. /Volumes/picaso/yams
|
|
// with 48k+ docs). Does NOT ingest — hammers search/list/get/status against the
|
|
// real corpus to measure read-path performance. Outputs:
|
|
// - Per-op latency traces (every operation recorded)
|
|
// - Live progress with rolling throughput
|
|
// - Memory watermarks (peak RSS, delta over time)
|
|
// - Error classification breakdown
|
|
// - Detailed JSONL with all latency traces for post-analysis
|
|
//
|
|
// Env overrides:
|
|
// YAMS_BENCH_DATA_DIR - required: existing corpus path
|
|
// YAMS_BENCH_SEARCH_CLIENTS - search thread count (default: 4)
|
|
// YAMS_BENCH_LIST_CLIENTS - list thread count (default: 4)
|
|
// YAMS_BENCH_GREP_CLIENTS - grep thread count (default: 2)
|
|
// YAMS_BENCH_STATUS_GET_CLIENTS - status/get thread count (default: 4)
|
|
// YAMS_BENCH_OPS_PER_CLIENT - ops per thread (default: 50)
|
|
// YAMS_BENCH_OP_TIMEOUT_S - per-op timeout in seconds (default: 60)
|
|
// YAMS_BENCH_STATUS_EVERY_N - status cadence in status/get threads (default: 4)
|
|
// YAMS_BENCH_CAT_EVERY_N - cat cadence among non-status status/get ops
|
|
// (default: 3)
|
|
// YAMS_BENCH_LIST_INFLIGHT_CAP - max concurrent list ops (0 = unlimited, default: 0)
|
|
// YAMS_BENCH_USAGE_PROFILE - mixed|extension_direct|external_agent_churn
|
|
// (default: mixed)
|
|
// YAMS_BENCH_MCP_SESSION_CHURN_EVERY_N - MCP only; every Nth request uses a
|
|
// transient MCP session (default: 6 for
|
|
// external_agent_churn, otherwise 0)
|
|
auto cfg = BenchConfig::fromEnv();
|
|
if (!cfg.dataDir) {
|
|
SKIP("Set YAMS_BENCH_DATA_DIR to an existing YAMS corpus to run this benchmark");
|
|
}
|
|
|
|
const bool extensionLikeProfile =
|
|
(cfg.usageProfile == "extension_direct" || cfg.usageProfile == "vscode_blackboard");
|
|
const bool externalAgentChurnProfile =
|
|
(cfg.usageProfile == "external_agent_churn" || cfg.usageProfile == "agent_churn");
|
|
|
|
// --- Configurable layout ---
|
|
const int defaultSearchClients = externalAgentChurnProfile ? 4 : (extensionLikeProfile ? 6 : 4);
|
|
const int defaultListClients = externalAgentChurnProfile ? 4 : (extensionLikeProfile ? 6 : 4);
|
|
const int defaultGrepClients = externalAgentChurnProfile ? 1 : (extensionLikeProfile ? 1 : 2);
|
|
const int defaultStatusGetClients =
|
|
externalAgentChurnProfile ? 12 : (extensionLikeProfile ? 10 : 4);
|
|
const int defaultOpsPerClient =
|
|
externalAgentChurnProfile ? 90 : (extensionLikeProfile ? 80 : 50);
|
|
const int defaultStatusEveryN = externalAgentChurnProfile ? 2 : (extensionLikeProfile ? 8 : 4);
|
|
const int defaultCatEveryN = externalAgentChurnProfile ? 14 : (extensionLikeProfile ? 10 : 3);
|
|
const int defaultListInflightCap =
|
|
externalAgentChurnProfile ? 1 : (extensionLikeProfile ? 2 : 0);
|
|
|
|
const int kSearchClients =
|
|
parseEnvIntClamped("YAMS_BENCH_SEARCH_CLIENTS", defaultSearchClients, 0, kMaxBenchClients);
|
|
const int kListClients =
|
|
parseEnvIntClamped("YAMS_BENCH_LIST_CLIENTS", defaultListClients, 0, kMaxBenchClients);
|
|
const int kGrepClients =
|
|
parseEnvIntClamped("YAMS_BENCH_GREP_CLIENTS", defaultGrepClients, 0, kMaxBenchClients);
|
|
const int kStatusGetClients = parseEnvIntClamped("YAMS_BENCH_STATUS_GET_CLIENTS",
|
|
defaultStatusGetClients, 0, kMaxBenchClients);
|
|
const int kTotalClients = checkedAddToInt(
|
|
{kSearchClients, kListClients, kGrepClients, kStatusGetClients}, kMaxBenchClients * 4);
|
|
const int kOpsPerClient = parseEnvIntClamped("YAMS_BENCH_OPS_PER_CLIENT", defaultOpsPerClient,
|
|
1, kMaxBenchOpsPerClient);
|
|
const int kOpTimeoutS = parseEnvIntClamped("YAMS_BENCH_OP_TIMEOUT_S", 60, 1, kMaxBenchSeconds);
|
|
const auto kOpTimeout = std::chrono::seconds(kOpTimeoutS);
|
|
const int kStatusEveryN = parseEnvIntClamped("YAMS_BENCH_STATUS_EVERY_N", defaultStatusEveryN,
|
|
1, kMaxBenchOpsPerClient);
|
|
const int kCatEveryN =
|
|
parseEnvIntClamped("YAMS_BENCH_CAT_EVERY_N", defaultCatEveryN, 1, kMaxBenchOpsPerClient);
|
|
const int kListInflightCap = parseEnvIntClamped("YAMS_BENCH_LIST_INFLIGHT_CAP",
|
|
defaultListInflightCap, 0, kMaxBenchClients);
|
|
const int kDiscoverPageSize =
|
|
parseEnvIntClamped("YAMS_BENCH_DISCOVER_PAGE_SIZE", 500, 1, kMaxBenchPageSize);
|
|
const int kDiscoverHashLimit =
|
|
parseEnvIntClamped("YAMS_BENCH_DISCOVER_HASH_LIMIT", 5000, 1, kMaxBenchHashLimit);
|
|
const int kHotspotWindowMs =
|
|
parseEnvIntClamped("YAMS_BENCH_HOTSPOT_WINDOW_MS", 1000, 1, kMaxBenchMilliseconds);
|
|
const bool useMcpPath = cfg.useMcpPath;
|
|
const bool useSessionGrep = parseEnvIntClamped("YAMS_BENCH_GREP_USE_SESSION", 0, 0, 1) != 0;
|
|
const int defaultMcpSessionChurnEveryN = (externalAgentChurnProfile && useMcpPath) ? 6 : 0;
|
|
const int kMcpSessionChurnEveryN =
|
|
parseEnvIntClamped("YAMS_BENCH_MCP_SESSION_CHURN_EVERY_N", defaultMcpSessionChurnEveryN, 0,
|
|
kMaxBenchOpsPerClient);
|
|
const size_t mcpPoolSize = static_cast<size_t>(parseEnvIntClamped(
|
|
"YAMS_BENCH_MCP_POOL_SIZE", std::max(1, kTotalClients), 1, kMaxBenchClients * 4));
|
|
|
|
std::cout << "\n=== Large Corpus Read Benchmark (instrumented) ===\n";
|
|
std::cout << " Data dir: " << cfg.dataDir->string() << "\n";
|
|
std::cout << " Layout: " << kSearchClients << " search + " << kListClients
|
|
<< " list + " << kGrepClients << " grep + " << kStatusGetClients
|
|
<< " status/get/cat = " << kTotalClients << " threads\n";
|
|
std::cout << " Ops/thread: " << kOpsPerClient << "\n";
|
|
std::cout << " Op timeout: " << kOpTimeoutS << "s\n";
|
|
std::cout << " Status every: " << kStatusEveryN << " op(s) on status/get/cat threads\n";
|
|
std::cout << " Cat every: " << kCatEveryN
|
|
<< " non-status op(s) on status/get/cat threads\n";
|
|
std::cout << " List cap: "
|
|
<< (kListInflightCap > 0 ? std::to_string(kListInflightCap) : "unlimited")
|
|
<< " in-flight\n";
|
|
std::cout << " Discover page: " << kDiscoverPageSize << " (limit " << kDiscoverHashLimit
|
|
<< ")\n";
|
|
std::cout << " Hotspot window: " << kHotspotWindowMs << "ms\n";
|
|
std::cout << " Usage profile: " << cfg.usageProfile << "\n";
|
|
std::cout << " Transport path: " << (useMcpPath ? "mcp(query-or-direct-tools)" : "daemon-ipc")
|
|
<< "\n";
|
|
if (useMcpPath) {
|
|
std::cout << " MCP pool size: " << mcpPoolSize << "\n";
|
|
std::cout << " MCP churn: every "
|
|
<< (kMcpSessionChurnEveryN > 0 ? std::to_string(kMcpSessionChurnEveryN)
|
|
: std::string("disabled"))
|
|
<< " request(s) uses a fresh MCP session\n";
|
|
}
|
|
if (kGrepClients > 0) {
|
|
std::cout << " Grep mode: "
|
|
<< (useSessionGrep ? "session hot path" : "global cold path") << "\n";
|
|
}
|
|
|
|
// --- Server-side timeout configuration ---
|
|
// Stay within TuneAdvisor range clamps:
|
|
// ipcTimeoutMs: [500, 600000]
|
|
// streamChunkTimeoutMs: [1000, 600000]
|
|
// searchBuildTimeout: unclamped
|
|
std::string timeoutStr =
|
|
std::to_string(std::min(checkedMulToInt(kOpTimeoutS, 2000, 600000), 600000));
|
|
#ifdef _WIN32
|
|
if (!std::getenv("YAMS_SEARCH_BUILD_TIMEOUT_MS"))
|
|
_putenv_s("YAMS_SEARCH_BUILD_TIMEOUT_MS", "120000");
|
|
if (!std::getenv("YAMS_IPC_TIMEOUT_MS"))
|
|
_putenv_s("YAMS_IPC_TIMEOUT_MS", timeoutStr.c_str());
|
|
if (!std::getenv("YAMS_STREAM_CHUNK_TIMEOUT_MS"))
|
|
_putenv_s("YAMS_STREAM_CHUNK_TIMEOUT_MS", timeoutStr.c_str());
|
|
#else
|
|
if (!std::getenv("YAMS_SEARCH_BUILD_TIMEOUT_MS"))
|
|
::setenv("YAMS_SEARCH_BUILD_TIMEOUT_MS", "120000", 0);
|
|
if (!std::getenv("YAMS_IPC_TIMEOUT_MS"))
|
|
::setenv("YAMS_IPC_TIMEOUT_MS", timeoutStr.c_str(), 0);
|
|
if (!std::getenv("YAMS_STREAM_CHUNK_TIMEOUT_MS"))
|
|
::setenv("YAMS_STREAM_CHUNK_TIMEOUT_MS", timeoutStr.c_str(), 0);
|
|
#endif
|
|
|
|
const char* ipcTimeoutEnv = std::getenv("YAMS_IPC_TIMEOUT_MS");
|
|
const char* streamTimeoutEnv = std::getenv("YAMS_STREAM_CHUNK_TIMEOUT_MS");
|
|
std::cout << " IPC timeout: " << (ipcTimeoutEnv ? ipcTimeoutEnv : "default") << " ms\n";
|
|
std::cout << " Stream timeout: " << (streamTimeoutEnv ? streamTimeoutEnv : "default")
|
|
<< " ms\n";
|
|
|
|
// --- Start daemon ---
|
|
auto opts = benchHarnessOptions(cfg);
|
|
DaemonHarness harness(opts);
|
|
REQUIRE(harness.start(std::chrono::seconds(300)));
|
|
std::this_thread::sleep_for(3s); // let async init settle
|
|
|
|
// Helper: create a client config with all timeouts aligned to kOpTimeout.
|
|
// The default headerTimeout (30s) was causing cascading 30s stalls because
|
|
// the client would time out waiting for the first response frame header.
|
|
auto makeClientConfig = [&](std::chrono::milliseconds timeout = std::chrono::milliseconds{0}) {
|
|
ClientConfig c;
|
|
c.socketPath = harness.socketPath();
|
|
c.autoStart = false;
|
|
auto t = timeout.count() > 0 ? timeout : kOpTimeout;
|
|
c.requestTimeout = t;
|
|
c.headerTimeout = t;
|
|
c.bodyTimeout = t;
|
|
return c;
|
|
};
|
|
|
|
// --- Discover corpus ---
|
|
auto discoverCfg = makeClientConfig();
|
|
DaemonClient discoverClient(discoverCfg);
|
|
std::shared_ptr<MCPPipelineClientPool> sharedMcpPool;
|
|
if (useMcpPath) {
|
|
sharedMcpPool = std::make_shared<MCPPipelineClientPool>(harness.socketPath(), mcpPoolSize);
|
|
}
|
|
|
|
std::atomic<uint64_t> mcpQueryCalls{0};
|
|
std::atomic<uint64_t> mcpTransientSessionCalls{0};
|
|
auto queryMcpStepForSlot = [&](size_t slot, const std::string& op,
|
|
const nlohmann::json& params) -> yams::Result<nlohmann::json> {
|
|
if (!sharedMcpPool) {
|
|
return yams::Error{yams::ErrorCode::InvalidState,
|
|
"MCP pool is not initialized for MCP benchmark path"};
|
|
}
|
|
const auto callNumber =
|
|
mcpQueryCalls.fetch_add(1, std::memory_order_relaxed) + static_cast<uint64_t>(1);
|
|
if (kMcpSessionChurnEveryN <= 0) {
|
|
return sharedMcpPool->queryStepForSlot(slot, op, params);
|
|
}
|
|
|
|
const bool useTransient = (callNumber % static_cast<uint64_t>(kMcpSessionChurnEveryN) == 0);
|
|
if (!useTransient) {
|
|
return sharedMcpPool->queryStepForSlot(slot, op, params);
|
|
}
|
|
|
|
mcpTransientSessionCalls.fetch_add(1, std::memory_order_relaxed);
|
|
try {
|
|
MCPPipelineClient transientClient(harness.socketPath());
|
|
return transientClient.queryStep(op, params);
|
|
} catch (const std::exception& e) {
|
|
return yams::Error{yams::ErrorCode::InvalidState,
|
|
std::string("Transient MCP session failed: ") + e.what()};
|
|
}
|
|
};
|
|
|
|
auto setupBenchmarkSession = [&](const std::string& sessionName) -> bool {
|
|
// Large-corpus grep benchmarks use sessions to exercise the hot path, not to scope
|
|
// results to the YAMS data directory itself. Adding a watched selector rooted at the
|
|
// storage dir narrows MCP grep to internal corpus files and stops matching the direct
|
|
// daemon hot-path workload.
|
|
if (useMcpPath) {
|
|
auto startRes = queryMcpStepForSlot(0, "session_start",
|
|
{{"name", sessionName},
|
|
{"description", "large corpus benchmark session"},
|
|
{"warm", false},
|
|
{"limit", 0}});
|
|
if (!startRes) {
|
|
WARN("session_start failed: " << startRes.error().message);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
MCPPipelineClient sessionClient(harness.socketPath());
|
|
auto startRes = sessionClient.queryStep("session_start",
|
|
{{"name", sessionName},
|
|
{"description", "large corpus benchmark session"},
|
|
{"warm", false},
|
|
{"limit", 0}});
|
|
if (!startRes) {
|
|
WARN("session_start failed: " << startRes.error().message);
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
std::vector<std::string> knownHashes;
|
|
{
|
|
auto discoverDirectPage = [&](std::int64_t offset, int& itemsInPage) -> bool {
|
|
if (offset > std::numeric_limits<int>::max()) {
|
|
WARN("Discovery list offset exceeds ListRequest range: " << offset);
|
|
return false;
|
|
}
|
|
ListRequest listReq;
|
|
listReq.limit = kDiscoverPageSize;
|
|
listReq.offset = static_cast<int>(offset);
|
|
auto listRes = yams::cli::run_sync(discoverClient.list(listReq), kOpTimeout);
|
|
if (!listRes) {
|
|
WARN("Discovery list failed at offset " << offset << ": "
|
|
<< listRes.error().message);
|
|
return false;
|
|
}
|
|
for (auto& item : listRes.value().items) {
|
|
if (!item.hash.empty()) {
|
|
knownHashes.push_back(item.hash);
|
|
++itemsInPage;
|
|
if (static_cast<int>(knownHashes.size()) >= kDiscoverHashLimit) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
};
|
|
|
|
bool useDirectDiscoveryFallback = false;
|
|
bool done = false;
|
|
for (std::int64_t offset = 0;
|
|
!done && static_cast<int>(knownHashes.size()) < kDiscoverHashLimit;
|
|
offset += static_cast<std::int64_t>(kDiscoverPageSize)) {
|
|
int itemsInPage = 0;
|
|
if (useMcpPath && !useDirectDiscoveryFallback) {
|
|
auto listRes = queryMcpStepForSlot(
|
|
0, "list",
|
|
{{"limit", kDiscoverPageSize}, {"offset", offset}, {"use_session", false}});
|
|
if (!listRes) {
|
|
WARN("Discovery list failed at offset " << offset << ": "
|
|
<< listRes.error().message);
|
|
break;
|
|
}
|
|
auto collectHashes = [&](const nlohmann::json& arr) {
|
|
if (!arr.is_array()) {
|
|
return;
|
|
}
|
|
for (const auto& item : arr) {
|
|
if (!item.is_object()) {
|
|
continue;
|
|
}
|
|
std::string hash;
|
|
if (item.contains("hash") && item["hash"].is_string()) {
|
|
hash = item["hash"].get<std::string>();
|
|
} else if (item.contains("id") && item["id"].is_string()) {
|
|
hash = item["id"].get<std::string>();
|
|
} else if (item.contains("document_hash") &&
|
|
item["document_hash"].is_string()) {
|
|
hash = item["document_hash"].get<std::string>();
|
|
}
|
|
|
|
if (!hash.empty()) {
|
|
knownHashes.push_back(std::move(hash));
|
|
++itemsInPage;
|
|
if (static_cast<int>(knownHashes.size()) >= kDiscoverHashLimit) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
const auto& payload = listRes.value();
|
|
if (payload.contains("items")) {
|
|
collectHashes(payload["items"]);
|
|
}
|
|
// MCP list handler returns "documents" key (not "items")
|
|
if (itemsInPage == 0 && payload.contains("documents")) {
|
|
collectHashes(payload["documents"]);
|
|
}
|
|
if (itemsInPage == 0 && payload.contains("results")) {
|
|
collectHashes(payload["results"]);
|
|
}
|
|
|
|
if (offset == 0 && itemsInPage == 0) {
|
|
int directItemsInPage = 0;
|
|
if (discoverDirectPage(offset, directItemsInPage) && directItemsInPage > 0) {
|
|
useDirectDiscoveryFallback = true;
|
|
itemsInPage = directItemsInPage;
|
|
INFO("Discovery fallback: using direct list() pagination because MCP list "
|
|
"did not return discoverable hashes.");
|
|
}
|
|
}
|
|
} else {
|
|
if (!discoverDirectPage(offset, itemsInPage)) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (itemsInPage < kDiscoverPageSize) {
|
|
done = true;
|
|
}
|
|
}
|
|
|
|
std::sort(knownHashes.begin(), knownHashes.end());
|
|
knownHashes.erase(std::unique(knownHashes.begin(), knownHashes.end()), knownHashes.end());
|
|
}
|
|
|
|
auto initSnap = DaemonSnapshot::capture(discoverClient);
|
|
std::cout << " Corpus docs: " << initSnap.documentsTotal << "\n";
|
|
std::cout << " Known hashes: " << knownHashes.size() << "\n";
|
|
std::cout << " Initial RSS: " << initSnap.memoryUsageMb << " MB\n";
|
|
|
|
if (knownHashes.empty() && initSnap.documentsTotal == 0) {
|
|
SKIP("Large-corpus benchmark requires a non-empty corpus: discovery returned 0 hashes or "
|
|
"daemon reports 0 documents. Verify YAMS_BENCH_DATA_DIR points to populated data.");
|
|
}
|
|
|
|
const std::string benchmarkSessionName =
|
|
"bench_large_corpus_" +
|
|
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count());
|
|
if (useSessionGrep) {
|
|
REQUIRE(setupBenchmarkSession(benchmarkSessionName));
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Phase 1: Single-client warmup — baseline latency without contention
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
std::cout << "\n--- Phase 1: Single-client warmup (5 ops each) ---\n";
|
|
{
|
|
auto warmCfg = makeClientConfig();
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
// Warm search
|
|
for (int i = 0; i < 5; ++i) {
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(
|
|
0, "search",
|
|
{{"query", "architecture design"}, {"limit", 10}, {"type", cfg.searchType}});
|
|
} else {
|
|
SearchRequest req;
|
|
req.query = "architecture design";
|
|
req.limit = 10;
|
|
req.searchType = cfg.searchType;
|
|
auto direct = yams::cli::run_sync(warmClient.search(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["results"] = nlohmann::json::array();
|
|
for (const auto& r : direct.value().results) {
|
|
j["results"].push_back({{"id", r.id}, {"path", r.path}});
|
|
}
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - t0)
|
|
.count();
|
|
std::cout << " search[" << i << "]: " << us / 1000 << "ms" << (res ? "" : " FAIL")
|
|
<< "\n";
|
|
}
|
|
// Warm list
|
|
for (int i = 0; i < 5; ++i) {
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
int items = 0;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(0, "list", {{"limit", 50}, {"use_session", false}});
|
|
if (res && res.value().contains("items") && res.value()["items"].is_array()) {
|
|
items = static_cast<int>(res.value()["items"].size());
|
|
} else if (res && res.value().contains("documents") &&
|
|
res.value()["documents"].is_array()) {
|
|
items = static_cast<int>(res.value()["documents"].size());
|
|
}
|
|
} else {
|
|
ListRequest req;
|
|
req.limit = 50;
|
|
auto direct = yams::cli::run_sync(warmClient.list(req), kOpTimeout);
|
|
if (direct) {
|
|
items = static_cast<int>(direct.value().items.size());
|
|
nlohmann::json j;
|
|
j["items"] = nlohmann::json::array();
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - t0)
|
|
.count();
|
|
std::cout << " list[" << i << "]: " << us / 1000 << "ms (" << items << " items)"
|
|
<< (res ? "" : " FAIL") << "\n";
|
|
}
|
|
// Warm get
|
|
if (!knownHashes.empty()) {
|
|
for (int i = 0; i < 5 && i < static_cast<int>(knownHashes.size()); ++i) {
|
|
GetRequest req;
|
|
req.hash = knownHashes[i];
|
|
req.metadataOnly = true;
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(
|
|
0, "get", {{"hash", knownHashes[i]}, {"include_content", false}});
|
|
} else {
|
|
auto direct = yams::cli::run_sync(warmClient.get(req), kOpTimeout);
|
|
if (direct) {
|
|
res = nlohmann::json::object();
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - t0)
|
|
.count();
|
|
std::cout << " get[" << i << "]: " << us / 1000 << "ms"
|
|
<< (res ? "" : " FAIL") << "\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Phase 2: Concurrent load with per-op tracing
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
std::cout << "\n--- Phase 2: Concurrent load (" << kTotalClients << " threads x "
|
|
<< kOpsPerClient << " ops) ---\n";
|
|
|
|
// Per-op trace record
|
|
struct OpTrace {
|
|
std::string opType; // "search", "list", "grep", "status", "get", "cat"
|
|
int threadId{0};
|
|
int opIndex{0};
|
|
int64_t latencyUs{0};
|
|
bool success{false};
|
|
std::string errorMsg;
|
|
int resultCount{0}; // items returned (search hits, list items)
|
|
int64_t wallClockMs{0}; // ms since benchmark start
|
|
};
|
|
|
|
// Monitor with sampler
|
|
auto monCfg = makeClientConfig(10s);
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, 1s);
|
|
|
|
// Per-op trace collectors (one per thread for lock-free append)
|
|
std::vector<std::vector<OpTrace>> perThreadTraces(kTotalClients);
|
|
|
|
// Shared counters
|
|
std::atomic<int> searchOps{0}, listOps{0}, grepOps{0}, statusOps{0}, getOps{0}, catOps{0};
|
|
std::atomic<int> searchFails{0}, listFails{0}, grepFails{0}, statusFails{0}, getFails{0},
|
|
catFails{0};
|
|
std::atomic<int> totalCompleted{0};
|
|
std::atomic<int> listInFlight{0};
|
|
const int totalExpected = checkedMulToInt(kTotalClients, kOpsPerClient);
|
|
std::mutex invalidHashMutex;
|
|
std::unordered_set<std::string> invalidHashes;
|
|
|
|
auto markHashInvalid = [&](const std::string& hash) {
|
|
if (hash.empty()) {
|
|
return;
|
|
}
|
|
std::lock_guard<std::mutex> lk(invalidHashMutex);
|
|
invalidHashes.insert(hash);
|
|
};
|
|
|
|
auto sampleKnownHash = [&](std::mt19937& rng) -> std::optional<std::string> {
|
|
if (knownHashes.empty()) {
|
|
return std::nullopt;
|
|
}
|
|
std::uniform_int_distribution<std::size_t> dist(0, knownHashes.size() - 1);
|
|
constexpr int kPickAttempts = 8;
|
|
for (int attempt = 0; attempt < kPickAttempts; ++attempt) {
|
|
std::string candidate = knownHashes[dist(rng)];
|
|
bool isInvalid = false;
|
|
{
|
|
std::lock_guard<std::mutex> lk(invalidHashMutex);
|
|
isInvalid = invalidHashes.find(candidate) != invalidHashes.end();
|
|
}
|
|
if (!isInvalid) {
|
|
return candidate;
|
|
}
|
|
}
|
|
return std::nullopt;
|
|
};
|
|
|
|
// Grep server-side telemetry sampled from grep responses
|
|
std::mutex grepServerStatsMutex;
|
|
std::vector<int64_t> grepExecMsSamples;
|
|
std::vector<int64_t> grepDocsScannedSamples;
|
|
std::vector<int64_t> grepLinesScannedSamples;
|
|
std::vector<int64_t> grepBytesScannedSamples;
|
|
std::vector<int64_t> grepContentRetrievalMsSamples;
|
|
std::vector<int64_t> grepMatchRenderUsSamples;
|
|
std::vector<int64_t> grepWorkerScanMsSamples;
|
|
std::vector<int64_t> grepRegexSearchCallsSamples;
|
|
std::vector<int64_t> grepBmhPrefilterSkipsSamples;
|
|
std::vector<int64_t> grepCandidateDiscoveryMsSamples;
|
|
std::vector<int64_t> grepPathFilteringMsSamples;
|
|
std::vector<int64_t> grepCandidateClassificationMsSamples;
|
|
std::vector<int64_t> grepMetadataBatchFetchMsSamples;
|
|
std::vector<int64_t> grepMetadataEnumerationMsSamples;
|
|
std::vector<int64_t> grepRegexScanMsSamples;
|
|
std::vector<int64_t> grepContentDecodeEstimateMsSamples;
|
|
std::vector<int64_t> grepWorkerCriticalTotalMsSamples;
|
|
std::vector<int64_t> grepWorkerCriticalRetrievalMsSamples;
|
|
std::vector<int64_t> grepWorkerCriticalRegexMsSamples;
|
|
std::vector<int64_t> grepWorkerCriticalOtherMsSamples;
|
|
|
|
// Search/list server-side telemetry sampled from responses
|
|
std::vector<int64_t> searchExecMsSamples;
|
|
std::vector<int64_t> searchDispatchServiceMsSamples;
|
|
std::vector<int64_t> searchDispatchServiceUsSamples;
|
|
std::vector<int64_t> searchDispatchMapSortMsSamples;
|
|
std::vector<int64_t> searchDispatchMapSortUsSamples;
|
|
std::vector<int64_t> searchDispatchFeedbackMsSamples;
|
|
std::vector<int64_t> searchDispatchFeedbackUsSamples;
|
|
std::vector<int64_t> searchDispatchResponseMsSamples;
|
|
std::vector<int64_t> searchDispatchResponseUsSamples;
|
|
std::vector<int64_t> searchDispatchTotalMsSamples;
|
|
std::vector<int64_t> searchDispatchTotalUsSamples;
|
|
std::vector<int64_t> searchTimingEmbeddingUsSamples;
|
|
std::vector<int64_t> searchTimingTextUsSamples;
|
|
std::vector<int64_t> searchTimingPathUsSamples;
|
|
std::vector<int64_t> searchTimingVectorUsSamples;
|
|
std::vector<int64_t> searchTimingEntityVectorUsSamples;
|
|
std::vector<int64_t> searchTimingCompressedAnnUsSamples;
|
|
std::vector<int64_t> searchTimingKgUsSamples;
|
|
std::vector<int64_t> searchTimingMetadataUsSamples;
|
|
std::vector<int64_t> searchTimingTagUsSamples;
|
|
std::vector<int64_t> searchTimingMultiVectorUsSamples;
|
|
std::vector<int64_t> searchTimingConceptsUsSamples;
|
|
std::vector<int64_t> searchTimingRerankerUsSamples;
|
|
std::vector<int64_t> searchTimingGraphRerankUsSamples;
|
|
std::vector<int64_t> searchSubphraseFtsHitSamples;
|
|
std::vector<int64_t> searchAggressiveFtsHitSamples;
|
|
std::vector<int64_t> searchGraphExpansionFtsHitSamples;
|
|
std::vector<int64_t> searchGraphDocExpansionFtsHitSamples;
|
|
std::vector<int64_t> searchMultiVectorRawHitSamples;
|
|
std::vector<int64_t> searchGraphVectorRawHitSamples;
|
|
std::vector<int64_t> searchGraphQueryNeighborSeedDocSamples;
|
|
std::vector<int64_t> listServiceExecMsSamples;
|
|
std::vector<int64_t> listDispatchServiceMsSamples;
|
|
std::vector<int64_t> listDispatchServiceUsSamples;
|
|
std::vector<int64_t> listDispatchMapMsSamples;
|
|
std::vector<int64_t> listDispatchMapUsSamples;
|
|
std::vector<int64_t> listDispatchTotalMsSamples;
|
|
std::vector<int64_t> listDispatchTotalUsSamples;
|
|
std::vector<nlohmann::json> searchDebugSamples;
|
|
|
|
auto collectSearchServerTelemetry = [&](const nlohmann::json& payload) {
|
|
const auto searchStats = [&]() -> nlohmann::json {
|
|
if (payload.contains("search_stats")) {
|
|
return payload["search_stats"];
|
|
}
|
|
if (payload.contains("searchStats")) {
|
|
return payload["searchStats"];
|
|
}
|
|
return nlohmann::json::object();
|
|
}();
|
|
|
|
std::lock_guard<std::mutex> lk(grepServerStatsMutex);
|
|
if (searchDebugSamples.size() < 3 && !searchStats.empty()) {
|
|
searchDebugSamples.push_back(searchStats);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(payload, {"execution_time_ms", "executionTimeMs"})) {
|
|
searchExecMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_service_ms"})) {
|
|
searchDispatchServiceMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_service_us"})) {
|
|
searchDispatchServiceUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_map_sort_ms"})) {
|
|
searchDispatchMapSortMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_map_sort_us"})) {
|
|
searchDispatchMapSortUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_feedback_ms"})) {
|
|
searchDispatchFeedbackMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_feedback_us"})) {
|
|
searchDispatchFeedbackUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_response_ms"})) {
|
|
searchDispatchResponseMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_response_us"})) {
|
|
searchDispatchResponseUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_total_ms"})) {
|
|
searchDispatchTotalMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_dispatch_total_us"})) {
|
|
searchDispatchTotalUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_embedding_us"})) {
|
|
searchTimingEmbeddingUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_text_us"})) {
|
|
searchTimingTextUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_path_us"})) {
|
|
searchTimingPathUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_vector_us"})) {
|
|
searchTimingVectorUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_entity_vector_us"})) {
|
|
searchTimingEntityVectorUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_compressed_ann_us"})) {
|
|
searchTimingCompressedAnnUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_kg_us"})) {
|
|
searchTimingKgUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_metadata_us"})) {
|
|
searchTimingMetadataUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_tag_us"})) {
|
|
searchTimingTagUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_multi_vector_us"})) {
|
|
searchTimingMultiVectorUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_concepts_us"})) {
|
|
searchTimingConceptsUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_reranker_us"})) {
|
|
searchTimingRerankerUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"timing_graph_rerank_us"})) {
|
|
searchTimingGraphRerankUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"subphrase_fts_hit_count"})) {
|
|
searchSubphraseFtsHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"aggressive_fts_hit_count"})) {
|
|
searchAggressiveFtsHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"graph_expansion_fts_hit_count"})) {
|
|
searchGraphExpansionFtsHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"graph_doc_expansion_fts_hit_count"})) {
|
|
searchGraphDocExpansionFtsHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"multi_vector_raw_hit_count"})) {
|
|
searchMultiVectorRawHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"graph_vector_raw_hit_count"})) {
|
|
searchGraphVectorRawHitSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"graph_query_neighbor_seed_docs"})) {
|
|
searchGraphQueryNeighborSeedDocSamples.push_back(*v);
|
|
}
|
|
};
|
|
|
|
auto collectListServerTelemetry = [&](const nlohmann::json& payload) {
|
|
const auto listStats = [&]() -> nlohmann::json {
|
|
if (payload.contains("list_stats")) {
|
|
return payload["list_stats"];
|
|
}
|
|
if (payload.contains("listStats")) {
|
|
return payload["listStats"];
|
|
}
|
|
return nlohmann::json::object();
|
|
}();
|
|
|
|
std::lock_guard<std::mutex> lk(grepServerStatsMutex);
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"service_execution_time_ms"})) {
|
|
listServiceExecMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_service_ms"})) {
|
|
listDispatchServiceMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_service_us"})) {
|
|
listDispatchServiceUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_map_ms"})) {
|
|
listDispatchMapMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_map_us"})) {
|
|
listDispatchMapUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_total_ms"})) {
|
|
listDispatchTotalMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(listStats, {"phase_dispatch_total_us"})) {
|
|
listDispatchTotalUsSamples.push_back(*v);
|
|
}
|
|
};
|
|
|
|
// Live progress reporter
|
|
std::atomic<bool> progressStop{false};
|
|
auto progressStart = std::chrono::steady_clock::now();
|
|
std::thread progressThread([&]() {
|
|
int lastCompleted = 0;
|
|
while (!progressStop.load()) {
|
|
std::this_thread::sleep_for(2s);
|
|
int cur = totalCompleted.load();
|
|
auto elapsed =
|
|
std::chrono::duration<double>(std::chrono::steady_clock::now() - progressStart)
|
|
.count();
|
|
double rate = cur > 0 ? cur / elapsed : 0.0;
|
|
int delta = cur - lastCompleted;
|
|
lastCompleted = cur;
|
|
auto snap = DaemonSnapshot::capture(monitorClient);
|
|
std::cout << " [" << std::fixed << std::setprecision(1) << elapsed << "s] " << cur
|
|
<< "/" << totalExpected << " ops (" << std::setprecision(1) << rate
|
|
<< " ops/s, +" << delta << ") "
|
|
<< "RSS=" << std::setprecision(0) << snap.memoryUsageMb << "MB "
|
|
<< "conns=" << snap.activeConnections << "/" << snap.maxConnections << " "
|
|
<< "pressure=" << snap.pressureLevel << "\n";
|
|
}
|
|
});
|
|
|
|
std::atomic<bool> go{false};
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(kTotalClients);
|
|
|
|
// Search queries
|
|
static const char* corpusQueries[] = {
|
|
"architecture design patterns", "security vulnerability analysis",
|
|
"memory management allocation", "network protocol implementation",
|
|
"database query optimization", "concurrent thread synchronization",
|
|
"error handling recovery", "configuration file parsing",
|
|
"build system compilation", "testing framework assertions",
|
|
};
|
|
|
|
static const char* corpusGrepPatterns[] = {
|
|
"TODO|FIXME", "error|exception", "thread|mutex|lock",
|
|
"config|settings", "benchmark|performance", "daemon|socket",
|
|
"search|index", "vector|embedding", "plugin|model",
|
|
"timeout|retry",
|
|
};
|
|
|
|
int threadIdx = 0;
|
|
|
|
// --- Search threads ---
|
|
for (int s = 0; s < kSearchClients; ++s) {
|
|
int tid = threadIdx++;
|
|
perThreadTraces[tid].reserve(kOpsPerClient);
|
|
threads.emplace_back([&, tid, s]() {
|
|
std::optional<DaemonClient> client;
|
|
if (!useMcpPath) {
|
|
auto ccfg = makeClientConfig();
|
|
client.emplace(ccfg);
|
|
}
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerClient; ++i) {
|
|
SearchRequest req;
|
|
req.query = corpusQueries[(s * kOpsPerClient + i) % 10];
|
|
req.limit = (i % 3 == 0) ? 5 : (i % 3 == 1) ? 10 : 25;
|
|
req.searchType = cfg.searchType;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "search",
|
|
{{"query", req.query}, {"limit", req.limit}, {"type", req.searchType}});
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->search(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["results"] = nlohmann::json::array();
|
|
for (const auto& r : direct.value().results) {
|
|
j["results"].push_back({{"id", r.id}});
|
|
}
|
|
j["execution_time_ms"] = direct.value().elapsed.count();
|
|
j["query_info"] = direct.value().queryInfo;
|
|
j["search_stats"] = direct.value().searchStats;
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart).count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "search";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
trace.success = true;
|
|
const auto& payload = res.value();
|
|
if (payload.contains("results") && payload["results"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["results"].size());
|
|
}
|
|
collectSearchServerTelemetry(payload);
|
|
searchOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
searchFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- List threads ---
|
|
for (int l = 0; l < kListClients; ++l) {
|
|
int tid = threadIdx++;
|
|
perThreadTraces[tid].reserve(kOpsPerClient);
|
|
threads.emplace_back([&, tid]() {
|
|
std::optional<DaemonClient> client;
|
|
if (!useMcpPath) {
|
|
auto ccfg = makeClientConfig();
|
|
client.emplace(ccfg);
|
|
}
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerClient; ++i) {
|
|
ListRequest req;
|
|
req.limit = (i % 4 == 0) ? 10 : (i % 4 == 1) ? 25 : (i % 4 == 2) ? 50 : 100;
|
|
|
|
bool listPermitAcquired = false;
|
|
if (kListInflightCap > 0) {
|
|
while (!listPermitAcquired) {
|
|
int cur = listInFlight.load(std::memory_order_relaxed);
|
|
if (cur < kListInflightCap) {
|
|
listPermitAcquired = listInFlight.compare_exchange_weak(
|
|
cur, cur + 1, std::memory_order_relaxed);
|
|
}
|
|
if (!listPermitAcquired) {
|
|
std::this_thread::yield();
|
|
}
|
|
}
|
|
}
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(static_cast<size_t>(tid), "list",
|
|
{{"limit", req.limit}, {"use_session", false}});
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->list(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["items"] = nlohmann::json::array();
|
|
for (const auto& it : direct.value().items) {
|
|
j["items"].push_back({{"hash", it.hash}});
|
|
}
|
|
j["query_info"] = direct.value().queryInfo;
|
|
j["list_stats"] = direct.value().listStats;
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
|
|
if (kListInflightCap > 0 && listPermitAcquired) {
|
|
listInFlight.fetch_sub(1, std::memory_order_relaxed);
|
|
}
|
|
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart).count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "list";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
const auto& payload = res.value();
|
|
trace.success = true;
|
|
if (payload.contains("items") && payload["items"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["items"].size());
|
|
} else if (payload.contains("documents") && payload["documents"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["documents"].size());
|
|
}
|
|
collectListServerTelemetry(payload);
|
|
listOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
listFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- Status/Get threads ---
|
|
for (int g = 0; g < kGrepClients; ++g) {
|
|
int tid = threadIdx++;
|
|
perThreadTraces[tid].reserve(kOpsPerClient);
|
|
threads.emplace_back([&, tid, g]() {
|
|
std::optional<DaemonClient> client;
|
|
if (!useMcpPath) {
|
|
auto ccfg = makeClientConfig();
|
|
client.emplace(ccfg);
|
|
}
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerClient; ++i) {
|
|
const char* pattern = corpusGrepPatterns[(g * kOpsPerClient + i) % 10];
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "grep",
|
|
{{"pattern", pattern},
|
|
{"max_count", (i % 2 == 0) ? 25 : 50},
|
|
{"ignore_case", (i % 3 == 0)},
|
|
{"context", i % 2},
|
|
{"use_session", useSessionGrep},
|
|
{"session", useSessionGrep ? benchmarkSessionName : ""}});
|
|
} else {
|
|
GrepRequest req;
|
|
req.pattern = pattern;
|
|
req.maxMatches = static_cast<size_t>((i % 2 == 0) ? 25 : 50);
|
|
req.caseInsensitive = (i % 3 == 0);
|
|
req.contextLines = i % 2;
|
|
req.useSession = useSessionGrep;
|
|
req.sessionName = useSessionGrep ? benchmarkSessionName : std::string{};
|
|
auto direct = yams::cli::run_sync(client->grep(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["match_count"] = direct.value().totalMatches;
|
|
j["file_count"] = direct.value().filesSearched;
|
|
j["regex_matches"] = direct.value().regexMatches;
|
|
j["semantic_matches"] = direct.value().semanticMatches;
|
|
j["execution_time_ms"] = direct.value().executionTimeMs;
|
|
j["query_info"] = direct.value().queryInfo;
|
|
j["search_stats"] = direct.value().searchStats;
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart).count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "grep";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
const auto& payload = res.value();
|
|
trace.success = true;
|
|
if (payload.contains("match_count") &&
|
|
payload["match_count"].is_number_integer()) {
|
|
trace.resultCount = payload["match_count"].get<int>();
|
|
}
|
|
|
|
const auto searchStats = [&]() -> nlohmann::json {
|
|
if (payload.contains("search_stats")) {
|
|
return payload["search_stats"];
|
|
}
|
|
if (payload.contains("searchStats")) {
|
|
return payload["searchStats"];
|
|
}
|
|
return nlohmann::json::object();
|
|
}();
|
|
|
|
std::lock_guard<std::mutex> lk(grepServerStatsMutex);
|
|
if (auto v =
|
|
jsonGetInt64AnyKey(payload, {"execution_time_ms", "executionTimeMs"})) {
|
|
grepExecMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"docs_scanned"})) {
|
|
grepDocsScannedSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"lines_scanned"})) {
|
|
grepLinesScannedSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"bytes_scanned"})) {
|
|
grepBytesScannedSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"content_retrieval_ms"})) {
|
|
grepContentRetrievalMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"match_render_us"})) {
|
|
grepMatchRenderUsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_worker_scan_ms"})) {
|
|
grepWorkerScanMsSamples.push_back(*v);
|
|
}
|
|
if (auto v =
|
|
jsonGetInt64AnyKey(searchStats, {"phase_candidate_discovery_ms"})) {
|
|
grepCandidateDiscoveryMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_path_filtering_ms"})) {
|
|
grepPathFilteringMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats,
|
|
{"phase_candidate_classification_ms"})) {
|
|
grepCandidateClassificationMsSamples.push_back(*v);
|
|
}
|
|
if (auto v =
|
|
jsonGetInt64AnyKey(searchStats, {"phase_metadata_batch_fetch_ms"})) {
|
|
grepMetadataBatchFetchMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"regex_scan_ms"})) {
|
|
grepRegexScanMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"content_decode_ms_estimate"})) {
|
|
grepContentDecodeEstimateMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"worker_critical_total_ms"})) {
|
|
grepWorkerCriticalTotalMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats,
|
|
{"worker_critical_content_retrieval_ms"})) {
|
|
grepWorkerCriticalRetrievalMsSamples.push_back(*v);
|
|
}
|
|
if (auto v =
|
|
jsonGetInt64AnyKey(searchStats, {"worker_critical_regex_scan_ms"})) {
|
|
grepWorkerCriticalRegexMsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"worker_critical_other_ms"})) {
|
|
grepWorkerCriticalOtherMsSamples.push_back(*v);
|
|
}
|
|
{
|
|
int64_t metadataEnumerationMs = 0;
|
|
bool hasMetadataEnumeration = false;
|
|
if (auto v =
|
|
jsonGetInt64AnyKey(searchStats, {"phase_candidate_discovery_ms"})) {
|
|
metadataEnumerationMs += *v;
|
|
hasMetadataEnumeration = true;
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"phase_path_filtering_ms"})) {
|
|
metadataEnumerationMs += *v;
|
|
hasMetadataEnumeration = true;
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats,
|
|
{"phase_candidate_classification_ms"})) {
|
|
metadataEnumerationMs += *v;
|
|
hasMetadataEnumeration = true;
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats,
|
|
{"phase_metadata_batch_fetch_ms"})) {
|
|
metadataEnumerationMs += *v;
|
|
hasMetadataEnumeration = true;
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"metadata_enumeration_ms"})) {
|
|
metadataEnumerationMs = *v;
|
|
hasMetadataEnumeration = true;
|
|
}
|
|
if (hasMetadataEnumeration) {
|
|
grepMetadataEnumerationMsSamples.push_back(metadataEnumerationMs);
|
|
}
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"regex_search_calls"})) {
|
|
grepRegexSearchCallsSamples.push_back(*v);
|
|
}
|
|
if (auto v = jsonGetInt64AnyKey(searchStats, {"bmh_prefilter_skips"})) {
|
|
grepBmhPrefilterSkipsSamples.push_back(*v);
|
|
}
|
|
grepOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
grepFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- Status/Get/Cat threads ---
|
|
for (int g = 0; g < kStatusGetClients; ++g) {
|
|
int tid = threadIdx++;
|
|
perThreadTraces[tid].reserve(kOpsPerClient);
|
|
threads.emplace_back([&, tid, g]() {
|
|
std::optional<DaemonClient> client;
|
|
if (!useMcpPath) {
|
|
auto ccfg = makeClientConfig();
|
|
client.emplace(ccfg);
|
|
}
|
|
std::mt19937 rng(static_cast<unsigned>(g * 997 + 17));
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < kOpsPerClient; ++i) {
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
if (extensionLikeProfile) {
|
|
const int slot = (i + g) % 20;
|
|
const bool doStatus = (slot == 0) || knownHashes.empty();
|
|
const bool doSearch = !doStatus && (slot >= 1 && slot <= 4);
|
|
const bool doList = !doStatus && !doSearch && (slot >= 5 && slot <= 10);
|
|
const bool doCat = !doStatus && !doSearch && !doList && (slot == 19);
|
|
|
|
if (doStatus) {
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(static_cast<size_t>(tid), "status",
|
|
nlohmann::json::object());
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->status(), kOpTimeout);
|
|
if (direct) {
|
|
res = nlohmann::json::object();
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "status";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
trace.success = true;
|
|
statusOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
statusFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
continue;
|
|
}
|
|
|
|
if (doSearch) {
|
|
SearchRequest req;
|
|
req.query = corpusQueries[(g * kOpsPerClient + i) % 10];
|
|
req.limit = 5;
|
|
req.searchType = cfg.searchType;
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(static_cast<size_t>(tid), "search",
|
|
{{"query", req.query},
|
|
{"limit", req.limit},
|
|
{"type", req.searchType}});
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->search(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["results"] = nlohmann::json::array();
|
|
for (const auto& r : direct.value().results) {
|
|
j["results"].push_back({{"id", r.id}});
|
|
}
|
|
j["execution_time_ms"] = direct.value().elapsed.count();
|
|
j["query_info"] = direct.value().queryInfo;
|
|
j["search_stats"] = direct.value().searchStats;
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "search";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
const auto& payload = res.value();
|
|
trace.success = true;
|
|
if (payload.contains("results") && payload["results"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["results"].size());
|
|
}
|
|
collectSearchServerTelemetry(payload);
|
|
searchOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
searchFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
continue;
|
|
}
|
|
|
|
if (doList) {
|
|
ListRequest req;
|
|
req.limit = ((i % 2) == 0) ? 10 : 25;
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res =
|
|
queryMcpStepForSlot(static_cast<size_t>(tid), "list",
|
|
{{"limit", req.limit}, {"use_session", false}});
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->list(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["items"] = nlohmann::json::array();
|
|
for (const auto& it : direct.value().items) {
|
|
j["items"].push_back({{"hash", it.hash}});
|
|
}
|
|
j["query_info"] = direct.value().queryInfo;
|
|
j["list_stats"] = direct.value().listStats;
|
|
res = j;
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "list";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
const auto& payload = res.value();
|
|
trace.success = true;
|
|
if (payload.contains("items") && payload["items"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["items"].size());
|
|
} else if (payload.contains("documents") &&
|
|
payload["documents"].is_array()) {
|
|
trace.resultCount = static_cast<int>(payload["documents"].size());
|
|
}
|
|
collectListServerTelemetry(payload);
|
|
listOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
listFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
continue;
|
|
}
|
|
|
|
auto runGetOrCat = [&](const std::string& hash,
|
|
bool requestContent) -> yams::Result<nlohmann::json> {
|
|
if (requestContent) {
|
|
if (useMcpPath) {
|
|
return queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "get",
|
|
{{"hash", hash}, {"include_content", true}});
|
|
}
|
|
CatRequest req;
|
|
req.hash = hash;
|
|
auto direct = yams::cli::run_sync(client->cat(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["has_content"] = direct.value().hasContent;
|
|
j["size"] = direct.value().size;
|
|
return j;
|
|
}
|
|
return yams::Error{direct.error()};
|
|
}
|
|
|
|
if (useMcpPath) {
|
|
return queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "get",
|
|
{{"hash", hash}, {"include_content", false}});
|
|
}
|
|
GetRequest req;
|
|
req.hash = hash;
|
|
req.metadataOnly = true;
|
|
auto direct = yams::cli::run_sync(client->get(req), kOpTimeout);
|
|
if (direct) {
|
|
return nlohmann::json::object();
|
|
}
|
|
return yams::Error{direct.error()};
|
|
};
|
|
|
|
const bool requestContent = doCat;
|
|
yams::Result<nlohmann::json> res =
|
|
yams::Error{yams::ErrorCode::NotFound, "No hash candidates available"};
|
|
constexpr int kHashRetryAttempts = 2;
|
|
for (int attempt = 0; attempt < kHashRetryAttempts; ++attempt) {
|
|
auto hashOpt = sampleKnownHash(rng);
|
|
if (!hashOpt.has_value()) {
|
|
break;
|
|
}
|
|
const auto& hash = *hashOpt;
|
|
res = runGetOrCat(hash, requestContent);
|
|
if (res) {
|
|
break;
|
|
}
|
|
if (isDocumentMissingFailure(res.error().message)) {
|
|
markHashInvalid(hash);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = doCat ? "cat" : "get";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
trace.success = true;
|
|
if (doCat) {
|
|
if (res.value().contains("size") && res.value()["size"].is_number()) {
|
|
trace.resultCount =
|
|
static_cast<int>(res.value()["size"].get<uint64_t>());
|
|
}
|
|
catOps.fetch_add(1);
|
|
} else {
|
|
getOps.fetch_add(1);
|
|
}
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
if (doCat) {
|
|
catFails.fetch_add(1);
|
|
} else {
|
|
getFails.fetch_add(1);
|
|
}
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
totalCompleted.fetch_add(1);
|
|
continue;
|
|
}
|
|
|
|
bool isStatus = (i % kStatusEveryN == 0) || knownHashes.empty();
|
|
|
|
if (isStatus) {
|
|
yams::Result<nlohmann::json> res;
|
|
if (useMcpPath) {
|
|
res = queryMcpStepForSlot(static_cast<size_t>(tid), "status",
|
|
nlohmann::json::object());
|
|
} else {
|
|
auto direct = yams::cli::run_sync(client->status(), kOpTimeout);
|
|
if (direct) {
|
|
res = nlohmann::json::object();
|
|
} else {
|
|
res = yams::Error{direct.error()};
|
|
}
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = "status";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
trace.success = true;
|
|
statusOps.fetch_add(1);
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
statusFails.fetch_add(1);
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
} else {
|
|
bool doCat = (i % kCatEveryN == 0);
|
|
auto runGetOrCat = [&](const std::string& hash,
|
|
bool requestContent) -> yams::Result<nlohmann::json> {
|
|
if (requestContent) {
|
|
if (useMcpPath) {
|
|
return queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "get",
|
|
{{"hash", hash}, {"include_content", true}});
|
|
}
|
|
CatRequest req;
|
|
req.hash = hash;
|
|
auto direct = yams::cli::run_sync(client->cat(req), kOpTimeout);
|
|
if (direct) {
|
|
nlohmann::json j;
|
|
j["has_content"] = direct.value().hasContent;
|
|
j["size"] = direct.value().size;
|
|
return j;
|
|
}
|
|
return yams::Error{direct.error()};
|
|
}
|
|
|
|
if (useMcpPath) {
|
|
return queryMcpStepForSlot(
|
|
static_cast<size_t>(tid), "get",
|
|
{{"hash", hash}, {"include_content", false}});
|
|
}
|
|
GetRequest req;
|
|
req.hash = hash;
|
|
req.metadataOnly = true;
|
|
auto direct = yams::cli::run_sync(client->get(req), kOpTimeout);
|
|
if (direct) {
|
|
return nlohmann::json::object();
|
|
}
|
|
return yams::Error{direct.error()};
|
|
};
|
|
|
|
const bool requestContent = doCat;
|
|
yams::Result<nlohmann::json> res =
|
|
yams::Error{yams::ErrorCode::NotFound, "No hash candidates available"};
|
|
constexpr int kHashRetryAttempts = 2;
|
|
for (int attempt = 0; attempt < kHashRetryAttempts; ++attempt) {
|
|
auto hashOpt = sampleKnownHash(rng);
|
|
if (!hashOpt.has_value()) {
|
|
break;
|
|
}
|
|
const auto& hash = *hashOpt;
|
|
res = runGetOrCat(hash, requestContent);
|
|
if (res) {
|
|
break;
|
|
}
|
|
if (isDocumentMissingFailure(res.error().message)) {
|
|
markHashInvalid(hash);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
auto wallMs =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - globalStart)
|
|
.count();
|
|
|
|
OpTrace trace;
|
|
trace.opType = doCat ? "cat" : "get";
|
|
trace.threadId = tid;
|
|
trace.opIndex = i;
|
|
trace.latencyUs = latUs;
|
|
trace.wallClockMs = wallMs;
|
|
if (res) {
|
|
trace.success = true;
|
|
if (doCat) {
|
|
if (res.value().contains("size") && res.value()["size"].is_number()) {
|
|
trace.resultCount =
|
|
static_cast<int>(res.value()["size"].get<uint64_t>());
|
|
}
|
|
catOps.fetch_add(1);
|
|
} else {
|
|
getOps.fetch_add(1);
|
|
}
|
|
} else {
|
|
trace.success = false;
|
|
trace.errorMsg = res.error().message;
|
|
if (doCat) {
|
|
catFails.fetch_add(1);
|
|
} else {
|
|
getFails.fetch_add(1);
|
|
}
|
|
}
|
|
perThreadTraces[tid].push_back(std::move(trace));
|
|
}
|
|
totalCompleted.fetch_add(1);
|
|
}
|
|
});
|
|
}
|
|
|
|
go.store(true);
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
progressStop.store(true);
|
|
progressThread.join();
|
|
sampler.stop();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// Analysis
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
|
|
// Aggregate latency vectors from per-thread traces
|
|
std::vector<int64_t> allSearchLat, allListLat, allGrepLat, allStatusLat, allGetLat, allCatLat;
|
|
std::vector<OpTrace> allTraces;
|
|
std::map<std::string, int> errorCategories;
|
|
|
|
for (auto& threadTraces : perThreadTraces) {
|
|
for (auto& tr : threadTraces) {
|
|
allTraces.push_back(tr);
|
|
if (tr.success) {
|
|
if (tr.opType == "search")
|
|
pushMetricSample(allSearchLat, tr.latencyUs);
|
|
else if (tr.opType == "list")
|
|
pushMetricSample(allListLat, tr.latencyUs);
|
|
else if (tr.opType == "grep")
|
|
pushMetricSample(allGrepLat, tr.latencyUs);
|
|
else if (tr.opType == "status")
|
|
pushMetricSample(allStatusLat, tr.latencyUs);
|
|
else if (tr.opType == "get")
|
|
pushMetricSample(allGetLat, tr.latencyUs);
|
|
else if (tr.opType == "cat")
|
|
pushMetricSample(allCatLat, tr.latencyUs);
|
|
} else {
|
|
errorCategories[classifyFailureMessage(tr.errorMsg)]++;
|
|
}
|
|
}
|
|
}
|
|
|
|
auto searchStats = PercentileStats::compute(allSearchLat);
|
|
auto listStats = PercentileStats::compute(allListLat);
|
|
auto grepStats = PercentileStats::compute(allGrepLat);
|
|
auto statusStats = PercentileStats::compute(allStatusLat);
|
|
auto getStats = PercentileStats::compute(allGetLat);
|
|
auto catStats = PercentileStats::compute(allCatLat);
|
|
|
|
auto summarizeSamples = [](const std::vector<int64_t>& values) {
|
|
json summary{{"count", values.size()},
|
|
{"min", 0},
|
|
{"p50", 0},
|
|
{"p90", 0},
|
|
{"p95", 0},
|
|
{"p99", 0},
|
|
{"max", 0},
|
|
{"mean", 0.0}};
|
|
if (values.empty()) {
|
|
return summary;
|
|
}
|
|
auto sortedValues = values;
|
|
std::sort(sortedValues.begin(), sortedValues.end());
|
|
const auto atPct = [&](double pct) {
|
|
const auto idx =
|
|
std::min(sortedValues.size() - 1,
|
|
static_cast<size_t>(pct * static_cast<double>(sortedValues.size())));
|
|
return sortedValues[idx];
|
|
};
|
|
sortedValues.erase(std::remove_if(sortedValues.begin(), sortedValues.end(),
|
|
[](int64_t value) { return value < 0; }),
|
|
sortedValues.end());
|
|
for (auto& value : sortedValues) {
|
|
value = clampMetricSample(value);
|
|
}
|
|
if (sortedValues.empty()) {
|
|
return summary;
|
|
}
|
|
std::sort(sortedValues.begin(), sortedValues.end());
|
|
const long double total = std::accumulate(
|
|
sortedValues.begin(), sortedValues.end(), 0.0L,
|
|
[](long double sum, int64_t value) { return sum + static_cast<long double>(value); });
|
|
const auto mean =
|
|
static_cast<double>(total / static_cast<long double>(sortedValues.size()));
|
|
summary["min"] = sortedValues.front();
|
|
summary["p50"] = atPct(0.50);
|
|
summary["p90"] = atPct(0.90);
|
|
summary["p95"] = atPct(0.95);
|
|
summary["p99"] = atPct(0.99);
|
|
summary["max"] = sortedValues.back();
|
|
summary["mean"] = mean;
|
|
return summary;
|
|
};
|
|
|
|
const json grepExecStats = summarizeSamples(grepExecMsSamples);
|
|
const json grepDocsScannedStats = summarizeSamples(grepDocsScannedSamples);
|
|
const json grepLinesScannedStats = summarizeSamples(grepLinesScannedSamples);
|
|
const json grepBytesScannedStats = summarizeSamples(grepBytesScannedSamples);
|
|
const json grepContentRetrievalStats = summarizeSamples(grepContentRetrievalMsSamples);
|
|
const json grepMatchRenderStats = summarizeSamples(grepMatchRenderUsSamples);
|
|
const json grepWorkerScanStats = summarizeSamples(grepWorkerScanMsSamples);
|
|
const json grepCandidateDiscoveryStats = summarizeSamples(grepCandidateDiscoveryMsSamples);
|
|
const json grepPathFilteringStats = summarizeSamples(grepPathFilteringMsSamples);
|
|
const json grepCandidateClassificationStats =
|
|
summarizeSamples(grepCandidateClassificationMsSamples);
|
|
const json grepMetadataBatchFetchStats = summarizeSamples(grepMetadataBatchFetchMsSamples);
|
|
const json grepMetadataEnumerationStats = summarizeSamples(grepMetadataEnumerationMsSamples);
|
|
const json grepRegexScanStats = summarizeSamples(grepRegexScanMsSamples);
|
|
const json grepContentDecodeEstimateStats =
|
|
summarizeSamples(grepContentDecodeEstimateMsSamples);
|
|
const json grepWorkerCriticalTotalStats = summarizeSamples(grepWorkerCriticalTotalMsSamples);
|
|
const json grepWorkerCriticalRetrievalStats =
|
|
summarizeSamples(grepWorkerCriticalRetrievalMsSamples);
|
|
const json grepWorkerCriticalRegexStats = summarizeSamples(grepWorkerCriticalRegexMsSamples);
|
|
const json grepWorkerCriticalOtherStats = summarizeSamples(grepWorkerCriticalOtherMsSamples);
|
|
const json grepRegexSearchCallsStats = summarizeSamples(grepRegexSearchCallsSamples);
|
|
const json grepBmhPrefilterSkipsStats = summarizeSamples(grepBmhPrefilterSkipsSamples);
|
|
const json searchExecStats = summarizeSamples(searchExecMsSamples);
|
|
const json searchDispatchServiceStats = summarizeSamples(searchDispatchServiceMsSamples);
|
|
const json searchDispatchServiceUsStats = summarizeSamples(searchDispatchServiceUsSamples);
|
|
const json searchDispatchMapSortStats = summarizeSamples(searchDispatchMapSortMsSamples);
|
|
const json searchDispatchMapSortUsStats = summarizeSamples(searchDispatchMapSortUsSamples);
|
|
const json searchDispatchFeedbackStats = summarizeSamples(searchDispatchFeedbackMsSamples);
|
|
const json searchDispatchFeedbackUsStats = summarizeSamples(searchDispatchFeedbackUsSamples);
|
|
const json searchDispatchResponseStats = summarizeSamples(searchDispatchResponseMsSamples);
|
|
const json searchDispatchResponseUsStats = summarizeSamples(searchDispatchResponseUsSamples);
|
|
const json searchDispatchTotalStats = summarizeSamples(searchDispatchTotalMsSamples);
|
|
const json searchDispatchTotalUsStats = summarizeSamples(searchDispatchTotalUsSamples);
|
|
const json searchTimingEmbeddingUsStats = summarizeSamples(searchTimingEmbeddingUsSamples);
|
|
const json searchTimingTextUsStats = summarizeSamples(searchTimingTextUsSamples);
|
|
const json searchTimingPathUsStats = summarizeSamples(searchTimingPathUsSamples);
|
|
const json searchTimingVectorUsStats = summarizeSamples(searchTimingVectorUsSamples);
|
|
const json searchTimingEntityVectorUsStats =
|
|
summarizeSamples(searchTimingEntityVectorUsSamples);
|
|
const json searchTimingCompressedAnnUsStats =
|
|
summarizeSamples(searchTimingCompressedAnnUsSamples);
|
|
const json searchTimingKgUsStats = summarizeSamples(searchTimingKgUsSamples);
|
|
const json searchTimingMetadataUsStats = summarizeSamples(searchTimingMetadataUsSamples);
|
|
const json searchTimingTagUsStats = summarizeSamples(searchTimingTagUsSamples);
|
|
const json searchTimingMultiVectorUsStats = summarizeSamples(searchTimingMultiVectorUsSamples);
|
|
const json searchTimingConceptsUsStats = summarizeSamples(searchTimingConceptsUsSamples);
|
|
const json searchTimingRerankerUsStats = summarizeSamples(searchTimingRerankerUsSamples);
|
|
const json searchTimingGraphRerankUsStats = summarizeSamples(searchTimingGraphRerankUsSamples);
|
|
const json searchSubphraseFtsHitStats = summarizeSamples(searchSubphraseFtsHitSamples);
|
|
const json searchAggressiveFtsHitStats = summarizeSamples(searchAggressiveFtsHitSamples);
|
|
const json searchGraphExpansionFtsHitStats =
|
|
summarizeSamples(searchGraphExpansionFtsHitSamples);
|
|
const json searchGraphDocExpansionFtsHitStats =
|
|
summarizeSamples(searchGraphDocExpansionFtsHitSamples);
|
|
const json searchMultiVectorRawHitStats = summarizeSamples(searchMultiVectorRawHitSamples);
|
|
const json searchGraphVectorRawHitStats = summarizeSamples(searchGraphVectorRawHitSamples);
|
|
const json searchGraphQueryNeighborSeedDocStats =
|
|
summarizeSamples(searchGraphQueryNeighborSeedDocSamples);
|
|
const json listServiceExecStats = summarizeSamples(listServiceExecMsSamples);
|
|
const json listDispatchServiceStats = summarizeSamples(listDispatchServiceMsSamples);
|
|
const json listDispatchServiceUsStats = summarizeSamples(listDispatchServiceUsSamples);
|
|
const json listDispatchMapStats = summarizeSamples(listDispatchMapMsSamples);
|
|
const json listDispatchMapUsStats = summarizeSamples(listDispatchMapUsSamples);
|
|
const json listDispatchTotalStats = summarizeSamples(listDispatchTotalMsSamples);
|
|
const json listDispatchTotalUsStats = summarizeSamples(listDispatchTotalUsSamples);
|
|
|
|
int totalOps = searchOps.load() + listOps.load() + grepOps.load() + statusOps.load() +
|
|
getOps.load() + catOps.load();
|
|
int totalFails = searchFails.load() + listFails.load() + grepFails.load() + statusFails.load() +
|
|
getFails.load() + catFails.load();
|
|
double opsPerSec = safeRate(static_cast<double>(totalOps), globalElapsed);
|
|
size_t invalidHashCount = 0;
|
|
{
|
|
std::lock_guard<std::mutex> lk(invalidHashMutex);
|
|
invalidHashCount = invalidHashes.size();
|
|
}
|
|
|
|
// Memory watermarks from time series
|
|
auto samples = sampler.getSamples();
|
|
double peakRss = 0.0, minRss = std::numeric_limits<double>::max();
|
|
for (auto& s : samples) {
|
|
peakRss = std::max(peakRss, s.snap.memoryUsageMb);
|
|
minRss = std::min(minRss, s.snap.memoryUsageMb);
|
|
}
|
|
if (samples.empty())
|
|
minRss = 0.0;
|
|
|
|
std::cout << "\n=== Large Corpus Results (instrumented) ===\n";
|
|
std::cout << " Corpus: " << finalSnap.documentsTotal << " docs\n";
|
|
std::cout << " Layout: " << kSearchClients << " search + " << kListClients
|
|
<< " list + " << kGrepClients << " grep + " << kStatusGetClients
|
|
<< " status/get/cat\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed
|
|
<< "s\n";
|
|
std::cout << " Total ops: " << totalOps << " (" << std::setprecision(1) << opsPerSec
|
|
<< " ops/s)\n";
|
|
std::cout << " Total failures: " << totalFails << "\n\n";
|
|
if (invalidHashCount > 0) {
|
|
std::cout << " Invalid hash candidates quarantined: " << invalidHashCount << "\n\n";
|
|
}
|
|
|
|
std::cout << " Op breakdown:\n";
|
|
std::cout << " Search: " << searchOps.load() << " ok, " << searchFails.load() << " fail\n";
|
|
std::cout << " List: " << listOps.load() << " ok, " << listFails.load() << " fail\n";
|
|
std::cout << " Grep: " << grepOps.load() << " ok, " << grepFails.load() << " fail\n";
|
|
std::cout << " Status: " << statusOps.load() << " ok, " << statusFails.load() << " fail\n";
|
|
std::cout << " Get: " << getOps.load() << " ok, " << getFails.load() << " fail\n\n";
|
|
std::cout << " Cat: " << catOps.load() << " ok, " << catFails.load() << " fail\n\n";
|
|
|
|
printPercentiles("Search latency", searchStats);
|
|
printPercentiles("List latency ", listStats);
|
|
printPercentiles("Grep latency ", grepStats);
|
|
printPercentiles("Status latency", statusStats);
|
|
printPercentiles("Get latency ", getStats);
|
|
printPercentiles("Cat latency ", catStats);
|
|
|
|
if (grepExecStats.value("count", 0) > 0 || grepWorkerScanStats.value("count", 0) > 0) {
|
|
std::cout << "\n Grep server telemetry:\n";
|
|
std::cout << " execution_ms: mean=" << grepExecStats.value("mean", 0.0)
|
|
<< " p95=" << grepExecStats.value("p95", 0)
|
|
<< " max=" << grepExecStats.value("max", 0) << "\n";
|
|
std::cout << " metadata_enumeration_ms: mean="
|
|
<< grepMetadataEnumerationStats.value("mean", 0.0)
|
|
<< " p95=" << grepMetadataEnumerationStats.value("p95", 0)
|
|
<< " max=" << grepMetadataEnumerationStats.value("max", 0) << "\n";
|
|
std::cout << " candidate_discovery_ms: mean="
|
|
<< grepCandidateDiscoveryStats.value("mean", 0.0)
|
|
<< " p95=" << grepCandidateDiscoveryStats.value("p95", 0) << "\n";
|
|
std::cout << " path_filtering_ms: mean=" << grepPathFilteringStats.value("mean", 0.0)
|
|
<< " p95=" << grepPathFilteringStats.value("p95", 0) << "\n";
|
|
std::cout << " candidate_classification_ms: mean="
|
|
<< grepCandidateClassificationStats.value("mean", 0.0)
|
|
<< " p95=" << grepCandidateClassificationStats.value("p95", 0) << "\n";
|
|
std::cout << " metadata_batch_fetch_ms: mean="
|
|
<< grepMetadataBatchFetchStats.value("mean", 0.0)
|
|
<< " p95=" << grepMetadataBatchFetchStats.value("p95", 0) << "\n";
|
|
std::cout << " worker_scan_ms: mean=" << grepWorkerScanStats.value("mean", 0.0)
|
|
<< " p95=" << grepWorkerScanStats.value("p95", 0)
|
|
<< " max=" << grepWorkerScanStats.value("max", 0) << "\n";
|
|
std::cout << " content_retrieval_ms: mean="
|
|
<< grepContentRetrievalStats.value("mean", 0.0)
|
|
<< " p95=" << grepContentRetrievalStats.value("p95", 0)
|
|
<< " max=" << grepContentRetrievalStats.value("max", 0) << "\n";
|
|
std::cout << " match_render_us: mean=" << grepMatchRenderStats.value("mean", 0.0)
|
|
<< " p95=" << grepMatchRenderStats.value("p95", 0)
|
|
<< " max=" << grepMatchRenderStats.value("max", 0) << "\n";
|
|
std::cout << " content_decode_ms_estimate: mean="
|
|
<< grepContentDecodeEstimateStats.value("mean", 0.0)
|
|
<< " p95=" << grepContentDecodeEstimateStats.value("p95", 0)
|
|
<< " max=" << grepContentDecodeEstimateStats.value("max", 0) << "\n";
|
|
std::cout << " regex_scan_ms: mean=" << grepRegexScanStats.value("mean", 0.0)
|
|
<< " p95=" << grepRegexScanStats.value("p95", 0)
|
|
<< " max=" << grepRegexScanStats.value("max", 0) << "\n";
|
|
std::cout << " worker_critical_total_ms: mean="
|
|
<< grepWorkerCriticalTotalStats.value("mean", 0.0)
|
|
<< " p95=" << grepWorkerCriticalTotalStats.value("p95", 0)
|
|
<< " max=" << grepWorkerCriticalTotalStats.value("max", 0) << "\n";
|
|
std::cout << " worker_critical_content_retrieval_ms: mean="
|
|
<< grepWorkerCriticalRetrievalStats.value("mean", 0.0)
|
|
<< " p95=" << grepWorkerCriticalRetrievalStats.value("p95", 0)
|
|
<< " max=" << grepWorkerCriticalRetrievalStats.value("max", 0) << "\n";
|
|
std::cout << " worker_critical_regex_scan_ms: mean="
|
|
<< grepWorkerCriticalRegexStats.value("mean", 0.0)
|
|
<< " p95=" << grepWorkerCriticalRegexStats.value("p95", 0)
|
|
<< " max=" << grepWorkerCriticalRegexStats.value("max", 0) << "\n";
|
|
std::cout << " worker_critical_other_ms: mean="
|
|
<< grepWorkerCriticalOtherStats.value("mean", 0.0)
|
|
<< " p95=" << grepWorkerCriticalOtherStats.value("p95", 0)
|
|
<< " max=" << grepWorkerCriticalOtherStats.value("max", 0) << "\n";
|
|
}
|
|
|
|
if (searchDispatchTotalStats.value("count", 0) > 0 ||
|
|
listDispatchTotalStats.value("count", 0) > 0) {
|
|
std::cout << "\n Search/list server telemetry:\n";
|
|
std::cout << " search_total_ms: mean=" << searchDispatchTotalStats.value("mean", 0.0)
|
|
<< " p95=" << searchDispatchTotalStats.value("p95", 0)
|
|
<< " max=" << searchDispatchTotalStats.value("max", 0) << "\n";
|
|
std::cout << " list_total_ms: mean=" << listDispatchTotalStats.value("mean", 0.0)
|
|
<< " p95=" << listDispatchTotalStats.value("p95", 0)
|
|
<< " max=" << listDispatchTotalStats.value("max", 0) << "\n";
|
|
auto printSearchComponentUs = [&](const char* label, const json& stats) {
|
|
if (stats.value("count", 0) <= 0) {
|
|
return;
|
|
}
|
|
std::cout << " " << label << ": mean=" << (stats.value("mean", 0.0) / 1000.0)
|
|
<< "ms p95=" << (static_cast<double>(stats.value("p95", 0)) / 1000.0)
|
|
<< "ms\n";
|
|
};
|
|
printSearchComponentUs("timing_embedding", searchTimingEmbeddingUsStats);
|
|
printSearchComponentUs("timing_text", searchTimingTextUsStats);
|
|
printSearchComponentUs("timing_path", searchTimingPathUsStats);
|
|
printSearchComponentUs("timing_vector", searchTimingVectorUsStats);
|
|
printSearchComponentUs("timing_entity_vector", searchTimingEntityVectorUsStats);
|
|
printSearchComponentUs("timing_compressed_ann", searchTimingCompressedAnnUsStats);
|
|
printSearchComponentUs("timing_kg", searchTimingKgUsStats);
|
|
printSearchComponentUs("timing_metadata", searchTimingMetadataUsStats);
|
|
printSearchComponentUs("timing_tag", searchTimingTagUsStats);
|
|
printSearchComponentUs("timing_multi_vector", searchTimingMultiVectorUsStats);
|
|
printSearchComponentUs("timing_concepts", searchTimingConceptsUsStats);
|
|
printSearchComponentUs("timing_reranker", searchTimingRerankerUsStats);
|
|
printSearchComponentUs("timing_graph_rerank", searchTimingGraphRerankUsStats);
|
|
if (searchGraphDocExpansionFtsHitStats.value("count", 0) > 0) {
|
|
std::cout << " graph_doc_expansion_fts_hit_count: mean="
|
|
<< searchGraphDocExpansionFtsHitStats.value("mean", 0.0)
|
|
<< " p95=" << searchGraphDocExpansionFtsHitStats.value("p95", 0) << "\n";
|
|
}
|
|
if (searchMultiVectorRawHitStats.value("count", 0) > 0) {
|
|
std::cout << " multi_vector_raw_hit_count: mean="
|
|
<< searchMultiVectorRawHitStats.value("mean", 0.0)
|
|
<< " p95=" << searchMultiVectorRawHitStats.value("p95", 0) << "\n";
|
|
}
|
|
if (searchGraphVectorRawHitStats.value("count", 0) > 0) {
|
|
std::cout << " graph_vector_raw_hit_count: mean="
|
|
<< searchGraphVectorRawHitStats.value("mean", 0.0)
|
|
<< " p95=" << searchGraphVectorRawHitStats.value("p95", 0) << "\n";
|
|
}
|
|
}
|
|
|
|
std::cout << "\n Memory:\n";
|
|
std::cout << " Initial: " << std::setprecision(0) << initSnap.memoryUsageMb << " MB\n";
|
|
std::cout << " Peak: " << peakRss << " MB\n";
|
|
std::cout << " Min: " << minRss << " MB\n";
|
|
std::cout << " Final: " << finalSnap.memoryUsageMb << " MB\n";
|
|
std::cout << " Delta: " << (finalSnap.memoryUsageMb - initSnap.memoryUsageMb) << " MB\n\n";
|
|
|
|
if (!errorCategories.empty()) {
|
|
std::cout << " Error categories:\n";
|
|
for (auto& [cat, count] : errorCategories) {
|
|
std::cout << " " << cat << ": " << count << "\n";
|
|
}
|
|
std::cout << "\n";
|
|
}
|
|
|
|
// Hotspot analysis: find slowest ops
|
|
std::sort(allTraces.begin(), allTraces.end(),
|
|
[](const OpTrace& a, const OpTrace& b) { return a.latencyUs > b.latencyUs; });
|
|
|
|
std::cout << " Top 10 slowest operations:\n";
|
|
for (int i = 0; i < 10 && i < static_cast<int>(allTraces.size()); ++i) {
|
|
auto& tr = allTraces[i];
|
|
std::cout << " " << (i + 1) << ". " << tr.opType << " t" << tr.threadId << "#"
|
|
<< tr.opIndex << ": " << tr.latencyUs / 1000 << "ms"
|
|
<< (tr.success ? "" : " FAIL:" + tr.errorMsg) << " @" << tr.wallClockMs << "ms\n";
|
|
}
|
|
std::cout << "\n";
|
|
|
|
// Thread-level hotspots
|
|
std::map<int, std::vector<int64_t>> perThreadSuccessLatUs;
|
|
std::map<int, int> perThreadFails;
|
|
for (const auto& tr : allTraces) {
|
|
if (tr.success) {
|
|
pushMetricSample(perThreadSuccessLatUs[tr.threadId], tr.latencyUs);
|
|
} else {
|
|
perThreadFails[tr.threadId]++;
|
|
}
|
|
}
|
|
struct ThreadHotspot {
|
|
int threadId{0};
|
|
int64_t p95Us{0};
|
|
double meanUs{0.0};
|
|
int failures{0};
|
|
size_t okCount{0};
|
|
};
|
|
std::vector<ThreadHotspot> threadHotspots;
|
|
threadHotspots.reserve(static_cast<size_t>(kTotalClients));
|
|
for (int tid = 0; tid < kTotalClients; ++tid) {
|
|
auto it = perThreadSuccessLatUs.find(tid);
|
|
std::vector<int64_t> lats =
|
|
(it != perThreadSuccessLatUs.end()) ? it->second : std::vector<int64_t>{};
|
|
auto stats = PercentileStats::compute(lats);
|
|
threadHotspots.push_back(
|
|
ThreadHotspot{tid, stats.p95, stats.mean, perThreadFails[tid], stats.count});
|
|
}
|
|
std::sort(threadHotspots.begin(), threadHotspots.end(),
|
|
[](const ThreadHotspot& a, const ThreadHotspot& b) {
|
|
if (a.p95Us != b.p95Us) {
|
|
return a.p95Us > b.p95Us;
|
|
}
|
|
return a.failures > b.failures;
|
|
});
|
|
|
|
std::cout << " Slowest threads (p95):\n";
|
|
for (int i = 0; i < 5 && i < static_cast<int>(threadHotspots.size()); ++i) {
|
|
const auto& th = threadHotspots[i];
|
|
std::cout << " t" << th.threadId << ": p95=" << th.p95Us / 1000
|
|
<< "ms mean=" << std::fixed << std::setprecision(1) << (th.meanUs / 1000.0)
|
|
<< "ms ok=" << th.okCount << " fail=" << th.failures << "\n";
|
|
}
|
|
std::cout << "\n";
|
|
|
|
// Time-window hotspots
|
|
struct WindowStats {
|
|
int ops{0};
|
|
int fails{0};
|
|
int64_t totalLatUs{0};
|
|
};
|
|
std::map<int, WindowStats> windows;
|
|
for (const auto& tr : allTraces) {
|
|
const auto boundedWallMs = clampMetricSample(tr.wallClockMs, kMaxBenchmarkWallMs);
|
|
const int bucket = clampInt64ToInt(boundedWallMs / std::max(1, kHotspotWindowMs), 0,
|
|
std::numeric_limits<int>::max());
|
|
auto& ws = windows[bucket];
|
|
ws.ops++;
|
|
ws.totalLatUs = checkedMetricAdd(ws.totalLatUs, tr.latencyUs);
|
|
if (!tr.success) {
|
|
ws.fails++;
|
|
}
|
|
}
|
|
struct WindowHotspot {
|
|
int bucket{0};
|
|
double avgMs{0.0};
|
|
int ops{0};
|
|
int fails{0};
|
|
};
|
|
std::vector<WindowHotspot> windowHotspots;
|
|
windowHotspots.reserve(windows.size());
|
|
for (const auto& [bucket, ws] : windows) {
|
|
if (!isHotspotWindowEligible(ws.ops, kTotalClients)) {
|
|
continue;
|
|
}
|
|
const double avgMs =
|
|
(ws.ops > 0) ? (static_cast<double>(ws.totalLatUs) / ws.ops / 1000.0) : 0.0;
|
|
windowHotspots.push_back(WindowHotspot{bucket, avgMs, ws.ops, ws.fails});
|
|
}
|
|
std::sort(windowHotspots.begin(), windowHotspots.end(),
|
|
[](const WindowHotspot& a, const WindowHotspot& b) {
|
|
if (a.avgMs != b.avgMs) {
|
|
return a.avgMs > b.avgMs;
|
|
}
|
|
return a.fails > b.fails;
|
|
});
|
|
|
|
std::cout << " Slowest time windows:\n";
|
|
for (int i = 0; i < 5 && i < static_cast<int>(windowHotspots.size()); ++i) {
|
|
const auto& hw = windowHotspots[i];
|
|
const int64_t startMs = checkedMetricMul(hw.bucket, kHotspotWindowMs, kMaxBenchmarkWallMs);
|
|
const int64_t endMs = checkedMetricAdd(startMs, kHotspotWindowMs, kMaxBenchmarkWallMs);
|
|
std::cout << " [" << startMs << ", " << endMs << ")ms"
|
|
<< " avg=" << std::fixed << std::setprecision(1) << hw.avgMs
|
|
<< "ms ops=" << hw.ops << " fail=" << hw.fails << "\n";
|
|
}
|
|
std::cout << "\n";
|
|
|
|
// Latency distribution (histogram buckets)
|
|
auto printHistogram = [](const std::string& label, std::vector<int64_t>& lats) {
|
|
if (lats.empty())
|
|
return;
|
|
std::sort(lats.begin(), lats.end());
|
|
// Buckets: <1ms, 1-10ms, 10-100ms, 100ms-1s, 1-10s, >10s
|
|
int b0 = 0, b1 = 0, b2 = 0, b3 = 0, b4 = 0, b5 = 0;
|
|
for (auto us : lats) {
|
|
if (us < 1000)
|
|
b0++;
|
|
else if (us < 10000)
|
|
b1++;
|
|
else if (us < 100000)
|
|
b2++;
|
|
else if (us < 1000000)
|
|
b3++;
|
|
else if (us < 10000000)
|
|
b4++;
|
|
else
|
|
b5++;
|
|
}
|
|
std::cout << " " << label << " histogram: <1ms=" << b0 << " 1-10ms=" << b1
|
|
<< " 10-100ms=" << b2 << " 100ms-1s=" << b3 << " 1-10s=" << b4 << " >10s=" << b5
|
|
<< "\n";
|
|
};
|
|
|
|
std::cout << " Latency distribution:\n";
|
|
printHistogram("Search", allSearchLat);
|
|
printHistogram("List ", allListLat);
|
|
printHistogram("Grep ", allGrepLat);
|
|
printHistogram("Status", allStatusLat);
|
|
printHistogram("Get ", allGetLat);
|
|
printHistogram("Cat ", allCatLat);
|
|
std::cout << "\n";
|
|
|
|
CHECK(totalOps > 0);
|
|
// Soft-fail on errors — we want the data even if some ops time out
|
|
if (totalFails > 0) {
|
|
WARN("Had " << totalFails << " failures out of " << totalExpected << " total ops");
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
// JSONL output with full traces
|
|
// ─────────────────────────────────────────────────────────────────────────
|
|
json timeSeries = json::array();
|
|
for (auto& s : samples) {
|
|
timeSeries.push_back(json{{"elapsed_ms", s.elapsedMs}, {"snapshot", s.snap.toJson()}});
|
|
}
|
|
|
|
// Per-op latency trace array (sorted by wall clock)
|
|
std::sort(allTraces.begin(), allTraces.end(),
|
|
[](const OpTrace& a, const OpTrace& b) { return a.wallClockMs < b.wallClockMs; });
|
|
json traceArray = json::array();
|
|
for (auto& tr : allTraces) {
|
|
json j{{"op", tr.opType}, {"tid", tr.threadId}, {"idx", tr.opIndex},
|
|
{"lat_us", tr.latencyUs}, {"ok", tr.success}, {"items", tr.resultCount},
|
|
{"wall_ms", tr.wallClockMs}};
|
|
if (!tr.errorMsg.empty())
|
|
j["error"] = tr.errorMsg;
|
|
traceArray.push_back(std::move(j));
|
|
}
|
|
|
|
json threadHotspotsJson = json::array();
|
|
for (const auto& th : threadHotspots) {
|
|
threadHotspotsJson.push_back(json{{"tid", th.threadId},
|
|
{"p95_us", th.p95Us},
|
|
{"mean_us", th.meanUs},
|
|
{"ok", th.okCount},
|
|
{"fail", th.failures}});
|
|
}
|
|
|
|
json windowHotspotsJson = json::array();
|
|
for (const auto& hw : windowHotspots) {
|
|
const int64_t startMs = static_cast<int64_t>(hw.bucket) * kHotspotWindowMs;
|
|
const int64_t endMs = startMs + kHotspotWindowMs;
|
|
windowHotspotsJson.push_back(json{{"start_ms", startMs},
|
|
{"end_ms", endMs},
|
|
{"avg_ms", hw.avgMs},
|
|
{"ops", hw.ops},
|
|
{"fail", hw.fails}});
|
|
}
|
|
|
|
const std::string recordTestName =
|
|
externalAgentChurnProfile ? "large_corpus_reads_churn" : "large_corpus_reads";
|
|
const double mcpTransientRatio = (mcpQueryCalls.load() > 0)
|
|
? static_cast<double>(mcpTransientSessionCalls.load()) /
|
|
static_cast<double>(mcpQueryCalls.load())
|
|
: 0.0;
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", recordTestName},
|
|
{"request_path", useMcpPath ? "mcp_query" : "daemon_ipc"},
|
|
{"backend", cfg.backend},
|
|
{"auth_mode", cfg.authMode},
|
|
{"data_dir", cfg.dataDir->string()},
|
|
{"num_clients", kTotalClients},
|
|
{"corpus_docs", finalSnap.documentsTotal},
|
|
{"layout", json{{"search_clients", kSearchClients},
|
|
{"list_clients", kListClients},
|
|
{"grep_clients", kGrepClients},
|
|
{"status_get_clients", kStatusGetClients},
|
|
{"ops_per_client", kOpsPerClient},
|
|
{"op_timeout_s", kOpTimeoutS},
|
|
{"status_every_n", kStatusEveryN},
|
|
{"cat_every_n", kCatEveryN},
|
|
{"list_inflight_cap", kListInflightCap},
|
|
{"mcp_session_churn_every_n", kMcpSessionChurnEveryN},
|
|
{"usage_profile", cfg.usageProfile}}},
|
|
{"total_ops", totalOps},
|
|
{"total_failures", totalFails},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"ops_per_sec", opsPerSec},
|
|
{"mcp", json{{"query_calls", mcpQueryCalls.load()},
|
|
{"transient_session_calls", mcpTransientSessionCalls.load()},
|
|
{"transient_session_ratio", mcpTransientRatio}}},
|
|
{"hash_pool",
|
|
json{{"known_hashes", knownHashes.size()}, {"invalid_quarantined", invalidHashCount}}},
|
|
{"search", json{{"ops", searchOps.load()},
|
|
{"fails", searchFails.load()},
|
|
{"latency", searchStats.toJson()}}},
|
|
{"list", json{{"ops", listOps.load()},
|
|
{"fails", listFails.load()},
|
|
{"latency", listStats.toJson()}}},
|
|
{"grep", json{{"ops", grepOps.load()},
|
|
{"fails", grepFails.load()},
|
|
{"latency", grepStats.toJson()}}},
|
|
{"status", json{{"ops", statusOps.load()},
|
|
{"fails", statusFails.load()},
|
|
{"latency", statusStats.toJson()}}},
|
|
{"get",
|
|
json{{"ops", getOps.load()}, {"fails", getFails.load()}, {"latency", getStats.toJson()}}},
|
|
{"cat",
|
|
json{{"ops", catOps.load()}, {"fails", catFails.load()}, {"latency", catStats.toJson()}}},
|
|
{"grep_server",
|
|
json{{"execution_time_ms", grepExecStats},
|
|
{"metadata_enumeration_ms", grepMetadataEnumerationStats},
|
|
{"phase_candidate_discovery_ms", grepCandidateDiscoveryStats},
|
|
{"phase_path_filtering_ms", grepPathFilteringStats},
|
|
{"phase_candidate_classification_ms", grepCandidateClassificationStats},
|
|
{"phase_metadata_batch_fetch_ms", grepMetadataBatchFetchStats},
|
|
{"docs_scanned", grepDocsScannedStats},
|
|
{"lines_scanned", grepLinesScannedStats},
|
|
{"bytes_scanned", grepBytesScannedStats},
|
|
{"content_retrieval_ms", grepContentRetrievalStats},
|
|
{"match_render_us", grepMatchRenderStats},
|
|
{"content_decode_ms_estimate", grepContentDecodeEstimateStats},
|
|
{"phase_worker_scan_ms", grepWorkerScanStats},
|
|
{"regex_scan_ms", grepRegexScanStats},
|
|
{"worker_critical_total_ms", grepWorkerCriticalTotalStats},
|
|
{"worker_critical_content_retrieval_ms", grepWorkerCriticalRetrievalStats},
|
|
{"worker_critical_regex_scan_ms", grepWorkerCriticalRegexStats},
|
|
{"worker_critical_other_ms", grepWorkerCriticalOtherStats},
|
|
{"regex_search_calls", grepRegexSearchCallsStats},
|
|
{"bmh_prefilter_skips", grepBmhPrefilterSkipsStats}}},
|
|
{"search_server",
|
|
json{{"execution_time_ms", searchExecStats},
|
|
{"phase_dispatch_service_ms", searchDispatchServiceStats},
|
|
{"phase_dispatch_service_us", searchDispatchServiceUsStats},
|
|
{"phase_dispatch_map_sort_ms", searchDispatchMapSortStats},
|
|
{"phase_dispatch_map_sort_us", searchDispatchMapSortUsStats},
|
|
{"phase_dispatch_feedback_ms", searchDispatchFeedbackStats},
|
|
{"phase_dispatch_feedback_us", searchDispatchFeedbackUsStats},
|
|
{"phase_dispatch_response_ms", searchDispatchResponseStats},
|
|
{"phase_dispatch_response_us", searchDispatchResponseUsStats},
|
|
{"phase_dispatch_total_ms", searchDispatchTotalStats},
|
|
{"phase_dispatch_total_us", searchDispatchTotalUsStats},
|
|
{"timing_embedding_us", searchTimingEmbeddingUsStats},
|
|
{"timing_text_us", searchTimingTextUsStats},
|
|
{"timing_path_us", searchTimingPathUsStats},
|
|
{"timing_vector_us", searchTimingVectorUsStats},
|
|
{"timing_entity_vector_us", searchTimingEntityVectorUsStats},
|
|
{"timing_compressed_ann_us", searchTimingCompressedAnnUsStats},
|
|
{"timing_kg_us", searchTimingKgUsStats},
|
|
{"timing_metadata_us", searchTimingMetadataUsStats},
|
|
{"timing_tag_us", searchTimingTagUsStats},
|
|
{"timing_multi_vector_us", searchTimingMultiVectorUsStats},
|
|
{"timing_concepts_us", searchTimingConceptsUsStats},
|
|
{"timing_reranker_us", searchTimingRerankerUsStats},
|
|
{"timing_graph_rerank_us", searchTimingGraphRerankUsStats},
|
|
{"subphrase_fts_hit_count", searchSubphraseFtsHitStats},
|
|
{"aggressive_fts_hit_count", searchAggressiveFtsHitStats},
|
|
{"graph_expansion_fts_hit_count", searchGraphExpansionFtsHitStats},
|
|
{"graph_doc_expansion_fts_hit_count", searchGraphDocExpansionFtsHitStats},
|
|
{"multi_vector_raw_hit_count", searchMultiVectorRawHitStats},
|
|
{"graph_vector_raw_hit_count", searchGraphVectorRawHitStats},
|
|
{"graph_query_neighbor_seed_docs", searchGraphQueryNeighborSeedDocStats}}},
|
|
{"search_debug_samples", searchDebugSamples},
|
|
{"list_server", json{{"service_execution_time_ms", listServiceExecStats},
|
|
{"phase_dispatch_service_ms", listDispatchServiceStats},
|
|
{"phase_dispatch_service_us", listDispatchServiceUsStats},
|
|
{"phase_dispatch_map_ms", listDispatchMapStats},
|
|
{"phase_dispatch_map_us", listDispatchMapUsStats},
|
|
{"phase_dispatch_total_ms", listDispatchTotalStats},
|
|
{"phase_dispatch_total_us", listDispatchTotalUsStats}}},
|
|
{"memory", json{{"initial_mb", initSnap.memoryUsageMb},
|
|
{"peak_mb", peakRss},
|
|
{"min_mb", minRss},
|
|
{"final_mb", finalSnap.memoryUsageMb},
|
|
{"delta_mb", finalSnap.memoryUsageMb - initSnap.memoryUsageMb}}},
|
|
{"error_categories", errorCategories},
|
|
{"hotspots", json{{"window_ms", kHotspotWindowMs},
|
|
{"threads", threadHotspotsJson},
|
|
{"windows", windowHotspotsJson}}},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"time_series", timeSeries},
|
|
{"op_traces", traceArray},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
|
|
std::cout << " Results written to: " << cfg.outputPath.string() << "\n\n";
|
|
}
|
|
|
|
// =============================================================================
|
|
// EMBEDDINGS-ENABLED BENCHMARK
|
|
// =============================================================================
|
|
|
|
TEST_CASE("Multi-client ingestion: embeddings pipeline", "[!benchmark][multi-client][embeddings]") {
|
|
// Full ingestion benchmark with real plugins loaded: ONNX embeddings,
|
|
// symbol extraction, entity recognition. Measures the complete pipeline
|
|
// including embedding generation which is the most expensive stage.
|
|
//
|
|
// Requires either:
|
|
// YAMS_BENCH_ENABLE_PLUGINS=1 with plugins and models available, or
|
|
// YAMS_BENCH_DATA_DIR pointing to a corpus (can combine both for
|
|
// ingest-into-existing-corpus testing).
|
|
auto cfg = BenchConfig::fromEnv();
|
|
if (!cfg.enablePlugins) {
|
|
SKIP("Set YAMS_BENCH_ENABLE_PLUGINS=1 to run the embeddings benchmark");
|
|
}
|
|
|
|
std::cout << "\n=== Embeddings Pipeline Benchmark ===\n";
|
|
std::cout << " Plugins: enabled\n";
|
|
std::cout << " Real model provider: " << (cfg.useRealModelProvider ? "yes" : "mock") << "\n";
|
|
if (cfg.pluginDir)
|
|
std::cout << " Plugin dir: " << cfg.pluginDir->string() << "\n";
|
|
if (cfg.dataDir)
|
|
std::cout << " Data dir: " << cfg.dataDir->string() << "\n";
|
|
|
|
constexpr int kWriterClients = 4;
|
|
constexpr int kSearchClients = 4;
|
|
constexpr int kTotalClients = kWriterClients + kSearchClients;
|
|
|
|
auto opts = benchHarnessOptions(cfg);
|
|
DaemonHarness harness(opts);
|
|
// Plugin loading + model warmup needs extra time
|
|
REQUIRE(harness.start(std::chrono::seconds(120)));
|
|
|
|
// Allow async init to complete (plugin loading, model warmup, connection pools)
|
|
std::this_thread::sleep_for(5s);
|
|
|
|
// Monitor
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 30s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
auto initSnap = DaemonSnapshot::capture(monitorClient);
|
|
std::cout << " Initial docs: " << initSnap.documentsTotal << "\n";
|
|
std::cout << " Docs per writer: " << cfg.docsPerClient << "\n";
|
|
|
|
TimeSeriesSampler sampler;
|
|
sampler.start(monitorClient, 1s);
|
|
|
|
// Latency collectors
|
|
std::mutex addLatMutex, searchLatMutex;
|
|
std::vector<int64_t> addLatencies, searchLatencies;
|
|
std::atomic<int> addOps{0}, searchOps{0};
|
|
std::atomic<int> addFails{0}, searchFails{0};
|
|
|
|
// Collect hashes for search/get during ingest
|
|
std::mutex hashMutex;
|
|
std::vector<std::string> ingestedHashes;
|
|
|
|
std::atomic<bool> go{false};
|
|
std::atomic<bool> writersFinished{false};
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(kTotalClients);
|
|
|
|
// Writer threads: ingest with embeddings pipeline active
|
|
for (int w = 0; w < kWriterClients; ++w) {
|
|
threads.emplace_back([&, w]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 60s; // embedding can be slow
|
|
DaemonClient client(ccfg);
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "embed_w" + std::to_string(w) + "_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(w, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "embeddings", "writer-" + std::to_string(w)};
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 60s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res && !res.value().hash.empty()) {
|
|
addOps.fetch_add(1);
|
|
{
|
|
std::lock_guard<std::mutex> lk(addLatMutex);
|
|
addLatencies.push_back(latUs);
|
|
}
|
|
{
|
|
std::lock_guard<std::mutex> lk(hashMutex);
|
|
ingestedHashes.push_back(res.value().hash);
|
|
}
|
|
} else {
|
|
addFails.fetch_add(1);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// Search threads: query while ingest + embeddings are running
|
|
for (int s = 0; s < kSearchClients; ++s) {
|
|
threads.emplace_back([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
while (!go.load())
|
|
std::this_thread::yield();
|
|
|
|
static const char* queries[] = {
|
|
"architecture design", "performance tuning parameters",
|
|
"ingestion pipeline stages", "resource governance pressure",
|
|
"IPC transport protocol",
|
|
};
|
|
|
|
// Keep searching until writers finish
|
|
int i = 0;
|
|
while (!writersFinished.load()) {
|
|
SearchRequest req;
|
|
req.query = queries[i % 5];
|
|
req.limit = 10;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.search(req), 30s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
searchOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(searchLatMutex);
|
|
searchLatencies.push_back(latUs);
|
|
} else {
|
|
searchFails.fetch_add(1);
|
|
}
|
|
++i;
|
|
}
|
|
});
|
|
}
|
|
|
|
go.store(true);
|
|
|
|
// Wait for writers, then signal searchers to stop
|
|
for (int w = 0; w < kWriterClients; ++w)
|
|
threads[w].join();
|
|
writersFinished.store(true);
|
|
|
|
// Join searchers
|
|
for (int s = kWriterClients; s < kTotalClients; ++s)
|
|
threads[s].join();
|
|
|
|
sampler.stop();
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
// Wait for embeddings pipeline to drain
|
|
bool drained =
|
|
waitForDrain(monitorClient, std::chrono::seconds(std::max(cfg.drainTimeoutSecs, 300)));
|
|
auto finalSnap = DaemonSnapshot::capture(monitorClient);
|
|
|
|
auto addStats = PercentileStats::compute(addLatencies);
|
|
auto searchStats = PercentileStats::compute(searchLatencies);
|
|
|
|
int totalOps = addOps.load() + searchOps.load();
|
|
int totalFails = addFails.load() + searchFails.load();
|
|
double addThroughput = safeRate(static_cast<double>(addOps.load()), globalElapsed);
|
|
double searchThroughput = safeRate(static_cast<double>(searchOps.load()), globalElapsed);
|
|
|
|
std::cout << "\n=== Embeddings Pipeline Results ===\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
std::cout << " Total ops: " << totalOps << " Failures: " << totalFails << "\n\n";
|
|
|
|
std::cout << " Ingest:\n";
|
|
std::cout << " Added: " << addOps.load() << " (" << std::setprecision(1) << addThroughput
|
|
<< " docs/s) Fails: " << addFails.load() << "\n";
|
|
printPercentiles(" Add latency ", addStats);
|
|
|
|
std::cout << "\n Search (concurrent with ingest):\n";
|
|
std::cout << " Queries: " << searchOps.load() << " (" << std::setprecision(1)
|
|
<< searchThroughput << " q/s) Fails: " << searchFails.load() << "\n";
|
|
printPercentiles(" Search latency", searchStats);
|
|
|
|
std::cout << "\n Pipeline:\n";
|
|
std::cout << " Queue drained: " << (drained ? "yes" : "NO") << "\n";
|
|
std::cout << " Final docs_total: " << finalSnap.documentsTotal << "\n";
|
|
std::cout << " Final memory: " << finalSnap.memoryUsageMb << " MB\n\n";
|
|
|
|
CHECK(addOps.load() > 0);
|
|
CHECK(drained);
|
|
|
|
// Emit JSONL
|
|
json timeSeries = json::array();
|
|
for (auto& s : sampler.getSamples()) {
|
|
timeSeries.push_back(json{{"elapsed_ms", s.elapsedMs}, {"snapshot", s.snap.toJson()}});
|
|
}
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "embeddings_pipeline"},
|
|
{"plugins_enabled", cfg.enablePlugins},
|
|
{"real_model_provider", cfg.useRealModelProvider},
|
|
{"data_dir", cfg.dataDir ? cfg.dataDir->string() : ""},
|
|
{"plugin_dir", cfg.pluginDir ? cfg.pluginDir->string() : ""},
|
|
{"writers", kWriterClients},
|
|
{"searchers", kSearchClients},
|
|
{"docs_per_writer", cfg.docsPerClient},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"add", json{{"ops", addOps.load()},
|
|
{"fails", addFails.load()},
|
|
{"throughput_per_sec", addThroughput},
|
|
{"latency", addStats.toJson()}}},
|
|
{"search", json{{"ops", searchOps.load()},
|
|
{"fails", searchFails.load()},
|
|
{"throughput_per_sec", searchThroughput},
|
|
{"latency", searchStats.toJson()}}},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"drained", drained},
|
|
{"daemon_snapshot_final", finalSnap.toJson()},
|
|
{"time_series", timeSeries},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: connection contention",
|
|
"[!benchmark][multi-client][contention]") {
|
|
// Stress test: many clients making rapid short-lived connections
|
|
auto cfg = BenchConfig::fromEnv();
|
|
int burstClients =
|
|
checkedMulToInt(cfg.numClients, 4, kMaxBenchClients * 4); // Higher contention
|
|
int opsPerClient = 20; // Shorter burst per client
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(harness.start(kStartTimeout));
|
|
|
|
// Monitor
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
|
|
(void)DaemonSnapshot::capture(monitorClient); // warmup connection
|
|
|
|
std::atomic<int> successCount{0};
|
|
std::atomic<int> failCount{0};
|
|
std::atomic<int> retryAfterCount{0};
|
|
std::mutex latMutex;
|
|
std::vector<int64_t> allLatencies;
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
std::vector<std::thread> threads;
|
|
for (int c = 0; c < burstClients; ++c) {
|
|
threads.emplace_back([&, c]() {
|
|
std::vector<int64_t> localLats;
|
|
for (int i = 0; i < opsPerClient; ++i) {
|
|
// Alternate between status, list, and add to test different op types
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
bool success = false;
|
|
|
|
if (i % 3 == 0) {
|
|
auto res = yams::cli::run_sync(client.status(), 10s);
|
|
success = static_cast<bool>(res);
|
|
if (res && res.value().retryAfterMs > 0)
|
|
retryAfterCount.fetch_add(1);
|
|
} else if (i % 3 == 1) {
|
|
ListRequest req;
|
|
req.limit = 5;
|
|
auto res = yams::cli::run_sync(client.list(req), 10s);
|
|
success = static_cast<bool>(res);
|
|
} else {
|
|
AddDocumentRequest req;
|
|
req.name = "burst_c" + std::to_string(c) + "_" + std::to_string(i) + ".md";
|
|
req.content = "Burst test content from client " + std::to_string(c);
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 10s);
|
|
success = static_cast<bool>(res);
|
|
}
|
|
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
localLats.push_back(latUs);
|
|
|
|
if (success)
|
|
successCount.fetch_add(1);
|
|
else
|
|
failCount.fetch_add(1);
|
|
}
|
|
|
|
std::lock_guard<std::mutex> lock(latMutex);
|
|
allLatencies.insert(allLatencies.end(), localLats.begin(), localLats.end());
|
|
});
|
|
}
|
|
|
|
for (auto& t : threads)
|
|
t.join();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
int totalOps = burstClients * opsPerClient;
|
|
double throughput = static_cast<double>(successCount.load()) / globalElapsed;
|
|
|
|
auto postSnap = DaemonSnapshot::capture(monitorClient);
|
|
auto latStats = PercentileStats::compute(allLatencies);
|
|
|
|
std::cout << "\n=== Connection Contention (N=" << burstClients << " burst clients) ===\n";
|
|
std::cout << " Total ops: " << totalOps << " (success=" << successCount.load()
|
|
<< " fail=" << failCount.load() << ")\n";
|
|
std::cout << " Backpressure signals (retryAfter>0): " << retryAfterCount.load() << "\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
std::cout << " Throughput: " << throughput << " ops/s\n";
|
|
printPercentiles("Op latency", latStats);
|
|
std::cout << " Peak connections: " << postSnap.maxConnections << "\n";
|
|
std::cout << " Forced closes: " << postSnap.requestsProcessed << " total reqs processed\n\n";
|
|
|
|
// Failures acceptable under extreme contention, but should be minority
|
|
double failRate = static_cast<double>(failCount.load()) / static_cast<double>(totalOps);
|
|
CHECK(failRate < 0.20); // Less than 20% failure acceptable under burst
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "connection_contention"},
|
|
{"burst_clients", burstClients},
|
|
{"ops_per_client", opsPerClient},
|
|
{"total_ops", totalOps},
|
|
{"success_count", successCount.load()},
|
|
{"fail_count", failCount.load()},
|
|
{"retry_after_count", retryAfterCount.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"throughput_ops_per_sec", throughput},
|
|
{"latency", latStats.toJson()},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: reader starvation under ingest load",
|
|
"[!benchmark][multi-client][starvation]") {
|
|
// Verify that readers are not starved when heavy ingestion is in progress
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(harness.start(kStartTimeout));
|
|
|
|
// Pre-seed so readers have something to query
|
|
{
|
|
ClientConfig warmCfg;
|
|
warmCfg.socketPath = harness.socketPath();
|
|
warmCfg.autoStart = false;
|
|
warmCfg.requestTimeout = 30s;
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
for (int i = 0; i < 30; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "preseed_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(0, i, cfg.docSizeBytes);
|
|
yams::cli::run_sync(warmClient.streamingAddDocument(req), 30s);
|
|
}
|
|
(void)waitForDrain(warmClient, 30s);
|
|
}
|
|
|
|
// Writer clients (heavy ingestion)
|
|
int numWriters = std::max(1, cfg.numClients - 1);
|
|
// Reader clients (continuous reads)
|
|
int numReaders = std::max(1, cfg.numClients / 2);
|
|
|
|
std::atomic<bool> done{false};
|
|
std::atomic<int> writerDocs{0};
|
|
std::atomic<int> readerOps{0};
|
|
std::atomic<int> readerFails{0};
|
|
std::mutex readerLatMutex;
|
|
std::vector<int64_t> readerLatencies;
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
// Launch writers
|
|
std::vector<std::thread> writerThreads;
|
|
for (int w = 0; w < numWriters; ++w) {
|
|
writerThreads.emplace_back([&, w]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "writer" + std::to_string(w) + "_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(w, i, cfg.docSizeBytes);
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
if (res)
|
|
writerDocs.fetch_add(1);
|
|
}
|
|
});
|
|
}
|
|
|
|
// Launch readers (run until writers finish)
|
|
std::vector<std::thread> readerThreads;
|
|
for (int r = 0; r < numReaders; ++r) {
|
|
readerThreads.emplace_back([&]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
|
|
while (!done.load()) {
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
ListRequest req;
|
|
req.limit = 10;
|
|
auto res = yams::cli::run_sync(client.list(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
readerOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lock(readerLatMutex);
|
|
readerLatencies.push_back(latUs);
|
|
} else {
|
|
readerFails.fetch_add(1);
|
|
}
|
|
|
|
// Small delay to avoid pure spin
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
});
|
|
}
|
|
|
|
// Wait for writers to finish
|
|
for (auto& t : writerThreads)
|
|
t.join();
|
|
done.store(true);
|
|
|
|
// Wait for readers to stop
|
|
for (auto& t : readerThreads)
|
|
t.join();
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
auto readerStats = PercentileStats::compute(readerLatencies);
|
|
|
|
std::cout << "\n=== Reader Starvation Test ===\n";
|
|
std::cout << " Writers: " << numWriters << " (ingested " << writerDocs.load() << " docs)\n";
|
|
std::cout << " Readers: " << numReaders << " (" << readerOps.load() << " ops, "
|
|
<< readerFails.load() << " fails)\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
printPercentiles("Reader latency", readerStats);
|
|
|
|
// Reader p99 should stay reasonable even under write load
|
|
// This threshold can be tuned based on baseline measurements
|
|
if (readerStats.count > 0) {
|
|
std::cout << " Reader throughput: "
|
|
<< safeRate(static_cast<double>(readerOps.load()), globalElapsed) << " ops/s\n\n";
|
|
}
|
|
|
|
CHECK(readerOps.load() > 0); // Readers must make progress
|
|
CHECK(readerFails.load() < readerOps.load()); // Most reads should succeed
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "reader_starvation"},
|
|
{"num_writers", numWriters},
|
|
{"num_readers", numReaders},
|
|
{"docs_per_writer", cfg.docsPerClient},
|
|
{"total_writer_docs", writerDocs.load()},
|
|
{"total_reader_ops", readerOps.load()},
|
|
{"total_reader_fails", readerFails.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"reader_latency", readerStats.toJson()},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|
|
|
|
TEST_CASE("Multi-client ingestion: file access under ingest load",
|
|
"[!benchmark][multi-client][access]") {
|
|
// Validate get/cat responsiveness while concurrent ingestion is running.
|
|
auto cfg = BenchConfig::fromEnv();
|
|
|
|
DaemonHarness harness(benchHarnessOptions());
|
|
REQUIRE(harness.start(kStartTimeout));
|
|
|
|
// Seed a baseline corpus so readers always have accessible hashes.
|
|
std::vector<std::string> knownHashes;
|
|
std::mutex knownHashesMutex;
|
|
{
|
|
ClientConfig warmCfg;
|
|
warmCfg.socketPath = harness.socketPath();
|
|
warmCfg.autoStart = false;
|
|
warmCfg.requestTimeout = 30s;
|
|
DaemonClient warmClient(warmCfg);
|
|
|
|
for (int i = 0; i < 64; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "access_preseed_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(700, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "access-preseed"};
|
|
auto res = yams::cli::run_sync(warmClient.streamingAddDocument(req), 30s);
|
|
if (res && !res.value().hash.empty()) {
|
|
std::lock_guard<std::mutex> lk(knownHashesMutex);
|
|
knownHashes.push_back(res.value().hash);
|
|
}
|
|
}
|
|
(void)waitForDrain(warmClient, 30s);
|
|
}
|
|
|
|
const int numWriters = std::max(1, cfg.numClients - std::max(1, cfg.numClients / 3));
|
|
const int numReaders = std::max(1, cfg.numClients / 3);
|
|
|
|
std::atomic<bool> done{false};
|
|
std::atomic<int> writerDocs{0};
|
|
std::atomic<int> readerOps{0};
|
|
std::atomic<int> readerFails{0};
|
|
std::atomic<int> readerAttempts{0};
|
|
std::atomic<int> getOps{0};
|
|
std::atomic<int> catOps{0};
|
|
std::mutex getLatMutex;
|
|
std::mutex catLatMutex;
|
|
std::vector<int64_t> getLatencies;
|
|
std::vector<int64_t> catLatencies;
|
|
|
|
auto pickHash = [&](std::mt19937& rng) -> std::string {
|
|
std::lock_guard<std::mutex> lk(knownHashesMutex);
|
|
if (knownHashes.empty()) {
|
|
return std::string{};
|
|
}
|
|
std::uniform_int_distribution<std::size_t> dist(0, knownHashes.size() - 1);
|
|
return knownHashes[dist(rng)];
|
|
};
|
|
|
|
auto globalStart = std::chrono::steady_clock::now();
|
|
|
|
std::vector<std::thread> writerThreads;
|
|
writerThreads.reserve(numWriters);
|
|
for (int w = 0; w < numWriters; ++w) {
|
|
writerThreads.emplace_back([&, w]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 30s;
|
|
DaemonClient client(ccfg);
|
|
|
|
for (int i = 0; i < cfg.docsPerClient; ++i) {
|
|
AddDocumentRequest req;
|
|
req.name = "access_writer" + std::to_string(w) + "_" + std::to_string(i) + ".md";
|
|
req.content = generateDocument(w, i, cfg.docSizeBytes);
|
|
req.tags = {"bench", "access", "writer-" + std::to_string(w)};
|
|
|
|
auto res = yams::cli::run_sync(client.streamingAddDocument(req), 30s);
|
|
if (res) {
|
|
writerDocs.fetch_add(1);
|
|
if (!res.value().hash.empty()) {
|
|
std::lock_guard<std::mutex> lk(knownHashesMutex);
|
|
knownHashes.push_back(res.value().hash);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
std::vector<std::thread> readerThreads;
|
|
readerThreads.reserve(numReaders);
|
|
for (int r = 0; r < numReaders; ++r) {
|
|
readerThreads.emplace_back([&, r]() {
|
|
ClientConfig ccfg;
|
|
ccfg.socketPath = harness.socketPath();
|
|
ccfg.autoStart = false;
|
|
ccfg.requestTimeout = 10s;
|
|
DaemonClient client(ccfg);
|
|
|
|
std::mt19937 rng(static_cast<unsigned>(r * 1009 + 17));
|
|
bool doGet = true;
|
|
|
|
while (!done.load()) {
|
|
std::string hash = pickHash(rng);
|
|
if (hash.empty()) {
|
|
std::this_thread::sleep_for(5ms);
|
|
continue;
|
|
}
|
|
readerAttempts.fetch_add(1);
|
|
|
|
// Alternate get (metadata-focused) and cat (content-focused)
|
|
if (doGet) {
|
|
GetRequest req;
|
|
req.hash = hash;
|
|
req.metadataOnly = true;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.get(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
getOps.fetch_add(1);
|
|
readerOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(getLatMutex);
|
|
getLatencies.push_back(latUs);
|
|
} else {
|
|
readerFails.fetch_add(1);
|
|
}
|
|
} else {
|
|
CatRequest req;
|
|
req.hash = hash;
|
|
|
|
auto t0 = std::chrono::steady_clock::now();
|
|
auto res = yams::cli::run_sync(client.cat(req), 10s);
|
|
auto t1 = std::chrono::steady_clock::now();
|
|
auto latUs =
|
|
std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0).count();
|
|
|
|
if (res) {
|
|
catOps.fetch_add(1);
|
|
readerOps.fetch_add(1);
|
|
std::lock_guard<std::mutex> lk(catLatMutex);
|
|
catLatencies.push_back(latUs);
|
|
} else {
|
|
readerFails.fetch_add(1);
|
|
}
|
|
}
|
|
doGet = !doGet;
|
|
|
|
std::this_thread::sleep_for(2ms);
|
|
}
|
|
});
|
|
}
|
|
|
|
for (auto& t : writerThreads)
|
|
t.join();
|
|
done.store(true);
|
|
|
|
for (auto& t : readerThreads)
|
|
t.join();
|
|
|
|
ClientConfig monCfg;
|
|
monCfg.socketPath = harness.socketPath();
|
|
monCfg.autoStart = false;
|
|
monCfg.requestTimeout = 10s;
|
|
DaemonClient monitorClient(monCfg);
|
|
bool drained = waitForDrain(monitorClient, std::chrono::seconds(cfg.drainTimeoutSecs));
|
|
|
|
auto globalEnd = std::chrono::steady_clock::now();
|
|
double globalElapsed = std::chrono::duration<double>(globalEnd - globalStart).count();
|
|
|
|
auto getStats = PercentileStats::compute(getLatencies);
|
|
auto catStats = PercentileStats::compute(catLatencies);
|
|
|
|
std::cout << "\n=== File Access Under Ingest Load ===\n";
|
|
std::cout << " Writers: " << numWriters << " (ingested " << writerDocs.load() << " docs)\n";
|
|
std::cout << " Readers: " << numReaders << " (ops=" << readerOps.load()
|
|
<< ", attempts=" << readerAttempts.load() << ", get=" << getOps.load()
|
|
<< ", cat=" << catOps.load() << ", fails=" << readerFails.load() << ")\n";
|
|
std::cout << " Wall time: " << std::fixed << std::setprecision(2) << globalElapsed << "s\n";
|
|
printPercentiles("Get latency", getStats);
|
|
printPercentiles("Cat latency", catStats);
|
|
std::cout << " Queue drained: " << (drained ? "yes" : "NO") << "\n\n";
|
|
|
|
CHECK(writerDocs.load() > 0);
|
|
CHECK(readerOps.load() > 0);
|
|
CHECK(readerFails.load() < readerAttempts.load());
|
|
CHECK(drained);
|
|
|
|
json record{
|
|
{"timestamp", isoTimestamp()},
|
|
{"test", "access_under_ingest_load"},
|
|
{"num_clients", cfg.numClients},
|
|
{"num_writers", numWriters},
|
|
{"num_readers", numReaders},
|
|
{"docs_per_writer", cfg.docsPerClient},
|
|
{"doc_size_bytes", cfg.docSizeBytes},
|
|
{"total_writer_docs", writerDocs.load()},
|
|
{"reader_attempts", readerAttempts.load()},
|
|
{"reader_ops", readerOps.load()},
|
|
{"get_ops", getOps.load()},
|
|
{"cat_ops", catOps.load()},
|
|
{"reader_fails", readerFails.load()},
|
|
{"elapsed_seconds", globalElapsed},
|
|
{"get_latency", getStats.toJson()},
|
|
{"cat_latency", catStats.toJson()},
|
|
{"drained", drained},
|
|
{"tuning_profile", cfg.tuningProfile},
|
|
};
|
|
emitJsonl(cfg.outputPath, record);
|
|
}
|