#include "glove/mcp/client.hpp" #include "glove/mcp/transport.hpp" #include "src/mcp/codec.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { constexpr std::size_t default_samples = 20; constexpr std::size_t default_iterations = 1'000; constexpr std::size_t warmup_iterations = 100; constexpr std::size_t minimum_samples = 2; constexpr std::size_t maximum_samples = 1'000; constexpr std::size_t maximum_iterations = 10'000'000; constexpr double nanoseconds_per_second = 1'000'000'000.0; constexpr double normal_95_percentile = 1.959963984540054; struct options { std::size_t samples = default_samples; std::size_t iterations = default_iterations; }; struct measurement { double mean_nanoseconds = 0.0; double standard_deviation_nanoseconds = 0.0; double ci95_lower_nanoseconds = 0.0; double ci95_upper_nanoseconds = 0.0; std::uint64_t checksum = 0; }; auto parse_count(std::string_view value, std::size_t minimum, std::size_t maximum) -> std::expected { std::size_t parsed = 0; const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || parsed < minimum || parsed > maximum) { return std::unexpected(std::string{"benchmark count is outside its allowed range"}); } return parsed; } auto parse_options(int argc, char** argv) -> std::expected { options parsed; for (int index = 1; index < argc; index += 2) { if (index + 1 >= argc) { return std::unexpected(std::string{"benchmark options require a value"}); } const std::string_view option{argv[index]}; const std::string_view value{argv[index + 1]}; if (option == "--samples") { auto count = parse_count(value, minimum_samples, maximum_samples); if (!count) { return std::unexpected(count.error()); } parsed.samples = *count; } else if (option == "--iterations") { auto count = parse_count(value, 1, maximum_iterations); if (!count) { return std::unexpected(count.error()); } parsed.iterations = *count; } else { return std::unexpected(std::string{"unknown benchmark option"}); } } return parsed; } template auto measure(const options& configured, Operation&& operation) -> std::expected { std::uint64_t checksum = 0; for (std::size_t iteration = 0; iteration < warmup_iterations; ++iteration) { auto result = operation(); if (!result) { return std::unexpected(result.error()); } checksum += *result; } std::vector sample_nanoseconds; sample_nanoseconds.reserve(configured.samples); for (std::size_t sample = 0; sample < configured.samples; ++sample) { const auto started = std::chrono::steady_clock::now(); for (std::size_t iteration = 0; iteration < configured.iterations; ++iteration) { auto result = operation(); if (!result) { return std::unexpected(result.error()); } checksum += *result; } const auto elapsed = std::chrono::steady_clock::now() - started; const auto elapsed_nanoseconds = std::chrono::duration{elapsed}.count(); sample_nanoseconds.push_back( elapsed_nanoseconds / static_cast(configured.iterations) ); } const double mean = std::accumulate(sample_nanoseconds.begin(), sample_nanoseconds.end(), 0.0) / static_cast(sample_nanoseconds.size()); double squared_deviation_sum = 0.0; for (const double sample : sample_nanoseconds) { const double deviation = sample - mean; squared_deviation_sum += deviation * deviation; } const double standard_deviation = std::sqrt(squared_deviation_sum / static_cast(sample_nanoseconds.size() - 1)); const double margin = normal_95_percentile * standard_deviation / std::sqrt(static_cast(sample_nanoseconds.size())); return measurement{ .mean_nanoseconds = mean, .standard_deviation_nanoseconds = standard_deviation, .ci95_lower_nanoseconds = mean - margin, .ci95_upper_nanoseconds = mean + margin, .checksum = checksum, }; } void write_result(std::string_view name, const options& configured, const measurement& measured) { std::cout << std::setprecision(17) << "{\"schema_version\":1,\"benchmark\":\"" << name << "\",\"samples\":" << configured.samples << ",\"iterations_per_sample\":" << configured.iterations << ",\"warmup_iterations\":" << warmup_iterations << ",\"latency_ns_mean\":" << measured.mean_nanoseconds << ",\"latency_ns_stddev\":" << measured.standard_deviation_nanoseconds << ",\"latency_ns_ci95_normal_lower\":" << measured.ci95_lower_nanoseconds << ",\"latency_ns_ci95_normal_upper\":" << measured.ci95_upper_nanoseconds << ",\"throughput_ops_per_second\":" << nanoseconds_per_second / measured.mean_nanoseconds << ",\"checksum\":" << measured.checksum << "}\n"; } auto codec_operation() -> std::expected { constexpr std::string_view frame = R"({"jsonrpc":"2.0","id":1,"result":{"tools":[{"name":"echo","description":"echoes its input","inputSchema":{"type":"object"}},{"name":"add","description":"adds two numbers","inputSchema":{"type":"object"}}]}})"; auto envelope = glove::mcp::codec::decode_response(frame); if (!envelope) { return std::unexpected(envelope.error()); } auto tools = glove::mcp::codec::parse_tools_list_result(*envelope); if (!tools) { return std::unexpected(tools.error()); } return tools->size(); } auto make_round_trip_client() -> std::unique_ptr { auto handler = [](std::string_view frame) -> std::optional { auto request = glove::mcp::codec::decode_request(frame); if (!request || !request->id || request->method != "tools/list") { return R"({"jsonrpc":"2.0","id":0,"error":{"code":-32600,"message":"invalid benchmark request"}})"; } auto response = glove::mcp::codec::encode_response_with_result( *request->id, R"({"tools":[{"name":"echo","description":"echoes its input","inputSchema":{"type":"object"}}]})" ); if (!response) { return R"({"jsonrpc":"2.0","id":0,"error":{"code":-32603,"message":"benchmark encoding failed"}})"; } return std::move(*response); }; return glove::mcp::make_client(glove::mcp::make_in_memory_transport(std::move(handler))); } auto run(const options& configured) -> std::expected { auto codec = measure(configured, codec_operation); if (!codec) { return std::unexpected(codec.error()); } write_result("mcp.codec.tools_list_decode", configured, *codec); auto client = make_round_trip_client(); auto round_trip = measure(configured, [&client]() -> std::expected { auto tools = client->list_tools(); if (!tools) { return std::unexpected(tools.error()); } return tools->size(); }); if (!round_trip) { return std::unexpected(round_trip.error()); } write_result("mcp.client.in_memory_tools_list_round_trip", configured, *round_trip); return {}; } } // namespace auto main(int argc, char** argv) -> int { auto configured = parse_options(argc, argv); if (!configured) { std::fprintf(stderr, "glove_mcp_benchmark: %s\n", configured.error().c_str()); return 2; } auto result = run(*configured); if (!result) { std::fprintf(stderr, "glove_mcp_benchmark: %s\n", result.error().c_str()); return 1; } return 0; }