#!/usr/bin/env python3 """ Summarize multi-client benchmark JSONL artifacts. Input records are produced by tests/benchmarks/multi_client_ingestion_bench.cpp. This tool computes KPI aggregates needed for iterative optimization loops. """ from __future__ import annotations import argparse import json import math import statistics from collections import defaultdict from pathlib import Path from typing import Any def _mean(values: list[float]) -> float: if not values: return 0.0 return float(statistics.fmean(values)) def _percentile(values: list[float], pct: float) -> float: if not values: return 0.0 sorted_values = sorted(values) idx = min(len(sorted_values) - 1, math.floor(pct * len(sorted_values))) return float(sorted_values[idx]) def _to_summary(values: list[float], unit: str) -> dict[str, Any]: if not values: return { "count": 0, "unit": unit, "mean": 0.0, "p50": 0.0, "p95": 0.0, "p99": 0.0, "min": 0.0, "max": 0.0, } return { "count": len(values), "unit": unit, "mean": _mean(values), "p50": _percentile(values, 0.50), "p95": _percentile(values, 0.95), "p99": _percentile(values, 0.99), "min": float(min(values)), "max": float(max(values)), } def _rate(numerator: float, denominator: float) -> float: if denominator <= 0: return 0.0 return numerator / denominator def _safe_float(value: Any, default: float = 0.0) -> float: try: return float(value) except (TypeError, ValueError): return default def _load_records(paths: list[Path]) -> list[dict[str, Any]]: records: list[dict[str, Any]] = [] for path in paths: if not path.exists(): raise FileNotFoundError(f"Input JSONL not found: {path}") with path.open("r", encoding="utf-8") as handle: for lineno, line in enumerate(handle, start=1): line = line.strip() if not line: continue try: row = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"{path}:{lineno}: invalid JSON: {exc}") from exc if isinstance(row, dict): row["_source_file"] = str(path) records.append(row) return records def summarize(records: list[dict[str, Any]], input_paths: list[Path]) -> dict[str, Any]: large_corpus: dict[str, dict[str, list[float]]] = {} large_corpus_churn: dict[str, dict[str, list[float]]] = {} large_usage_profiles: dict[str, dict[str, int]] = defaultdict( lambda: defaultdict(int) ) large_churn_usage_profiles: dict[str, dict[str, int]] = defaultdict( lambda: defaultdict(int) ) contention_metrics: dict[str, list[float]] = { "throughput_ops_per_sec": [], "fail_rate": [], "retry_after_rate": [], "latency_p95_ms": [], "success_rate": [], } case_buckets: dict[ tuple[str, str, str, str, str, str, int], dict[str, list[float]] ] = {} def get_large_bucket( target: dict[str, dict[str, list[float]]], request_path: str ) -> dict[str, list[float]]: if request_path not in target: target[request_path] = { "ops_per_sec": [], "fail_rate": [], "connection_drop_rate": [], "document_not_found_rate": [], "content_not_found_rate": [], "invalid_hash_quarantined": [], "search_p95_ms": [], "list_p95_ms": [], "grep_p95_ms": [], "status_p95_ms": [], "get_p95_ms": [], "cat_p95_ms": [], "search_fail_rate": [], "list_fail_rate": [], "grep_fail_rate": [], "status_fail_rate": [], "get_fail_rate": [], "cat_fail_rate": [], "search_ops_per_sec": [], "list_ops_per_sec": [], "grep_ops_per_sec": [], "status_ops_per_sec": [], "get_ops_per_sec": [], "cat_ops_per_sec": [], "op_trace_count": [], "grep_server_execution_time_ms_p95": [], "grep_server_phase_worker_scan_ms_p95": [], "grep_server_content_retrieval_ms_p95": [], "grep_server_worker_critical_total_ms_p95": [], "grep_server_worker_critical_content_retrieval_ms_p95": [], "grep_server_worker_critical_regex_scan_ms_p95": [], "grep_server_worker_critical_other_ms_p95": [], "grep_server_docs_scanned_mean": [], "grep_server_lines_scanned_mean": [], "grep_server_bytes_scanned_mean": [], "search_server_execution_time_ms_p95": [], "search_server_phase_dispatch_service_ms_p95": [], "search_server_phase_dispatch_service_us_p95": [], "search_server_phase_dispatch_map_sort_ms_p95": [], "search_server_phase_dispatch_map_sort_us_p95": [], "search_server_phase_dispatch_feedback_ms_p95": [], "search_server_phase_dispatch_feedback_us_p95": [], "search_server_phase_dispatch_response_ms_p95": [], "search_server_phase_dispatch_response_us_p95": [], "search_server_phase_dispatch_total_ms_p95": [], "search_server_phase_dispatch_total_us_p95": [], "list_server_service_execution_time_ms_p95": [], "list_server_phase_dispatch_service_ms_p95": [], "list_server_phase_dispatch_service_us_p95": [], "list_server_phase_dispatch_map_ms_p95": [], "list_server_phase_dispatch_map_us_p95": [], "list_server_phase_dispatch_total_ms_p95": [], "list_server_phase_dispatch_total_us_p95": [], } return target[request_path] def get_case_bucket( key: tuple[str, str, str, str, str, str, int], ) -> dict[str, list[float]]: if key not in case_buckets: case_buckets[key] = { "ops_per_sec": [], "fail_rate": [], "latency_p95_ms": [], } return case_buckets[key] for row in records: test_name = str(row.get("test", "")) run_phase = str(row.get("run_phase", "unknown")) run_transport = str( row.get("run_transport", row.get("request_path", "unknown")) ) usage_profile = str(row.get("layout", {}).get("usage_profile", "unknown")) backend = str(row.get("run_backend", row.get("backend", "unknown"))) layout = ( row.get("layout", {}) if isinstance(row.get("layout", {}), dict) else {} ) num_clients = int( _safe_float( row.get( "num_clients", _safe_float(layout.get("search_clients", 0.0)) + _safe_float(layout.get("list_clients", 0.0)) + _safe_float(layout.get("grep_clients", 0.0)) + _safe_float(layout.get("status_get_clients", 0.0)), ), 0.0, ) ) if test_name in {"large_corpus_reads", "large_corpus_reads_churn"}: is_churn = test_name == "large_corpus_reads_churn" request_path = str(row.get("request_path", "unknown")) bucket = get_large_bucket( large_corpus_churn if is_churn else large_corpus, request_path ) elapsed_seconds = _safe_float(row.get("elapsed_seconds", 0.0)) total_ops = _safe_float(row.get("total_ops", 0.0)) total_failures = _safe_float(row.get("total_failures", 0.0)) error_categories = row.get("error_categories", {}) connection_drop = _safe_float(error_categories.get("connection_drop", 0.0)) document_not_found = _safe_float( error_categories.get("document_not_found", 0.0) ) content_not_found = _safe_float( error_categories.get("content_not_found", 0.0) ) hash_pool = row.get("hash_pool", {}) bucket["ops_per_sec"].append(_safe_float(row.get("ops_per_sec", 0.0))) bucket["fail_rate"].append(_rate(total_failures, total_ops)) bucket["connection_drop_rate"].append(_rate(connection_drop, total_ops)) bucket["document_not_found_rate"].append( _rate(document_not_found, total_ops) ) bucket["content_not_found_rate"].append(_rate(content_not_found, total_ops)) bucket["invalid_hash_quarantined"].append( _safe_float(hash_pool.get("invalid_quarantined", 0.0)) ) for op in ("search", "list", "grep", "status", "get", "cat"): op_obj = row.get(op, {}) op_ops = _safe_float(op_obj.get("ops", 0.0)) op_fails = _safe_float(op_obj.get("fails", 0.0)) op_p95_ms = ( _safe_float(op_obj.get("latency", {}).get("p95_us", 0.0)) / 1000.0 ) bucket[f"{op}_p95_ms"].append(op_p95_ms) bucket[f"{op}_fail_rate"].append(_rate(op_fails, op_ops)) bucket[f"{op}_ops_per_sec"].append(_rate(op_ops, elapsed_seconds)) traces = row.get("op_traces", []) bucket["op_trace_count"].append( float(len(traces)) if isinstance(traces, list) else 0.0 ) grep_server = row.get("grep_server", {}) bucket["grep_server_execution_time_ms_p95"].append( _safe_float(grep_server.get("execution_time_ms", {}).get("p95", 0.0)) ) bucket["grep_server_phase_worker_scan_ms_p95"].append( _safe_float(grep_server.get("phase_worker_scan_ms", {}).get("p95", 0.0)) ) bucket["grep_server_content_retrieval_ms_p95"].append( _safe_float(grep_server.get("content_retrieval_ms", {}).get("p95", 0.0)) ) bucket["grep_server_worker_critical_total_ms_p95"].append( _safe_float( grep_server.get("worker_critical_total_ms", {}).get("p95", 0.0) ) ) bucket["grep_server_worker_critical_content_retrieval_ms_p95"].append( _safe_float( grep_server.get("worker_critical_content_retrieval_ms", {}).get( "p95", 0.0 ) ) ) bucket["grep_server_worker_critical_regex_scan_ms_p95"].append( _safe_float( grep_server.get("worker_critical_regex_scan_ms", {}).get("p95", 0.0) ) ) bucket["grep_server_worker_critical_other_ms_p95"].append( _safe_float( grep_server.get("worker_critical_other_ms", {}).get("p95", 0.0) ) ) bucket["grep_server_docs_scanned_mean"].append( _safe_float(grep_server.get("docs_scanned", {}).get("mean", 0.0)) ) bucket["grep_server_lines_scanned_mean"].append( _safe_float(grep_server.get("lines_scanned", {}).get("mean", 0.0)) ) bucket["grep_server_bytes_scanned_mean"].append( _safe_float(grep_server.get("bytes_scanned", {}).get("mean", 0.0)) ) search_server = row.get("search_server", {}) bucket["search_server_execution_time_ms_p95"].append( _safe_float(search_server.get("execution_time_ms", {}).get("p95", 0.0)) ) bucket["search_server_phase_dispatch_service_ms_p95"].append( _safe_float( search_server.get("phase_dispatch_service_ms", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_service_us_p95"].append( _safe_float( search_server.get("phase_dispatch_service_us", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_map_sort_ms_p95"].append( _safe_float( search_server.get("phase_dispatch_map_sort_ms", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_map_sort_us_p95"].append( _safe_float( search_server.get("phase_dispatch_map_sort_us", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_feedback_ms_p95"].append( _safe_float( search_server.get("phase_dispatch_feedback_ms", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_feedback_us_p95"].append( _safe_float( search_server.get("phase_dispatch_feedback_us", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_response_ms_p95"].append( _safe_float( search_server.get("phase_dispatch_response_ms", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_response_us_p95"].append( _safe_float( search_server.get("phase_dispatch_response_us", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_total_ms_p95"].append( _safe_float( search_server.get("phase_dispatch_total_ms", {}).get("p95", 0.0) ) ) bucket["search_server_phase_dispatch_total_us_p95"].append( _safe_float( search_server.get("phase_dispatch_total_us", {}).get("p95", 0.0) ) ) list_server = row.get("list_server", {}) bucket["list_server_service_execution_time_ms_p95"].append( _safe_float( list_server.get("service_execution_time_ms", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_service_ms_p95"].append( _safe_float( list_server.get("phase_dispatch_service_ms", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_service_us_p95"].append( _safe_float( list_server.get("phase_dispatch_service_us", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_map_ms_p95"].append( _safe_float( list_server.get("phase_dispatch_map_ms", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_map_us_p95"].append( _safe_float( list_server.get("phase_dispatch_map_us", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_total_ms_p95"].append( _safe_float( list_server.get("phase_dispatch_total_ms", {}).get("p95", 0.0) ) ) bucket["list_server_phase_dispatch_total_us_p95"].append( _safe_float( list_server.get("phase_dispatch_total_us", {}).get("p95", 0.0) ) ) if is_churn: large_churn_usage_profiles[request_path][usage_profile] += 1 else: large_usage_profiles[request_path][usage_profile] += 1 case_key = ( run_phase, test_name, request_path, run_transport, usage_profile, backend, num_clients, ) c_bucket = get_case_bucket(case_key) c_bucket["ops_per_sec"].append(_safe_float(row.get("ops_per_sec", 0.0))) c_bucket["fail_rate"].append(_rate(total_failures, total_ops)) c_bucket["latency_p95_ms"].append( _safe_float(row.get("search", {}).get("latency", {}).get("p95_us", 0.0)) / 1000.0 ) elif test_name == "connection_contention": total_ops = _safe_float(row.get("total_ops", 0.0)) fail_count = _safe_float(row.get("fail_count", 0.0)) retry_after_count = _safe_float(row.get("retry_after_count", 0.0)) success_count = _safe_float(row.get("success_count", 0.0)) throughput = _safe_float(row.get("throughput_ops_per_sec", 0.0)) fail_rate = _rate(fail_count, total_ops) retry_after_rate = _rate(retry_after_count, total_ops) success_rate = _rate(success_count, total_ops) latency_p95_ms = ( _safe_float(row.get("latency", {}).get("p95_us", 0.0)) / 1000.0 ) contention_metrics["throughput_ops_per_sec"].append(throughput) contention_metrics["fail_rate"].append(fail_rate) contention_metrics["retry_after_rate"].append(retry_after_rate) contention_metrics["latency_p95_ms"].append(latency_p95_ms) contention_metrics["success_rate"].append(success_rate) case_key = ( run_phase, test_name, "n/a", run_transport, usage_profile, backend, num_clients, ) c_bucket = get_case_bucket(case_key) c_bucket["ops_per_sec"].append(throughput) c_bucket["fail_rate"].append(fail_rate) c_bucket["latency_p95_ms"].append(latency_p95_ms) # Backward-compatible summaries def _build_large_compat( source: dict[str, dict[str, list[float]]], ) -> dict[str, Any]: out: dict[str, Any] = {} for request_path, metrics in source.items(): out[request_path] = { "ops_per_sec": _to_summary(metrics["ops_per_sec"], "ops/s"), "fail_rate": _to_summary(metrics["fail_rate"], "ratio"), "connection_drop_rate": _to_summary( metrics["connection_drop_rate"], "ratio" ), "document_not_found_rate": _to_summary( metrics["document_not_found_rate"], "ratio" ), "content_not_found_rate": _to_summary( metrics["content_not_found_rate"], "ratio" ), "search_p95_ms": _to_summary(metrics["search_p95_ms"], "ms"), "list_p95_ms": _to_summary(metrics["list_p95_ms"], "ms"), } return out def _build_large_v2( source: dict[str, dict[str, list[float]]], usage_profiles: dict[str, dict[str, int]], ) -> dict[str, Any]: out: dict[str, Any] = {} for request_path, metrics in source.items(): out[request_path] = { "ops_per_sec": _to_summary(metrics["ops_per_sec"], "ops/s"), "fail_rate": _to_summary(metrics["fail_rate"], "ratio"), "connection_drop_rate": _to_summary( metrics["connection_drop_rate"], "ratio" ), "document_not_found_rate": _to_summary( metrics["document_not_found_rate"], "ratio" ), "content_not_found_rate": _to_summary( metrics["content_not_found_rate"], "ratio" ), "invalid_hash_quarantined": _to_summary( metrics["invalid_hash_quarantined"], "count" ), "usage_profiles": dict(sorted(usage_profiles[request_path].items())), "search": { "p95_ms": _to_summary(metrics["search_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["search_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["search_ops_per_sec"], "ops/s"), }, "list": { "p95_ms": _to_summary(metrics["list_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["list_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["list_ops_per_sec"], "ops/s"), }, "grep": { "p95_ms": _to_summary(metrics["grep_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["grep_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["grep_ops_per_sec"], "ops/s"), }, "status": { "p95_ms": _to_summary(metrics["status_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["status_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["status_ops_per_sec"], "ops/s"), }, "get": { "p95_ms": _to_summary(metrics["get_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["get_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["get_ops_per_sec"], "ops/s"), }, "cat": { "p95_ms": _to_summary(metrics["cat_p95_ms"], "ms"), "fail_rate": _to_summary(metrics["cat_fail_rate"], "ratio"), "ops_per_sec": _to_summary(metrics["cat_ops_per_sec"], "ops/s"), }, "op_trace_count": _to_summary(metrics["op_trace_count"], "count"), "grep_server": { "execution_time_ms_p95": _to_summary( metrics["grep_server_execution_time_ms_p95"], "ms" ), "phase_worker_scan_ms_p95": _to_summary( metrics["grep_server_phase_worker_scan_ms_p95"], "ms" ), "content_retrieval_ms_p95": _to_summary( metrics["grep_server_content_retrieval_ms_p95"], "ms" ), "worker_critical_total_ms_p95": _to_summary( metrics["grep_server_worker_critical_total_ms_p95"], "ms" ), "worker_critical_content_retrieval_ms_p95": _to_summary( metrics["grep_server_worker_critical_content_retrieval_ms_p95"], "ms", ), "worker_critical_regex_scan_ms_p95": _to_summary( metrics["grep_server_worker_critical_regex_scan_ms_p95"], "ms" ), "worker_critical_other_ms_p95": _to_summary( metrics["grep_server_worker_critical_other_ms_p95"], "ms" ), "docs_scanned_mean": _to_summary( metrics["grep_server_docs_scanned_mean"], "count" ), "lines_scanned_mean": _to_summary( metrics["grep_server_lines_scanned_mean"], "count" ), "bytes_scanned_mean": _to_summary( metrics["grep_server_bytes_scanned_mean"], "bytes" ), }, "search_server": { "execution_time_ms_p95": _to_summary( metrics["search_server_execution_time_ms_p95"], "ms" ), "phase_dispatch_service_ms_p95": _to_summary( metrics["search_server_phase_dispatch_service_ms_p95"], "ms" ), "phase_dispatch_service_us_p95": _to_summary( metrics["search_server_phase_dispatch_service_us_p95"], "us" ), "phase_dispatch_map_sort_ms_p95": _to_summary( metrics["search_server_phase_dispatch_map_sort_ms_p95"], "ms" ), "phase_dispatch_map_sort_us_p95": _to_summary( metrics["search_server_phase_dispatch_map_sort_us_p95"], "us" ), "phase_dispatch_feedback_ms_p95": _to_summary( metrics["search_server_phase_dispatch_feedback_ms_p95"], "ms" ), "phase_dispatch_feedback_us_p95": _to_summary( metrics["search_server_phase_dispatch_feedback_us_p95"], "us" ), "phase_dispatch_response_ms_p95": _to_summary( metrics["search_server_phase_dispatch_response_ms_p95"], "ms" ), "phase_dispatch_response_us_p95": _to_summary( metrics["search_server_phase_dispatch_response_us_p95"], "us" ), "phase_dispatch_total_ms_p95": _to_summary( metrics["search_server_phase_dispatch_total_ms_p95"], "ms" ), "phase_dispatch_total_us_p95": _to_summary( metrics["search_server_phase_dispatch_total_us_p95"], "us" ), }, "list_server": { "service_execution_time_ms_p95": _to_summary( metrics["list_server_service_execution_time_ms_p95"], "ms" ), "phase_dispatch_service_ms_p95": _to_summary( metrics["list_server_phase_dispatch_service_ms_p95"], "ms" ), "phase_dispatch_service_us_p95": _to_summary( metrics["list_server_phase_dispatch_service_us_p95"], "us" ), "phase_dispatch_map_ms_p95": _to_summary( metrics["list_server_phase_dispatch_map_ms_p95"], "ms" ), "phase_dispatch_map_us_p95": _to_summary( metrics["list_server_phase_dispatch_map_us_p95"], "us" ), "phase_dispatch_total_ms_p95": _to_summary( metrics["list_server_phase_dispatch_total_ms_p95"], "ms" ), "phase_dispatch_total_us_p95": _to_summary( metrics["list_server_phase_dispatch_total_us_p95"], "us" ), }, } return out large_corpus_compat = _build_large_compat(large_corpus) large_corpus_v2 = _build_large_v2(large_corpus, large_usage_profiles) large_corpus_churn_compat = _build_large_compat(large_corpus_churn) large_corpus_churn_v2 = _build_large_v2( large_corpus_churn, large_churn_usage_profiles ) contention_compat = { "throughput_ops_per_sec": _to_summary( contention_metrics["throughput_ops_per_sec"], "ops/s" ), "fail_rate": _to_summary(contention_metrics["fail_rate"], "ratio"), "retry_after_rate": _to_summary( contention_metrics["retry_after_rate"], "ratio" ), "latency_p95_ms": _to_summary(contention_metrics["latency_p95_ms"], "ms"), } contention_v2 = { **contention_compat, "success_rate": _to_summary(contention_metrics["success_rate"], "ratio"), } case_breakdown: list[dict[str, Any]] = [] for key in sorted(case_buckets.keys()): ( run_phase, test_name, request_path, run_transport, usage_profile, backend, num_clients, ) = key metrics = case_buckets[key] case_breakdown.append( { "phase": run_phase, "test": test_name, "request_path": request_path, "transport": run_transport, "usage_profile": usage_profile, "backend": backend, "num_clients": num_clients, "ops_per_sec": _to_summary(metrics["ops_per_sec"], "ops/s"), "fail_rate": _to_summary(metrics["fail_rate"], "ratio"), "latency_p95_ms": _to_summary(metrics["latency_p95_ms"], "ms"), } ) out = { "schema_version": "multi_client_summary_v2", "record_count": len(records), "inputs": [str(p) for p in input_paths], "large_corpus_reads": large_corpus_compat, "large_corpus_reads_churn": large_corpus_churn_compat, "connection_contention": contention_compat, "large_corpus_reads_v2": large_corpus_v2, "large_corpus_reads_churn_v2": large_corpus_churn_v2, "connection_contention_v2": contention_v2, "case_breakdown": case_breakdown, } return out def main() -> int: parser = argparse.ArgumentParser( description="Summarize multi-client benchmark JSONL runs" ) parser.add_argument( "--input", type=Path, action="append", required=True, help="JSONL file path (repeatable)", ) parser.add_argument( "--output", type=Path, required=True, help="Summary JSON output path" ) args = parser.parse_args() records = _load_records(args.input) summary = summarize(records, args.input) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", encoding="utf-8") as handle: json.dump(summary, handle, indent=2) print(f"Wrote summary: {args.output}") return 0 if __name__ == "__main__": raise SystemExit(main())