#pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #if __APPLE__ #include #endif namespace yams::test { class BenchmarkTracker { public: struct BenchmarkResult { std::string name; double value; std::string unit; std::chrono::system_clock::time_point timestamp; std::map environment; std::map metrics; // Additional metrics std::map attributes; // String-valued metadata nlohmann::json toJSON() const { nlohmann::json j; j["name"] = name; j["value"] = value; j["unit"] = unit; j["timestamp"] = std::chrono::duration_cast(timestamp.time_since_epoch()) .count(); j["environment"] = environment; j["metrics"] = metrics; if (!attributes.empty()) { j["attributes"] = attributes; } return j; } static BenchmarkResult fromJSON(const nlohmann::json& j) { BenchmarkResult result; result.name = j["name"]; result.value = j["value"]; result.unit = j["unit"]; result.timestamp = std::chrono::system_clock::time_point( std::chrono::milliseconds(j["timestamp"].get())); result.environment = j["environment"].get>(); if (j.contains("metrics")) { result.metrics = j["metrics"].get>(); } if (j.contains("attributes")) { result.attributes = j["attributes"].get>(); } return result; } }; BenchmarkTracker(const std::filesystem::path& dataPath = "benchmark_results.json") : dataPath_(dataPath) { loadHistory(); captureEnvironment(); } ~BenchmarkTracker() { saveHistory(); } // Force writing the history JSON to disk immediately. // This is useful for suites that want to archive the history file before exit. void flushHistory() { saveHistory(); } void recordResult(const BenchmarkResult& result) { results_.push_back(result); // Update history if (history_.find(result.name) == history_.end()) { history_[result.name] = std::vector(); } history_[result.name].push_back(result); // Check for regression immediately if (detectRegression(result.name)) { std::cerr << "WARNING: Performance regression detected for " << result.name << std::endl; } } // Best-effort snapshot copy of results/history/environment. // Note: history is also persisted on destruction. void snapshotTo(const std::filesystem::path& outPath) const { nlohmann::json data; data["environment"] = currentEnvironment_; data["results"] = nlohmann::json::array(); for (const auto& r : results_) { data["results"].push_back(r.toJSON()); } data["history"] = nlohmann::json::object(); for (const auto& [name, results] : history_) { data["history"][name] = nlohmann::json::array(); for (const auto& r : results) { data["history"][name].push_back(r.toJSON()); } } std::error_code ec; std::filesystem::create_directories(outPath.parent_path(), ec); std::ofstream file(outPath); file << data.dump(2); } std::vector getHistory(const std::string& name, size_t limit = 100) { if (history_.find(name) == history_.end()) { return {}; } auto& results = history_[name]; size_t start = results.size() > limit ? results.size() - limit : 0; return std::vector(results.begin() + start, results.end()); } bool detectRegression(const std::string& name, double threshold = 0.1) { auto history = getHistory(name, 10); if (history.size() < 2) { return false; // Not enough data } // Compare latest with average of previous results double latest = history.back().value; double sum = 0; for (size_t i = 0; i < history.size() - 1; ++i) { sum += history[i].value; } double average = sum / (history.size() - 1); // Check if latest is significantly worse (higher for time, lower for throughput) if (history.back().unit.find("ms") != std::string::npos || history.back().unit.find("sec") != std::string::npos) { // Time-based metric: higher is worse return (latest - average) / average > threshold; } else { // Throughput metric: lower is worse return (average - latest) / average > threshold; } } void generateReport(const std::filesystem::path& outputPath) { nlohmann::json report; report["generated_at"] = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); report["environment"] = currentEnvironment_; // Summary statistics nlohmann::json summary; for (const auto& [name, results] : history_) { if (results.empty()) continue; nlohmann::json benchSummary; benchSummary["name"] = name; benchSummary["runs"] = results.size(); // Calculate statistics double sum = 0, min = results[0].value, max = results[0].value; for (const auto& r : results) { sum += r.value; min = std::min(min, r.value); max = std::max(max, r.value); } benchSummary["average"] = sum / results.size(); benchSummary["min"] = min; benchSummary["max"] = max; benchSummary["latest"] = results.back().value; benchSummary["unit"] = results.back().unit; // Detect trend if (results.size() >= 3) { double firstHalf = 0, secondHalf = 0; size_t mid = results.size() / 2; for (size_t i = 0; i < mid; ++i) { firstHalf += results[i].value; } for (size_t i = mid; i < results.size(); ++i) { secondHalf += results[i].value; } firstHalf /= mid; secondHalf /= (results.size() - mid); double change = (secondHalf - firstHalf) / firstHalf; benchSummary["trend"] = change > 0.05 ? "degrading" : change < -0.05 ? "improving" : "stable"; benchSummary["trend_percent"] = change * 100; } summary[name] = benchSummary; } report["summary"] = summary; report["results"] = nlohmann::json::array(); for (const auto& result : results_) { report["results"].push_back(result.toJSON()); } // Write report std::ofstream file(outputPath); file << report.dump(2); file.close(); } void generateMarkdownReport(const std::filesystem::path& outputPath) { std::ofstream file(outputPath); file << "# Benchmark Report\n\n"; file << "Generated: " << getCurrentTimestamp() << "\n\n"; file << "## Environment\n\n"; for (const auto& [key, value] : currentEnvironment_) { file << "- **" << key << "**: " << value << "\n"; } file << "\n"; file << "## Results Summary\n\n"; file << "| Benchmark | Latest | Average | Min | Max | Trend | Runs |\n"; file << "|-----------|--------|---------|-----|-----|-------|------|\n"; for (const auto& [name, results] : history_) { if (results.empty()) continue; double sum = 0, min = results[0].value, max = results[0].value; for (const auto& r : results) { sum += r.value; min = std::min(min, r.value); max = std::max(max, r.value); } double avg = sum / results.size(); std::string trend = "→"; if (results.size() >= 3) { double recent = results.back().value; double change = (recent - avg) / avg; if (change > 0.05) trend = "↗️"; else if (change < -0.05) trend = "↘️"; } file << "| " << name << " | " << formatValue(results.back().value, results.back().unit) << " | " << formatValue(avg, results.back().unit) << " | " << formatValue(min, results.back().unit) << " | " << formatValue(max, results.back().unit) << " | " << trend << " | " << results.size() << " |\n"; } file << "\n## Regression Analysis\n\n"; bool hasRegression = false; for (const auto& [name, _] : history_) { if (detectRegression(name)) { hasRegression = true; file << "⚠️ **" << name << "**: Performance regression detected\n"; } } if (!hasRegression) { file << "✅ No performance regressions detected\n"; } file.close(); } // Compare with baseline struct ComparisonResult { std::string benchmark; double baseline; double current; double percentChange; bool regression; }; std::vector compareWithBaseline(const std::filesystem::path& baselinePath) { std::vector comparisons; // Load baseline std::ifstream baselineFile(baselinePath); if (!baselineFile.is_open()) { return comparisons; } nlohmann::json baseline = nlohmann::json::parse(baselineFile); baselineFile.close(); // Compare each benchmark for (const auto& result : results_) { if (baseline["summary"].contains(result.name)) { ComparisonResult comp; comp.benchmark = result.name; comp.baseline = baseline["summary"][result.name]["average"]; comp.current = result.value; comp.percentChange = ((comp.current - comp.baseline) / comp.baseline) * 100; // Determine if regression based on metric type if (result.unit.find("ms") != std::string::npos || result.unit.find("sec") != std::string::npos) { comp.regression = comp.percentChange > 10; // 10% slower } else { comp.regression = comp.percentChange < -10; // 10% less throughput } comparisons.push_back(comp); } } return comparisons; } private: std::filesystem::path dataPath_; std::vector results_; std::map> history_; std::map currentEnvironment_; void loadHistory() { if (!std::filesystem::exists(dataPath_)) { return; } std::ifstream file(dataPath_); if (!file.is_open()) { return; } try { nlohmann::json data = nlohmann::json::parse(file); if (data.contains("history")) { for (const auto& [name, results] : data["history"].items()) { history_[name] = std::vector(); for (const auto& r : results) { history_[name].push_back(BenchmarkResult::fromJSON(r)); } } } } catch (const std::exception& e) { std::cerr << "Failed to load benchmark history: " << e.what() << std::endl; } file.close(); } void saveHistory() { nlohmann::json data; data["history"] = nlohmann::json::object(); for (const auto& [name, results] : history_) { data["history"][name] = nlohmann::json::array(); for (const auto& r : results) { data["history"][name].push_back(r.toJSON()); } } std::ofstream file(dataPath_); file << data.dump(2); file.close(); } void captureEnvironment() { // Capture system information currentEnvironment_["platform"] = #ifdef _WIN32 "Windows"; #elif __APPLE__ "macOS"; #elif __linux__ "Linux"; #else "Unknown"; #endif currentEnvironment_["compiler"] = #ifdef __clang__ "Clang " + std::to_string(__clang_major__); #elif __GNUC__ "GCC " + std::to_string(__GNUC__); #elif _MSC_VER "MSVC " + std::to_string(_MSC_VER); #else "Unknown"; #endif currentEnvironment_["build_type"] = #ifdef NDEBUG "Release"; #else "Debug"; #endif currentEnvironment_["cpu_cores"] = std::to_string(std::thread::hardware_concurrency()); #if __APPLE__ { char buf[256]; std::size_t len = sizeof(buf); if (::sysctlbyname("machdep.cpu.brand_string", buf, &len, nullptr, 0) == 0 && len > 0) { currentEnvironment_["cpu_model"] = std::string(buf); } } { std::uint64_t mem = 0; std::size_t len = sizeof(mem); if (::sysctlbyname("hw.memsize", &mem, &len, nullptr, 0) == 0 && mem > 0) { currentEnvironment_["mem_bytes"] = std::to_string(mem); } } #endif } std::string getCurrentTimestamp() { auto now = std::chrono::system_clock::now(); auto time_t = std::chrono::system_clock::to_time_t(now); std::stringstream ss; ss << std::put_time(std::localtime(&time_t), "%Y-%m-%d %H:%M:%S"); return ss.str(); } std::string formatValue(double value, const std::string& unit) { std::stringstream ss; ss << std::fixed << std::setprecision(2) << value << " " << unit; return ss.str(); } }; } // namespace yams::test