yams/tests/integration/search/fsm_tuning_benchmark_integration_test.cpp

// SPDX-License-Identifier: GPL-3.0-or-later
// Copyright 2025 YAMS Contributors

/**
 * @file fsm_tuning_benchmark_integration_test.cpp
 * @brief Integration tests for FSM tuning with internal benchmark validation
 *
 * Phase 4 of Epic yams-7ez4: Tests the complete loop between:
 * - CorpusStats collection from real corpus
 * - SearchTuner FSM state selection based on corpus metrics
 * - InternalBenchmark validation of tuned search quality
 * - BenchmarkComparison for regression/improvement detection
 *
 * These tests validate that the adaptive tuning system works correctly
 * with real documents and the actual search infrastructure.
 */

#include "tests/integration/daemon/test_async_helpers.h"
#include "tests/integration/daemon/test_daemon_harness.h"

#include <catch2/catch_approx.hpp>
#include <catch2/catch_test_macros.hpp>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
#include <catch2/matchers/catch_matchers_string.hpp>

#include <yams/daemon/client/daemon_client.h>
#include <yams/metadata/metadata_repository.h>
#include <yams/search/internal_benchmark.h>
#include <yams/search/search_engine.h>
#include <yams/search/search_tuner.h>
#include <yams/storage/corpus_stats.h>

#include <array>
#include <chrono>
#include <cmath>
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>

// Windows daemon IPC tests are unstable due to socket shutdown race conditions
#ifdef _WIN32
#define SKIP_ON_WINDOWS_DAEMON()                                                                   \
    SKIP("Daemon IPC tests unstable on Windows - see windows-daemon-ipc-plan.md")
#else
#define SKIP_ON_WINDOWS_DAEMON() ((void)0)
#endif

using namespace yams;
using namespace yams::daemon;
using namespace yams::search;
using namespace yams::storage;
using namespace yams::metadata;
using namespace std::chrono_literals;
using Catch::Approx;
using Catch::Matchers::ContainsSubstring;

namespace yams::test {

// =============================================================================
// Test Fixture: Provides daemon + search infrastructure for integration tests
// =============================================================================

class FSMTuningIntegrationFixture {
public:
    FSMTuningIntegrationFixture() : harness_(makeHarnessOptions()) {
        // Start daemon with full search infrastructure
        bool started = false;
        for (int attempt = 0; attempt < 3; ++attempt) {
            started = harness_.start(std::chrono::seconds(30));
            if (started) {
                break;
            }
            harness_.stop();
            std::this_thread::sleep_for(std::chrono::milliseconds(250));
        }
        if (!started) {
            throw std::runtime_error("Failed to start daemon for FSM tuning integration tests");
        }

        // Create daemon client
        ClientConfig clientCfg;
        clientCfg.socketPath = harness_.socketPath();
        clientCfg.connectTimeout = 5s;
        clientCfg.autoStart = false;
        client_ = std::make_unique<DaemonClient>(clientCfg);

        // Initial delay for GlobalIOContext threads to stabilize
        std::this_thread::sleep_for(std::chrono::milliseconds(200));

        // Connect to daemon with retry logic for GlobalIOContext timing
        Result<void> connectResult;
        constexpr int maxRetries = 3;
        for (int attempt = 0; attempt < maxRetries; ++attempt) {
            connectResult = cli::run_sync(client_->connect(), 5s);
            if (connectResult) {
                break;
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(200));
        }
        if (!connectResult) {
            throw std::runtime_error("Failed to connect to daemon: " +
                                     connectResult.error().message);
        }

        // Wait for daemon to be ready
        auto statusResult = cli::run_sync(client_->status(), 5s);
        if (!statusResult) {
            throw std::runtime_error("Failed to get daemon status: " +
                                     statusResult.error().message);
        }
    }

    ~FSMTuningIntegrationFixture() {
        if (client_) {
            client_->disconnect();
        }
        harness_.stop();
    }

    // Ingest a test file into the daemon
    Result<std::string> ingestFile(const std::string& filename, const std::string& content) {
        // Create temp file
        auto filePath = harness_.dataDir() / filename;
        std::ofstream ofs(filePath);
        if (!ofs) {
            return Error{ErrorCode::IOError, "Failed to create test file"};
        }
        ofs << content;
        ofs.close();

        // Ingest via daemon client using streaming add
        AddDocumentRequest req;
        req.path = filePath.string();
        req.tags = {"test", "integration"};

        auto result = cli::run_sync(client_->streamingAddDocument(req), 10s);
        if (!result) {
            return Error{result.error().code, result.error().message};
        }

        return result.value().hash;
    }

    // Execute search via daemon client
    Result<daemon::SearchResponse> executeSearch(const SearchRequest& req) {
        return cli::run_sync(client_->search(req), 10s);
    }

    DaemonClient* client() { return client_.get(); }
    const std::filesystem::path& dataDir() const { return harness_.dataDir(); }

private:
    static DaemonHarness::Options makeHarnessOptions() {
        DaemonHarness::Options opts;
        opts.enableModelProvider = false;
        opts.useMockModelProvider = false;
        return opts;
    }

    DaemonHarness harness_;
    std::unique_ptr<DaemonClient> client_;
};

// =============================================================================
// Helper: Generate test documents with different characteristics
// =============================================================================

std::string generateCodeDocument(size_t functions = 5) {
    std::string code = R"(// Auto-generated test code file
#include <vector>
#include <string>
#include <algorithm>

namespace yams::test {

)";

    for (size_t i = 0; i < functions; ++i) {
        code += "/**\n";
        code += " * @brief Function " + std::to_string(i) + " for testing search indexing\n";
        code += " * @param input The input vector to process\n";
        code += " * @return Processed result vector\n";
        code += " */\n";
        code += "std::vector<int> processFunction" + std::to_string(i) + "(";
        code += "const std::vector<int>& input) {\n";
        code += "    std::vector<int> result;\n";
        code += "    result.reserve(input.size());\n";
        code += "    for (const auto& val : input) {\n";
        code += "        result.push_back(val * " + std::to_string(i + 1) + ");\n";
        code += "    }\n";
        code += "    return result;\n";
        code += "}\n\n";
    }

    code += "} // namespace yams::test\n";
    return code;
}

std::string generateProseDocument(size_t paragraphs = 3) {
    std::string prose;

    for (size_t i = 0; i < paragraphs; ++i) {
        prose += "# Section " + std::to_string(i + 1) + "\n\n";
        prose += "This is paragraph " + std::to_string(i + 1) + " of the test document. ";
        prose += "It contains information about search engine optimization and "
                 "retrieval quality. ";
        prose += "The document discusses various topics including indexing, "
                 "ranking algorithms, ";
        prose += "and benchmark evaluation metrics like MRR and Recall@K.\n\n";
        prose += "Additional context about information retrieval systems and their ";
        prose += "performance characteristics in different corpus environments.\n\n";
    }

    return prose;
}

// =============================================================================
// Test: CorpusStats collection from real indexed documents
// =============================================================================

TEST_CASE("FSM Tuning Integration: CorpusStats reflects ingested documents",
          "[integration][search][fsm][corpus_stats]") {
    SKIP_ON_WINDOWS_DAEMON();
    FSMTuningIntegrationFixture fixture;

    // Ingest some code files
    for (int i = 0; i < 3; ++i) {
        auto result =
            fixture.ingestFile("test_code_" + std::to_string(i) + ".cpp", generateCodeDocument(5));
        REQUIRE(result.has_value());
    }

    // Ingest some prose files
    for (int i = 0; i < 2; ++i) {
        auto result =
            fixture.ingestFile("test_doc_" + std::to_string(i) + ".md", generateProseDocument(3));
        REQUIRE(result.has_value());
    }

    // Wait for indexing to complete
    std::this_thread::sleep_for(1s);

    // Verify corpus stats via status
    auto statusResult = cli::run_sync(fixture.client()->status(), 5s);
    REQUIRE(statusResult.has_value());

    // The daemon should report document count
    INFO("Daemon status after ingestion verified");
}

// =============================================================================
// Test: SearchTuner produces correct state for different corpus profiles
// =============================================================================

TEST_CASE("FSM Tuning Integration: SearchTuner state transitions",
          "[integration][search][fsm][tuner]") {
    // State selection and usable normalized output are contracts; the exact
    // benchmark-driven profile weights are not.
    const auto checkWeights = [](const SearchTuner& tuner) {
        const auto params = tuner.getParams();
        const std::array weights = {
            params.weights.text.value,     params.weights.simeonText.value,
            params.weights.vector.value,   params.weights.entityVector.value,
            params.weights.pathTree.value, params.weights.kg.value,
            params.weights.tag.value,      params.weights.metadata.value,
        };
        float total = 0.0f;
        for (std::size_t i = 0; i < weights.size(); ++i) {
            CAPTURE(i, weights[i]);
            CHECK(std::isfinite(weights[i]));
            CHECK(weights[i] >= 0.0f);
            total += weights[i];
        }
        CHECK(total == Approx(1.0f));
    };

    SECTION("Code-heavy corpus selects CODE state") {
        CorpusStats stats;
        stats.docCount = 500;
        stats.codeRatio = 0.85;
        stats.proseRatio = 0.10;
        stats.symbolDensity = 0.5;
        stats.pathDepthAvg = 3.0;
        stats.tagCoverage = 0.2;

        SearchTuner tuner(stats);

        // With code ratio > 0.7 and doc count < 1000, should be SMALL_CODE
        CHECK(tuner.currentState() == TuningState::SMALL_CODE);
        checkWeights(tuner);
    }

    SECTION("Prose-heavy corpus selects PROSE state") {
        CorpusStats stats;
        stats.docCount = 500;
        stats.codeRatio = 0.10;
        stats.proseRatio = 0.85;
        stats.symbolDensity = 0.01;
        stats.pathDepthAvg = 2.0;
        stats.pathRelativeDepthAvg = 3.0; // Deep nesting disqualifies isScientific
        stats.nativeSymbolDensity = 0.0;
        stats.tagCoverage = 0.3;

        SearchTuner tuner(stats);

        // With prose ratio > 0.7 and doc count < 1000, should be SMALL_PROSE
        CHECK(tuner.currentState() == TuningState::SMALL_PROSE);
        checkWeights(tuner);
    }

    SECTION("Scientific corpus (prose without path/tag structure)") {
        CorpusStats stats;
        stats.docCount = 1500; // Must be >= 1000 (not small) for SCIENTIFIC
        stats.codeRatio = 0.05;
        stats.proseRatio = 0.90;
        stats.symbolDensity = 0.01;
        stats.pathDepthAvg = 1.0;
        stats.pathRelativeDepthAvg = 0.5; // Flat structure triggers isScientific
        stats.nativeSymbolDensity = 0.01; // Low code structure
        stats.tagCoverage = 0.05;         // Minimal tags

        SearchTuner tuner(stats);

        CHECK(tuner.currentState() == TuningState::SCIENTIFIC);
        checkWeights(tuner);

        // This corpus has no usable structure signals. These zeros express
        // capability gates, not a preferred text/vector/Simeon blend.
        const auto params = tuner.getParams();
        CHECK(params.weights.pathTree.value == 0.0f);
        CHECK(params.weights.kg.value == 0.0f);
        CHECK(params.weights.tag.value == 0.0f);
    }

    SECTION("Large mixed corpus") {
        CorpusStats stats;
        stats.docCount = 5000;
        stats.codeRatio = 0.45;
        stats.proseRatio = 0.40;
        stats.symbolDensity = 0.2;
        stats.pathDepthAvg = 4.0;
        stats.tagCoverage = 0.5;

        SearchTuner tuner(stats);

        // Mixed content, neither code nor prose dominant
        CHECK((tuner.currentState() == TuningState::MIXED));
        checkWeights(tuner);
    }

    SECTION("Minimal corpus") {
        CorpusStats stats;
        stats.docCount = 50; // < 100
        stats.codeRatio = 0.50;
        stats.proseRatio = 0.30;
        stats.symbolDensity = 0.1;
        stats.pathDepthAvg = 2.0;
        stats.tagCoverage = 0.1;

        SearchTuner tuner(stats);

        CHECK((tuner.currentState() == TuningState::MINIMAL));
        checkWeights(tuner);
    }
}

// =============================================================================
// Test: TunedParams apply correctly to SearchEngineConfig
// =============================================================================

TEST_CASE("FSM Tuning Integration: TunedParams apply to config",
          "[integration][search][fsm][config]") {
    CorpusStats stats;
    stats.docCount = 2000;
    stats.codeRatio = 0.75;
    stats.proseRatio = 0.15;
    stats.symbolDensity = 0.4;
    stats.pathDepthAvg = 4.0;
    stats.tagCoverage = 0.3;

    SearchTuner tuner(stats);
    auto config = tuner.getConfig();

    // Verify config has tuned weights applied
    CHECK((config.textWeight == tuner.getParams().weights.text.value));
    CHECK((config.simeonTextWeight == tuner.getParams().weights.simeonText.value));
    CHECK((config.vectorWeight == tuner.getParams().weights.vector.value));
    CHECK((config.entityVectorWeight == tuner.getParams().weights.entityVector.value));
    CHECK((config.rrfK == static_cast<float>(tuner.getRrfK())));
    CHECK((config.pathTreeWeight == tuner.getParams().weights.pathTree.value));
    CHECK((config.kgWeight == tuner.getParams().weights.kg.value));
    CHECK((config.tagWeight == tuner.getParams().weights.tag.value));
    CHECK((config.metadataWeight == tuner.getParams().weights.metadata.value));

    // Verify JSON serialization for observability
    auto json = tuner.toJson();
    CHECK(json.contains("state"));
    CHECK(json.contains("params"));
    CHECK(json.contains("reason"));
    CHECK((json["state"].get<std::string>() == tuningStateToString(tuner.currentState())));
}

// =============================================================================
// Test: BenchmarkComparison detects regression and improvement
// =============================================================================

TEST_CASE("FSM Tuning Integration: BenchmarkComparison detection",
          "[integration][search][fsm][benchmark][comparison]") {
    SECTION("Detects significant improvement") {
        BenchmarkResults baseline;
        baseline.mrr = 0.60f;
        baseline.recallAtK = 0.55f;
        baseline.latency.meanMs = 50.0;

        BenchmarkResults current;
        current.mrr = 0.75f; // +0.15 improvement
        current.recallAtK = 0.70f;
        current.latency.meanMs = 45.0;

        auto comparison = InternalBenchmark::compare(baseline, current, 0.05f);

        CHECK((comparison.isImprovement == true));
        CHECK((comparison.isRegression == false));
        CHECK((comparison.mrrDelta == Approx(0.15f)));
        CHECK((comparison.recallDelta == Approx(0.15f)));
        CHECK_THAT(comparison.summary, ContainsSubstring("IMPROVEMENT"));
    }

    SECTION("Detects significant regression") {
        BenchmarkResults baseline;
        baseline.mrr = 0.80f;
        baseline.recallAtK = 0.75f;
        baseline.latency.meanMs = 40.0;

        BenchmarkResults current;
        current.mrr = 0.65f; // -0.15 regression
        current.recallAtK = 0.60f;
        current.latency.meanMs = 60.0;

        auto comparison = InternalBenchmark::compare(baseline, current, 0.05f);

        CHECK((comparison.isRegression == true));
        CHECK((comparison.isImprovement == false));
        CHECK((comparison.mrrDelta == Approx(-0.15f)));
        CHECK_THAT(comparison.summary, ContainsSubstring("REGRESSION"));
    }

    SECTION("No significant change within threshold") {
        BenchmarkResults baseline;
        baseline.mrr = 0.70f;
        baseline.recallAtK = 0.65f;
        baseline.latency.meanMs = 50.0;

        BenchmarkResults current;
        current.mrr = 0.72f; // +0.02, below 0.05 threshold
        current.recallAtK = 0.66f;
        current.latency.meanMs = 48.0;

        auto comparison = InternalBenchmark::compare(baseline, current, 0.05f);

        CHECK((comparison.isRegression == false));
        CHECK((comparison.isImprovement == false));
        CHECK((comparison.mrrDelta == Approx(0.02f)));
    }

    SECTION("Handles first run (no baseline)") {
        BenchmarkResults baseline;
        baseline.mrr = 0.0f; // No previous data
        baseline.queriesRun = 0;

        BenchmarkResults current;
        current.mrr = 0.75f;
        current.recallAtK = 0.70f;
        current.queriesRun = 100;

        auto comparison = InternalBenchmark::compare(baseline, current, 0.05f);

        // First run should not be flagged as regression
        CHECK((comparison.isRegression == false));
        // Could be flagged as improvement from 0
        CHECK((comparison.mrrDelta == Approx(0.75f)));
    }
}

// =============================================================================
// Test: BenchmarkResults summary format for CLI output
// =============================================================================

TEST_CASE("FSM Tuning Integration: BenchmarkResults summary",
          "[integration][search][fsm][benchmark][summary]") {
    BenchmarkResults results;
    results.mrr = 0.756f;
    results.recallAtK = 0.680f;
    results.k = 10;
    results.latency.meanMs = 42.5;
    results.latency.p95Ms = 85.3;
    results.queriesRun = 100;
    results.queriesSucceeded = 98;
    results.queriesFailed = 2;
    results.totalTime = std::chrono::milliseconds(4500);
    results.tuningState = "LARGE_CODE";

    auto summary = results.summary();

    // Verify summary contains key metrics
    CHECK_THAT(summary, ContainsSubstring("MRR"));
    CHECK_THAT(summary, ContainsSubstring("0.756") || ContainsSubstring("0.76"));
    CHECK_THAT(summary, ContainsSubstring("Recall"));
    CHECK_THAT(summary, ContainsSubstring("queries"));
}

// =============================================================================
// Test: End-to-end FSM tuning flow with real daemon
// =============================================================================

TEST_CASE("FSM Tuning Integration: End-to-end tuning validation",
          "[integration][search][fsm][e2e]") {
    SKIP_ON_WINDOWS_DAEMON();
    std::unique_ptr<FSMTuningIntegrationFixture> fixture;
    std::string fixtureError;
    for (int attempt = 0; attempt < 3; ++attempt) {
        try {
            fixture = std::make_unique<FSMTuningIntegrationFixture>();
            break;
        } catch (const std::exception& e) {
            fixtureError = e.what();
            std::this_thread::sleep_for(300ms);
        }
    }
    if (!fixture) {
        SKIP("Skipping e2e tuning validation due to daemon startup instability: " + fixtureError);
    }

    // 1. Ingest documents to create a code-heavy corpus
    INFO("Ingesting test documents...");
    for (int i = 0; i < 5; ++i) {
        auto result = fixture->ingestFile("e2e_code_" + std::to_string(i) + ".cpp",
                                          generateCodeDocument(3 + i));
        REQUIRE(result.has_value());
    }

    // 2. Wait for indexing
    std::this_thread::sleep_for(2s);

    // 3. Execute a search to verify the tuned engine works
    SearchRequest req;
    req.query = "processFunction vector result";
    req.searchType = "hybrid";
    req.limit = 10;
    req.timeout = 5s;

    auto searchResult = fixture->executeSearch(req);

    // Verify search completes (may return empty on mock provider)
    if (searchResult) {
        INFO("Search returned " << searchResult.value().results.size() << " results");
        // If we have results, they should be from our test files
        for (const auto& hit : searchResult.value().results) {
            INFO("Result: " << hit.path << " score=" << hit.score);
        }
    } else {
        // Mock provider may cause semantic portion to fail
        INFO("Search error (expected with mock): " << searchResult.error().message);
    }
}

// =============================================================================
// Test: Tuner state reason provides debugging info
// =============================================================================

TEST_CASE("FSM Tuning Integration: State reason explains selection",
          "[integration][search][fsm][debug]") {
    CorpusStats stats;
    stats.docCount = 500;
    stats.codeRatio = 0.85;
    stats.proseRatio = 0.10;
    stats.symbolDensity = 0.5;
    stats.pathDepthAvg = 3.0;
    stats.tagCoverage = 0.2;

    SearchTuner tuner(stats);

    // State reason should explain why this state was selected
    const auto& reason = tuner.stateReason();
    CHECK(!reason.empty());

    // Should mention the dominant corpus characteristic
    // (Implementation may vary, but should provide useful debug info)
    INFO("State reason: " << reason);
}

} // namespace yams::test