brick/tests/benchmarks/test_runner.py

#!/usr/bin/env python3
"""Behavior tests for Brick's artifact-first benchmark runner."""

from __future__ import annotations

import importlib.util
import pathlib
import subprocess
import tempfile
import unittest

HERE = pathlib.Path(__file__).resolve().parent
SPEC = importlib.util.spec_from_file_location(
    "brick_benchmark_runner", HERE / "runner.py"
)
assert SPEC is not None and SPEC.loader is not None
runner = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(runner)


class RunnerTests(unittest.TestCase):
    def test_source_labels_detect_missing_symbols_and_negative_file_edges(self) -> None:
        model = {
            "nodes": [
                {"id": "file", "kind": "file", "path": "js/negative.js", "name": "negative.js"},
                {"id": "real", "kind": "function", "path": "js/negative.js", "name": "real", "parent": "file"},
                {"id": "ghost", "kind": "external", "language": "javascript", "name": "ghost"},
            ],
            "edges": [{"source": "real", "target": "ghost", "kind": "calls"}],
        }
        labels = {"symbols": [["js/negative.js::real", "function"], ["js/positive.js::missing", "function"]], "relationships": []}
        result = runner.source_language_quality(model, labels)
        self.assertFalse(result["symbols"]["exact"])
        self.assertEqual(result["symbols"]["recall"], 0.5)
        self.assertFalse(result["relationships"]["exact"])
        self.assertEqual(len(result["relationships"]["false_positives"]), 1)
        self.assertFalse(result["by_language"]["js"]["relationships"]["exact"])

    def test_git_provenance_fingerprints_dirty_corpus_state(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            repository = pathlib.Path(directory)
            subprocess.run(["git", "init", "--quiet"], cwd=repository, check=True)
            subprocess.run(
                ["git", "config", "user.email", "brick-tests@example.invalid"],
                cwd=repository,
                check=True,
            )
            subprocess.run(
                ["git", "config", "user.name", "Brick Tests"],
                cwd=repository,
                check=True,
            )
            subprocess.run(
                ["git", "config", "commit.gpgsign", "false"], cwd=repository, check=True
            )
            tracked = repository / "tracked.cpp"
            tracked.write_text("int value = 1;\n", encoding="utf-8")
            subprocess.run(["git", "add", "tracked.cpp"], cwd=repository, check=True)
            subprocess.run(
                ["git", "commit", "--quiet", "-m", "fixture"],
                cwd=repository,
                check=True,
            )

            clean = runner.git_provenance(repository)
            self.assertFalse(clean["dirty"])
            self.assertIsNone(clean["working_tree_fingerprint"])

            tracked.write_text("int value = 2;\n", encoding="utf-8")
            (repository / "untracked.dart").write_text(
                "void main() {}\n", encoding="utf-8"
            )
            dirty = runner.git_provenance(repository)
            self.assertTrue(dirty["dirty"])
            self.assertEqual(dirty["revision"], clean["revision"])
            self.assertRegex(dirty["working_tree_fingerprint"], r"^[0-9a-f]{64}$")

            (repository / "untracked.dart").write_text(
                "void changed() {}\n", encoding="utf-8"
            )
            changed = runner.git_provenance(repository)
            self.assertNotEqual(
                changed["working_tree_fingerprint"], dirty["working_tree_fingerprint"]
            )

    def test_rejects_plan_name_path_traversal(self) -> None:
        with self.assertRaisesRegex(ValueError, "invalid benchmark plan name"):
            runner.load_plan("../lexical_smoke")

    def test_external_corpus_and_build_arguments_are_explicit(self) -> None:
        corpus = pathlib.Path("external-yams").resolve()
        plan = {"corpus": "tests/fixtures/mixed", "build_args": ["--exclude", "site"]}
        command = runner.command_for(
            plan,
            corpus,
            pathlib.Path("graph.json"),
            pathlib.Path("building.html"),
        )
        self.assertIn(str(corpus), command)
        self.assertEqual(command[-2:], ["--exclude", "site"])

    def test_max_cycle_expectation_allows_non_exact_real_corpus_gate(self) -> None:
        metrics = {
            "symbol_count": 12000,
            "dependency_edge_count": 90000,
            "cycle_count": 17,
        }
        expected = {
            "min_symbols": 10000,
            "min_dependency_edges": 80000,
            "max_cycles": 25,
        }
        assertions = runner.fixture_assertions(metrics, expected)
        self.assertEqual(
            assertions,
            {"symbols": True, "dependencies": True, "max_cycles": True},
        )

    def test_feature_flags_are_passed_before_cargo_argument_separator(self) -> None:
        plan = {"cargo_features": ["tree-sitter-p4-pilot"]}
        command = runner.command_for(
            plan,
            pathlib.Path("corpus"),
            pathlib.Path("graph.json"),
            pathlib.Path("graph.html"),
        )
        self.assertEqual(
            command[:6],
            [
                "cargo",
                "run",
                "--quiet",
                "--features",
                "tree-sitter-p4-pilot",
                "--",
            ],
        )

    def test_p4_relationship_quality_counts_clean_negatives(self) -> None:
        model = {
            "nodes": [
                {"id": "control:1", "name": "Pilot", "path": "pilot.p4"},
                {
                    "id": "table:1",
                    "name": "routes",
                    "path": "pilot.p4",
                    "parent": "control:1",
                },
                {
                    "id": "action:1",
                    "name": "forward",
                    "path": "pilot.p4",
                    "parent": "control:1",
                },
                {
                    "id": "action:2",
                    "name": "ghost",
                    "path": "pilot.p4",
                    "parent": "control:1",
                },
            ],
            "edges": [
                {
                    "source": "table:1",
                    "target": "action:1",
                    "attributes": {"p4.relation": "table-action"},
                },
                {
                    "source": "table:1",
                    "target": "action:2",
                    "attributes": {"p4.relation": "table-action"},
                },
            ],
        }
        quality = runner.p4_relationship_quality(
            model,
            [
                {
                    "source_path": "pilot.p4",
                    "source_owner": "Pilot",
                    "source": "routes",
                    "target_path": "pilot.p4",
                    "target_owner": "Pilot",
                    "target": "forward",
                    "relation": "table-action",
                }
            ],
        )
        self.assertEqual(quality["true_positives"], 1)
        self.assertEqual(quality["false_positives"], 1)
        self.assertEqual(quality["false_negatives"], 0)
        self.assertEqual(quality["precision"], 0.5)
        self.assertEqual(quality["recall"], 1.0)

    def test_p4_expectations_assert_exact_nested_metric_contract(self) -> None:
        metrics = {
            "symbol_count": 5,
            "dependency_edge_count": 6,
            "cycle_count": 1,
            "p4": {"parser_cycle_count": 1, "unreachable_state_count": 0},
        }
        expected = {
            "min_symbols": 5,
            "min_dependency_edges": 6,
            "cycles": 1,
            "p4": {"parser_cycle_count": 1, "unreachable_state_count": 1},
        }
        self.assertEqual(
            runner.fixture_assertions(metrics, expected),
            {
                "symbols": True,
                "dependencies": True,
                "cycles": True,
                "p4.parser_cycle_count": True,
                "p4.unreachable_state_count": False,
            },
        )


if __name__ == "__main__":
    unittest.main()