feat: establish first-class lexical language parity #2
4 changed files with 85 additions and 2 deletions
test(bench): gate language parity independently
commit
68c760e089
|
|
@ -5,6 +5,7 @@ xplan and Fold's optimization harness.
|
||||||
|
|
||||||
python3 tests/benchmarks/runner.py self-test
|
python3 tests/benchmarks/runner.py self-test
|
||||||
python3 tests/benchmarks/runner.py run lexical_smoke
|
python3 tests/benchmarks/runner.py run lexical_smoke
|
||||||
|
python3 tests/benchmarks/runner.py run language_parity
|
||||||
python3 tests/benchmarks/runner.py run yams_architecture --corpus /path/to/yams
|
python3 tests/benchmarks/runner.py run yams_architecture --corpus /path/to/yams
|
||||||
|
|
||||||
Each run writes a resolved plan, per-repeat graph/HTML artifacts, summary JSON,
|
Each run writes a resolved plan, per-repeat graph/HTML artifacts, summary JSON,
|
||||||
|
|
@ -13,6 +14,11 @@ across repeats. The seed corpus has explicit positive cycle and relationship
|
||||||
expectations; it is not representative enough for performance or language-wide
|
expectations; it is not representative enough for performance or language-wide
|
||||||
precision claims.
|
precision claims.
|
||||||
|
|
||||||
|
The `language_parity` plan repeats the labeled language corpus three times,
|
||||||
|
requires stable graph identities, and gates minimum file and symbol evidence for
|
||||||
|
each advertised language independently. Python and Swift counts characterize
|
||||||
|
their secondary tier; they do not promote those profiles.
|
||||||
|
|
||||||
The `yams_architecture` plan is a real-corpus integration benchmark. It records
|
The `yams_architecture` plan is a real-corpus integration benchmark. It records
|
||||||
three stable-identity scans and excludes YAMS's checked-in `site` output with
|
three stable-identity scans and excludes YAMS's checked-in `site` output with
|
||||||
Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
|
Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
|
||||||
|
|
|
||||||
31
tests/benchmarks/plans/language_parity.json
Normal file
31
tests/benchmarks/plans/language_parity.json
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"name": "language_parity",
|
||||||
|
"corpus": "tests/fixtures/languages",
|
||||||
|
"repeats": 3,
|
||||||
|
"expect": {
|
||||||
|
"min_symbols": 35,
|
||||||
|
"min_dependency_edges": 35,
|
||||||
|
"max_cycles": 2,
|
||||||
|
"language_min_symbols": {
|
||||||
|
"dart": 7,
|
||||||
|
"rust": 8,
|
||||||
|
"c": 5,
|
||||||
|
"cpp": 8,
|
||||||
|
"javascript": 8,
|
||||||
|
"typescript": 9,
|
||||||
|
"python": 4,
|
||||||
|
"swift": 2
|
||||||
|
},
|
||||||
|
"language_min_files": {
|
||||||
|
"dart": 1,
|
||||||
|
"rust": 2,
|
||||||
|
"c": 1,
|
||||||
|
"cpp": 1,
|
||||||
|
"javascript": 5,
|
||||||
|
"typescript": 3,
|
||||||
|
"python": 2,
|
||||||
|
"swift": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -131,7 +131,9 @@ def command_for(
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
|
def fixture_assertions(
|
||||||
|
metrics: dict, expected: dict, model: dict | None = None
|
||||||
|
) -> dict[str, bool]:
|
||||||
assertions = {
|
assertions = {
|
||||||
"symbols": metrics["symbol_count"] >= expected["min_symbols"],
|
"symbols": metrics["symbol_count"] >= expected["min_symbols"],
|
||||||
"dependencies": metrics["dependency_edge_count"]
|
"dependencies": metrics["dependency_edge_count"]
|
||||||
|
|
@ -141,6 +143,24 @@ def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
|
||||||
assertions["cycles"] = metrics["cycle_count"] == expected["cycles"]
|
assertions["cycles"] = metrics["cycle_count"] == expected["cycles"]
|
||||||
if "max_cycles" in expected:
|
if "max_cycles" in expected:
|
||||||
assertions["max_cycles"] = metrics["cycle_count"] <= expected["max_cycles"]
|
assertions["max_cycles"] = metrics["cycle_count"] <= expected["max_cycles"]
|
||||||
|
nodes = [] if model is None else model.get("nodes", [])
|
||||||
|
for language, minimum in expected.get("language_min_symbols", {}).items():
|
||||||
|
assertions[f"{language}_symbols"] = (
|
||||||
|
sum(
|
||||||
|
node.get("language") == language
|
||||||
|
and node.get("kind") in {"class", "function"}
|
||||||
|
for node in nodes
|
||||||
|
)
|
||||||
|
>= minimum
|
||||||
|
)
|
||||||
|
for language, minimum in expected.get("language_min_files", {}).items():
|
||||||
|
assertions[f"{language}_files"] = (
|
||||||
|
sum(
|
||||||
|
node.get("language") == language and node.get("kind") == "file"
|
||||||
|
for node in nodes
|
||||||
|
)
|
||||||
|
>= minimum
|
||||||
|
)
|
||||||
return assertions
|
return assertions
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -215,7 +235,7 @@ def run_plan(
|
||||||
|
|
||||||
metrics = samples[0]["metrics"]
|
metrics = samples[0]["metrics"]
|
||||||
expected = plan["expect"]
|
expected = plan["expect"]
|
||||||
assertions = fixture_assertions(metrics, expected)
|
assertions = fixture_assertions(metrics, expected, model)
|
||||||
if not all(assertions.values()):
|
if not all(assertions.values()):
|
||||||
raise AssertionError(f"fixture expectations failed: {assertions}; metrics={metrics}")
|
raise AssertionError(f"fixture expectations failed: {assertions}; metrics={metrics}")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,32 @@ class RunnerTests(unittest.TestCase):
|
||||||
{"symbols": True, "dependencies": True, "max_cycles": True},
|
{"symbols": True, "dependencies": True, "max_cycles": True},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_language_expectations_cannot_be_satisfied_by_another_language(self) -> None:
|
||||||
|
metrics = {"symbol_count": 3, "dependency_edge_count": 2, "cycle_count": 0}
|
||||||
|
expected = {
|
||||||
|
"min_symbols": 2,
|
||||||
|
"min_dependency_edges": 1,
|
||||||
|
"cycles": 0,
|
||||||
|
"language_min_symbols": {"dart": 2, "rust": 2},
|
||||||
|
"language_min_files": {"dart": 1, "rust": 1},
|
||||||
|
}
|
||||||
|
model = {
|
||||||
|
"nodes": [
|
||||||
|
{"kind": "file", "language": "dart"},
|
||||||
|
{"kind": "class", "language": "dart"},
|
||||||
|
{"kind": "function", "language": "dart"},
|
||||||
|
{"kind": "file", "language": "rust"},
|
||||||
|
{"kind": "function", "language": "rust"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
assertions = runner.fixture_assertions(metrics, expected, model)
|
||||||
|
|
||||||
|
self.assertTrue(assertions["dart_symbols"])
|
||||||
|
self.assertTrue(assertions["dart_files"])
|
||||||
|
self.assertFalse(assertions["rust_symbols"])
|
||||||
|
self.assertTrue(assertions["rust_files"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue