feat: establish first-class lexical language parity #2
4 changed files with 85 additions and 2 deletions
test(bench): gate language parity independently
commit
68c760e089
|
|
@ -5,6 +5,7 @@ xplan and Fold's optimization harness.
|
|||
|
||||
python3 tests/benchmarks/runner.py self-test
|
||||
python3 tests/benchmarks/runner.py run lexical_smoke
|
||||
python3 tests/benchmarks/runner.py run language_parity
|
||||
python3 tests/benchmarks/runner.py run yams_architecture --corpus /path/to/yams
|
||||
|
||||
Each run writes a resolved plan, per-repeat graph/HTML artifacts, summary JSON,
|
||||
|
|
@ -13,6 +14,11 @@ across repeats. The seed corpus has explicit positive cycle and relationship
|
|||
expectations; it is not representative enough for performance or language-wide
|
||||
precision claims.
|
||||
|
||||
The `language_parity` plan repeats the labeled language corpus three times,
|
||||
requires stable graph identities, and gates minimum file and symbol evidence for
|
||||
each advertised language independently. Python and Swift counts characterize
|
||||
their secondary tier; they do not promote those profiles.
|
||||
|
||||
The `yams_architecture` plan is a real-corpus integration benchmark. It records
|
||||
three stable-identity scans and excludes YAMS's checked-in `site` output with
|
||||
Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's
|
||||
|
|
|
|||
31
tests/benchmarks/plans/language_parity.json
Normal file
31
tests/benchmarks/plans/language_parity.json
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"schema_version": 1,
|
||||
"name": "language_parity",
|
||||
"corpus": "tests/fixtures/languages",
|
||||
"repeats": 3,
|
||||
"expect": {
|
||||
"min_symbols": 35,
|
||||
"min_dependency_edges": 35,
|
||||
"max_cycles": 2,
|
||||
"language_min_symbols": {
|
||||
"dart": 7,
|
||||
"rust": 8,
|
||||
"c": 5,
|
||||
"cpp": 8,
|
||||
"javascript": 8,
|
||||
"typescript": 9,
|
||||
"python": 4,
|
||||
"swift": 2
|
||||
},
|
||||
"language_min_files": {
|
||||
"dart": 1,
|
||||
"rust": 2,
|
||||
"c": 1,
|
||||
"cpp": 1,
|
||||
"javascript": 5,
|
||||
"typescript": 3,
|
||||
"python": 2,
|
||||
"swift": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -131,7 +131,9 @@ def command_for(
|
|||
]
|
||||
|
||||
|
||||
def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
|
||||
def fixture_assertions(
|
||||
metrics: dict, expected: dict, model: dict | None = None
|
||||
) -> dict[str, bool]:
|
||||
assertions = {
|
||||
"symbols": metrics["symbol_count"] >= expected["min_symbols"],
|
||||
"dependencies": metrics["dependency_edge_count"]
|
||||
|
|
@ -141,6 +143,24 @@ def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
|
|||
assertions["cycles"] = metrics["cycle_count"] == expected["cycles"]
|
||||
if "max_cycles" in expected:
|
||||
assertions["max_cycles"] = metrics["cycle_count"] <= expected["max_cycles"]
|
||||
nodes = [] if model is None else model.get("nodes", [])
|
||||
for language, minimum in expected.get("language_min_symbols", {}).items():
|
||||
assertions[f"{language}_symbols"] = (
|
||||
sum(
|
||||
node.get("language") == language
|
||||
and node.get("kind") in {"class", "function"}
|
||||
for node in nodes
|
||||
)
|
||||
>= minimum
|
||||
)
|
||||
for language, minimum in expected.get("language_min_files", {}).items():
|
||||
assertions[f"{language}_files"] = (
|
||||
sum(
|
||||
node.get("language") == language and node.get("kind") == "file"
|
||||
for node in nodes
|
||||
)
|
||||
>= minimum
|
||||
)
|
||||
return assertions
|
||||
|
||||
|
||||
|
|
@ -215,7 +235,7 @@ def run_plan(
|
|||
|
||||
metrics = samples[0]["metrics"]
|
||||
expected = plan["expect"]
|
||||
assertions = fixture_assertions(metrics, expected)
|
||||
assertions = fixture_assertions(metrics, expected, model)
|
||||
if not all(assertions.values()):
|
||||
raise AssertionError(f"fixture expectations failed: {assertions}; metrics={metrics}")
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,32 @@ class RunnerTests(unittest.TestCase):
|
|||
{"symbols": True, "dependencies": True, "max_cycles": True},
|
||||
)
|
||||
|
||||
def test_language_expectations_cannot_be_satisfied_by_another_language(self) -> None:
|
||||
metrics = {"symbol_count": 3, "dependency_edge_count": 2, "cycle_count": 0}
|
||||
expected = {
|
||||
"min_symbols": 2,
|
||||
"min_dependency_edges": 1,
|
||||
"cycles": 0,
|
||||
"language_min_symbols": {"dart": 2, "rust": 2},
|
||||
"language_min_files": {"dart": 1, "rust": 1},
|
||||
}
|
||||
model = {
|
||||
"nodes": [
|
||||
{"kind": "file", "language": "dart"},
|
||||
{"kind": "class", "language": "dart"},
|
||||
{"kind": "function", "language": "dart"},
|
||||
{"kind": "file", "language": "rust"},
|
||||
{"kind": "function", "language": "rust"},
|
||||
]
|
||||
}
|
||||
|
||||
assertions = runner.fixture_assertions(metrics, expected, model)
|
||||
|
||||
self.assertTrue(assertions["dart_symbols"])
|
||||
self.assertTrue(assertions["dart_files"])
|
||||
self.assertFalse(assertions["rust_symbols"])
|
||||
self.assertTrue(assertions["rust_files"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue