feat: establish first-class lexical language parity #2

Merged
trevon merged 14 commits from feat/language-parity-foundation into bench/yams-architecture-reproducibility 2026-08-30 22:09:40 +00:00
4 changed files with 85 additions and 2 deletions
Showing only changes of commit 68c760e089 - Show all commits

test(bench): gate language parity independently

trvon 2026-08-30 15:55:11 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -5,6 +5,7 @@ xplan and Fold's optimization harness.
python3 tests/benchmarks/runner.py self-test
python3 tests/benchmarks/runner.py run lexical_smoke
python3 tests/benchmarks/runner.py run language_parity
python3 tests/benchmarks/runner.py run yams_architecture --corpus /path/to/yams
Each run writes a resolved plan, per-repeat graph/HTML artifacts, summary JSON,
@ -13,6 +14,11 @@ across repeats. The seed corpus has explicit positive cycle and relationship
expectations; it is not representative enough for performance or language-wide
precision claims.
The `language_parity` plan repeats the labeled language corpus three times,
requires stable graph identities, and gates minimum file and symbol evidence for
each advertised language independently. Python and Swift counts characterize
their secondary tier; they do not promote those profiles.
The `yams_architecture` plan is a real-corpus integration benchmark. It records
three stable-identity scans and excludes YAMS's checked-in `site` output with
Brick's explicit `--exclude` option. Validate each resulting graph with YAMS's

View file

@ -0,0 +1,31 @@
{
"schema_version": 1,
"name": "language_parity",
"corpus": "tests/fixtures/languages",
"repeats": 3,
"expect": {
"min_symbols": 35,
"min_dependency_edges": 35,
"max_cycles": 2,
"language_min_symbols": {
"dart": 7,
"rust": 8,
"c": 5,
"cpp": 8,
"javascript": 8,
"typescript": 9,
"python": 4,
"swift": 2
},
"language_min_files": {
"dart": 1,
"rust": 2,
"c": 1,
"cpp": 1,
"javascript": 5,
"typescript": 3,
"python": 2,
"swift": 1
}
}
}

View file

@ -131,7 +131,9 @@ def command_for(
]
def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
def fixture_assertions(
metrics: dict, expected: dict, model: dict | None = None
) -> dict[str, bool]:
assertions = {
"symbols": metrics["symbol_count"] >= expected["min_symbols"],
"dependencies": metrics["dependency_edge_count"]
@ -141,6 +143,24 @@ def fixture_assertions(metrics: dict, expected: dict) -> dict[str, bool]:
assertions["cycles"] = metrics["cycle_count"] == expected["cycles"]
if "max_cycles" in expected:
assertions["max_cycles"] = metrics["cycle_count"] <= expected["max_cycles"]
nodes = [] if model is None else model.get("nodes", [])
for language, minimum in expected.get("language_min_symbols", {}).items():
assertions[f"{language}_symbols"] = (
sum(
node.get("language") == language
and node.get("kind") in {"class", "function"}
for node in nodes
)
>= minimum
)
for language, minimum in expected.get("language_min_files", {}).items():
assertions[f"{language}_files"] = (
sum(
node.get("language") == language and node.get("kind") == "file"
for node in nodes
)
>= minimum
)
return assertions
@ -215,7 +235,7 @@ def run_plan(
metrics = samples[0]["metrics"]
expected = plan["expect"]
assertions = fixture_assertions(metrics, expected)
assertions = fixture_assertions(metrics, expected, model)
if not all(assertions.values()):
raise AssertionError(f"fixture expectations failed: {assertions}; metrics={metrics}")

View file

@ -77,6 +77,32 @@ class RunnerTests(unittest.TestCase):
{"symbols": True, "dependencies": True, "max_cycles": True},
)
def test_language_expectations_cannot_be_satisfied_by_another_language(self) -> None:
metrics = {"symbol_count": 3, "dependency_edge_count": 2, "cycle_count": 0}
expected = {
"min_symbols": 2,
"min_dependency_edges": 1,
"cycles": 0,
"language_min_symbols": {"dart": 2, "rust": 2},
"language_min_files": {"dart": 1, "rust": 1},
}
model = {
"nodes": [
{"kind": "file", "language": "dart"},
{"kind": "class", "language": "dart"},
{"kind": "function", "language": "dart"},
{"kind": "file", "language": "rust"},
{"kind": "function", "language": "rust"},
]
}
assertions = runner.fixture_assertions(metrics, expected, model)
self.assertTrue(assertions["dart_symbols"])
self.assertTrue(assertions["dart_files"])
self.assertFalse(assertions["rust_symbols"])
self.assertTrue(assertions["rust_files"])
if __name__ == "__main__":
unittest.main()