e30764fb19
Add negative_cases_from_universe to expand an explicit file-by-rule universe into labeled negatives (enabling TP/FP/TN/FN, not just TP/FP), plus project-eval harness updates in run_project_eval and the score/score_fp/quality-gate helpers.
51 lines
1.6 KiB
Python
51 lines
1.6 KiB
Python
"""Shared, dependency-free evaluation primitives for Fold benchmarks.
|
|
|
|
Runners collect artifacts. This package owns canonical finding identities,
|
|
ground-truth matching, metrics, and repeated-run stability so those semantics do
|
|
not drift between benchmark entry points.
|
|
"""
|
|
|
|
from .confirmation import summarize_confirmation_results
|
|
from .dispositions import compare_dispositions, validate_disposition_baseline
|
|
from .metamorphic import compare_metamorphic_findings
|
|
from .metrics import (
|
|
aggregate_classification_metrics,
|
|
classification_metrics,
|
|
distribution_summary,
|
|
paired_comparison,
|
|
)
|
|
from .performance import compare_performance_lanes, optimization_surface
|
|
from .project import (
|
|
load_project_manifest,
|
|
manifest_fingerprint,
|
|
validate_project_manifest,
|
|
)
|
|
from .schema import validate_benchmark_case, validate_run_artifact
|
|
from .scoring import (
|
|
negative_cases_from_universe,
|
|
score_grouped_findings,
|
|
score_labeled_cases,
|
|
)
|
|
from .stability import stability_summary
|
|
|
|
__all__ = [
|
|
"aggregate_classification_metrics",
|
|
"classification_metrics",
|
|
"compare_dispositions",
|
|
"compare_metamorphic_findings",
|
|
"compare_performance_lanes",
|
|
"distribution_summary",
|
|
"load_project_manifest",
|
|
"manifest_fingerprint",
|
|
"negative_cases_from_universe",
|
|
"optimization_surface",
|
|
"paired_comparison",
|
|
"score_grouped_findings",
|
|
"score_labeled_cases",
|
|
"stability_summary",
|
|
"summarize_confirmation_results",
|
|
"validate_benchmark_case",
|
|
"validate_disposition_baseline",
|
|
"validate_project_manifest",
|
|
"validate_run_artifact",
|
|
]
|