fix: stop verify, reports and briefs from claiming more than pipe checked #1
8 changed files with 163 additions and 3 deletions
feat(report): state scan coverage in every check, guide and suggest output
A report listed findings without saying how much of the project they were
drawn from, so "0 findings" from 3 of 40 files looked like a clean project -
and an agent handed a brief had no way to know otherwise.
check, guide and suggest JSON now carry a `coverage` block
{discovered, selected, resolved, failed, skipped, complete}. The guide brief,
the markdown report, the suggest agent plan and both text outputs open with a
one-line coverage note, which says PARTIAL whenever a discovered file was not
analysed.
The note lives in the brief's header, which budget trimming never touches. The
"Coverage (what was actually examined)" caveats section was the first thing
dropped under a tight budget, silently; when it no longer fits, a one-line
pointer now says how many caveats were left out.
commit
749aa53348
|
|
@ -1,7 +1,7 @@
|
|||
/// pipe — static state-flow analysis for Dart and Flutter applications.
|
||||
library;
|
||||
|
||||
export 'src/analysis.dart' show Analyzer, FileOutcome;
|
||||
export 'src/analysis.dart' show Analyzer, FileOutcome, ScanCoverage;
|
||||
export 'src/errors.dart' show PipeException;
|
||||
export 'src/guide.dart'
|
||||
show
|
||||
|
|
|
|||
|
|
@ -35,6 +35,78 @@ class FileOutcome {
|
|||
List<Diagnostic> get diagnostics => unit?.diagnostics ?? const <Diagnostic>[];
|
||||
}
|
||||
|
||||
/// How much of a project an analysis actually examined.
|
||||
///
|
||||
/// Every report carries this, because a finding count means nothing without it:
|
||||
/// "0 findings" from 3 of 40 files is not a clean project, and a reader (or an
|
||||
/// agent) handed only the findings cannot tell.
|
||||
class ScanCoverage {
|
||||
const ScanCoverage({
|
||||
required this.discovered,
|
||||
required this.selected,
|
||||
required this.resolved,
|
||||
required this.failed,
|
||||
required this.skipped,
|
||||
});
|
||||
|
||||
factory ScanCoverage.of({
|
||||
required int discovered,
|
||||
required List<FileOutcome> outcomes,
|
||||
}) {
|
||||
var resolved = 0;
|
||||
var failed = 0;
|
||||
var skipped = 0;
|
||||
for (final outcome in outcomes) {
|
||||
if (outcome.resolved) {
|
||||
resolved++;
|
||||
} else if (outcome.skipped) {
|
||||
skipped++;
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
return ScanCoverage(
|
||||
discovered: discovered,
|
||||
selected: outcomes.length,
|
||||
resolved: resolved,
|
||||
failed: failed,
|
||||
skipped: skipped,
|
||||
);
|
||||
}
|
||||
|
||||
/// Dart files found under the scanned directories.
|
||||
final int discovered;
|
||||
|
||||
/// Files handed to the analyzer, after --no-include-generated and --limit.
|
||||
final int selected;
|
||||
final int resolved;
|
||||
final int failed;
|
||||
|
||||
/// Files the analyzer did not cover: excluded by analysis_options, or
|
||||
/// outside every analysis context.
|
||||
final int skipped;
|
||||
|
||||
bool get complete => resolved == discovered;
|
||||
|
||||
Map<String, Object?> toJson() => {
|
||||
'discovered': discovered,
|
||||
'selected': selected,
|
||||
'resolved': resolved,
|
||||
'failed': failed,
|
||||
'skipped': skipped,
|
||||
'complete': complete,
|
||||
};
|
||||
|
||||
/// One line, for the top of a brief or plan.
|
||||
String describe() {
|
||||
final notSelected = discovered - selected;
|
||||
return 'Coverage: $resolved of $discovered discovered file(s) analysed '
|
||||
'($failed failed, $skipped skipped by analysis_options or context'
|
||||
'${notSelected > 0 ? ', $notSelected not selected' : ''})'
|
||||
'${complete ? '.' : ' - results are PARTIAL.'}';
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the analyzer contexts for one target project.
|
||||
///
|
||||
/// Mirrors the lifecycle Flutter's own tooling uses: one collection rooted at
|
||||
|
|
|
|||
|
|
@ -728,6 +728,8 @@ class GuideCommand extends Command<int> {
|
|||
stdout.writeln(
|
||||
_encode({
|
||||
'root': guidance.root,
|
||||
if (analysis.report.coverage != null)
|
||||
'coverage': analysis.report.coverage!.toJson(),
|
||||
'summary': guidance.summary,
|
||||
'omittedFindings': guidance.omittedFindings,
|
||||
'brief': guidance.brief,
|
||||
|
|
|
|||
|
|
@ -295,6 +295,7 @@ Guidance buildGuidance({
|
|||
'bySeverity': report.countBySeverity,
|
||||
'byRule': report.countByRule,
|
||||
'suppressed': report.suppressed,
|
||||
if (report.coverage != null) 'coverage': report.coverage!.toJson(),
|
||||
};
|
||||
|
||||
// --- brief -------------------------------------------------------------
|
||||
|
|
@ -329,9 +330,13 @@ Guidance buildGuidance({
|
|||
'by rule: ${shownRules.map((e) => '${e.key} ${e.value}').join(', ')}'
|
||||
'${hiddenRules > 0 ? ', +$hiddenRules more rule(s)' : ''}';
|
||||
|
||||
// Coverage sits in the header, which is never trimmed: a finding count read
|
||||
// without it cannot be told apart from a clean result.
|
||||
final coverageLine = report.coverage?.describe();
|
||||
final baseHeader = StringBuffer()
|
||||
..writeln('# pipe brief — $root')
|
||||
..writeln()
|
||||
..write(coverageLine == null ? '' : '$coverageLine\n')
|
||||
..writeln(summaryLine)
|
||||
..writeln(aggregate)
|
||||
..writeln()
|
||||
|
|
@ -475,13 +480,26 @@ Guidance buildGuidance({
|
|||
for (final item in caveats) {
|
||||
note.writeln('- $item');
|
||||
}
|
||||
if (buffer.length + note.length < budgetChars) buffer.write(note);
|
||||
if (buffer.length + note.length < budgetChars) {
|
||||
buffer.write(note);
|
||||
} else {
|
||||
// Never drop the fact that caveats exist: a rule that was skipped reads
|
||||
// exactly like a rule that found nothing. Like the omission note, this
|
||||
// line is worth a small overflow.
|
||||
buffer
|
||||
..writeln()
|
||||
..writeln(
|
||||
'${caveats.length} coverage caveat(s) (rules skipped or scoped) '
|
||||
'omitted for space; run with --format md to read them.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// --- markdown ----------------------------------------------------------
|
||||
final markdown = StringBuffer()
|
||||
..writeln('# pipe report — $root')
|
||||
..writeln()
|
||||
..write(coverageLine == null ? '' : '$coverageLine\n\n')
|
||||
..writeln(
|
||||
'${flow.unions.length} blocs/cubits, ${summary['stateVariants']} state '
|
||||
'variants, ${report.findings.length} findings '
|
||||
|
|
|
|||
|
|
@ -221,11 +221,15 @@ class FindingReport {
|
|||
required this.findings,
|
||||
required this.suppressed,
|
||||
this.notes = const [],
|
||||
this.coverage,
|
||||
});
|
||||
|
||||
final String root;
|
||||
final List<Finding> findings;
|
||||
|
||||
/// Which files the findings were drawn from, when known.
|
||||
final ScanCoverage? coverage;
|
||||
|
||||
/// Rules skipped or downgraded, with the reason. Never silently drop signal.
|
||||
final List<String> suppressed;
|
||||
|
||||
|
|
@ -270,6 +274,7 @@ class FindingReport {
|
|||
|
||||
Map<String, Object?> toJson() => {
|
||||
'root': root,
|
||||
if (coverage != null) 'coverage': coverage!.toJson(),
|
||||
'totals': {
|
||||
'findings': findings.length,
|
||||
'new': newFindings.length,
|
||||
|
|
@ -290,6 +295,7 @@ class FindingReport {
|
|||
' baselined=${baselinedFindings.length} $countBySeverity',
|
||||
)
|
||||
..writeln('by rule $countByRule');
|
||||
if (coverage != null) buffer.writeln('files ${coverage!.describe()}');
|
||||
if (suppressed.isNotEmpty) {
|
||||
buffer.writeln('suppressed');
|
||||
for (final note in suppressed) {
|
||||
|
|
@ -855,6 +861,10 @@ FindingReport runChecks({
|
|||
findings: resolved,
|
||||
suppressed: suppressed,
|
||||
notes: notes,
|
||||
coverage: ScanCoverage.of(
|
||||
discovered: project.dartFiles.length,
|
||||
outcomes: outcomes,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import 'analysis.dart';
|
||||
import 'rules.dart';
|
||||
|
||||
/// One fixable root cause: a single action that resolves one or more findings.
|
||||
|
|
@ -76,6 +77,7 @@ class WorkPlan {
|
|||
required this.files,
|
||||
required this.totalFindings,
|
||||
required this.totalCauses,
|
||||
this.coverage,
|
||||
});
|
||||
|
||||
final String root;
|
||||
|
|
@ -83,6 +85,9 @@ class WorkPlan {
|
|||
final int totalFindings;
|
||||
final int totalCauses;
|
||||
|
||||
/// Which files the plan was drawn from, when known.
|
||||
final ScanCoverage? coverage;
|
||||
|
||||
/// Global ordering of causes, highest impact first.
|
||||
List<FixAction> get ordered {
|
||||
final all = [for (final plan in files) ...plan.actions];
|
||||
|
|
@ -92,6 +97,7 @@ class WorkPlan {
|
|||
|
||||
Map<String, Object?> toJson() => {
|
||||
'root': root,
|
||||
if (coverage != null) 'coverage': coverage!.toJson(),
|
||||
'totalFindings': totalFindings,
|
||||
'totalCauses': totalCauses,
|
||||
'files': [for (final plan in files) plan.toJson()],
|
||||
|
|
@ -101,6 +107,7 @@ class WorkPlan {
|
|||
String describe({int topFiles = 10, int topActions = 20}) {
|
||||
final buffer = StringBuffer()
|
||||
..writeln('root $root')
|
||||
..write(coverage == null ? '' : 'files ${coverage!.describe()}\n')
|
||||
..writeln(
|
||||
'plan $totalFindings findings in $totalCauses root cause(s)'
|
||||
' across ${files.length} file(s)',
|
||||
|
|
@ -146,6 +153,7 @@ class WorkPlan {
|
|||
final buffer = StringBuffer()
|
||||
..writeln('# pipe plan — $root')
|
||||
..writeln()
|
||||
..write(coverage == null ? '' : '${coverage!.describe()}\n')
|
||||
..writeln(
|
||||
'$totalFindings findings collapse to $totalCauses root cause(s) across '
|
||||
'${files.length} file(s). Work them in this order.',
|
||||
|
|
@ -373,5 +381,6 @@ WorkPlan buildWorkPlan({required String root, required FindingReport report}) {
|
|||
files: files,
|
||||
totalFindings: newFindings.length,
|
||||
totalCauses: order.length,
|
||||
coverage: report.coverage,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,7 +76,10 @@ void main() {
|
|||
|
||||
test('respects the character budget and reports what it dropped', () {
|
||||
final tight = guidance(budget: 700, maxFindings: 50);
|
||||
expect(tight.brief.length, lessThanOrEqualTo(700 + 200));
|
||||
// The header (now including the coverage line) and the omission and
|
||||
// caveat notes are never trimmed, so a budget this tight overflows by
|
||||
// their size; everything optional is dropped.
|
||||
expect(tight.brief.length, lessThanOrEqualTo(700 + 450));
|
||||
expect(tight.omittedFindings, greaterThan(0));
|
||||
expect(tight.brief, contains('omitted'));
|
||||
});
|
||||
|
|
@ -119,6 +122,20 @@ void main() {
|
|||
}
|
||||
});
|
||||
|
||||
test('coverage leads the brief and survives any budget', () {
|
||||
// A finding count read without coverage cannot be told apart from a clean
|
||||
// result, so the coverage line lives in the header, which is never trimmed.
|
||||
final coverage = analysis.report.coverage!;
|
||||
expect(coverage.resolved, greaterThan(0));
|
||||
expect(analysis.report.toJson()['coverage'], containsPair('failed', 0));
|
||||
for (final budget in [200, 700, 6000]) {
|
||||
final brief = guidance(budget: budget, maxFindings: 50).brief;
|
||||
expect(brief, contains(coverage.describe()), reason: 'budget $budget');
|
||||
}
|
||||
expect(guidance().summary['coverage'], coverage.toJson());
|
||||
expect(guidance().markdown, contains(coverage.describe()));
|
||||
});
|
||||
|
||||
test(
|
||||
'yams emission reports a reason instead of throwing when absent',
|
||||
() async {
|
||||
|
|
|
|||
|
|
@ -246,4 +246,36 @@ void main() {
|
|||
expect(block, isNot(contains('omitted')));
|
||||
expect(plan.describe(), contains('By file'));
|
||||
});
|
||||
|
||||
test('the plan states its coverage, whatever the budget', () {
|
||||
const coverage = ScanCoverage(
|
||||
discovered: 40,
|
||||
selected: 40,
|
||||
resolved: 37,
|
||||
failed: 1,
|
||||
skipped: 2,
|
||||
);
|
||||
final plan = buildWorkPlan(
|
||||
root: '/root',
|
||||
report: FindingReport(
|
||||
root: '/root',
|
||||
findings: [
|
||||
for (var i = 0; i < 30; i++)
|
||||
_finding(
|
||||
rule: ruleDeadHandler,
|
||||
severity: PipeSeverity.info,
|
||||
file: 'lib/f$i.dart',
|
||||
line: 1,
|
||||
data: {'event': 'Event$i'},
|
||||
),
|
||||
],
|
||||
suppressed: const [],
|
||||
coverage: coverage,
|
||||
),
|
||||
);
|
||||
final block = plan.toAgentBlock(budgetChars: 300, maxActions: 30);
|
||||
expect(block, contains('37 of 40 discovered file(s) analysed'));
|
||||
expect(block, contains('PARTIAL'));
|
||||
expect(plan.toJson()['coverage'], coverage.toJson());
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue