merge/audit-and-state-map #7
4 changed files with 193 additions and 4 deletions
fix(cli): refuse subset baselines and scope verify to --rule
Two more ways a partial scan produced a false claim, the same hazard class
as the --limit and unreadable-file cases verify already refuses:
- `verify --rule X` compared the fresh X-only report against the whole
baseline, so every accepted finding from any other rule was printed as
RESOLVED ("these fixes are confirmed") when it had merely not been
checked. The baseline is now filtered to the selected rules.
- `--write-baseline` (check and guide) wrote whatever the run produced.
Under --rule or --limit, or with files that could not be analysed, the
baseline silently omitted every finding outside that subset, and the
next full `check --baseline` reported them all as new. It now refuses:
--rule/--limit before analysis, unreadable files after it.
Each is covered by a test that fails against the previous cli.dart
(subset_scan_test.dart, and the unreadable-file case in
unreadable_test.dart). README documents the refusal.
commit
c065576e96
|
|
@ -110,6 +110,12 @@ pipe check . --write-baseline .pipe-baseline.json # accept current state
|
|||
pipe check . --baseline .pipe-baseline.json --fail-on warning
|
||||
```
|
||||
|
||||
`--write-baseline` refuses a partial scan (`--rule`, `--limit`, or files that
|
||||
could not be analysed): a baseline written from a subset omits every finding
|
||||
outside it, and the next full run would report them all as new. `verify --rule`
|
||||
likewise compares only that rule's accepted findings, instead of calling every
|
||||
other rule's findings resolved.
|
||||
|
||||
`--fail-on none|info|warning|error` (default `none`) fails the build when a
|
||||
**new** finding at or above that severity is present. Baselined findings never
|
||||
trip the gate, so a ratcheted repo stays green while a regression fails.
|
||||
|
|
|
|||
|
|
@ -106,9 +106,43 @@ Set<String>? _readBaseline(ArgResults results) {
|
|||
return decodeBaseline(file.readAsStringSync());
|
||||
}
|
||||
|
||||
void _writeBaseline(ArgResults results, String root, List<Finding> findings) {
|
||||
/// A baseline is the set of findings a repo accepts, so it must come from a
|
||||
/// complete scan with every rule on. Written from a subset, every finding
|
||||
/// outside that subset is missing from it, and the next full `check --baseline`
|
||||
/// reports all of them as new. Refused before analysis so the flags fail fast.
|
||||
void _refuseSubsetBaseline(ArgResults results) {
|
||||
if (results['write-baseline'] == null) return;
|
||||
final narrowing = [
|
||||
if ((results['rule'] as List<String>).isNotEmpty) '--rule',
|
||||
if (results['limit'] != null) '--limit',
|
||||
];
|
||||
if (narrowing.isEmpty) return;
|
||||
throw PipeException(
|
||||
'--write-baseline needs a complete scan: with ${narrowing.join(' and ')} '
|
||||
'the baseline would omit every finding outside that subset',
|
||||
remedy: 'drop ${narrowing.join(' and ')} when writing a baseline',
|
||||
);
|
||||
}
|
||||
|
||||
void _writeBaseline(
|
||||
ArgResults results,
|
||||
ProjectAnalysis analysis,
|
||||
List<Finding> findings,
|
||||
) {
|
||||
final path = results['write-baseline'] as String?;
|
||||
if (path == null) return;
|
||||
// Findings in a file that could not be analysed are absent, not fixed.
|
||||
final unresolved = analysis.run.outcomes
|
||||
.where((outcome) => !outcome.resolved && !outcome.skipped)
|
||||
.length;
|
||||
if (unresolved > 0) {
|
||||
throw PipeException(
|
||||
'--write-baseline needs a complete scan: $unresolved file(s) could not '
|
||||
'be analysed, so their findings would be missing from the baseline',
|
||||
remedy: 'run `pipe index` to see why they failed, then write it again',
|
||||
);
|
||||
}
|
||||
final root = analysis.project.root;
|
||||
File(path).writeAsStringSync(encodeBaseline(root, findings));
|
||||
stdout.writeln(
|
||||
'wrote baseline to $path (${findings.length} findings accepted)',
|
||||
|
|
@ -353,6 +387,7 @@ class CheckCommand extends Command<int> {
|
|||
@override
|
||||
Future<int> run() async {
|
||||
final results = argResults!;
|
||||
_refuseSubsetBaseline(results);
|
||||
final project = await loadProject(
|
||||
_target(results),
|
||||
directories: results['dir'] as List<String>,
|
||||
|
|
@ -377,7 +412,7 @@ class CheckCommand extends Command<int> {
|
|||
final top = readPositiveInt(results, 'top', fallback: 40);
|
||||
stdout.write(report.describe(top: top));
|
||||
}
|
||||
_writeBaseline(results, project.root, report.findings);
|
||||
_writeBaseline(results, analysis, report.findings);
|
||||
final threshold = switch (results['fail-on'] as String) {
|
||||
'error' => PipeSeverity.error,
|
||||
'warning' => PipeSeverity.warning,
|
||||
|
|
@ -545,9 +580,17 @@ class VerifyCommand extends Command<int> {
|
|||
);
|
||||
}
|
||||
|
||||
// With --rule, only the selected rules ran, so an accepted finding from
|
||||
// any other rule is merely unchecked. Comparing against the whole baseline
|
||||
// printed every one of them as RESOLVED.
|
||||
final report = verifyAgainstBaseline(
|
||||
report: analysis.report,
|
||||
baseline: entries,
|
||||
baseline: selected.isEmpty
|
||||
? entries
|
||||
: [
|
||||
for (final entry in entries)
|
||||
if (selected.contains(entry.rule)) entry,
|
||||
],
|
||||
);
|
||||
if (results['format'] as String == 'json') {
|
||||
stdout.writeln(_encode(report.toJson()));
|
||||
|
|
@ -691,6 +734,7 @@ class GuideCommand extends Command<int> {
|
|||
@override
|
||||
Future<int> run() async {
|
||||
final results = argResults!;
|
||||
_refuseSubsetBaseline(results);
|
||||
final project = await loadProject(
|
||||
_target(results),
|
||||
directories: results['dir'] as List<String>,
|
||||
|
|
@ -750,7 +794,7 @@ class GuideCommand extends Command<int> {
|
|||
}
|
||||
stdout.writeln('persisted to yams as $name');
|
||||
}
|
||||
_writeBaseline(results, project.root, analysis.report.findings);
|
||||
_writeBaseline(results, analysis, analysis.report.findings);
|
||||
} finally {
|
||||
await analysis.dispose();
|
||||
}
|
||||
|
|
|
|||
109
test/subset_scan_test.dart
Normal file
109
test/subset_scan_test.dart
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:pipe/pipe.dart';
|
||||
import 'package:pipe/src/cli.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
/// Collects what a command writes to stdout.
|
||||
class _CapturedStdout implements Stdout {
|
||||
final StringBuffer buffer = StringBuffer();
|
||||
|
||||
@override
|
||||
void write(Object? object) => buffer.write(object);
|
||||
|
||||
@override
|
||||
void writeln([Object? object = '']) => buffer.writeln(object);
|
||||
|
||||
@override
|
||||
dynamic noSuchMethod(Invocation invocation) => null;
|
||||
}
|
||||
|
||||
Future<(int?, String)> _run(List<String> args) async {
|
||||
final out = _CapturedStdout();
|
||||
final code = await IOOverrides.runZoned(
|
||||
() => PipeRunner().run(args),
|
||||
stdout: () => out,
|
||||
);
|
||||
return (code, out.buffer.toString());
|
||||
}
|
||||
|
||||
const String _fixture = 'test/fixtures/stuck_loading';
|
||||
|
||||
void main() {
|
||||
late String baseline;
|
||||
|
||||
setUpAll(() async {
|
||||
final dir = Directory.systemTemp.createTempSync('pipe-subset');
|
||||
baseline = '${dir.path}/baseline.json';
|
||||
final (code, _) = await _run([
|
||||
'check',
|
||||
_fixture,
|
||||
'--write-baseline',
|
||||
baseline,
|
||||
'-c',
|
||||
'1',
|
||||
]);
|
||||
expect(code, 0);
|
||||
});
|
||||
|
||||
test('verify --rule does not call other rules\' findings resolved', () async {
|
||||
// Only dead-handler ran, so accepted findings from every other rule were
|
||||
// unchecked, not fixed. Comparing against the whole baseline reported each
|
||||
// of them as RESOLVED - a false claim about someone's fixes.
|
||||
final entries = decodeBaselineEntries(File(baseline).readAsStringSync());
|
||||
expect(
|
||||
entries.map((entry) => entry.rule).toSet(),
|
||||
hasLength(greaterThan(1)),
|
||||
reason: 'the control needs findings from more than the selected rule',
|
||||
);
|
||||
|
||||
final (_, output) = await _run([
|
||||
'verify',
|
||||
_fixture,
|
||||
'--baseline',
|
||||
baseline,
|
||||
'--rule',
|
||||
ruleDeadHandler,
|
||||
'--format',
|
||||
'json',
|
||||
'-c',
|
||||
'1',
|
||||
]);
|
||||
final report = jsonDecode(output) as Map<String, Object?>;
|
||||
expect(report['resolved'], 0);
|
||||
expect(report['new'], 0);
|
||||
expect(
|
||||
report['baselineTotal'],
|
||||
entries.where((entry) => entry.rule == ruleDeadHandler).length,
|
||||
);
|
||||
});
|
||||
|
||||
for (final narrowing in [
|
||||
['--rule', ruleDeadHandler],
|
||||
['--limit', '3'],
|
||||
]) {
|
||||
test('--write-baseline refuses ${narrowing.first}', () async {
|
||||
final target = '${File(baseline).parent.path}/subset.json';
|
||||
await expectLater(
|
||||
_run([
|
||||
'check',
|
||||
_fixture,
|
||||
...narrowing,
|
||||
'--write-baseline',
|
||||
target,
|
||||
'-c',
|
||||
'1',
|
||||
]),
|
||||
throwsA(
|
||||
isA<PipeException>().having(
|
||||
(error) => error.toString(),
|
||||
'message',
|
||||
contains('complete scan'),
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(File(target).existsSync(), isFalse);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -72,6 +72,36 @@ void main() {
|
|||
);
|
||||
});
|
||||
|
||||
test('--write-baseline refuses a partially readable project', () async {
|
||||
// The unread file's findings would be missing from the accepted set, so the
|
||||
// next complete scan would report them all as new.
|
||||
final root = unreadableCorpus('write-partial', 'lib/item_bloc.dart');
|
||||
if (root == null) {
|
||||
markTestSkipped('chmod 000 does not restrict this process');
|
||||
return;
|
||||
}
|
||||
|
||||
final baseline = '$root/baseline.json';
|
||||
await expectLater(
|
||||
PipeRunner().run([
|
||||
'check',
|
||||
root,
|
||||
'--write-baseline',
|
||||
baseline,
|
||||
'-c',
|
||||
'1',
|
||||
]),
|
||||
throwsA(
|
||||
isA<PipeException>().having(
|
||||
(error) => error.toString(),
|
||||
'message',
|
||||
contains('complete scan'),
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(File(baseline).existsSync(), isFalse);
|
||||
});
|
||||
|
||||
test('recall keeps evaluating cases whose evidence is readable', () async {
|
||||
// A repo with one unreadable file must not lose coverage: a hit stands,
|
||||
// because evidence that was read is evidence. Skipping every case for the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue