c065576e96
Two more ways a partial scan produced a false claim, the same hazard class
as the --limit and unreadable-file cases verify already refuses:
- `verify --rule X` compared the fresh X-only report against the whole
baseline, so every accepted finding from any other rule was printed as
RESOLVED ("these fixes are confirmed") when it had merely not been
checked. The baseline is now filtered to the selected rules.
- `--write-baseline` (check and guide) wrote whatever the run produced.
Under --rule or --limit, or with files that could not be analysed, the
baseline silently omitted every finding outside that subset, and the
next full `check --baseline` reported them all as new. It now refuses:
--rule/--limit before analysis, unreadable files after it.
Each is covered by a test that fails against the previous cli.dart
(subset_scan_test.dart, and the unreadable-file case in
unreadable_test.dart). README documents the refusal.
109 lines
2.8 KiB
Dart
109 lines
2.8 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:pipe/pipe.dart';
|
|
import 'package:pipe/src/cli.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
/// Collects what a command writes to stdout.
|
|
class _CapturedStdout implements Stdout {
|
|
final StringBuffer buffer = StringBuffer();
|
|
|
|
@override
|
|
void write(Object? object) => buffer.write(object);
|
|
|
|
@override
|
|
void writeln([Object? object = '']) => buffer.writeln(object);
|
|
|
|
@override
|
|
dynamic noSuchMethod(Invocation invocation) => null;
|
|
}
|
|
|
|
Future<(int?, String)> _run(List<String> args) async {
|
|
final out = _CapturedStdout();
|
|
final code = await IOOverrides.runZoned(
|
|
() => PipeRunner().run(args),
|
|
stdout: () => out,
|
|
);
|
|
return (code, out.buffer.toString());
|
|
}
|
|
|
|
const String _fixture = 'test/fixtures/stuck_loading';
|
|
|
|
void main() {
|
|
late String baseline;
|
|
|
|
setUpAll(() async {
|
|
final dir = Directory.systemTemp.createTempSync('pipe-subset');
|
|
baseline = '${dir.path}/baseline.json';
|
|
final (code, _) = await _run([
|
|
'check',
|
|
_fixture,
|
|
'--write-baseline',
|
|
baseline,
|
|
'-c',
|
|
'1',
|
|
]);
|
|
expect(code, 0);
|
|
});
|
|
|
|
test('verify --rule does not call other rules\' findings resolved', () async {
|
|
// Only dead-handler ran, so accepted findings from every other rule were
|
|
// unchecked, not fixed. Comparing against the whole baseline reported each
|
|
// of them as RESOLVED - a false claim about someone's fixes.
|
|
final entries = decodeBaselineEntries(File(baseline).readAsStringSync());
|
|
expect(
|
|
entries.map((entry) => entry.rule).toSet(),
|
|
hasLength(greaterThan(1)),
|
|
reason: 'the control needs findings from more than the selected rule',
|
|
);
|
|
|
|
final (_, output) = await _run([
|
|
'verify',
|
|
_fixture,
|
|
'--baseline',
|
|
baseline,
|
|
'--rule',
|
|
ruleDeadHandler,
|
|
'--format',
|
|
'json',
|
|
'-c',
|
|
'1',
|
|
]);
|
|
final report = jsonDecode(output) as Map<String, Object?>;
|
|
expect(report['resolved'], 0);
|
|
expect(report['new'], 0);
|
|
expect(
|
|
report['baselineTotal'],
|
|
entries.where((entry) => entry.rule == ruleDeadHandler).length,
|
|
);
|
|
});
|
|
|
|
for (final narrowing in [
|
|
['--rule', ruleDeadHandler],
|
|
['--limit', '3'],
|
|
]) {
|
|
test('--write-baseline refuses ${narrowing.first}', () async {
|
|
final target = '${File(baseline).parent.path}/subset.json';
|
|
await expectLater(
|
|
_run([
|
|
'check',
|
|
_fixture,
|
|
...narrowing,
|
|
'--write-baseline',
|
|
target,
|
|
'-c',
|
|
'1',
|
|
]),
|
|
throwsA(
|
|
isA<PipeException>().having(
|
|
(error) => error.toString(),
|
|
'message',
|
|
contains('complete scan'),
|
|
),
|
|
),
|
|
);
|
|
expect(File(target).existsSync(), isFalse);
|
|
});
|
|
}
|
|
}
|