Two more ways a partial scan produced a false claim, the same hazard class
as the --limit and unreadable-file cases verify already refuses:
- `verify --rule X` compared the fresh X-only report against the whole
baseline, so every accepted finding from any other rule was printed as
RESOLVED ("these fixes are confirmed") when it had merely not been
checked. The baseline is now filtered to the selected rules.
- `--write-baseline` (check and guide) wrote whatever the run produced.
Under --rule or --limit, or with files that could not be analysed, the
baseline silently omitted every finding outside that subset, and the
next full `check --baseline` reported them all as new. It now refuses:
--rule/--limit before analysis, unreadable files after it.
Each is covered by a test that fails against the previous cli.dart
(subset_scan_test.dart, and the unreadable-file case in
unreadable_test.dart). README documents the refusal.
167 lines
5.2 KiB
Dart
167 lines
5.2 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:pipe/pipe.dart';
|
|
import 'package:pipe/src/cli.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
void _copyTree(Directory from, Directory to) {
|
|
to.createSync(recursive: true);
|
|
for (final entity in from.listSync(recursive: true)) {
|
|
final relative = entity.path.substring(from.path.length + 1);
|
|
final target = '${to.path}/$relative';
|
|
if (entity is Directory) {
|
|
Directory(target).createSync(recursive: true);
|
|
} else if (entity is File) {
|
|
File(target).writeAsBytesSync(entity.readAsBytesSync());
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A copy of the fixture in which one file cannot be read.
|
|
///
|
|
/// Returns null when `chmod 000` does not actually restrict this process (for
|
|
/// example when the suite runs as root), so the caller can skip honestly instead
|
|
/// of passing vacuously.
|
|
String? unreadableCorpus(String label, String lockedRelativePath) {
|
|
final temp = Directory.systemTemp.createTempSync('pipe-$label');
|
|
_copyTree(Directory('test/fixtures/stuck_loading'), temp);
|
|
final locked = File('${temp.path}/$lockedRelativePath');
|
|
Process.runSync('chmod', ['000', locked.path]);
|
|
try {
|
|
locked.readAsStringSync();
|
|
return null;
|
|
} on FileSystemException {
|
|
return temp.path;
|
|
}
|
|
}
|
|
|
|
void main() {
|
|
test('verify refuses to claim fixes from a partially readable project', () async {
|
|
// Findings in a file that could not be read are absent from the fresh report,
|
|
// so verify called them RESOLVED: with two unreadable files it printed
|
|
// "resolved 8 no longer reported - these fixes are confirmed" for findings it
|
|
// had simply not read, and a claim about someone's fixes must not rest on
|
|
// that. Same hazard as --limit, which verify already refuses.
|
|
final root = unreadableCorpus('verify-partial', 'lib/item_bloc.dart');
|
|
if (root == null) {
|
|
markTestSkipped('chmod 000 does not restrict this process');
|
|
return;
|
|
}
|
|
|
|
final baseline = '$root/baseline.json';
|
|
final written = await PipeRunner().run([
|
|
'check',
|
|
'test/fixtures/stuck_loading',
|
|
'--write-baseline',
|
|
baseline,
|
|
'-c',
|
|
'1',
|
|
]);
|
|
expect(written, 0);
|
|
|
|
await expectLater(
|
|
PipeRunner().run(['verify', root, '--baseline', baseline, '-c', '1']),
|
|
throwsA(
|
|
isA<PipeException>().having(
|
|
(error) => error.toString(),
|
|
'message',
|
|
contains('complete scan'),
|
|
),
|
|
),
|
|
);
|
|
});
|
|
|
|
test('--write-baseline refuses a partially readable project', () async {
|
|
// The unread file's findings would be missing from the accepted set, so the
|
|
// next complete scan would report them all as new.
|
|
final root = unreadableCorpus('write-partial', 'lib/item_bloc.dart');
|
|
if (root == null) {
|
|
markTestSkipped('chmod 000 does not restrict this process');
|
|
return;
|
|
}
|
|
|
|
final baseline = '$root/baseline.json';
|
|
await expectLater(
|
|
PipeRunner().run([
|
|
'check',
|
|
root,
|
|
'--write-baseline',
|
|
baseline,
|
|
'-c',
|
|
'1',
|
|
]),
|
|
throwsA(
|
|
isA<PipeException>().having(
|
|
(error) => error.toString(),
|
|
'message',
|
|
contains('complete scan'),
|
|
),
|
|
),
|
|
);
|
|
expect(File(baseline).existsSync(), isFalse);
|
|
});
|
|
|
|
test('recall keeps evaluating cases whose evidence is readable', () async {
|
|
// A repo with one unreadable file must not lose coverage: a hit stands,
|
|
// because evidence that was read is evidence. Skipping every case for the
|
|
// repo silently dropped the regression guard for cases that were perfectly
|
|
// judgeable - a worse failure than the false MISSED it replaced.
|
|
final root = unreadableCorpus('recall-overskip', 'lib/item_bloc.dart');
|
|
if (root == null) {
|
|
markTestSkipped('chmod 000 does not restrict this process');
|
|
return;
|
|
}
|
|
|
|
final manifest = '$root/manifest.json';
|
|
File(manifest).writeAsStringSync(
|
|
const JsonEncoder.withIndent(' ').convert({
|
|
'version': 1,
|
|
'cases': [
|
|
{
|
|
// Evidence lives in the unreadable file: cannot be judged.
|
|
'id': 'evidence-in-unreadable-file',
|
|
'repo': root,
|
|
'rule': 'dead-handler',
|
|
'match': {'event': 'LoadItems'},
|
|
},
|
|
{
|
|
// Evidence lives in a readable file: must still be evaluated.
|
|
'id': 'evidence-in-readable-file',
|
|
'repo': root,
|
|
'rule': 'dead-handler',
|
|
'match': {'event': 'NeverDispatched'},
|
|
},
|
|
],
|
|
}),
|
|
);
|
|
|
|
final report = await runRecall(
|
|
manifestPath: manifest,
|
|
root: root,
|
|
concurrency: 1,
|
|
);
|
|
|
|
expect(
|
|
report.skippedCases.single.testCase.id,
|
|
'evidence-in-unreadable-file',
|
|
);
|
|
expect(
|
|
report.skippedCases.single.skipped,
|
|
contains('could not be analysed'),
|
|
);
|
|
expect(report.missed, isEmpty);
|
|
expect(
|
|
report.evaluated.map((outcome) => outcome.testCase.id),
|
|
contains('evidence-in-readable-file'),
|
|
reason:
|
|
'a readable case must not be skipped because a sibling file failed',
|
|
);
|
|
expect(
|
|
report.evaluated
|
|
.where((outcome) => outcome.found)
|
|
.map((outcome) => outcome.testCase.id),
|
|
contains('evidence-in-readable-file'),
|
|
);
|
|
});
|
|
}
|