merge/audit-and-state-map #7

Merged
trevon merged 15 commits from merge/audit-and-state-map into main 2026-09-25 00:07:04 +00:00
4 changed files with 193 additions and 4 deletions
Showing only changes of commit c065576e96 - Show all commits

fix(cli): refuse subset baselines and scope verify to --rule

Two more ways a partial scan produced a false claim, the same hazard class
as the --limit and unreadable-file cases verify already refuses:

- `verify --rule X` compared the fresh X-only report against the whole
  baseline, so every accepted finding from any other rule was printed as
  RESOLVED ("these fixes are confirmed") when it had merely not been
  checked. The baseline is now filtered to the selected rules.

- `--write-baseline` (check and guide) wrote whatever the run produced.
  Under --rule or --limit, or with files that could not be analysed, the
  baseline silently omitted every finding outside that subset, and the
  next full `check --baseline` reported them all as new. It now refuses:
  --rule/--limit before analysis, unreadable files after it.

Each is covered by a test that fails against the previous cli.dart
(subset_scan_test.dart, and the unreadable-file case in
unreadable_test.dart). README documents the refusal.
trvon 2026-09-22 13:30:41 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -110,6 +110,12 @@ pipe check . --write-baseline .pipe-baseline.json # accept current state
pipe check . --baseline .pipe-baseline.json --fail-on warning
```
`--write-baseline` refuses a partial scan (`--rule`, `--limit`, or files that
could not be analysed): a baseline written from a subset omits every finding
outside it, and the next full run would report them all as new. `verify --rule`
likewise compares only that rule's accepted findings, instead of calling every
other rule's findings resolved.
`--fail-on none|info|warning|error` (default `none`) fails the build when a
**new** finding at or above that severity is present. Baselined findings never
trip the gate, so a ratcheted repo stays green while a regression fails.

View file

@ -106,9 +106,43 @@ Set<String>? _readBaseline(ArgResults results) {
return decodeBaseline(file.readAsStringSync());
}
void _writeBaseline(ArgResults results, String root, List<Finding> findings) {
/// A baseline is the set of findings a repo accepts, so it must come from a
/// complete scan with every rule on. Written from a subset, every finding
/// outside that subset is missing from it, and the next full `check --baseline`
/// reports all of them as new. Refused before analysis so the flags fail fast.
void _refuseSubsetBaseline(ArgResults results) {
if (results['write-baseline'] == null) return;
final narrowing = [
if ((results['rule'] as List<String>).isNotEmpty) '--rule',
if (results['limit'] != null) '--limit',
];
if (narrowing.isEmpty) return;
throw PipeException(
'--write-baseline needs a complete scan: with ${narrowing.join(' and ')} '
'the baseline would omit every finding outside that subset',
remedy: 'drop ${narrowing.join(' and ')} when writing a baseline',
);
}
void _writeBaseline(
ArgResults results,
ProjectAnalysis analysis,
List<Finding> findings,
) {
final path = results['write-baseline'] as String?;
if (path == null) return;
// Findings in a file that could not be analysed are absent, not fixed.
final unresolved = analysis.run.outcomes
.where((outcome) => !outcome.resolved && !outcome.skipped)
.length;
if (unresolved > 0) {
throw PipeException(
'--write-baseline needs a complete scan: $unresolved file(s) could not '
'be analysed, so their findings would be missing from the baseline',
remedy: 'run `pipe index` to see why they failed, then write it again',
);
}
final root = analysis.project.root;
File(path).writeAsStringSync(encodeBaseline(root, findings));
stdout.writeln(
'wrote baseline to $path (${findings.length} findings accepted)',
@ -353,6 +387,7 @@ class CheckCommand extends Command<int> {
@override
Future<int> run() async {
final results = argResults!;
_refuseSubsetBaseline(results);
final project = await loadProject(
_target(results),
directories: results['dir'] as List<String>,
@ -377,7 +412,7 @@ class CheckCommand extends Command<int> {
final top = readPositiveInt(results, 'top', fallback: 40);
stdout.write(report.describe(top: top));
}
_writeBaseline(results, project.root, report.findings);
_writeBaseline(results, analysis, report.findings);
final threshold = switch (results['fail-on'] as String) {
'error' => PipeSeverity.error,
'warning' => PipeSeverity.warning,
@ -545,9 +580,17 @@ class VerifyCommand extends Command<int> {
);
}
// With --rule, only the selected rules ran, so an accepted finding from
// any other rule is merely unchecked. Comparing against the whole baseline
// printed every one of them as RESOLVED.
final report = verifyAgainstBaseline(
report: analysis.report,
baseline: entries,
baseline: selected.isEmpty
? entries
: [
for (final entry in entries)
if (selected.contains(entry.rule)) entry,
],
);
if (results['format'] as String == 'json') {
stdout.writeln(_encode(report.toJson()));
@ -691,6 +734,7 @@ class GuideCommand extends Command<int> {
@override
Future<int> run() async {
final results = argResults!;
_refuseSubsetBaseline(results);
final project = await loadProject(
_target(results),
directories: results['dir'] as List<String>,
@ -750,7 +794,7 @@ class GuideCommand extends Command<int> {
}
stdout.writeln('persisted to yams as $name');
}
_writeBaseline(results, project.root, analysis.report.findings);
_writeBaseline(results, analysis, analysis.report.findings);
} finally {
await analysis.dispose();
}

109
test/subset_scan_test.dart Normal file
View file

@ -0,0 +1,109 @@
import 'dart:convert';
import 'dart:io';
import 'package:pipe/pipe.dart';
import 'package:pipe/src/cli.dart';
import 'package:test/test.dart';
/// Collects what a command writes to stdout.
class _CapturedStdout implements Stdout {
final StringBuffer buffer = StringBuffer();
@override
void write(Object? object) => buffer.write(object);
@override
void writeln([Object? object = '']) => buffer.writeln(object);
@override
dynamic noSuchMethod(Invocation invocation) => null;
}
Future<(int?, String)> _run(List<String> args) async {
final out = _CapturedStdout();
final code = await IOOverrides.runZoned(
() => PipeRunner().run(args),
stdout: () => out,
);
return (code, out.buffer.toString());
}
const String _fixture = 'test/fixtures/stuck_loading';
void main() {
late String baseline;
setUpAll(() async {
final dir = Directory.systemTemp.createTempSync('pipe-subset');
baseline = '${dir.path}/baseline.json';
final (code, _) = await _run([
'check',
_fixture,
'--write-baseline',
baseline,
'-c',
'1',
]);
expect(code, 0);
});
test('verify --rule does not call other rules\' findings resolved', () async {
// Only dead-handler ran, so accepted findings from every other rule were
// unchecked, not fixed. Comparing against the whole baseline reported each
// of them as RESOLVED - a false claim about someone's fixes.
final entries = decodeBaselineEntries(File(baseline).readAsStringSync());
expect(
entries.map((entry) => entry.rule).toSet(),
hasLength(greaterThan(1)),
reason: 'the control needs findings from more than the selected rule',
);
final (_, output) = await _run([
'verify',
_fixture,
'--baseline',
baseline,
'--rule',
ruleDeadHandler,
'--format',
'json',
'-c',
'1',
]);
final report = jsonDecode(output) as Map<String, Object?>;
expect(report['resolved'], 0);
expect(report['new'], 0);
expect(
report['baselineTotal'],
entries.where((entry) => entry.rule == ruleDeadHandler).length,
);
});
for (final narrowing in [
['--rule', ruleDeadHandler],
['--limit', '3'],
]) {
test('--write-baseline refuses ${narrowing.first}', () async {
final target = '${File(baseline).parent.path}/subset.json';
await expectLater(
_run([
'check',
_fixture,
...narrowing,
'--write-baseline',
target,
'-c',
'1',
]),
throwsA(
isA<PipeException>().having(
(error) => error.toString(),
'message',
contains('complete scan'),
),
),
);
expect(File(target).existsSync(), isFalse);
});
}
}

View file

@ -72,6 +72,36 @@ void main() {
);
});
test('--write-baseline refuses a partially readable project', () async {
// The unread file's findings would be missing from the accepted set, so the
// next complete scan would report them all as new.
final root = unreadableCorpus('write-partial', 'lib/item_bloc.dart');
if (root == null) {
markTestSkipped('chmod 000 does not restrict this process');
return;
}
final baseline = '$root/baseline.json';
await expectLater(
PipeRunner().run([
'check',
root,
'--write-baseline',
baseline,
'-c',
'1',
]),
throwsA(
isA<PipeException>().having(
(error) => error.toString(),
'message',
contains('complete scan'),
),
),
);
expect(File(baseline).existsSync(), isFalse);
});
test('recall keeps evaluating cases whose evidence is readable', () async {
// A repo with one unreadable file must not lose coverage: a hit stands,
// because evidence that was read is evidence. Skipping every case for the