fix/nuwayze-dogfood-findings #11
4 changed files with 147 additions and 10 deletions
feat(verify): default --baseline to <path>/.pipe-baseline.json
`pipe verify .` failed with "verify needs --baseline" even in a repo
whose baseline sits where the README's CI recipe writes it. nuwayze's
scripts/testing/pipe_state_flow_gate.sh commits exactly that file,
.pipe-baseline.json at the repo root, so every verify there had to
repeat the path.
Without --baseline, verify now reads <path>/.pipe-baseline.json when it
exists and says so on stderr ("pipe: no --baseline given; using ..."),
so a run never silently picks a file nobody named. With neither, the
error names both remedies: pass --baseline, or write the conventional
file with `pipe check <path> --write-baseline <path>/.pipe-baseline.json`
before the fix. pipe had no baseline-name constant; defaultBaselineName
is now the one place it is spelled.
Tests (test/verify_default_baseline_test.dart): an explicit --baseline
wins and announces nothing; the conventional file is used and
announced; with neither the error carries both remedies; and end to
end, `pipe verify <copy>` against a baseline written at the copy's
root reads it (--expect-resolved fails on the 28 still-present
findings). README and docs/usage.md show the shorter form.
On nuwayze-w3, `pipe verify /Users/.../nuwayze-w3` with no flag reads
its .pipe-baseline.json (65 accepted findings). 225 tests pass.
commit
adf898be19
|
|
@ -40,7 +40,7 @@ In CI, accept today's findings once and fail only on new ones:
|
|||
```bash
|
||||
pipe check . --write-baseline .pipe-baseline.json
|
||||
pipe check . --baseline .pipe-baseline.json --fail-on warning
|
||||
pipe verify . --baseline .pipe-baseline.json --expect-resolved # after a fix
|
||||
pipe verify . --expect-resolved # after a fix; reads .pipe-baseline.json
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
| `pipe check <path>` | Ranked findings with evidence and remediation. |
|
||||
| `pipe suggest <path>` | Findings collapsed to root causes and ranked, globally and per file. |
|
||||
| `pipe guide <path>` | A prompt-sized brief for an agent; `--yams` persists it to the yams corpus. |
|
||||
| `pipe verify <path> --baseline F` | Which accepted findings were resolved, remain, or are new. |
|
||||
| `pipe verify <path> [--baseline F]` | Which accepted findings were resolved, remain, or are new. |
|
||||
| `pipe recall --root <dir>` | How many labelled known bugs pipe detects (see [rules](rules.md#recall)). |
|
||||
| `pipe queries <path>` | Every Firestore query the code can issue, as typed records; an inventory, no rules (see [queries](queries.md)). |
|
||||
|
||||
|
|
@ -109,6 +109,10 @@ RESOLVED
|
|||
+ lib/flag_cubit.dart:25 loadUnguarded [loading-flag-not-reset]
|
||||
```
|
||||
|
||||
Without `--baseline`, `verify` reads `<path>/.pipe-baseline.json`, the file the
|
||||
CI recipe above writes, and says so on stderr; with neither, it fails and names
|
||||
both. So `pipe verify .` is enough in a repo that follows the recipe.
|
||||
|
||||
`--expect-resolved` exits non-zero while anything accepted is still reported or
|
||||
was not scanned. `verify` only calls a finding resolved when it looked for it:
|
||||
|
||||
|
|
|
|||
|
|
@ -625,6 +625,36 @@ class RecallCommand extends Command<int> {
|
|||
}
|
||||
}
|
||||
|
||||
/// The conventional baseline file name, at the project root. It is what the
|
||||
/// README's CI recipe writes, and what `verify` reads when `--baseline` is
|
||||
/// omitted.
|
||||
const String defaultBaselineName = '.pipe-baseline.json';
|
||||
|
||||
/// The baseline `verify` compares against: [explicit] when given, else
|
||||
/// [defaultBaselineName] under the project [target] when that exists. Using
|
||||
/// the default is announced on [log] (stderr), so a run never silently picks a
|
||||
/// file the user did not name.
|
||||
String resolveVerifyBaseline(
|
||||
String? explicit,
|
||||
String target, {
|
||||
StringSink? log,
|
||||
}) {
|
||||
if (explicit != null) return explicit;
|
||||
final conventional = p.join(target, defaultBaselineName);
|
||||
if (File(conventional).existsSync()) {
|
||||
(log ?? stderr).writeln('pipe: no --baseline given; using $conventional');
|
||||
return conventional;
|
||||
}
|
||||
throw PipeException(
|
||||
'verify needs a baseline: no --baseline given and no $defaultBaselineName '
|
||||
'at the project root ($conventional)',
|
||||
remedy:
|
||||
'pass --baseline <file> with the findings you fixed, or write '
|
||||
'$conventional with `pipe check $target --write-baseline '
|
||||
'$conventional` before the fix',
|
||||
);
|
||||
}
|
||||
|
||||
/// Report which accepted findings are gone after a fix.
|
||||
class VerifyCommand extends Command<int> {
|
||||
VerifyCommand() {
|
||||
|
|
@ -632,7 +662,9 @@ class VerifyCommand extends Command<int> {
|
|||
argParser
|
||||
..addOption(
|
||||
'baseline',
|
||||
help: 'Baseline of accepted findings to verify against (required).',
|
||||
help:
|
||||
'Baseline of accepted findings to verify against '
|
||||
'(default: <path>/$defaultBaselineName when it exists).',
|
||||
)
|
||||
..addMultiOption('rule', help: 'Limit to specific rules (repeatable).')
|
||||
..addOption('top', defaultsTo: '25', help: 'Rows per section.')
|
||||
|
|
@ -653,13 +685,10 @@ class VerifyCommand extends Command<int> {
|
|||
@override
|
||||
Future<int> run() async {
|
||||
final results = argResults!;
|
||||
final baselinePath = results['baseline'] as String?;
|
||||
if (baselinePath == null) {
|
||||
throw PipeException(
|
||||
'verify needs --baseline',
|
||||
remedy: 'pass the baseline that contains the findings you fixed',
|
||||
);
|
||||
}
|
||||
final baselinePath = resolveVerifyBaseline(
|
||||
results['baseline'] as String?,
|
||||
_target(results),
|
||||
);
|
||||
// Verifying a partial scan is unsound, and silently so: with --limit the
|
||||
// findings that were never scanned are absent from the fresh report, which
|
||||
// `verify` reports as RESOLVED - claiming fixes that were never made.
|
||||
|
|
|
|||
104
test/verify_default_baseline_test.dart
Normal file
104
test/verify_default_baseline_test.dart
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import 'dart:io';
|
||||
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:pipe/pipe.dart';
|
||||
import 'package:pipe/src/cli.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
/// A copy of the rule fixture, so a baseline can be written at its root.
|
||||
String _fixtureCopy() {
|
||||
final from = Directory('test/fixtures/stuck_loading');
|
||||
final to = Directory.systemTemp.createTempSync('pipe-verify-default');
|
||||
for (final entity in from.listSync(recursive: true)) {
|
||||
final target = p.join(to.path, p.relative(entity.path, from: from.path));
|
||||
if (entity is Directory) {
|
||||
Directory(target).createSync(recursive: true);
|
||||
} else if (entity is File) {
|
||||
File(target)
|
||||
..createSync(recursive: true)
|
||||
..writeAsBytesSync(entity.readAsBytesSync());
|
||||
}
|
||||
}
|
||||
return to.path;
|
||||
}
|
||||
|
||||
void main() {
|
||||
// `pipe verify .` used to fail with "verify needs --baseline" even in a repo
|
||||
// whose baseline sits at the conventional path the README's CI recipe
|
||||
// writes (nuwayze's scripts/testing/pipe_state_flow_gate.sh).
|
||||
group('resolveVerifyBaseline', () {
|
||||
late Directory temp;
|
||||
|
||||
setUp(() => temp = Directory.systemTemp.createTempSync('pipe-verify-bl'));
|
||||
tearDown(() => temp.deleteSync(recursive: true));
|
||||
|
||||
test('an explicit --baseline wins, with nothing announced', () {
|
||||
File(p.join(temp.path, defaultBaselineName)).writeAsStringSync('{}');
|
||||
final log = StringBuffer();
|
||||
expect(
|
||||
resolveVerifyBaseline('other.json', temp.path, log: log),
|
||||
'other.json',
|
||||
);
|
||||
expect(log.toString(), isEmpty);
|
||||
});
|
||||
|
||||
test('without --baseline the conventional file is used and announced', () {
|
||||
final conventional = p.join(temp.path, defaultBaselineName);
|
||||
File(conventional).writeAsStringSync('{}');
|
||||
final log = StringBuffer();
|
||||
expect(resolveVerifyBaseline(null, temp.path, log: log), conventional);
|
||||
expect(log.toString(), contains('using $conventional'));
|
||||
});
|
||||
|
||||
test('without either, the error names both remedies', () {
|
||||
expect(
|
||||
() => resolveVerifyBaseline(null, temp.path, log: StringBuffer()),
|
||||
throwsA(
|
||||
isA<PipeException>()
|
||||
.having((e) => e.message, 'message', contains('--baseline'))
|
||||
.having(
|
||||
(e) => e.remedy,
|
||||
'remedy',
|
||||
allOf(
|
||||
contains('--baseline'),
|
||||
contains(p.join(temp.path, defaultBaselineName)),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('pipe verify <path> reads <path>/.pipe-baseline.json', () async {
|
||||
final root = _fixtureCopy();
|
||||
try {
|
||||
final baseline = p.join(root, defaultBaselineName);
|
||||
expect(
|
||||
await PipeRunner().run([
|
||||
'check',
|
||||
root,
|
||||
'--write-baseline',
|
||||
baseline,
|
||||
'-c',
|
||||
'1',
|
||||
]),
|
||||
0,
|
||||
);
|
||||
// Nothing changed since the baseline, so every accepted finding remains
|
||||
// and --expect-resolved fails: proof the default baseline was read,
|
||||
// since an empty one would have nothing to expect.
|
||||
expect(
|
||||
await PipeRunner().run([
|
||||
'verify',
|
||||
root,
|
||||
'--expect-resolved',
|
||||
'-c',
|
||||
'1',
|
||||
]),
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
Directory(root).deleteSync(recursive: true);
|
||||
}
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue