fix(guide,suggest): word agent guidance no stronger than the evidence #2

Merged
trevon merged 13 commits from docs/honest-agent-guidance into main 2026-09-22 21:19:58 +00:00
6 changed files with 107 additions and 5 deletions
Showing only changes of commit ea9d51078a - Show all commits

fix(guide,suggest): make verify hints commands that can be pasted as-is

Every brief entry and plan action ended with `verify: pipe check --rule X`.
Pasted, that checks the current directory, over lib/ only, against no
baseline - not the run that produced the finding. Run from anywhere but the
project root, or after a scan with --dir or --baseline, it answered a
different question.

The hint is now built by checkCommand(): `pipe check <root> --rule X`, plus
`--baseline <path>` and every `--dir` when the run used them, with paths
shell-quoted when needed.
trvon 2026-09-22 12:02:22 -06:00
Signed by: trevon
GPG key ID: 38686EA797D7A8A7

View file

@ -7,6 +7,7 @@ export 'src/guide.dart'
show show
Guidance, Guidance,
buildGuidance, buildGuidance,
checkCommand,
emitToYams, emitToYams,
findYams, findYams,
gitHead, gitHead,

View file

@ -651,6 +651,8 @@ class SuggestCommand extends Command<int> {
root: project.root, root: project.root,
report: analysis.report, report: analysis.report,
policy: _readPolicy(results), policy: _readPolicy(results),
baselinePath: results['baseline'] as String?,
directories: project.directories,
); );
// Read every numeric option up front. Each was previously read inside the // Read every numeric option up front. Each was previously read inside the
// one output branch that uses it, so `suggest --top abc --format agent` // one output branch that uses it, so `suggest --top abc --format agent`
@ -745,6 +747,7 @@ class GuideCommand extends Command<int> {
report: analysis.report, report: analysis.report,
policy: _readPolicy(results), policy: _readPolicy(results),
scannedDirectories: project.directories, scannedDirectories: project.directories,
baselinePath: results['baseline'] as String?,
budgetChars: readPositiveInt(results, 'budget', fallback: 6000), budgetChars: readPositiveInt(results, 'budget', fallback: 6000),
maxFindings: readPositiveInt(results, 'max-findings', fallback: 12), maxFindings: readPositiveInt(results, 'max-findings', fallback: 12),
); );

View file

@ -265,6 +265,7 @@ String _groupBlock(
int index, int index,
FixPolicy policy, FixPolicy policy,
String scope, String scope,
String Function(String rule) verifyFor,
) { ) {
final head = group.first; final head = group.first;
final buffer = StringBuffer(); final buffer = StringBuffer();
@ -324,7 +325,7 @@ String _groupBlock(
for (final item in head.evidence.take(1)) { for (final item in head.evidence.take(1)) {
buffer.writeln(' $_evidenceLabel: ${dataSpan(item)}'); buffer.writeln(' $_evidenceLabel: ${dataSpan(item)}');
} }
buffer.writeln(' verify: pipe check --rule ${head.rule}'); buffer.writeln(' verify: ${verifyFor(head.rule)}');
return buffer.toString(); return buffer.toString();
} }
@ -337,8 +338,15 @@ Guidance buildGuidance({
int maxFindings = 12, int maxFindings = 12,
FixPolicy policy = FixPolicy.wireUp, FixPolicy policy = FixPolicy.wireUp,
List<String> scannedDirectories = const ['lib'], List<String> scannedDirectories = const ['lib'],
String? baselinePath,
}) { }) {
final scope = scannedDirectories.join(', '); final scope = scannedDirectories.join(', ');
String verifyFor(String rule) => checkCommand(
root: root,
rule: rule,
baseline: baselinePath,
directories: scannedDirectories,
);
final ranked = rankFindings(report.newFindings); final ranked = rankFindings(report.newFindings);
final summary = <String, Object?>{ final summary = <String, Object?>{
'root': root, 'root': root,
@ -438,7 +446,13 @@ Guidance buildGuidance({
for (var i = 0; i < groups.length; i++) { for (var i = 0; i < groups.length; i++) {
final block = selected.length >= maxFindings final block = selected.length >= maxFindings
? null ? null
: _groupBlock(groups[i], selected.length + 1, policy, scope); : _groupBlock(
groups[i],
selected.length + 1,
policy,
scope,
verifyFor,
);
if (block == null || if (block == null ||
runningLength + block.length + noteReserve > budgetChars) { runningLength + block.length + noteReserve > budgetChars) {
// Stop at the first entry that does not fit. Skipping it and packing the // Stop at the first entry that does not fit. Skipping it and packing the
@ -647,6 +661,32 @@ Guidance buildGuidance({
); );
} }
/// A `pipe check` command that re-runs one rule the way an analysis ran: same
/// root, same baseline, same --dir scope. The hint used to be a bare
/// `pipe check --rule X`, which checks the current directory against no
/// baseline over lib/ only - not what produced the finding.
String checkCommand({
required String root,
required String rule,
String? baseline,
List<String> directories = const ['lib'],
}) {
final parts = ['pipe', 'check', _shellQuote(root), '--rule', rule];
if (baseline != null) parts.addAll(['--baseline', _shellQuote(baseline)]);
final defaultScope = directories.length == 1 && directories.single == 'lib';
if (!defaultScope) {
for (final directory in directories) {
parts.addAll(['--dir', _shellQuote(directory)]);
}
}
return parts.join(' ');
}
String _shellQuote(String value) =>
RegExp(r'^[A-Za-z0-9_./:=@+-]+$').hasMatch(value)
? value
: "'${value.replaceAll("'", r"'\''")}'";
/// Metadata for a report persisted to yams: what produced it, from which /// Metadata for a report persisted to yams: what produced it, from which
/// project, when, and - when the project is a git checkout - at which commit, /// project, when, and - when the project is a git checkout - at which commit,
/// so a report read back from the corpus can be dated and reproduced. /// so a report read back from the corpus can be dated and reproduced.

View file

@ -17,7 +17,8 @@ class FixAction {
required this.why, required this.why,
this.file = '', this.file = '',
this.line, this.line,
}); String? verify,
}) : verify = verify ?? checkCommand(root: '.', rule: rule);
final String rule; final String rule;
final PipeSeverity severity; final PipeSeverity severity;
@ -43,7 +44,9 @@ class FixAction {
final String file; final String file;
final int? line; final int? line;
String get verify => 'pipe check --rule $rule'; /// A command that re-checks this rule the way the plan was produced, so it
/// can be pasted as-is.
final String verify;
Map<String, Object?> toJson() => { Map<String, Object?> toJson() => {
'rule': rule, 'rule': rule,
@ -409,6 +412,8 @@ WorkPlan buildWorkPlan({
required String root, required String root,
required FindingReport report, required FindingReport report,
FixPolicy policy = FixPolicy.wireUp, FixPolicy policy = FixPolicy.wireUp,
String? baselinePath,
List<String> directories = const ['lib'],
}) { }) {
final newFindings = report.newFindings; final newFindings = report.newFindings;
final order = <String>[]; final order = <String>[];
@ -447,6 +452,12 @@ WorkPlan buildWorkPlan({
why: _whyOf(group, score, policy), why: _whyOf(group, score, policy),
file: file, file: file,
line: head.line, line: head.line,
verify: checkCommand(
root: root,
rule: head.rule,
baseline: baselinePath,
directories: directories,
),
); );
byFile.putIfAbsent(file, () => []).add(action); byFile.putIfAbsent(file, () => []).add(action);
fileScores.update(file, (value) => value + score, ifAbsent: () => score); fileScores.update(file, (value) => value + score, ifAbsent: () => score);

View file

@ -159,6 +159,31 @@ void main() {
); );
}); });
test('verify hints are commands that can be pasted as-is', () {
// `pipe check --rule X` checked the current directory, over lib/, against
// no baseline - not the run that produced the brief.
final brief = buildGuidance(
root: '/work/app',
flow: analysis.flow,
report: analysis.report,
baselinePath: 'ci/base.json',
scannedDirectories: const ['lib', 'test'],
).brief;
expect(
brief,
matches(
RegExp(
r'verify: pipe check /work/app --rule [a-z-]+ '
r'--baseline ci/base\.json --dir lib --dir test',
),
),
);
expect(
checkCommand(root: '/work/app', rule: ruleDeadHandler),
'pipe check /work/app --rule dead-handler',
);
});
test('yams metadata records provenance, not a stale owner', () { test('yams metadata records provenance, not a stale owner', () {
final metadata = yamsMetadata( final metadata = yamsMetadata(
project: 'app', project: 'app',

View file

@ -144,7 +144,7 @@ void main() {
expect(action.why, startsWith('${action.score} = ')); expect(action.why, startsWith('${action.score} = '));
expect(action.why, contains('warning 50')); expect(action.why, contains('warning 50'));
expect(action.why, contains('user-visible failure')); expect(action.why, contains('user-visible failure'));
expect(action.verify, 'pipe check --rule $ruleEmittedNotRendered'); expect(action.verify, 'pipe check /root --rule $ruleEmittedNotRendered');
expect(action.action, isNotEmpty); expect(action.action, isNotEmpty);
}); });
@ -394,4 +394,26 @@ void main() {
expect(action.resolves, 3); expect(action.resolves, 3);
expect(action.action, contains('dispatch each of Alpha, Beta, Gamma')); expect(action.action, contains('dispatch each of Alpha, Beta, Gamma'));
}); });
test('the verify hint reruns the check the way the plan was made', () {
final plan = buildWorkPlan(
root: '/my app',
report: _report([
_finding(
rule: ruleDeadHandler,
severity: PipeSeverity.info,
file: 'lib/a.dart',
line: 1,
data: {'event': 'A'},
),
]),
baselinePath: '.pipe-baseline.json',
directories: const ['lib', 'test'],
);
expect(
plan.ordered.single.verify,
"pipe check '/my app' --rule dead-handler "
'--baseline .pipe-baseline.json --dir lib --dir test',
);
});
} }