strawberry/tests/test_cli_replay.py
trvon be237d6040
Findings: reconcile evidence and harden policy rebuilds
Preserve independent support, typed rule namespaces and stable process lifetimes. Share authoritative managed-set reconciliation across daemon and offline correlation, load offline policy without seeding/migration, and release resources after shutdown-flush exceptions. Keep retirement history, transaction ownership and fault/allocation regressions; no whole-set crash-atomicity or live acceptance claim.
2026-11-03 06:55:47 -07:00

389 lines
19 KiB
Python

#!/usr/bin/env python3
"""Actual CLI contracts on isolated synthetic stores; no training or live RPC."""
import hashlib
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
CLI = Path(sys.argv[1]).resolve()
FIXTURE = Path(sys.argv[2]).resolve()
def hashes(folder):
return {str(p.relative_to(folder)): hashlib.sha256(p.read_bytes()).hexdigest()
for p in sorted(folder.rglob('*')) if p.is_file()}
def crc32c(data):
"""Reference Castagnoli CRC for deliberately mutated fixture headers."""
c = 0xFFFFFFFF
for byte in data:
c ^= byte
for _ in range(8):
c = (c >> 1) ^ (0x82F63B78 if c & 1 else 0)
return c ^ 0xFFFFFFFF
class ReplayCli(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(prefix='strawberry_cli_replay_')
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.store = self.root / 'store'
p = subprocess.run([str(FIXTURE), str(self.store)], capture_output=True,
text=True, timeout=30)
self.assertEqual(p.returncode, 0, p.stderr)
self.reference = json.loads(p.stdout.splitlines()[-1])
self.config = self.write_config('full.conf', self.store)
self.original = hashes(self.store)
self.addCleanup(self.assert_original_unchanged)
def assert_original_unchanged(self):
self.assertEqual(hashes(self.store), self.original)
self.assertFalse((self.root / 'models').exists())
def write_config(self, name, store):
p = self.root / name
p.write_text(f'db_path = {store}\nmodel_dir = {self.root / "models"}\n'
f'control_socket_path = {self.root / "no-control.sock"}\n')
return p
def cli(self, *args, config=None):
return subprocess.run([str(CLI), *map(str, args), '--config',
str(config or self.config)], capture_output=True,
text=True, timeout=30)
def clone(self, name, omit=None):
out = self.root / name
out.mkdir()
for p in self.store.rglob('*'):
rel = p.relative_to(self.store)
if omit and omit(rel):
continue
dest = out / rel
if p.is_dir():
dest.mkdir(exist_ok=True)
elif p.is_file():
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(p.read_bytes())
return out
def assert_failure(self, p, code=None):
self.assertNotEqual(p.returncode, 0, p.stdout)
if code is not None:
self.assertEqual(p.returncode, code, p.stderr)
self.assertEqual(p.stdout, '')
self.assertTrue(p.stderr.strip())
def test_unavailable_dictionary_is_not_an_empty_stream(self):
for stream in ('evidence', 'telemetry'):
copy = self.clone('unknown-dictionary-' + stream)
segment = next((copy / stream).glob('seg-*.log'))
data = bytearray(segment.read_bytes())
self.assertEqual(data[37], 1) # zstd, not raw
self.assertEqual(int.from_bytes(data[56:60], 'little'), crc32c(data[32:56]))
data[48:52] = (0x5EED).to_bytes(4, 'little')
data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little')
segment.write_bytes(data)
before = hashes(copy)
for flags in ((), ('--training-summary',)):
with self.subTest(stream=stream, flags=flags):
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 3)
self.assertEqual(hashes(copy), before)
def test_nontruncated_block_header_errors_are_invalid(self):
for stream in ('evidence', 'telemetry'):
for kind in ('magic', 'header_crc', 'version', 'codec', 'size'):
copy = self.clone('bad-block-header-' + stream + '-' + kind)
segment = next((copy / stream).glob('seg-*.log'))
data = bytearray(segment.read_bytes())
if kind == 'magic':
data[32] ^= 0x80
elif kind == 'header_crc':
data[56] ^= 0x80
else:
if kind == 'version':
data[36] = 255
elif kind == 'codec':
data[37] = 255
else:
data[40:44] = (1 << 31).to_bytes(4, 'little')
data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little')
segment.write_bytes(data)
before = hashes(copy)
for flags in ((), ('--training-summary',)):
with self.subTest(stream=stream, kind=kind, flags=flags):
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1)
self.assertEqual(hashes(copy), before)
def test_verifier_is_one_json_document(self):
p = self.cli('store', 'verify', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertTrue(j['ok'])
self.assertEqual(j['records'], 8)
def test_verifier_notes_do_not_assert_complete_replay_coverage(self):
copy = self.clone('verify-partial', lambda rel: rel.parts[0] == 'telemetry')
config = self.write_config('partial.conf', copy)
before = hashes(copy)
p = self.cli('store', 'verify', '--json', config=config)
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertTrue(j['ok']) # existing best-effort verifier contract, intentionally unchanged
self.assertTrue(j['notes'])
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
self.assertEqual(hashes(copy), before)
def test_summary_without_explicit_source_does_not_discover_live_daemon(self):
config = self.root / 'no-source.conf'
config.write_text(f'control_socket_path = {self.root / "no-control.sock"}\n')
p = self.cli('replay', '--training-summary', '--json', config=config)
self.assert_failure(p, 3)
self.assertIn('explicit snapshot', p.stderr)
def test_missing_listed_stream_is_not_empty_success(self):
for stream in ('evidence', 'telemetry'):
with self.subTest(stream=stream):
copy = self.clone('missing-' + stream, lambda rel, stream=stream: rel.parts[0] == stream)
before = hashes(copy)
p = self.cli('replay', copy, '--warmup', '0', '--json')
self.assert_failure(p, 3)
self.assertEqual(hashes(copy), before)
def test_missing_listed_file_is_not_empty_success(self):
for stream in ('evidence', 'telemetry'):
with self.subTest(stream=stream):
files = list((self.store / stream).glob('seg-*.log'))
self.assertTrue(files)
rel = files[0].relative_to(self.store)
copy = self.clone('file-' + stream, lambda p, rel=rel: p == rel)
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 3)
def test_invalid_header_cannot_yield_partial_replay(self):
copy = self.clone('invalid-header')
segment = next((copy / 'telemetry').glob('seg-*.log'))
data = bytearray(segment.read_bytes())
data[0] ^= 0x80
segment.write_bytes(data)
before = hashes(copy)
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 1)
self.assertEqual(hashes(copy), before)
def test_corrupt_later_block_cannot_export_valid_prefix(self):
copy = self.clone('invalid-later-block')
segment = next((copy / 'telemetry').glob('seg-*.log'))
data = bytearray(segment.read_bytes())
# Segment header 32 bytes, codec header 28; stored length at +12.
stored = int.from_bytes(data[44:48], 'little')
second_payload = 32 + 28 + stored + 28
self.assertLess(second_payload, len(data))
data[second_payload] ^= 0x80
segment.write_bytes(data)
before = hashes(copy)
for flags in ((), ('--training-summary',)):
with self.subTest(flags=flags):
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1)
self.assertEqual(hashes(copy), before)
def test_malformed_label_cannot_export_telemetry_or_examples(self):
store = self.root / 'malformed-label'
p = subprocess.run([str(FIXTURE), str(store), 'malformed-label'], capture_output=True,
text=True, timeout=30)
self.assertEqual(p.returncode, 0, p.stderr)
before = hashes(store)
for flags in ((), ('--training-summary',)):
self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1)
self.assertEqual(hashes(store), before)
def test_sealed_footer_and_fallback_are_distinct(self):
for corrupt_footer in (False, True):
with self.subTest(corrupt_footer=corrupt_footer):
store = self.root / ('sealed-' + str(corrupt_footer))
p = subprocess.run([str(FIXTURE), str(store), 'sealed'], capture_output=True,
text=True, timeout=30)
self.assertEqual(p.returncode, 0, p.stderr)
if corrupt_footer:
segment = sorted((store / 'telemetry').glob('seg-*.log'))[0]
data = segment.read_bytes()
self.assertGreater(len(data), 32)
segment.write_bytes(data[:-1])
before = hashes(store)
for flags in ((), ('--training-summary',)):
self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1)
else:
self.assertTrue((store / 'MANIFEST.prev').is_file())
(store / 'MANIFEST').rename(store / 'MANIFEST.saved')
before = hashes(store)
p = self.cli('replay', store, '--warmup', '0', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(json.loads(p.stdout)['records'], 6)
self.assert_failure(self.cli('replay', store, '--training-summary', '--json'), 3)
self.assertEqual(hashes(store), before)
def test_active_tail_is_normal_for_replay_not_snapshot_summary(self):
copy = self.clone('active-tail')
segment = next((copy / 'telemetry').glob('seg-*.log'))
segment.write_bytes(segment.read_bytes() + b'x')
before = hashes(copy)
p = self.cli('replay', copy, '--warmup', '0', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(json.loads(p.stdout)['records'], 6)
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
self.assertEqual(hashes(copy), before)
def test_default_samples_are_explicitly_uncollected(self):
p = self.cli('replay', self.store, '--warmup', '0', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertEqual(j['windows'], 4)
self.assertFalse(j['examples_collected'])
self.assertEqual(j['examples_kept'], 0)
def test_summary_matches_existing_collector_and_is_aggregate_only(self):
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertEqual(j['mode'], 'training_summary')
self.assertEqual(j['assessment'], 'not_assessed')
self.assertFalse(j['telemetry_empty'])
self.assertFalse(j['sample_empty'])
s = j['sample']
self.assertTrue(s['collected'])
for key in ('eligible', 'eligible_positive', 'retained', 'retained_positive'):
self.assertEqual(s[key], self.reference[key])
self.assertEqual(s['retained_zero_label'], 2)
self.assertEqual(s['entity_cap'], 500)
self.assertEqual(s['entities'], 1)
self.assertEqual(s['max_entity_examples'], 4)
self.assertEqual(s['max_entity_share'], 1.0)
self.assertEqual(j['labels']['records'], 2)
self.assertEqual(j['coverage']['detected_windows'], 2)
self.assertEqual(j['coverage']['late_detected_windows'], 1)
for marker in ('synthetic', 'SyntheticTeacher', 'SyntheticQuarantine',
'root_node', 'root_exe', 'root_shape', 'top_windows',
'a' * 64, 'd' * 64, str(self.root)):
self.assertNotIn(marker, p.stdout)
def test_summary_semantics_are_deterministic_timings_are_separate(self):
results = []
for _ in range(2):
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
timings = j.pop('timings')
self.assertEqual(timings['unit'], 'wall_microseconds')
for key in ('replay', 'summary_build'):
self.assertEqual(timings[key]['count'], 1)
self.assertEqual(timings[key]['sum'], timings[key]['max'])
results.append(j)
self.assertEqual(results[0], results[1])
def test_summary_human_is_not_json_or_host_identifiers(self):
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary')
self.assertEqual(p.returncode, 0, p.stderr)
self.assertIn('retained', p.stdout.lower())
self.assertNotIn('synthetic', p.stdout.lower())
self.assertNotIn(str(self.root), p.stdout)
with self.assertRaises(json.JSONDecodeError):
json.loads(p.stdout)
def test_warmup_empty_sample_is_not_empty_telemetry(self):
p = self.cli('replay', self.store, '--training-summary', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertFalse(j['telemetry_empty'])
self.assertTrue(j['sample_empty'])
self.assertEqual(j['sample']['retained'], 0)
self.assertIsNone(j['sample']['max_entity_share'])
def test_complete_empty_store_is_explicit(self):
empty = self.root / 'empty'
p = subprocess.run([str(FIXTURE), str(empty), 'empty'], capture_output=True,
text=True, timeout=30)
self.assertEqual(p.returncode, 0, p.stderr)
before = hashes(empty)
p = self.cli('replay', empty, '--training-summary', '--json')
self.assertEqual(p.returncode, 0, p.stderr)
j = json.loads(p.stdout)
self.assertTrue(j['telemetry_empty'])
self.assertTrue(j['sample_empty'])
self.assertEqual(hashes(empty), before)
def test_summary_rejects_missing_stream(self):
copy = self.clone('summary-missing', lambda rel: rel.parts[0] == 'evidence')
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
def test_summary_conflicts_fail_before_opening_inputs(self):
for flags in (('--model', 'missing.sbm'), ('--inject', 'missing.trace'),
('--policy-eval',), ('--top', '20'), ('--inject-copies', '5')):
with self.subTest(flags=flags):
self.assert_failure(self.cli('replay', self.root / 'not-a-store',
'--training-summary', '--json', *flags), 2)
self.assert_failure(self.cli('--training-summary', '--json'), 2)
# Legacy flag form must not dispatch status instead of diagnostics.
# The source guard runs before config or control-socket discovery.
self.assert_failure(self.cli('--replay', self.store, '--training-summary', '--status', '--json'), 2)
def test_summary_requires_current_manifest(self):
copy = self.clone('fragments', lambda rel: rel.name.startswith('MANIFEST'))
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
def offline_demo(self):
store = self.root / 'offline-demo'
config = self.write_config('offline-demo.conf', store)
policy = self.root / 'offline-policy.json'
config.write_text(config.read_text() + f'policy_path = {policy}\n')
p = self.cli('--seed-demo-trace', config=config)
self.assertEqual(p.returncode, 0, p.stderr)
return store, config, policy
def test_offline_correlate_respects_existing_policy_without_writing_it(self):
_, config, policy = self.offline_demo()
# Legacy policy can be read without migration/default-feed writes.
value = {'version': 1, 'feeds': [], 'allowlist': [
{'id': 'fixture-rule', 'kind': 'rule', 'value': 'SyntheticDemoRule', 'rule': ''}]}
policy.write_text(json.dumps(value))
before = policy.read_bytes()
p = self.cli('--correlate', '--json', config=config)
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(json.loads(p.stdout)['findings'], [])
self.assertEqual(policy.read_bytes(), before)
listed = self.cli('findings', 'list', '--json', config=config)
self.assertEqual(listed.returncode, 0, listed.stderr)
self.assertEqual(json.loads(listed.stdout)['findings'], [])
value['allowlist'] = []
policy.write_text(json.dumps(value))
p = self.cli('--correlate', '--json', config=config)
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(len(json.loads(p.stdout)['findings']), 1) # raw demo evidence restored
def test_offline_correlate_invalid_policy_fails_before_store_mutation(self):
store, config, policy = self.offline_demo()
policy.write_text('{invalid')
before = hashes(store)
p = self.cli('--correlate', '--json', config=config)
self.assert_failure(p, 3)
self.assertEqual(hashes(store), before)
self.assertEqual(policy.read_text(), '{invalid')
policy.unlink() # missing means no allowlist, never seed a policy merely to correlate
p = self.cli('--correlate', '--json', config=config)
self.assertEqual(p.returncode, 0, p.stderr)
self.assertEqual(len(json.loads(p.stdout)['findings']), 1)
self.assertFalse(policy.exists())
def test_help_and_completions_show_the_mode(self):
p = subprocess.run([str(CLI), 'replay', '--help'], capture_output=True,
text=True, timeout=10)
self.assertEqual(p.returncode, 0)
self.assertIn('--training-summary', p.stdout)
p = subprocess.run([str(CLI), 'completion', 'bash'], capture_output=True,
text=True, timeout=10)
self.assertIn('--training-summary', p.stdout)
if __name__ == '__main__':
unittest.main(argv=[sys.argv[0]], verbosity=2)