#!/usr/bin/env python3 """Actual CLI contracts on isolated synthetic stores; no training or live RPC.""" import hashlib import json import subprocess import sys import tempfile import unittest from pathlib import Path CLI = Path(sys.argv[1]).resolve() FIXTURE = Path(sys.argv[2]).resolve() def hashes(folder): return {str(p.relative_to(folder)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(folder.rglob('*')) if p.is_file()} def crc32c(data): """Reference Castagnoli CRC for deliberately mutated fixture headers.""" c = 0xFFFFFFFF for byte in data: c ^= byte for _ in range(8): c = (c >> 1) ^ (0x82F63B78 if c & 1 else 0) return c ^ 0xFFFFFFFF class ReplayCli(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory(prefix='strawberry_cli_replay_') self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) self.store = self.root / 'store' p = subprocess.run([str(FIXTURE), str(self.store)], capture_output=True, text=True, timeout=30) self.assertEqual(p.returncode, 0, p.stderr) self.reference = json.loads(p.stdout.splitlines()[-1]) self.config = self.write_config('full.conf', self.store) self.original = hashes(self.store) self.addCleanup(self.assert_original_unchanged) def assert_original_unchanged(self): self.assertEqual(hashes(self.store), self.original) self.assertFalse((self.root / 'models').exists()) def write_config(self, name, store): p = self.root / name p.write_text(f'db_path = {store}\nmodel_dir = {self.root / "models"}\n' f'control_socket_path = {self.root / "no-control.sock"}\n') return p def cli(self, *args, config=None): return subprocess.run([str(CLI), *map(str, args), '--config', str(config or self.config)], capture_output=True, text=True, timeout=30) def clone(self, name, omit=None): out = self.root / name out.mkdir() for p in self.store.rglob('*'): rel = p.relative_to(self.store) if omit and omit(rel): continue dest = out / rel if p.is_dir(): dest.mkdir(exist_ok=True) elif p.is_file(): dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(p.read_bytes()) return out def assert_failure(self, p, code=None): self.assertNotEqual(p.returncode, 0, p.stdout) if code is not None: self.assertEqual(p.returncode, code, p.stderr) self.assertEqual(p.stdout, '') self.assertTrue(p.stderr.strip()) def test_unavailable_dictionary_is_not_an_empty_stream(self): for stream in ('evidence', 'telemetry'): copy = self.clone('unknown-dictionary-' + stream) segment = next((copy / stream).glob('seg-*.log')) data = bytearray(segment.read_bytes()) self.assertEqual(data[37], 1) # zstd, not raw self.assertEqual(int.from_bytes(data[56:60], 'little'), crc32c(data[32:56])) data[48:52] = (0x5EED).to_bytes(4, 'little') data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little') segment.write_bytes(data) before = hashes(copy) for flags in ((), ('--training-summary',)): with self.subTest(stream=stream, flags=flags): self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 3) self.assertEqual(hashes(copy), before) def test_nontruncated_block_header_errors_are_invalid(self): for stream in ('evidence', 'telemetry'): for kind in ('magic', 'header_crc', 'version', 'codec', 'size'): copy = self.clone('bad-block-header-' + stream + '-' + kind) segment = next((copy / stream).glob('seg-*.log')) data = bytearray(segment.read_bytes()) if kind == 'magic': data[32] ^= 0x80 elif kind == 'header_crc': data[56] ^= 0x80 else: if kind == 'version': data[36] = 255 elif kind == 'codec': data[37] = 255 else: data[40:44] = (1 << 31).to_bytes(4, 'little') data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little') segment.write_bytes(data) before = hashes(copy) for flags in ((), ('--training-summary',)): with self.subTest(stream=stream, kind=kind, flags=flags): self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1) self.assertEqual(hashes(copy), before) def test_verifier_is_one_json_document(self): p = self.cli('store', 'verify', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertTrue(j['ok']) self.assertEqual(j['records'], 8) def test_verifier_notes_do_not_assert_complete_replay_coverage(self): copy = self.clone('verify-partial', lambda rel: rel.parts[0] == 'telemetry') config = self.write_config('partial.conf', copy) before = hashes(copy) p = self.cli('store', 'verify', '--json', config=config) self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertTrue(j['ok']) # existing best-effort verifier contract, intentionally unchanged self.assertTrue(j['notes']) self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3) self.assertEqual(hashes(copy), before) def test_summary_without_explicit_source_does_not_discover_live_daemon(self): config = self.root / 'no-source.conf' config.write_text(f'control_socket_path = {self.root / "no-control.sock"}\n') p = self.cli('replay', '--training-summary', '--json', config=config) self.assert_failure(p, 3) self.assertIn('explicit snapshot', p.stderr) def test_missing_listed_stream_is_not_empty_success(self): for stream in ('evidence', 'telemetry'): with self.subTest(stream=stream): copy = self.clone('missing-' + stream, lambda rel, stream=stream: rel.parts[0] == stream) before = hashes(copy) p = self.cli('replay', copy, '--warmup', '0', '--json') self.assert_failure(p, 3) self.assertEqual(hashes(copy), before) def test_missing_listed_file_is_not_empty_success(self): for stream in ('evidence', 'telemetry'): with self.subTest(stream=stream): files = list((self.store / stream).glob('seg-*.log')) self.assertTrue(files) rel = files[0].relative_to(self.store) copy = self.clone('file-' + stream, lambda p, rel=rel: p == rel) self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 3) def test_invalid_header_cannot_yield_partial_replay(self): copy = self.clone('invalid-header') segment = next((copy / 'telemetry').glob('seg-*.log')) data = bytearray(segment.read_bytes()) data[0] ^= 0x80 segment.write_bytes(data) before = hashes(copy) self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 1) self.assertEqual(hashes(copy), before) def test_corrupt_later_block_cannot_export_valid_prefix(self): copy = self.clone('invalid-later-block') segment = next((copy / 'telemetry').glob('seg-*.log')) data = bytearray(segment.read_bytes()) # Segment header 32 bytes, codec header 28; stored length at +12. stored = int.from_bytes(data[44:48], 'little') second_payload = 32 + 28 + stored + 28 self.assertLess(second_payload, len(data)) data[second_payload] ^= 0x80 segment.write_bytes(data) before = hashes(copy) for flags in ((), ('--training-summary',)): with self.subTest(flags=flags): self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1) self.assertEqual(hashes(copy), before) def test_malformed_label_cannot_export_telemetry_or_examples(self): store = self.root / 'malformed-label' p = subprocess.run([str(FIXTURE), str(store), 'malformed-label'], capture_output=True, text=True, timeout=30) self.assertEqual(p.returncode, 0, p.stderr) before = hashes(store) for flags in ((), ('--training-summary',)): self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1) self.assertEqual(hashes(store), before) def test_sealed_footer_and_fallback_are_distinct(self): for corrupt_footer in (False, True): with self.subTest(corrupt_footer=corrupt_footer): store = self.root / ('sealed-' + str(corrupt_footer)) p = subprocess.run([str(FIXTURE), str(store), 'sealed'], capture_output=True, text=True, timeout=30) self.assertEqual(p.returncode, 0, p.stderr) if corrupt_footer: segment = sorted((store / 'telemetry').glob('seg-*.log'))[0] data = segment.read_bytes() self.assertGreater(len(data), 32) segment.write_bytes(data[:-1]) before = hashes(store) for flags in ((), ('--training-summary',)): self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1) else: self.assertTrue((store / 'MANIFEST.prev').is_file()) (store / 'MANIFEST').rename(store / 'MANIFEST.saved') before = hashes(store) p = self.cli('replay', store, '--warmup', '0', '--json') self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout)['records'], 6) self.assert_failure(self.cli('replay', store, '--training-summary', '--json'), 3) self.assertEqual(hashes(store), before) def test_active_tail_is_normal_for_replay_not_snapshot_summary(self): copy = self.clone('active-tail') segment = next((copy / 'telemetry').glob('seg-*.log')) segment.write_bytes(segment.read_bytes() + b'x') before = hashes(copy) p = self.cli('replay', copy, '--warmup', '0', '--json') self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout)['records'], 6) self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3) self.assertEqual(hashes(copy), before) def test_default_samples_are_explicitly_uncollected(self): p = self.cli('replay', self.store, '--warmup', '0', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertEqual(j['windows'], 4) self.assertFalse(j['examples_collected']) self.assertEqual(j['examples_kept'], 0) def test_summary_matches_existing_collector_and_is_aggregate_only(self): p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertEqual(j['mode'], 'training_summary') self.assertEqual(j['assessment'], 'not_assessed') self.assertFalse(j['telemetry_empty']) self.assertFalse(j['sample_empty']) s = j['sample'] self.assertTrue(s['collected']) for key in ('eligible', 'eligible_positive', 'retained', 'retained_positive'): self.assertEqual(s[key], self.reference[key]) self.assertEqual(s['retained_zero_label'], 2) self.assertEqual(s['entity_cap'], 500) self.assertEqual(s['entities'], 1) self.assertEqual(s['max_entity_examples'], 4) self.assertEqual(s['max_entity_share'], 1.0) self.assertEqual(j['labels']['records'], 2) self.assertEqual(j['coverage']['detected_windows'], 2) self.assertEqual(j['coverage']['late_detected_windows'], 1) for marker in ('synthetic', 'SyntheticTeacher', 'SyntheticQuarantine', 'root_node', 'root_exe', 'root_shape', 'top_windows', 'a' * 64, 'd' * 64, str(self.root)): self.assertNotIn(marker, p.stdout) def test_summary_semantics_are_deterministic_timings_are_separate(self): results = [] for _ in range(2): p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) timings = j.pop('timings') self.assertEqual(timings['unit'], 'wall_microseconds') for key in ('replay', 'summary_build'): self.assertEqual(timings[key]['count'], 1) self.assertEqual(timings[key]['sum'], timings[key]['max']) results.append(j) self.assertEqual(results[0], results[1]) def test_summary_human_is_not_json_or_host_identifiers(self): p = self.cli('replay', self.store, '--warmup', '0', '--training-summary') self.assertEqual(p.returncode, 0, p.stderr) self.assertIn('retained', p.stdout.lower()) self.assertNotIn('synthetic', p.stdout.lower()) self.assertNotIn(str(self.root), p.stdout) with self.assertRaises(json.JSONDecodeError): json.loads(p.stdout) def test_warmup_empty_sample_is_not_empty_telemetry(self): p = self.cli('replay', self.store, '--training-summary', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertFalse(j['telemetry_empty']) self.assertTrue(j['sample_empty']) self.assertEqual(j['sample']['retained'], 0) self.assertIsNone(j['sample']['max_entity_share']) def test_complete_empty_store_is_explicit(self): empty = self.root / 'empty' p = subprocess.run([str(FIXTURE), str(empty), 'empty'], capture_output=True, text=True, timeout=30) self.assertEqual(p.returncode, 0, p.stderr) before = hashes(empty) p = self.cli('replay', empty, '--training-summary', '--json') self.assertEqual(p.returncode, 0, p.stderr) j = json.loads(p.stdout) self.assertTrue(j['telemetry_empty']) self.assertTrue(j['sample_empty']) self.assertEqual(hashes(empty), before) def test_summary_rejects_missing_stream(self): copy = self.clone('summary-missing', lambda rel: rel.parts[0] == 'evidence') self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3) def test_summary_conflicts_fail_before_opening_inputs(self): for flags in (('--model', 'missing.sbm'), ('--inject', 'missing.trace'), ('--policy-eval',), ('--top', '20'), ('--inject-copies', '5')): with self.subTest(flags=flags): self.assert_failure(self.cli('replay', self.root / 'not-a-store', '--training-summary', '--json', *flags), 2) self.assert_failure(self.cli('--training-summary', '--json'), 2) # Legacy flag form must not dispatch status instead of diagnostics. # The source guard runs before config or control-socket discovery. self.assert_failure(self.cli('--replay', self.store, '--training-summary', '--status', '--json'), 2) def test_summary_requires_current_manifest(self): copy = self.clone('fragments', lambda rel: rel.name.startswith('MANIFEST')) self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3) def offline_demo(self): store = self.root / 'offline-demo' config = self.write_config('offline-demo.conf', store) policy = self.root / 'offline-policy.json' config.write_text(config.read_text() + f'policy_path = {policy}\n') p = self.cli('--seed-demo-trace', config=config) self.assertEqual(p.returncode, 0, p.stderr) return store, config, policy def test_offline_correlate_respects_existing_policy_without_writing_it(self): _, config, policy = self.offline_demo() # Legacy policy can be read without migration/default-feed writes. value = {'version': 1, 'feeds': [], 'allowlist': [ {'id': 'fixture-rule', 'kind': 'rule', 'value': 'SyntheticDemoRule', 'rule': ''}]} policy.write_text(json.dumps(value)) before = policy.read_bytes() p = self.cli('--correlate', '--json', config=config) self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout)['findings'], []) self.assertEqual(policy.read_bytes(), before) listed = self.cli('findings', 'list', '--json', config=config) self.assertEqual(listed.returncode, 0, listed.stderr) self.assertEqual(json.loads(listed.stdout)['findings'], []) value['allowlist'] = [] policy.write_text(json.dumps(value)) p = self.cli('--correlate', '--json', config=config) self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(len(json.loads(p.stdout)['findings']), 1) # raw demo evidence restored def test_offline_correlate_invalid_policy_fails_before_store_mutation(self): store, config, policy = self.offline_demo() policy.write_text('{invalid') before = hashes(store) p = self.cli('--correlate', '--json', config=config) self.assert_failure(p, 3) self.assertEqual(hashes(store), before) self.assertEqual(policy.read_text(), '{invalid') policy.unlink() # missing means no allowlist, never seed a policy merely to correlate p = self.cli('--correlate', '--json', config=config) self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(len(json.loads(p.stdout)['findings']), 1) self.assertFalse(policy.exists()) def test_help_and_completions_show_the_mode(self): p = subprocess.run([str(CLI), 'replay', '--help'], capture_output=True, text=True, timeout=10) self.assertEqual(p.returncode, 0) self.assertIn('--training-summary', p.stdout) p = subprocess.run([str(CLI), 'completion', 'bash'], capture_output=True, text=True, timeout=10) self.assertIn('--training-summary', p.stdout) if __name__ == '__main__': unittest.main(argv=[sys.argv[0]], verbosity=2)