Preserve independent support, typed rule namespaces and stable process lifetimes. Share authoritative managed-set reconciliation across daemon and offline correlation, load offline policy without seeding/migration, and release resources after shutdown-flush exceptions. Keep retirement history, transaction ownership and fault/allocation regressions; no whole-set crash-atomicity or live acceptance claim.
389 lines
19 KiB
Python
389 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Actual CLI contracts on isolated synthetic stores; no training or live RPC."""
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
CLI = Path(sys.argv[1]).resolve()
|
|
FIXTURE = Path(sys.argv[2]).resolve()
|
|
|
|
|
|
def hashes(folder):
|
|
return {str(p.relative_to(folder)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(folder.rglob('*')) if p.is_file()}
|
|
|
|
|
|
def crc32c(data):
|
|
"""Reference Castagnoli CRC for deliberately mutated fixture headers."""
|
|
c = 0xFFFFFFFF
|
|
for byte in data:
|
|
c ^= byte
|
|
for _ in range(8):
|
|
c = (c >> 1) ^ (0x82F63B78 if c & 1 else 0)
|
|
return c ^ 0xFFFFFFFF
|
|
|
|
|
|
class ReplayCli(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(prefix='strawberry_cli_replay_')
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.store = self.root / 'store'
|
|
p = subprocess.run([str(FIXTURE), str(self.store)], capture_output=True,
|
|
text=True, timeout=30)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.reference = json.loads(p.stdout.splitlines()[-1])
|
|
self.config = self.write_config('full.conf', self.store)
|
|
self.original = hashes(self.store)
|
|
self.addCleanup(self.assert_original_unchanged)
|
|
|
|
def assert_original_unchanged(self):
|
|
self.assertEqual(hashes(self.store), self.original)
|
|
self.assertFalse((self.root / 'models').exists())
|
|
|
|
def write_config(self, name, store):
|
|
p = self.root / name
|
|
p.write_text(f'db_path = {store}\nmodel_dir = {self.root / "models"}\n'
|
|
f'control_socket_path = {self.root / "no-control.sock"}\n')
|
|
return p
|
|
|
|
def cli(self, *args, config=None):
|
|
return subprocess.run([str(CLI), *map(str, args), '--config',
|
|
str(config or self.config)], capture_output=True,
|
|
text=True, timeout=30)
|
|
|
|
def clone(self, name, omit=None):
|
|
out = self.root / name
|
|
out.mkdir()
|
|
for p in self.store.rglob('*'):
|
|
rel = p.relative_to(self.store)
|
|
if omit and omit(rel):
|
|
continue
|
|
dest = out / rel
|
|
if p.is_dir():
|
|
dest.mkdir(exist_ok=True)
|
|
elif p.is_file():
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(p.read_bytes())
|
|
return out
|
|
|
|
def assert_failure(self, p, code=None):
|
|
self.assertNotEqual(p.returncode, 0, p.stdout)
|
|
if code is not None:
|
|
self.assertEqual(p.returncode, code, p.stderr)
|
|
self.assertEqual(p.stdout, '')
|
|
self.assertTrue(p.stderr.strip())
|
|
|
|
def test_unavailable_dictionary_is_not_an_empty_stream(self):
|
|
for stream in ('evidence', 'telemetry'):
|
|
copy = self.clone('unknown-dictionary-' + stream)
|
|
segment = next((copy / stream).glob('seg-*.log'))
|
|
data = bytearray(segment.read_bytes())
|
|
self.assertEqual(data[37], 1) # zstd, not raw
|
|
self.assertEqual(int.from_bytes(data[56:60], 'little'), crc32c(data[32:56]))
|
|
data[48:52] = (0x5EED).to_bytes(4, 'little')
|
|
data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little')
|
|
segment.write_bytes(data)
|
|
before = hashes(copy)
|
|
for flags in ((), ('--training-summary',)):
|
|
with self.subTest(stream=stream, flags=flags):
|
|
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 3)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_nontruncated_block_header_errors_are_invalid(self):
|
|
for stream in ('evidence', 'telemetry'):
|
|
for kind in ('magic', 'header_crc', 'version', 'codec', 'size'):
|
|
copy = self.clone('bad-block-header-' + stream + '-' + kind)
|
|
segment = next((copy / stream).glob('seg-*.log'))
|
|
data = bytearray(segment.read_bytes())
|
|
if kind == 'magic':
|
|
data[32] ^= 0x80
|
|
elif kind == 'header_crc':
|
|
data[56] ^= 0x80
|
|
else:
|
|
if kind == 'version':
|
|
data[36] = 255
|
|
elif kind == 'codec':
|
|
data[37] = 255
|
|
else:
|
|
data[40:44] = (1 << 31).to_bytes(4, 'little')
|
|
data[56:60] = crc32c(data[32:56]).to_bytes(4, 'little')
|
|
segment.write_bytes(data)
|
|
before = hashes(copy)
|
|
for flags in ((), ('--training-summary',)):
|
|
with self.subTest(stream=stream, kind=kind, flags=flags):
|
|
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_verifier_is_one_json_document(self):
|
|
p = self.cli('store', 'verify', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertTrue(j['ok'])
|
|
self.assertEqual(j['records'], 8)
|
|
|
|
def test_verifier_notes_do_not_assert_complete_replay_coverage(self):
|
|
copy = self.clone('verify-partial', lambda rel: rel.parts[0] == 'telemetry')
|
|
config = self.write_config('partial.conf', copy)
|
|
before = hashes(copy)
|
|
p = self.cli('store', 'verify', '--json', config=config)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertTrue(j['ok']) # existing best-effort verifier contract, intentionally unchanged
|
|
self.assertTrue(j['notes'])
|
|
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_summary_without_explicit_source_does_not_discover_live_daemon(self):
|
|
config = self.root / 'no-source.conf'
|
|
config.write_text(f'control_socket_path = {self.root / "no-control.sock"}\n')
|
|
p = self.cli('replay', '--training-summary', '--json', config=config)
|
|
self.assert_failure(p, 3)
|
|
self.assertIn('explicit snapshot', p.stderr)
|
|
|
|
def test_missing_listed_stream_is_not_empty_success(self):
|
|
for stream in ('evidence', 'telemetry'):
|
|
with self.subTest(stream=stream):
|
|
copy = self.clone('missing-' + stream, lambda rel, stream=stream: rel.parts[0] == stream)
|
|
before = hashes(copy)
|
|
p = self.cli('replay', copy, '--warmup', '0', '--json')
|
|
self.assert_failure(p, 3)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_missing_listed_file_is_not_empty_success(self):
|
|
for stream in ('evidence', 'telemetry'):
|
|
with self.subTest(stream=stream):
|
|
files = list((self.store / stream).glob('seg-*.log'))
|
|
self.assertTrue(files)
|
|
rel = files[0].relative_to(self.store)
|
|
copy = self.clone('file-' + stream, lambda p, rel=rel: p == rel)
|
|
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 3)
|
|
|
|
def test_invalid_header_cannot_yield_partial_replay(self):
|
|
copy = self.clone('invalid-header')
|
|
segment = next((copy / 'telemetry').glob('seg-*.log'))
|
|
data = bytearray(segment.read_bytes())
|
|
data[0] ^= 0x80
|
|
segment.write_bytes(data)
|
|
before = hashes(copy)
|
|
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json'), 1)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_corrupt_later_block_cannot_export_valid_prefix(self):
|
|
copy = self.clone('invalid-later-block')
|
|
segment = next((copy / 'telemetry').glob('seg-*.log'))
|
|
data = bytearray(segment.read_bytes())
|
|
# Segment header 32 bytes, codec header 28; stored length at +12.
|
|
stored = int.from_bytes(data[44:48], 'little')
|
|
second_payload = 32 + 28 + stored + 28
|
|
self.assertLess(second_payload, len(data))
|
|
data[second_payload] ^= 0x80
|
|
segment.write_bytes(data)
|
|
before = hashes(copy)
|
|
for flags in ((), ('--training-summary',)):
|
|
with self.subTest(flags=flags):
|
|
self.assert_failure(self.cli('replay', copy, '--warmup', '0', '--json', *flags), 1)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_malformed_label_cannot_export_telemetry_or_examples(self):
|
|
store = self.root / 'malformed-label'
|
|
p = subprocess.run([str(FIXTURE), str(store), 'malformed-label'], capture_output=True,
|
|
text=True, timeout=30)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
before = hashes(store)
|
|
for flags in ((), ('--training-summary',)):
|
|
self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1)
|
|
self.assertEqual(hashes(store), before)
|
|
|
|
def test_sealed_footer_and_fallback_are_distinct(self):
|
|
for corrupt_footer in (False, True):
|
|
with self.subTest(corrupt_footer=corrupt_footer):
|
|
store = self.root / ('sealed-' + str(corrupt_footer))
|
|
p = subprocess.run([str(FIXTURE), str(store), 'sealed'], capture_output=True,
|
|
text=True, timeout=30)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
if corrupt_footer:
|
|
segment = sorted((store / 'telemetry').glob('seg-*.log'))[0]
|
|
data = segment.read_bytes()
|
|
self.assertGreater(len(data), 32)
|
|
segment.write_bytes(data[:-1])
|
|
before = hashes(store)
|
|
for flags in ((), ('--training-summary',)):
|
|
self.assert_failure(self.cli('replay', store, '--warmup', '0', '--json', *flags), 1)
|
|
else:
|
|
self.assertTrue((store / 'MANIFEST.prev').is_file())
|
|
(store / 'MANIFEST').rename(store / 'MANIFEST.saved')
|
|
before = hashes(store)
|
|
p = self.cli('replay', store, '--warmup', '0', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertEqual(json.loads(p.stdout)['records'], 6)
|
|
self.assert_failure(self.cli('replay', store, '--training-summary', '--json'), 3)
|
|
self.assertEqual(hashes(store), before)
|
|
|
|
def test_active_tail_is_normal_for_replay_not_snapshot_summary(self):
|
|
copy = self.clone('active-tail')
|
|
segment = next((copy / 'telemetry').glob('seg-*.log'))
|
|
segment.write_bytes(segment.read_bytes() + b'x')
|
|
before = hashes(copy)
|
|
p = self.cli('replay', copy, '--warmup', '0', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertEqual(json.loads(p.stdout)['records'], 6)
|
|
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
|
|
self.assertEqual(hashes(copy), before)
|
|
|
|
def test_default_samples_are_explicitly_uncollected(self):
|
|
p = self.cli('replay', self.store, '--warmup', '0', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertEqual(j['windows'], 4)
|
|
self.assertFalse(j['examples_collected'])
|
|
self.assertEqual(j['examples_kept'], 0)
|
|
|
|
def test_summary_matches_existing_collector_and_is_aggregate_only(self):
|
|
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertEqual(j['mode'], 'training_summary')
|
|
self.assertEqual(j['assessment'], 'not_assessed')
|
|
self.assertFalse(j['telemetry_empty'])
|
|
self.assertFalse(j['sample_empty'])
|
|
s = j['sample']
|
|
self.assertTrue(s['collected'])
|
|
for key in ('eligible', 'eligible_positive', 'retained', 'retained_positive'):
|
|
self.assertEqual(s[key], self.reference[key])
|
|
self.assertEqual(s['retained_zero_label'], 2)
|
|
self.assertEqual(s['entity_cap'], 500)
|
|
self.assertEqual(s['entities'], 1)
|
|
self.assertEqual(s['max_entity_examples'], 4)
|
|
self.assertEqual(s['max_entity_share'], 1.0)
|
|
self.assertEqual(j['labels']['records'], 2)
|
|
self.assertEqual(j['coverage']['detected_windows'], 2)
|
|
self.assertEqual(j['coverage']['late_detected_windows'], 1)
|
|
for marker in ('synthetic', 'SyntheticTeacher', 'SyntheticQuarantine',
|
|
'root_node', 'root_exe', 'root_shape', 'top_windows',
|
|
'a' * 64, 'd' * 64, str(self.root)):
|
|
self.assertNotIn(marker, p.stdout)
|
|
|
|
def test_summary_semantics_are_deterministic_timings_are_separate(self):
|
|
results = []
|
|
for _ in range(2):
|
|
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
timings = j.pop('timings')
|
|
self.assertEqual(timings['unit'], 'wall_microseconds')
|
|
for key in ('replay', 'summary_build'):
|
|
self.assertEqual(timings[key]['count'], 1)
|
|
self.assertEqual(timings[key]['sum'], timings[key]['max'])
|
|
results.append(j)
|
|
self.assertEqual(results[0], results[1])
|
|
|
|
def test_summary_human_is_not_json_or_host_identifiers(self):
|
|
p = self.cli('replay', self.store, '--warmup', '0', '--training-summary')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertIn('retained', p.stdout.lower())
|
|
self.assertNotIn('synthetic', p.stdout.lower())
|
|
self.assertNotIn(str(self.root), p.stdout)
|
|
with self.assertRaises(json.JSONDecodeError):
|
|
json.loads(p.stdout)
|
|
|
|
def test_warmup_empty_sample_is_not_empty_telemetry(self):
|
|
p = self.cli('replay', self.store, '--training-summary', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertFalse(j['telemetry_empty'])
|
|
self.assertTrue(j['sample_empty'])
|
|
self.assertEqual(j['sample']['retained'], 0)
|
|
self.assertIsNone(j['sample']['max_entity_share'])
|
|
|
|
def test_complete_empty_store_is_explicit(self):
|
|
empty = self.root / 'empty'
|
|
p = subprocess.run([str(FIXTURE), str(empty), 'empty'], capture_output=True,
|
|
text=True, timeout=30)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
before = hashes(empty)
|
|
p = self.cli('replay', empty, '--training-summary', '--json')
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
j = json.loads(p.stdout)
|
|
self.assertTrue(j['telemetry_empty'])
|
|
self.assertTrue(j['sample_empty'])
|
|
self.assertEqual(hashes(empty), before)
|
|
|
|
def test_summary_rejects_missing_stream(self):
|
|
copy = self.clone('summary-missing', lambda rel: rel.parts[0] == 'evidence')
|
|
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
|
|
|
|
def test_summary_conflicts_fail_before_opening_inputs(self):
|
|
for flags in (('--model', 'missing.sbm'), ('--inject', 'missing.trace'),
|
|
('--policy-eval',), ('--top', '20'), ('--inject-copies', '5')):
|
|
with self.subTest(flags=flags):
|
|
self.assert_failure(self.cli('replay', self.root / 'not-a-store',
|
|
'--training-summary', '--json', *flags), 2)
|
|
self.assert_failure(self.cli('--training-summary', '--json'), 2)
|
|
# Legacy flag form must not dispatch status instead of diagnostics.
|
|
# The source guard runs before config or control-socket discovery.
|
|
self.assert_failure(self.cli('--replay', self.store, '--training-summary', '--status', '--json'), 2)
|
|
|
|
def test_summary_requires_current_manifest(self):
|
|
copy = self.clone('fragments', lambda rel: rel.name.startswith('MANIFEST'))
|
|
self.assert_failure(self.cli('replay', copy, '--training-summary', '--json'), 3)
|
|
|
|
def offline_demo(self):
|
|
store = self.root / 'offline-demo'
|
|
config = self.write_config('offline-demo.conf', store)
|
|
policy = self.root / 'offline-policy.json'
|
|
config.write_text(config.read_text() + f'policy_path = {policy}\n')
|
|
p = self.cli('--seed-demo-trace', config=config)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
return store, config, policy
|
|
|
|
def test_offline_correlate_respects_existing_policy_without_writing_it(self):
|
|
_, config, policy = self.offline_demo()
|
|
# Legacy policy can be read without migration/default-feed writes.
|
|
value = {'version': 1, 'feeds': [], 'allowlist': [
|
|
{'id': 'fixture-rule', 'kind': 'rule', 'value': 'SyntheticDemoRule', 'rule': ''}]}
|
|
policy.write_text(json.dumps(value))
|
|
before = policy.read_bytes()
|
|
p = self.cli('--correlate', '--json', config=config)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertEqual(json.loads(p.stdout)['findings'], [])
|
|
self.assertEqual(policy.read_bytes(), before)
|
|
listed = self.cli('findings', 'list', '--json', config=config)
|
|
self.assertEqual(listed.returncode, 0, listed.stderr)
|
|
self.assertEqual(json.loads(listed.stdout)['findings'], [])
|
|
value['allowlist'] = []
|
|
policy.write_text(json.dumps(value))
|
|
p = self.cli('--correlate', '--json', config=config)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertEqual(len(json.loads(p.stdout)['findings']), 1) # raw demo evidence restored
|
|
|
|
def test_offline_correlate_invalid_policy_fails_before_store_mutation(self):
|
|
store, config, policy = self.offline_demo()
|
|
policy.write_text('{invalid')
|
|
before = hashes(store)
|
|
p = self.cli('--correlate', '--json', config=config)
|
|
self.assert_failure(p, 3)
|
|
self.assertEqual(hashes(store), before)
|
|
self.assertEqual(policy.read_text(), '{invalid')
|
|
policy.unlink() # missing means no allowlist, never seed a policy merely to correlate
|
|
p = self.cli('--correlate', '--json', config=config)
|
|
self.assertEqual(p.returncode, 0, p.stderr)
|
|
self.assertEqual(len(json.loads(p.stdout)['findings']), 1)
|
|
self.assertFalse(policy.exists())
|
|
|
|
def test_help_and_completions_show_the_mode(self):
|
|
p = subprocess.run([str(CLI), 'replay', '--help'], capture_output=True,
|
|
text=True, timeout=10)
|
|
self.assertEqual(p.returncode, 0)
|
|
self.assertIn('--training-summary', p.stdout)
|
|
p = subprocess.run([str(CLI), 'completion', 'bash'], capture_output=True,
|
|
text=True, timeout=10)
|
|
self.assertIn('--training-summary', p.stdout)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main(argv=[sys.argv[0]], verbosity=2)
|