9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
321 lines
15 KiB
Python
321 lines
15 KiB
Python
"""CPU-only v2 codec bounds, crash prefixes, budget admission and copy parity."""
|
|
import copy
|
|
import gzip
|
|
import json
|
|
import lzma
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from types import MappingProxyType
|
|
|
|
import pytest
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import (
|
|
MAX_CHUNK_RAW_BYTES, MAX_CHUNK_RECORDS, MAX_RECORD_BYTES, TRACE_FILENAME,
|
|
XZ_DECODER_MEMORY_BYTES, plain,
|
|
)
|
|
import test_contact_cycle_sim_evaluation_audit as audit_tests
|
|
from test_contact_cycle_sim_evaluation_audit import ROOT, protocol, records
|
|
|
|
# Reuse the source-named CPU-only fixtures without loading task registration.
|
|
recording = audit_tests.recording
|
|
offline_reader = audit_tests.offline_reader
|
|
|
|
|
|
def decoded_members(path):
|
|
data = path.read_bytes()
|
|
while data:
|
|
decoder = lzma.LZMADecompressor(memlimit=XZ_DECODER_MEMORY_BYTES)
|
|
raw = decoder.decompress(data)
|
|
assert decoder.eof
|
|
yield raw
|
|
data = decoder.unused_data
|
|
|
|
|
|
def make_recorder(recording, tmp_path, budget=lambda *args: None):
|
|
return recording.EvaluationRecorder(protocol(), 0, tmp_path / 'trial',
|
|
budget_check=budget, supplemental=lambda _: {})
|
|
|
|
|
|
def test_plain_and_copied_exact_immutable_fast_path_parity(recording):
|
|
class SubInt(int):
|
|
pass
|
|
value = {'bool': True, 'int': 123, 'float': -.0, 'none': None, 'str': 'x',
|
|
'subclass': SubInt(42), 'list': [1, {'nested': [2., 3]}], 'tuple': (7, 8)}
|
|
def old_plain(v):
|
|
if hasattr(v, 'items'):
|
|
return {k: old_plain(x) for k, x in v.items()}
|
|
if isinstance(v, (list, tuple)):
|
|
return [old_plain(x) for x in v]
|
|
return copy.deepcopy(v)
|
|
for fn in (plain, recording.copied):
|
|
result = fn(value)
|
|
assert result == old_plain(value)
|
|
assert type(result['subclass']) is SubInt
|
|
assert math.copysign(1., result['float']) == -1.
|
|
result['list'][1]['nested'][0] = 100
|
|
assert value['list'][1]['nested'][0] == 2.
|
|
assert math.isnan(fn(float('nan')))
|
|
assert plain(MappingProxyType(value)) == old_plain(value)
|
|
|
|
|
|
def test_chunk_record_bound_final_partial_and_exact_order(recording, tmp_path):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
original = [{'kind': 'payload', 'sequence': i} for i in range(67)]
|
|
for record in original:
|
|
recorder._write(record)
|
|
assert len(recorder.pending) <= MAX_CHUNK_RECORDS
|
|
assert len(recorder.pending) == 3
|
|
terminal = {'kind': 'terminal', 'runtime_stop': None,
|
|
'external_failure': {'causes': ['cpu_test'], 'fatal': True, 'context': {}}, 'recording_faults': []}
|
|
recorder._write(terminal, stage='finalize')
|
|
members = list(decoded_members(recorder.directory / TRACE_FILENAME))
|
|
assert [len(raw.splitlines()) for raw in members] == [64, 4]
|
|
assert [json.loads(line) for raw in members for line in raw.splitlines()] == original + [terminal]
|
|
assert not recorder.pending and recorder.pending_raw_bytes == 0
|
|
recorder.stream.close()
|
|
|
|
|
|
def test_chunk_raw_bound_is_separate_from_unchanged_record_limit(recording, tmp_path):
|
|
assert MAX_RECORD_BYTES == 1024 * 1024
|
|
recorder = make_recorder(recording, tmp_path)
|
|
overhead = len((json.dumps({'kind': 'payload', 'data': ''}, separators=(',', ':')) + '\n').encode())
|
|
record = {'kind': 'payload', 'data': 'x' * (MAX_RECORD_BYTES - overhead)}
|
|
for _ in range(9):
|
|
recorder._write(record)
|
|
assert recorder.pending_raw_bytes <= MAX_CHUNK_RAW_BYTES
|
|
assert recorder.pending_raw_bytes == MAX_RECORD_BYTES
|
|
recorder._flush()
|
|
assert [len(raw) for raw in decoded_members(recorder.directory / TRACE_FILENAME)] == [8 * MAX_RECORD_BYTES,
|
|
MAX_RECORD_BYTES]
|
|
record['data'] += 'x'
|
|
with pytest.raises(ValueError, match='record exceeds'):
|
|
recorder._write(record)
|
|
recorder.stream.close()
|
|
|
|
|
|
@pytest.mark.parametrize('kind', ['initial', 'seam', 'step_result', 'terminal', 'capture_fault'])
|
|
def test_checkpoint_kinds_flush_pending_immediately(recording, tmp_path, kind):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
recorder._write({'kind': 'payload'})
|
|
assert recorder.written_bytes == 0
|
|
recorder._write({'kind': kind})
|
|
assert not recorder.pending
|
|
assert len(list(decoded_members(recorder.directory / TRACE_FILENAME))) == 1
|
|
recorder.stream.close()
|
|
|
|
|
|
def test_first_physical_failure_persisted_before_callback_return(recording, tmp_path):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
original = []
|
|
for record in records(protocol(), end=2):
|
|
if record['kind'] == 'tick' and record['state']['substeps'] == 2:
|
|
record['state']['root'][0] = .031
|
|
record['state']['ik_target'][0] -= .031
|
|
original.append(copy.deepcopy(record))
|
|
causes = recorder._record(record)
|
|
assert 'P1_root_horizontal' in causes
|
|
assert not recorder.pending
|
|
assert [json.loads(line) for raw in decoded_members(recorder.directory / TRACE_FILENAME)
|
|
for line in raw.splitlines()] == original
|
|
assert not (recorder.directory / 'metrics.json').exists()
|
|
recorder.stream.close()
|
|
|
|
|
|
def test_finalize_external_failure_flushes_partial_chunk_without_ack(recording, tmp_path):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
original = list(records(protocol(), end=2))
|
|
for record in original:
|
|
assert not recorder._record(record)
|
|
assert recorder.pending
|
|
ack = []
|
|
report = recorder.finalize(external_failure={'causes': ['cpu_test'], 'fatal': True, 'context': {}},
|
|
acknowledge=ack.append)
|
|
assert not ack and not report['functional_passed']
|
|
raw = [json.loads(line) for member in decoded_members(recorder.directory / TRACE_FILENAME)
|
|
for line in member.splitlines()]
|
|
assert raw[:-1] == original and raw[-1]['kind'] == 'terminal'
|
|
|
|
|
|
@pytest.mark.parametrize('deny', [False, True])
|
|
def test_exact_finalized_member_admission_includes_header_footer(recording, tmp_path, deny):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
record = {'kind': 'payload', 'data': 'abc' * 2000}
|
|
recorder._write(record)
|
|
raw = b''.join(recorder.pending)
|
|
expected = lzma.compress(raw, preset=0)
|
|
calls = []
|
|
def budget(stage, pending, written):
|
|
calls.append((stage, pending, written))
|
|
assert pending == len(expected) and written == 0
|
|
if pending > len(expected) - int(deny):
|
|
raise OSError('one byte over aggregate cap')
|
|
recorder.budget_check = budget
|
|
if deny:
|
|
with pytest.raises(OSError, match='aggregate'):
|
|
recorder._flush()
|
|
assert (recorder.directory / TRACE_FILENAME).stat().st_size == 0
|
|
assert b''.join(recorder.pending) == raw
|
|
else:
|
|
recorder._flush()
|
|
assert (recorder.directory / TRACE_FILENAME).read_bytes() == expected
|
|
assert len(calls) == 1
|
|
recorder.stream.close()
|
|
|
|
|
|
def test_pending_reservation_on_every_record_and_handled_cleanup(recording, tmp_path):
|
|
recorder = make_recorder(recording, tmp_path)
|
|
calls = []
|
|
def budget(stage, pending, written):
|
|
calls.append((stage, pending, written))
|
|
if stage == 'record':
|
|
assert pending >= recorder.pending_raw_bytes
|
|
raise OSError('aggregate pending reservation denied')
|
|
recorder.budget_check = budget
|
|
# Admission fails with evidence still available for the cleanup allowance.
|
|
assert recorder._record(next(records(protocol(), end=1)))
|
|
assert recorder.pending and recorder.written_bytes == 0
|
|
ack = []
|
|
recorder.finalize(external_failure={'causes': ['storage_test'], 'fatal': True, 'context': {}},
|
|
acknowledge=ack.append)
|
|
assert not ack
|
|
assert len(calls) >= 4 # per-record reservation, final reservation, member, metrics
|
|
raw = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes())
|
|
assert [json.loads(line)['kind'] for line in raw.splitlines()] == ['initial', 'terminal']
|
|
|
|
|
|
def test_forced_exit_loses_only_uncommitted_chunk_no_terminal(tmp_path, offline_reader):
|
|
directory = tmp_path / 'crashed'
|
|
code = '''
|
|
import os, sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path.cwd() / 'scripts'))
|
|
from profile_cycle_sim_evidence import modules
|
|
p, a, b, r = modules()
|
|
protocol = p.load_protocol(Path('configs/contact_cycle/continuous_eval_p1_v1.json'))
|
|
recorder = r.EvaluationRecorder(protocol, 0, Path(sys.argv[1]), budget_check=lambda *a: None, supplemental=lambda _: {})
|
|
recorder._write({'kind': 'initial', 'saved': True})
|
|
recorder._write({'kind': 'tick', 'uncommitted': True})
|
|
os._exit(23)
|
|
'''
|
|
child = subprocess.run([sys.executable, '-c', code, str(directory)], cwd=ROOT,
|
|
env={**os.environ, 'CUDA_VISIBLE_DEVICES': ''}, timeout=30)
|
|
assert child.returncode == 23
|
|
lines = list(offline_reader.trace_lines(directory / TRACE_FILENAME, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf))
|
|
assert [json.loads(line) for line in lines] == [{'kind': 'initial', 'saved': True}]
|
|
assert not (directory / 'metrics.json').exists()
|
|
|
|
|
|
@pytest.mark.parametrize('codec', ['gzip', 'xz'])
|
|
def test_reader_old_and_new_exact_record_and_total_decoded_bounds(offline_reader, tmp_path, codec):
|
|
path = tmp_path / ('trace.jsonl.gz' if codec == 'gzip' else TRACE_FILENAME)
|
|
lines = [b'a' * (MAX_RECORD_BYTES - 1) + b'\n', b'b\n', b'c' * 200000 + b'\n']
|
|
encode = (lambda raw: gzip.compress(raw, mtime=0)) if codec == 'gzip' else (lambda raw: lzma.compress(raw, preset=0))
|
|
path.write_bytes(b''.join(encode(line) for line in lines) if codec == 'gzip' else encode(b''.join(lines)))
|
|
kwargs = dict(compressed=True, max_bytes=2**20, max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf)
|
|
assert list(offline_reader.trace_lines(path, **kwargs)) == lines
|
|
with pytest.raises(ValueError, match='decoded_limit'):
|
|
list(offline_reader.trace_lines(path, **kwargs, max_decoded_bytes=sum(map(len, lines)) - 1))
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['memory', 'records', 'raw_bytes', 'record_bytes', 'split_record', 'truncated', 'corrupt'])
|
|
def test_xz_reader_rejects_bounds_and_damage(offline_reader, tmp_path, fault):
|
|
raw, preset = b'{"kind":"terminal"}\n', 0
|
|
if fault == 'memory':
|
|
preset = 6 # 8MiB dictionary + overhead exceeds the decoder's 8MiB cap.
|
|
elif fault == 'records':
|
|
raw = b'{}\n' * 65
|
|
elif fault == 'raw_bytes':
|
|
raw = (b'x' * (MAX_RECORD_BYTES - 1) + b'\n') * 9
|
|
elif fault == 'record_bytes':
|
|
raw = b'x' * MAX_RECORD_BYTES + b'\n'
|
|
data = lzma.compress(raw, preset=preset)
|
|
if fault == 'split_record':
|
|
data = lzma.compress(raw[:5], preset=0) + lzma.compress(raw[5:], preset=0)
|
|
elif fault == 'truncated':
|
|
data = data[:-1]
|
|
elif fault == 'corrupt':
|
|
data = data[:-8] + bytes([data[-8] ^ 1]) + data[-7:]
|
|
path = tmp_path / TRACE_FILENAME
|
|
path.write_bytes(data)
|
|
with pytest.raises((ValueError, lzma.LZMAError, EOFError)):
|
|
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf))
|
|
|
|
|
|
def test_xz_deadline_checked_while_draining_output_and_empty_streams(offline_reader, tmp_path, monkeypatch):
|
|
path = tmp_path / TRACE_FILENAME
|
|
path.write_bytes(lzma.compress(b'a' * (MAX_RECORD_BYTES - 1) + b'\n', preset=0))
|
|
now = 0
|
|
def clock():
|
|
nonlocal now
|
|
now += 1
|
|
return now
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', clock)
|
|
with pytest.raises(ValueError, match='budget_audit'):
|
|
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=MAX_RECORD_BYTES, deadline=10))
|
|
path.write_bytes(lzma.compress(b'', preset=0) * 100)
|
|
now = 0
|
|
with pytest.raises(ValueError, match='budget_audit'):
|
|
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=MAX_RECORD_BYTES, deadline=10))
|
|
|
|
|
|
def test_chunk_repeated_retained_records_compress_without_loss(recording, tmp_path):
|
|
# Saved rows are optional external evidence; the standalone report covers both.
|
|
path = Path('/home/cen/sf-runs/p1-03/launch-1/trial-01/trace.v1.jsonl.gz')
|
|
if not path.is_file():
|
|
pytest.skip('retained row1 unavailable')
|
|
recorder = make_recorder(recording, tmp_path)
|
|
with gzip.open(path, 'rb') as source:
|
|
original = [source.readline() for _ in range(81)]
|
|
for raw in original:
|
|
recorder._write(json.loads(raw))
|
|
recorder._flush()
|
|
trace = recorder.directory / TRACE_FILENAME
|
|
assert lzma.decompress(trace.read_bytes()) == b''.join(original)
|
|
legacy_bytes = sum(len(gzip.compress(raw, compresslevel=1, mtime=0)) for raw in original)
|
|
assert trace.stat().st_size < legacy_bytes / 4
|
|
recorder.stream.close()
|
|
|
|
|
|
@pytest.mark.parametrize('tail', ['missing_terminal', 'truncated_terminal'])
|
|
def test_complete_physical_prefix_without_intact_terminal_never_passes(
|
|
recording, offline_reader, tmp_path, monkeypatch, tail):
|
|
from dataclasses import asdict
|
|
|
|
from test_contact_cycle_sim_evaluation_audit import Stop
|
|
|
|
p = protocol('P2')
|
|
recorder = recording.EvaluationRecorder(p, 0, tmp_path / 'trial',
|
|
budget_check=lambda *args: None, supplemental=lambda _: {})
|
|
stop = plain(asdict(Stop(physics_ticks=4500, causes=(), reason='protocol_complete',
|
|
clock=plain(asdict(p.evidence_clock(0, 4500))), completed_controls=900,
|
|
partial_control_ticks=0, completed_clock_cycles=1, all_sim_steps_attempted=4502,
|
|
all_sim_steps_completed=4502, rollout_steps=4500, composer_buffer_replacements=4500)))
|
|
for record in records(p):
|
|
recorder._write(record)
|
|
if record['kind'] == 'tick' and record['state']['substeps'] % 5 == 0:
|
|
g = record['state']['substeps']
|
|
recorder._write({'kind': 'step_result', 'ticks_advanced': 5, 'observed_tick': g,
|
|
'reward': [0.], 'has_next_observation': g != 4500, 'stop': stop if g == 4500 else None})
|
|
if tail == 'truncated_terminal':
|
|
recorder._write({'kind': 'terminal', 'runtime_stop': stop, 'external_failure': None, 'recording_faults': []})
|
|
recorder.stream.close()
|
|
trace = recorder.directory / TRACE_FILENAME
|
|
if tail == 'truncated_terminal':
|
|
with trace.open('r+b') as stream:
|
|
stream.truncate(trace.stat().st_size - 4)
|
|
output = tmp_path / 'offline.json'
|
|
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
|
|
str(ROOT / 'configs/contact_cycle/bounded_diagnostic_p2_v1.json'), '--trial-index', '0',
|
|
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**26), '--wall-seconds', '30'])
|
|
assert offline_reader.main() == 1
|
|
report = json.loads(output.read_text())
|
|
assert report['observed_physics_ticks'] == 4500 and report['protocol_complete']
|
|
assert not report['functional_passed']
|
|
assert all(c.startswith(('recording_missing_terminal', 'recording_offline:')) for c in report['all_causes'])
|