Files
SkillFactory/tests/test_contact_cycle_sim_evidence_codec.py
T
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

321 lines
15 KiB
Python

"""CPU-only v2 codec bounds, crash prefixes, budget admission and copy parity."""
import copy
import gzip
import json
import lzma
import math
import os
import subprocess
import sys
from pathlib import Path
from types import MappingProxyType
import pytest
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import (
MAX_CHUNK_RAW_BYTES, MAX_CHUNK_RECORDS, MAX_RECORD_BYTES, TRACE_FILENAME,
XZ_DECODER_MEMORY_BYTES, plain,
)
import test_contact_cycle_sim_evaluation_audit as audit_tests
from test_contact_cycle_sim_evaluation_audit import ROOT, protocol, records
# Reuse the source-named CPU-only fixtures without loading task registration.
recording = audit_tests.recording
offline_reader = audit_tests.offline_reader
def decoded_members(path):
data = path.read_bytes()
while data:
decoder = lzma.LZMADecompressor(memlimit=XZ_DECODER_MEMORY_BYTES)
raw = decoder.decompress(data)
assert decoder.eof
yield raw
data = decoder.unused_data
def make_recorder(recording, tmp_path, budget=lambda *args: None):
return recording.EvaluationRecorder(protocol(), 0, tmp_path / 'trial',
budget_check=budget, supplemental=lambda _: {})
def test_plain_and_copied_exact_immutable_fast_path_parity(recording):
class SubInt(int):
pass
value = {'bool': True, 'int': 123, 'float': -.0, 'none': None, 'str': 'x',
'subclass': SubInt(42), 'list': [1, {'nested': [2., 3]}], 'tuple': (7, 8)}
def old_plain(v):
if hasattr(v, 'items'):
return {k: old_plain(x) for k, x in v.items()}
if isinstance(v, (list, tuple)):
return [old_plain(x) for x in v]
return copy.deepcopy(v)
for fn in (plain, recording.copied):
result = fn(value)
assert result == old_plain(value)
assert type(result['subclass']) is SubInt
assert math.copysign(1., result['float']) == -1.
result['list'][1]['nested'][0] = 100
assert value['list'][1]['nested'][0] == 2.
assert math.isnan(fn(float('nan')))
assert plain(MappingProxyType(value)) == old_plain(value)
def test_chunk_record_bound_final_partial_and_exact_order(recording, tmp_path):
recorder = make_recorder(recording, tmp_path)
original = [{'kind': 'payload', 'sequence': i} for i in range(67)]
for record in original:
recorder._write(record)
assert len(recorder.pending) <= MAX_CHUNK_RECORDS
assert len(recorder.pending) == 3
terminal = {'kind': 'terminal', 'runtime_stop': None,
'external_failure': {'causes': ['cpu_test'], 'fatal': True, 'context': {}}, 'recording_faults': []}
recorder._write(terminal, stage='finalize')
members = list(decoded_members(recorder.directory / TRACE_FILENAME))
assert [len(raw.splitlines()) for raw in members] == [64, 4]
assert [json.loads(line) for raw in members for line in raw.splitlines()] == original + [terminal]
assert not recorder.pending and recorder.pending_raw_bytes == 0
recorder.stream.close()
def test_chunk_raw_bound_is_separate_from_unchanged_record_limit(recording, tmp_path):
assert MAX_RECORD_BYTES == 1024 * 1024
recorder = make_recorder(recording, tmp_path)
overhead = len((json.dumps({'kind': 'payload', 'data': ''}, separators=(',', ':')) + '\n').encode())
record = {'kind': 'payload', 'data': 'x' * (MAX_RECORD_BYTES - overhead)}
for _ in range(9):
recorder._write(record)
assert recorder.pending_raw_bytes <= MAX_CHUNK_RAW_BYTES
assert recorder.pending_raw_bytes == MAX_RECORD_BYTES
recorder._flush()
assert [len(raw) for raw in decoded_members(recorder.directory / TRACE_FILENAME)] == [8 * MAX_RECORD_BYTES,
MAX_RECORD_BYTES]
record['data'] += 'x'
with pytest.raises(ValueError, match='record exceeds'):
recorder._write(record)
recorder.stream.close()
@pytest.mark.parametrize('kind', ['initial', 'seam', 'step_result', 'terminal', 'capture_fault'])
def test_checkpoint_kinds_flush_pending_immediately(recording, tmp_path, kind):
recorder = make_recorder(recording, tmp_path)
recorder._write({'kind': 'payload'})
assert recorder.written_bytes == 0
recorder._write({'kind': kind})
assert not recorder.pending
assert len(list(decoded_members(recorder.directory / TRACE_FILENAME))) == 1
recorder.stream.close()
def test_first_physical_failure_persisted_before_callback_return(recording, tmp_path):
recorder = make_recorder(recording, tmp_path)
original = []
for record in records(protocol(), end=2):
if record['kind'] == 'tick' and record['state']['substeps'] == 2:
record['state']['root'][0] = .031
record['state']['ik_target'][0] -= .031
original.append(copy.deepcopy(record))
causes = recorder._record(record)
assert 'P1_root_horizontal' in causes
assert not recorder.pending
assert [json.loads(line) for raw in decoded_members(recorder.directory / TRACE_FILENAME)
for line in raw.splitlines()] == original
assert not (recorder.directory / 'metrics.json').exists()
recorder.stream.close()
def test_finalize_external_failure_flushes_partial_chunk_without_ack(recording, tmp_path):
recorder = make_recorder(recording, tmp_path)
original = list(records(protocol(), end=2))
for record in original:
assert not recorder._record(record)
assert recorder.pending
ack = []
report = recorder.finalize(external_failure={'causes': ['cpu_test'], 'fatal': True, 'context': {}},
acknowledge=ack.append)
assert not ack and not report['functional_passed']
raw = [json.loads(line) for member in decoded_members(recorder.directory / TRACE_FILENAME)
for line in member.splitlines()]
assert raw[:-1] == original and raw[-1]['kind'] == 'terminal'
@pytest.mark.parametrize('deny', [False, True])
def test_exact_finalized_member_admission_includes_header_footer(recording, tmp_path, deny):
recorder = make_recorder(recording, tmp_path)
record = {'kind': 'payload', 'data': 'abc' * 2000}
recorder._write(record)
raw = b''.join(recorder.pending)
expected = lzma.compress(raw, preset=0)
calls = []
def budget(stage, pending, written):
calls.append((stage, pending, written))
assert pending == len(expected) and written == 0
if pending > len(expected) - int(deny):
raise OSError('one byte over aggregate cap')
recorder.budget_check = budget
if deny:
with pytest.raises(OSError, match='aggregate'):
recorder._flush()
assert (recorder.directory / TRACE_FILENAME).stat().st_size == 0
assert b''.join(recorder.pending) == raw
else:
recorder._flush()
assert (recorder.directory / TRACE_FILENAME).read_bytes() == expected
assert len(calls) == 1
recorder.stream.close()
def test_pending_reservation_on_every_record_and_handled_cleanup(recording, tmp_path):
recorder = make_recorder(recording, tmp_path)
calls = []
def budget(stage, pending, written):
calls.append((stage, pending, written))
if stage == 'record':
assert pending >= recorder.pending_raw_bytes
raise OSError('aggregate pending reservation denied')
recorder.budget_check = budget
# Admission fails with evidence still available for the cleanup allowance.
assert recorder._record(next(records(protocol(), end=1)))
assert recorder.pending and recorder.written_bytes == 0
ack = []
recorder.finalize(external_failure={'causes': ['storage_test'], 'fatal': True, 'context': {}},
acknowledge=ack.append)
assert not ack
assert len(calls) >= 4 # per-record reservation, final reservation, member, metrics
raw = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes())
assert [json.loads(line)['kind'] for line in raw.splitlines()] == ['initial', 'terminal']
def test_forced_exit_loses_only_uncommitted_chunk_no_terminal(tmp_path, offline_reader):
directory = tmp_path / 'crashed'
code = '''
import os, sys
from pathlib import Path
sys.path.insert(0, str(Path.cwd() / 'scripts'))
from profile_cycle_sim_evidence import modules
p, a, b, r = modules()
protocol = p.load_protocol(Path('configs/contact_cycle/continuous_eval_p1_v1.json'))
recorder = r.EvaluationRecorder(protocol, 0, Path(sys.argv[1]), budget_check=lambda *a: None, supplemental=lambda _: {})
recorder._write({'kind': 'initial', 'saved': True})
recorder._write({'kind': 'tick', 'uncommitted': True})
os._exit(23)
'''
child = subprocess.run([sys.executable, '-c', code, str(directory)], cwd=ROOT,
env={**os.environ, 'CUDA_VISIBLE_DEVICES': ''}, timeout=30)
assert child.returncode == 23
lines = list(offline_reader.trace_lines(directory / TRACE_FILENAME, compressed=True, max_bytes=2**20,
max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf))
assert [json.loads(line) for line in lines] == [{'kind': 'initial', 'saved': True}]
assert not (directory / 'metrics.json').exists()
@pytest.mark.parametrize('codec', ['gzip', 'xz'])
def test_reader_old_and_new_exact_record_and_total_decoded_bounds(offline_reader, tmp_path, codec):
path = tmp_path / ('trace.jsonl.gz' if codec == 'gzip' else TRACE_FILENAME)
lines = [b'a' * (MAX_RECORD_BYTES - 1) + b'\n', b'b\n', b'c' * 200000 + b'\n']
encode = (lambda raw: gzip.compress(raw, mtime=0)) if codec == 'gzip' else (lambda raw: lzma.compress(raw, preset=0))
path.write_bytes(b''.join(encode(line) for line in lines) if codec == 'gzip' else encode(b''.join(lines)))
kwargs = dict(compressed=True, max_bytes=2**20, max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf)
assert list(offline_reader.trace_lines(path, **kwargs)) == lines
with pytest.raises(ValueError, match='decoded_limit'):
list(offline_reader.trace_lines(path, **kwargs, max_decoded_bytes=sum(map(len, lines)) - 1))
@pytest.mark.parametrize('fault', ['memory', 'records', 'raw_bytes', 'record_bytes', 'split_record', 'truncated', 'corrupt'])
def test_xz_reader_rejects_bounds_and_damage(offline_reader, tmp_path, fault):
raw, preset = b'{"kind":"terminal"}\n', 0
if fault == 'memory':
preset = 6 # 8MiB dictionary + overhead exceeds the decoder's 8MiB cap.
elif fault == 'records':
raw = b'{}\n' * 65
elif fault == 'raw_bytes':
raw = (b'x' * (MAX_RECORD_BYTES - 1) + b'\n') * 9
elif fault == 'record_bytes':
raw = b'x' * MAX_RECORD_BYTES + b'\n'
data = lzma.compress(raw, preset=preset)
if fault == 'split_record':
data = lzma.compress(raw[:5], preset=0) + lzma.compress(raw[5:], preset=0)
elif fault == 'truncated':
data = data[:-1]
elif fault == 'corrupt':
data = data[:-8] + bytes([data[-8] ^ 1]) + data[-7:]
path = tmp_path / TRACE_FILENAME
path.write_bytes(data)
with pytest.raises((ValueError, lzma.LZMAError, EOFError)):
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
max_record_bytes=MAX_RECORD_BYTES, deadline=math.inf))
def test_xz_deadline_checked_while_draining_output_and_empty_streams(offline_reader, tmp_path, monkeypatch):
path = tmp_path / TRACE_FILENAME
path.write_bytes(lzma.compress(b'a' * (MAX_RECORD_BYTES - 1) + b'\n', preset=0))
now = 0
def clock():
nonlocal now
now += 1
return now
monkeypatch.setattr(offline_reader.time, 'monotonic', clock)
with pytest.raises(ValueError, match='budget_audit'):
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
max_record_bytes=MAX_RECORD_BYTES, deadline=10))
path.write_bytes(lzma.compress(b'', preset=0) * 100)
now = 0
with pytest.raises(ValueError, match='budget_audit'):
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
max_record_bytes=MAX_RECORD_BYTES, deadline=10))
def test_chunk_repeated_retained_records_compress_without_loss(recording, tmp_path):
# Saved rows are optional external evidence; the standalone report covers both.
path = Path('/home/cen/sf-runs/p1-03/launch-1/trial-01/trace.v1.jsonl.gz')
if not path.is_file():
pytest.skip('retained row1 unavailable')
recorder = make_recorder(recording, tmp_path)
with gzip.open(path, 'rb') as source:
original = [source.readline() for _ in range(81)]
for raw in original:
recorder._write(json.loads(raw))
recorder._flush()
trace = recorder.directory / TRACE_FILENAME
assert lzma.decompress(trace.read_bytes()) == b''.join(original)
legacy_bytes = sum(len(gzip.compress(raw, compresslevel=1, mtime=0)) for raw in original)
assert trace.stat().st_size < legacy_bytes / 4
recorder.stream.close()
@pytest.mark.parametrize('tail', ['missing_terminal', 'truncated_terminal'])
def test_complete_physical_prefix_without_intact_terminal_never_passes(
recording, offline_reader, tmp_path, monkeypatch, tail):
from dataclasses import asdict
from test_contact_cycle_sim_evaluation_audit import Stop
p = protocol('P2')
recorder = recording.EvaluationRecorder(p, 0, tmp_path / 'trial',
budget_check=lambda *args: None, supplemental=lambda _: {})
stop = plain(asdict(Stop(physics_ticks=4500, causes=(), reason='protocol_complete',
clock=plain(asdict(p.evidence_clock(0, 4500))), completed_controls=900,
partial_control_ticks=0, completed_clock_cycles=1, all_sim_steps_attempted=4502,
all_sim_steps_completed=4502, rollout_steps=4500, composer_buffer_replacements=4500)))
for record in records(p):
recorder._write(record)
if record['kind'] == 'tick' and record['state']['substeps'] % 5 == 0:
g = record['state']['substeps']
recorder._write({'kind': 'step_result', 'ticks_advanced': 5, 'observed_tick': g,
'reward': [0.], 'has_next_observation': g != 4500, 'stop': stop if g == 4500 else None})
if tail == 'truncated_terminal':
recorder._write({'kind': 'terminal', 'runtime_stop': stop, 'external_failure': None, 'recording_faults': []})
recorder.stream.close()
trace = recorder.directory / TRACE_FILENAME
if tail == 'truncated_terminal':
with trace.open('r+b') as stream:
stream.truncate(trace.stat().st_size - 4)
output = tmp_path / 'offline.json'
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
str(ROOT / 'configs/contact_cycle/bounded_diagnostic_p2_v1.json'), '--trial-index', '0',
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**26), '--wall-seconds', '30'])
assert offline_reader.main() == 1
report = json.loads(output.read_text())
assert report['observed_physics_ticks'] == 4500 and report['protocol_complete']
assert not report['functional_passed']
assert all(c.startswith(('recording_missing_terminal', 'recording_offline:')) for c in report['all_causes'])