"""CPU synthetic evidence only: no simulator, actor, checkpoint or optimizer execution.""" import copy import gzip import importlib.util import json import lzma import math import sys from dataclasses import asdict, dataclass, field from pathlib import Path from types import SimpleNamespace import pytest import torch from SkillFactory.legs.contact_cycle_sim.evaluation_audit import ( BOOLS, SCALARS, TRACE_FILENAME, VECTORS, TrialAudit, matrix_report, plain, sub, ) from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import load_protocol ROOT = Path(__file__).resolve().parents[1] def protocol(stage='P1'): return load_protocol(ROOT / 'configs/contact_cycle' / ( 'continuous_eval_p1_v1.json' if stage == 'P1' else 'bounded_diagnostic_p2_v1.json')) def state(p, trial, clock, refreshes=1): anchors = {'anchor_a_world': [0., 0., .022], 'root_origin_world': [0., 0., .3], 'initial_root_world': [0., 0., .3]} c = p.command(trial, clock, anchor_a_world=anchors['anchor_a_world'], root_origin_world=anchors['root_origin_world'], weight_N=40.) g = clock.global_tick s = {k: 0. for k in SCALARS} s.update({k: False for k in BOOLS}) s.update({k: [0.] * n for k, n in VECTORS.items()}) s.update(weight=40., time=g * .002, substeps=g, anchor_height=.3, anchors=anchors, body_names=['virtual_payload', 'FL_thigh', 'FL_calf'], foot=list(c.target_world_m), normal=[0., 0., 40. if c.phase in (0, 5) else 16. if c.phase == 1 else 0.], root=[0., 0., .3, 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], gap=c.target_world_m[2] - .022, q_ik=[refreshes * .001] * 3, refreshes=refreshes, contact=c.phase in (0, 1, 5), acquired=g >= 5, armed=g >= 5, separated=c.phase in (2, 3), onset_phase=2 if c.phase in (2, 3) else -1, edge_phase=-1, ik_target=sub(c.target_world_m, [0., 0., .3]), scenario=p.trial(trial).scenario_id, event_program=plain(p.document['events']), external=[[0.] * 6 for _ in range(3)], guide=[[0.] * 6 for _ in range(3)], auxiliary=[[0.] * 6 for _ in range(3)], sensor_delay=[[0.] * 18], action_delay=[[0.] * 3], rng={'test_rng': [1, 2, 3]}, actuator_state={}, applied=None, counters={'reset_calls': 3, 'pose_write_calls': 1, 'composer_buffer_replacements': g, 'initialization_steps': 2, 'rollout_steps': g, 'attempted_steps': g + 2, 'completed_steps': g + 2}, reward_components={k: 0. for k in ('task', 'effort', 'action_rate', 'root_position', 'root_velocity', 'residual')}) s['events'] = [{'fired': False, 'completed': False, 'start': 0., 'envelope': 0., 'integral': 0.} for _ in s['event_program']] return s, plain(asdict(c)) def records(p, trial=0, end=None): total = p.trial(trial).total_ticks if end is None else end clock = p.next_command_clock(trial, 0) s, command = state(p, trial, clock) refclock = plain(asdict(clock)) event = dict(kind='initial', trial_index=trial, clock=refclock, command=command, held_control_index=None, action_origin_tick=None, reference_used_clock=None, reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[]) yield {'kind': 'initial', 'event': event, 'state': s} refreshes = 1 for g in range(1, total + 1): if (g - 1) % 5 == 0: learned = p.trial(trial).controller_kind == 'learned_policy' yield {'kind': 'control', 'origin_tick': g - 1, 'observation': [0.] * 46, 'action': [0.] * 3, 'mean': [0.] * 3 if learned else None, 'std': [.1] * 3 if learned else None} clock = p.evidence_clock(trial, g) s, command = state(p, trial, clock, refreshes) event = dict(kind='tick', trial_index=trial, clock=plain(asdict(clock)), command=command, held_control_index=(g-1)//5, action_origin_tick=5*((g-1)//5), reference_used_clock=refclock, reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[]) record = {'kind': 'tick', 'event': event, 'state': s} yield record if g % 5 == 0 and g < p.trial(trial).total_ticks: if g % 4500 == 0: yield {**copy.deepcopy(record), 'kind': 'seam'} clock = p.next_command_clock(trial, g, seam_admitted=True) _, command = state(p, trial, clock) refreshes += 1 refclock = plain(asdict(clock)) s = copy.deepcopy(s) s.update(refreshes=refreshes, q_ik=[refreshes * .001] * 3, ik_target=sub(command['target_world_m'], s['root'][:3])) yield {'kind': 'next_reference', 'state': s, 'event': { **event, 'kind': 'next_reference', 'clock': refclock, 'command': command, 'held_control_index': None, 'action_origin_tick': None, 'reference_next_clock': refclock}} def run(p=None, trial=0, end=None, mutate=None): p = protocol() if p is None else p audit = TrialAudit(p, trial) for record in records(p, trial, end): if mutate: mutate(record) causes = audit.consume(record) if causes: break return audit def test_complete_three_cycles_and_clock_prefix(): audit = run() report = audit.report() assert report['first_failure'] is None assert report['functional_passed'] and report['passed_prefix_cycles'] == 3 assert report['completed_clock_cycles'] == 3 assert all(c['metrics']['phase_counts'] == [1499, 500, 300, 700, 400, 1101] for c in report['cycles']) assert all(s['carry_pre'] == s['carry_post'] and s['post'] is not None for s in report['seams']) assert not report['accepted_skill'] and report['optimizer_steps'] == 0 def test_incomplete_tail_not_passed_and_later_gates_not_evaluated(): report = run(end=4512).report() assert not report['functional_passed'] and report['passed_prefix_cycles'] == 1 assert report['completed_clock_cycles'] == 1 assert report['cycles'][1]['gates']['landing_position'] == 'not_evaluated' assert report['cycles'][2]['status'] == 'not_evaluated' @pytest.mark.parametrize('field,value', [('root', [1.] * 13), ('q_ik', [0.] * 3), ('sensor_delay', [[1.] * 18]), ('contact', False), ('rng', {'changed': [4]}), ('action_delay', [[1.] * 3])]) def test_seam_fingerprint_rejects_teleport_and_memory_reset(field, value): def mutate(r): if r['kind'] == 'seam': r['state'][field] = value audit = run(mutate=mutate) assert audit.first_failure['global_tick'] == 4500 assert any('recording' in c for c in audit.first_failure['causes']) @pytest.mark.parametrize('kind', ['duplicate', 'skip', 'time_wrap', 'reset_counter']) def test_clock_and_reset_faults(kind): p = protocol() audit = TrialAudit(p, 0) stream = iter(records(p, end=3)) for _ in range(3): r = next(stream) assert not audit.consume(r) r = copy.deepcopy(r) if kind == 'duplicate' else next(stream) if kind == 'skip': r['event']['clock']['global_tick'] = 3 elif kind == 'time_wrap': r['state']['time'] = 0 elif kind == 'reset_counter': r['state']['counters']['reset_calls'] += 1 assert audit.consume(r) assert audit.first_failure @pytest.mark.parametrize('field,value', [('normal', [0., 0., float('nan')]), ('tilt', float('inf')), ('foot', [0., 0.]), ('q', None)]) def test_nonfinite_and_malformed_first_offending_tick(field, value): def mutate(r): if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 2: r['state'][field] = value audit = run(end=3, mutate=mutate) assert audit.first_failure['global_tick'] == 2 assert not audit.report()['functional_passed'] @pytest.mark.parametrize('value,passed', [(.03, True), (.03000001, False)]) def test_exact_root_threshold(value, passed): def mutate(r): if r['kind'] == 'tick': r['state']['root'][0] = value r['state']['ik_target'][0] -= value audit = run(end=1, mutate=mutate) assert (audit.first_failure is None) == passed def test_overlapping_failures_preserved(): def mutate(r): if r['kind'] == 'tick': r['state'].update(tilt=.7, joint_excess=.02, limit_ratio=2.) audit = run(end=1, mutate=mutate) assert {'tilt', 'hardware_joint_limit', 'hardware_effort_velocity'} <= set(audit.first_failure['causes']) @pytest.mark.parametrize('valid_ticks,passed', [(4, False), (5, True)]) def test_lift_dwell_exact_deadline(valid_ticks, passed): def mutate(r): if r['kind'] == 'tick' and r['event']['command']['phase'] == 2: t = r['event']['clock']['local_tick'] r['state']['gap'] = .022 if t > 2260 - valid_ticks else .021 audit = run(end=2260, mutate=mutate) assert (audit.first_failure is None) == passed assert audit.margins['lift_dwell'] == passed if not passed: assert audit.first_failure['global_tick'] == 2260 def test_lift_onset_unload_not_laundered(): def mutate(r): if r['kind'] == 'tick' and r['event']['command']['phase'] == 2: r['state']['onset_phase'] = 1 audit = run(end=2260, mutate=mutate) assert 'P1_lift_dwell' in audit.first_failure['causes'] @pytest.mark.parametrize('bad_tick,passed', [(4450, True), (4451, False)]) def test_final_reload_dwell_boundary(bad_tick, passed): def mutate(r): if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == bad_tick: r['state']['normal'][2] = 31.9 audit = run(end=4500, mutate=mutate) assert (audit.first_failure is None) == passed assert audit.cycles[0]['P1_margins']['reload_dwell'] == passed def test_support_and_unload_stop_at_window_last_sample(): def mutate(r): if r['kind'] == 'tick' and r['event']['command']['phase'] == 1: r['state']['normal'][2] = 29. audit = run(end=2100, mutate=mutate) assert audit.first_failure['global_tick'] == 1999 assert audit.gates['unloaded'] is True assert audit.margins['unload_ratio'] is False assert audit.gates['real_flight'] == 'not_evaluated' def test_P2_stricter_margins_are_diagnostic_not_suppressed_functional_failures(): def mutate(r): if r['kind'] in ('initial', 'tick', 'next_reference'): r['state']['root'][0] = .04 r['state']['ik_target'][0] = r['event']['command']['target_world_m'][0] - .04 report = run(protocol('P2'), mutate=mutate).report() assert report['functional_passed'] assert report['cycles'][0]['P1_margins']['root_horizontal'] is False assert report['cycles'][0]['P1_margins_role'] == 'diagnostic_only' def applied_zero(r, p, trial): """Pre-step capture with zero envelope, used before the event should start.""" s = r['state'] events = [] for i in p.trial(trial).event_indices: program = s['event_program'][i] events.append(dict(event_index=i, body_index=s['body_names'].index(program['body']), envelope=0., force_world=[0.] * 3, couple_world=[0.] * 3, point_world=program['point_m'], com_world=[0.] * 3, frame_quat_xyzw=[0., 0., 0., 1.], point_frame_quat_xyzw=[0., 0., 0., 1.], point_frame_origin_world=[0.] * 3)) s['applied'] = dict(format='skillfactory.contact-cycle-applied-wrench.v1', trial_index=trial, global_tick=r['event']['clock']['global_tick'], events=events, total_com_wrench=[[0.] * 6 for _ in s['body_names']]) @pytest.mark.parametrize('fault', ['missing', 'stale', 'frame', 'moment']) def test_required_prephysics_applied_evidence(fault): p, trial = protocol('P2'), 4 def mutate(r): if r['kind'] == 'tick': applied_zero(r, p, trial) a = r['state']['applied'] if fault == 'missing': r['state']['applied'] = None elif fault == 'stale': a['global_tick'] = 0 elif fault == 'frame': a['events'][0]['frame_quat_xyzw'] = [0., 0., 1., 0.] else: a['total_com_wrench'][0][3] = .01 audit = run(p, trial, end=1, mutate=mutate) assert any('recording_applied' in c for c in audit.first_failure['causes']) def test_undelivered_time_event_fails_at_first_delivery_deadline(): p, trial = protocol('P2'), 4 # Use a direct event-window probe so original functional windows are irrelevant. audit = TrialAudit(p, trial) initial = next(records(p, trial, end=0)) assert not audit.consume(initial) audit.event_previous['state']['time'] = 7.5 r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick') clock = p.evidence_clock(trial, 3751) r['event']['clock'] = plain(asdict(clock)) applied_zero(r, p, trial) audit.last = r causes = audit._events(r['state'], clock) assert 'event_delivery:0' in causes assert audit.delivery[0]['expected'] and audit.delivery[0]['missed'] assert audit.delivery[0]['expected_area_s'] == 6. def test_matrix_includes_all_not_run_rows_and_does_not_assume_pairing(): result = matrix_report(protocol('P2'), []) assert len(result['rows']) == 20 and len(result['pairs']) == 10 assert all(r['status'] == 'not_run' for r in result['rows']) assert all(p['status'] == 'not_evaluated' for p in result['pairs']) @pytest.fixture def recording(): path = ROOT / 'source/SkillFactory/SkillFactory/tasks/manager_based/contact_cycle_sim/evaluation_recording.py' spec = importlib.util.spec_from_file_location('_cpu_evaluation_recording', path) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module @dataclass class Stop: trial_index: int = 0 physics_ticks: int = 2 causes: tuple = ('swing_clearance',) reason: str = 'failure' fatal: bool = False clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2}) completed_controls: int = 0 partial_control_ticks: int = 2 completed_clock_cycles: int = 0 all_sim_steps_attempted: int = 4 all_sim_steps_completed: int = 4 initialization_steps: int = 2 rollout_steps: int = 2 reset_calls: int = 3 pose_write_calls: int = 1 composer_buffer_replacements: int = 2 def recorder_prefix(recording, tmp_path): budgets = [] recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence', budget_check=lambda *args: budgets.append(args), supplemental=lambda s: {}) for r in records(protocol(), end=2): assert not recorder._record(r) stop = Stop() assert not recorder.after_step(SimpleNamespace(ticks_advanced=2, observation=None, reward=None, stop=stop)) return recorder, stop, budgets def test_partial_control_durable_persistence_then_exact_ack(recording, tmp_path): recorder, stop, budgets = recorder_prefix(recording, tmp_path) acknowledgements = [] def acknowledge(value): assert (recorder.directory / 'metrics.json').is_file() acknowledgements.append(value) report = recorder.finalize(stop=stop, acknowledge=acknowledge) assert acknowledgements == [stop] assert report['first_failure']['global_tick'] == 2 decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode() raw = [json.loads(x) for x in decoded.splitlines()] assert len([r for r in raw if r['kind'] == 'tick']) == 2 assert raw[-2]['ticks_advanced'] == 2 and raw[-2]['reward'] is None assert raw[-1]['kind'] == 'terminal' and not report['functional_passed'] assert any(b[0] == 'finalize' for b in budgets) def test_io_error_never_acknowledges(recording, tmp_path, monkeypatch): recorder, stop, _ = recorder_prefix(recording, tmp_path) acknowledgements = [] def fail(_): raise OSError('disk fault') monkeypatch.setattr(recording.os, 'fsync', fail) with pytest.raises(OSError, match='disk fault'): recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert not acknowledgements and recorder.faults assert (recorder.directory / TRACE_FILENAME).exists() recorder.stream.close() def test_external_failure_terminal_no_fake_tick_and_no_ack(recording, tmp_path): recorder, stop, _ = recorder_prefix(recording, tmp_path) acknowledgements = [] report = recorder.finalize(stop=stop, external_failure={ 'causes': ['identity_changed'], 'fatal': True, 'context': {'stage': 'after_step'}}, acknowledge=acknowledgements.append) assert not acknowledgements assert report['observed_physics_ticks'] == 2 assert 'identity_changed' in report['first_failure']['causes'] def test_nonfinite_raw_and_budget_fault_are_retained(recording, tmp_path): recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence', budget_check=lambda *args: None, supplemental=lambda s: {}) for r in records(protocol(), end=1): if r['kind'] == 'tick': r['state']['tilt'] = math.nan causes = recorder._record(r) assert causes recorder.finalize(external_failure={'causes': ['recording_nonfinite'], 'fatal': True, 'context': {}}) assert '"nonfinite":"nan"' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode() def test_tensor_copy_is_independent(recording): tensor = torch.ones(1, 3) result = recording.copied({'x': tensor}) tensor.zero_() assert result == {'x': [[1., 1., 1.]]} def test_incomplete_endpoint_gates_return_immediate_stop(): p = protocol() audit = TrialAudit(p, 0) for record in records(p, end=4500): if record['kind'] == 'tick' and record['event']['clock']['global_tick'] == 4500: audit.gates['real_flight'] = 'not_evaluated' causes = audit.consume(record) assert 'recording_incomplete_cycle_gates' in causes break assert not audit.consume(record) assert audit.first_failure['global_tick'] == 4500 assert not audit.report()['cycles'][0]['passed'] def test_no_event_P1_requires_zero_wrench_not_fabricated_event_capture(): assert run(end=1).first_failure is None def mutate(r): if r['kind'] == 'tick': r['state']['external'][0][0] = .01 audit = run(end=1, mutate=mutate) assert 'identity_unexpected_external_wrench' in audit.first_failure['causes'] @pytest.mark.parametrize('fault', ['trial', 'point_frame', 'free_couple', 'moment_arm']) def test_nonzero_applied_evidence_checks_frame_and_COM_moment(fault): from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross, rotate p, trial = protocol('P2'), 8 audit = TrialAudit(p, trial) r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick') applied_zero(r, p, trial) s = r['state'] e = s['applied']['events'][0] program = s['event_program'][1] q = [0., 0., math.sqrt(.5), math.sqrt(.5)] e.update(envelope=.5, frame_quat_xyzw=q, point_frame_quat_xyzw=q, point_frame_origin_world=[.2, .3, .4], com_world=[.1, .2, .3]) s['events'][1]['envelope'] = .5 e['point_world'] = [a+b for a, b in zip(e['point_frame_origin_world'], rotate(q, program['point_m']))] e['force_world'] = [v*.5 for v in rotate(q, program['force_N'])] e['couple_world'] = [v*.5 for v in rotate(q, program['couple_Nm'])] wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'], cross(sub(e['point_world'], e['com_world']), e['force_world']))] s['external'][e['body_index']] = wrench s['applied']['total_com_wrench'][e['body_index']] = wrench.copy() clock = p.evidence_clock(trial, 1) assert audit._applied(s, clock)[1] == e if fault == 'trial': s['applied']['trial_index'] = 4 elif fault == 'point_frame': e['point_frame_quat_xyzw'] = [0., 0., 0., 1.] elif fault == 'free_couple': e['couple_world'][0] += .01 else: e['com_world'][0] += .01 with pytest.raises(ValueError, match='applied'): audit._applied(s, clock) def test_trigger_provenance_rejects_fabricated_early_fire(): p, trial = protocol('P2'), 4 def mutate(r): if r['kind'] == 'tick': applied_zero(r, p, trial) r['state']['events'][0]['fired'] = True audit = run(p, trial, end=1, mutate=mutate) assert 'recording_event_trigger_or_envelope:0' in audit.first_failure['causes'] def frozen_schedule_mutator(p, trial, *, edge_tick=None, inspect=None): from SkillFactory.legs.contact_cycle.mechanics import WrenchSchedule from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross schedule = WrenchSchedule(plain(p.document['events']), 1, 'cpu') def mutate(r): if r['kind'] != 'tick': return s = r['state'] g = r['event']['clock']['global_tick'] clock = p.next_command_clock(trial, 0) if g == 1 else p.evidence_clock(trial, g-1) command = p.command(trial, clock, anchor_a_world=s['anchors']['anchor_a_world'], root_origin_world=s['anchors']['root_origin_world'], weight_N=40.) # Match the actual frozen runtime: integer substeps multiplied by dt # in float32, NOT a Python double product subsequently cast to float32. time = torch.tensor([g-1]) * .002 s['time'] = float((torch.tensor([g]) * .002)[0]) s['liftoff'] = g == edge_tick and s['scenario'] == 2 s['touchdown'] = g == edge_tick and s['scenario'] == 3 s['edge_phase'] = 2 if s['liftoff'] else -1 schedule.step(time, torch.tensor([command.phase]), torch.tensor([g-1 == edge_tick and s['scenario'] == 2]), torch.tensor([g-1 == edge_tick and s['scenario'] == 3]), torch.tensor([s['scenario']]), .002) if inspect: inspect(g, time, schedule) for i, event in enumerate(s['events']): event.update(fired=bool(schedule.fired[0, i]), completed=bool(schedule.completed[0, i]), start=float(schedule.start[0, i]), envelope=float(schedule.envelope[0, i]), integral=float(schedule.integral[0, i])) applied_zero(r, p, trial) for e in s['applied']['events']: i, body = e['event_index'], e['body_index'] e['envelope'] = s['events'][i]['envelope'] program = s['event_program'][i] e['force_world'] = [v*e['envelope'] for v in program['force_N']] e['couple_world'] = [v*e['envelope'] for v in program['couple_Nm']] wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'], cross(sub(e['point_world'], e['com_world']), e['force_world']))] s['external'][body] = wrench s['applied']['total_com_wrench'][body] = wrench.copy() return mutate def test_frozen_time_schedule_complete_delivery_and_impulses(): p, trial = protocol('P2'), 4 report = run(p, trial, mutate=frozen_schedule_mutator(p, trial)).report() assert report['first_failure'] is None assert report['functional_passed'] and report['protocol_complete'] event = report['delivery'][0] assert event['status'] is True and event['actual_applied_evidence_available'] assert event['force_impulse_Ns'] == pytest.approx([18., -24., -12.], abs=1e-4) assert event['couple_impulse_Nms'] == pytest.approx([1.2, .6, -.6], abs=1e-5) @pytest.mark.parametrize('fault', ['partial_reward', 'next_observation', 'duplicate_step', 'terminal_counters']) def test_partial_and_terminal_records_fail_closed(recording, tmp_path, fault): recorder, stop, _ = recorder_prefix(recording, tmp_path) if fault == 'terminal_counters': stop.all_sim_steps_completed += 1 else: bad = copy.deepcopy(recorder.audit.step_results[-1]) if fault == 'partial_reward': bad['reward'] = [1.] elif fault == 'next_observation': bad['has_next_observation'] = True assert recorder.audit.consume(bad) acknowledgements = [] report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert not acknowledgements assert any(c.startswith('recording') for c in report['all_causes']) def test_recording_fault_after_earlier_failure_is_not_hidden(recording, tmp_path): recorder, stop, _ = recorder_prefix(recording, tmp_path) recorder.audit.fail(('recording_late_fault',), kind='terminal', tick=3) acknowledgements = [] report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert report['first_failure']['global_tick'] == 2 assert 'recording_late_fault' in report['all_causes'] assert not acknowledgements def test_capture_fault_reports_actual_tick_without_fabricating_sample(recording, tmp_path, monkeypatch): recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence', budget_check=lambda *args: None, supplemental=lambda s: {}) for record in records(protocol(), end=1): assert not recorder._record(record) @dataclass class Event: kind: str = 'tick' clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2}) causes: tuple = ('hardware_joint_limit',) def broken(*args): raise ValueError('accessor failed') monkeypatch.setattr(recording, 'snapshot', broken) assert recorder(Event(), None) report = recorder.finalize(external_failure={'causes': ['recording_capture'], 'fatal': True, 'context': {}}) assert report['first_failure']['global_tick'] == 2 assert report['observed_physics_ticks'] == 1 assert 'hardware_joint_limit' in report['first_failure']['causes'] assert 'capture_fault' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode() def test_offline_reaudit_matches_live_partial_report(recording, tmp_path): import subprocess recorder, stop, _ = recorder_prefix(recording, tmp_path) live = recorder.finalize(stop=stop) output = tmp_path / 'offline.json' result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'), '--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0', '--trace', str(recorder.directory / TRACE_FILENAME), '--output', str(output), '--max-trace-bytes', '1048576', '--wall-seconds', '20'], capture_output=True, text=True) assert result.returncode == 1, result.stderr offline = json.loads(output.read_text()) for key in ('first_failure', 'all_causes', 'cycles', 'functional_passed', 'protocol_complete', 'terminal'): assert offline[key] == live[key] assert offline['new_simulation_steps'] == offline['new_optimizer_steps'] == 0 def test_matrix_mismatched_pair_never_publishes_controller_delta(): p = protocol('P2') rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': {'anchor': i}, 'functional_passed': True} for i in (0, 1)] pair = matrix_report(p, rows)['pairs'][0] assert pair['status'] == 'invalid_initial_pair' assert pair['initial_exactly_equal'] is False assert pair['both_functional_passed'] is False assert pair['learned_minus_reference'] == 'not_evaluated' def test_matrix_absent_pair_snapshots_are_not_equal_evidence(): p = protocol('P2') rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': None} for i in (0, 1)] report = matrix_report(p, rows) assert report['pairs'][0]['status'] == 'not_evaluated' def test_xz_lossless_members_and_exact_inclusive_byte_accounting(recording, tmp_path): recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence', budget_check=lambda *args: None, supplemental=lambda s: {}) original = list(records(protocol(), end=2)) for record in original: assert not recorder._record(record) assert (recorder.directory / TRACE_FILENAME).stat().st_size == recorder.written_bytes # The initial is a checkpoint; control/ticks remain bounded and uncommitted. assert len(recorder.pending) == len(original) - 1 assert [json.loads(x) for x in lzma.decompress( (recorder.directory / TRACE_FILENAME).read_bytes()).splitlines()] == original[:1] recorder._flush() assert not recorder.pending and recorder.pending_raw_bytes == 0 decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).splitlines() assert [json.loads(line) for line in decoded] == original recorder.stream.close() @pytest.mark.parametrize('stage', ['record', 'terminal_member', 'metrics', 'close', 'compression']) def test_xz_storage_failures_preserve_complete_prefix_never_ack(recording, tmp_path, monkeypatch, stage): recorder, stop, _ = recorder_prefix(recording, tmp_path) path = recorder.directory / TRACE_FILENAME prefix = path.read_bytes() before = [json.loads(line) for line in lzma.decompress(prefix).splitlines()] finalize_calls = 0 def budget(where, pending, written): nonlocal finalize_calls assert written == path.stat().st_size if where == 'finalize': finalize_calls += 1 fail = ((stage == 'record' and where == 'record') or (stage == 'terminal_member' and finalize_calls == 2) or (stage == 'metrics' and finalize_calls == 3)) if fail: # Deny by one byte: the pending count must INCLUDE XZ header/check/footer. assert pending > 18 raise OSError('aggregate byte budget includes complete compressed member') recorder.budget_check = budget if stage == 'compression': def broken_compression(*args, **kwargs): raise OSError('compression fault') monkeypatch.setattr(recording.lzma, 'compress', broken_compression) if stage == 'close': stream = recorder.stream class CloseFault: def __getattr__(self, key): return getattr(stream, key) def close(self): raise OSError('close fault') recorder.stream = CloseFault() acknowledgements = [] if stage == 'record': assert recorder.after_step(SimpleNamespace(ticks_advanced=0, observation=None, reward=None, stop=stop)) recorder.finalize(stop=stop, acknowledge=acknowledgements.append) else: with pytest.raises(OSError): recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert not acknowledgements and recorder.faults decoded = [json.loads(line) for line in lzma.decompress(path.read_bytes()).splitlines()] assert decoded[:len(before)] == before if stage == 'close': stream.close() else: recorder.stream.close() @pytest.mark.parametrize('fault', ['truncated', 'corrupt', 'oversized', 'missing_terminal']) def test_offline_xz_rejects_damaged_or_incomplete_tail(recording, tmp_path, fault): import subprocess recorder, stop, _ = recorder_prefix(recording, tmp_path) trace = recorder.directory / TRACE_FILENAME if fault == 'missing_terminal': recorder.stream.close() else: recorder.finalize(stop=stop) data = trace.read_bytes() if fault == 'truncated': trace.write_bytes(data[:-4]) elif fault == 'corrupt': trace.write_bytes(data[:-8] + bytes([data[-8] ^ 1]) + data[-7:]) elif fault == 'oversized': trace.write_bytes(data + lzma.compress(b'x' * (1024 * 1024 + 1) + b'\n')) output = tmp_path / 'offline.json' result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'), '--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0', '--trace', str(trace), '--output', str(output), '--max-trace-bytes', '1048576', '--wall-seconds', '20'], capture_output=True, text=True) assert result.returncode == 1, result.stderr report = json.loads(output.read_text()) assert not report['functional_passed'] assert any(c.startswith(('recording_offline', 'recording_missing_terminal')) for c in report['all_causes']) assert report['observed_physics_ticks'] == 2 @pytest.mark.parametrize('value', [None, [], 'not a state']) def test_malformed_state_container_is_a_recorded_failure(value): def mutate(r): if r['kind'] == 'tick': r['state'] = value audit = run(end=1, mutate=mutate) assert 'recording_invalid:state_not_object' in audit.first_failure['causes'] assert audit.first_failure['global_tick'] == 1 def test_protocol_complete_is_not_a_passing_final_endpoint(): def mutate(r): if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 13500: r['state']['tilt'] = .7 report = run(mutate=mutate).report() assert report['protocol_complete'] and report['completed_clock_cycles'] == 3 assert report['passed_prefix_cycles'] == 2 and not report['functional_passed'] assert report['first_failure']['global_tick'] == 13500 def test_xz_short_write_retains_readable_complete_record_prefix(recording, tmp_path): recorder, stop, _ = recorder_prefix(recording, tmp_path) path = recorder.directory / TRACE_FILENAME prefix = path.read_bytes() stream = recorder.stream class ShortWrite: def __getattr__(self, key): return getattr(stream, key) def write(self, data): return stream.write(data[:len(data)//2]) recorder.stream = ShortWrite() acknowledgements = [] with pytest.raises(OSError, match='Short evidence write'): recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert not acknowledgements assert path.read_bytes().startswith(prefix) expected = lzma.decompress(prefix).splitlines() with lzma.open(path, 'rb') as reader: assert [reader.readline().rstrip(b'\n') for _ in expected] == expected with pytest.raises(EOFError): reader.read() stream.close() def test_frozen_phase_schedule_float32_completion_uses_actual_start(): p, trial = protocol('P2'), 12 seen = [] def inspect(g, time, schedule): if g == 3351: start = float(schedule.start[0, 2]) seen.append((start, float(time[0]), float(time[0] - schedule.start[0, 2]))) assert not schedule.completed[0, 2] audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=3399, inspect=inspect)) assert seen == [pytest.approx((6.1000003815, 6.7000002861, .5999999046), abs=1e-10)] assert audit.first_failure is None assert audit.report()['functional_passed'] assert all(audit.delivery[i]['status'] is True for i in (2, 3)) @pytest.mark.parametrize('fault', ['late_completion', 'area']) def test_phase_schedule_completion_slack_does_not_relax_delivery(fault): p, trial = protocol('P2'), 12 schedule = frozen_schedule_mutator(p, trial) def mutate(r): schedule(r) if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 3352: event = r['state']['events'][2] if fault == 'late_completion': event['completed'] = False else: event['integral'] -= .003 audit = run(p, trial, mutate=mutate) assert audit.first_failure['global_tick'] == 3352 assert 'event_delivery:2' in audit.first_failure['causes'] @pytest.mark.parametrize('trial,event,closure', [(12, 3, 3400), (8, 1, 2300)]) @pytest.mark.parametrize('edge_offset', [-1, 0, 1, None]) def test_edge_trigger_phase_closure_respects_last_pre_step(trial, event, closure, edge_offset): p = protocol('P2') edge_tick = None if edge_offset is None else closure + edge_offset audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=edge_tick)) if edge_offset == -1: assert audit.first_failure is None assert audit.delivery[event]['status'] is True assert audit.delivery[event]['absolute_start_s'] >= (closure - 1) * .002 else: # Includes boundary/reload-first landing (or swing-first liftoff), # which cannot be consumed by a scheduler still in the eligible phase. assert audit.first_failure['global_tick'] == closure assert audit.first_failure['causes'] == ['event_delivery:' + str(event)] assert audit.delivery[event]['missed'] def test_delayed_liftoff_is_not_a_missing_trigger_at_phase_closure(): p, trial = protocol('P2'), 8 audit = run(p, trial, end=2300, mutate=frozen_schedule_mutator(p, trial, edge_tick=2299)) assert audit.first_failure is None event = audit.delivery[1] assert event['fired'] and not event['completed'] and not event['missed'] assert event['absolute_start_s'] > 4.6 assert event['status'] == 'not_evaluated' def test_recorder_requires_existing_parent_without_creating_ancestors(recording, tmp_path): directory = tmp_path / 'missing' / 'nested' / 'trial' with pytest.raises(FileNotFoundError): recording.EvaluationRecorder(protocol(), 0, directory, budget_check=lambda *args: None, supplemental=lambda s: {}) assert not (tmp_path / 'missing').exists() @pytest.mark.parametrize('fail_at', [None, 3, 4]) def test_recorder_directory_and_parent_fsync_before_ack(recording, tmp_path, monkeypatch, fail_at): import os import stat recorder, stop, _ = recorder_prefix(recording, tmp_path) fsync, calls, acknowledgements = os.fsync, [], [] def sync(fd): calls.append((os.fstat(fd).st_ino, stat.S_ISDIR(os.fstat(fd).st_mode))) assert not acknowledgements if len(calls) == fail_at: raise OSError('directory fsync failed') fsync(fd) monkeypatch.setattr(recording.os, 'fsync', sync) if fail_at: with pytest.raises(OSError, match='directory fsync failed'): recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert not acknowledgements and not recorder.finalized and recorder.faults recorder.stream.close() else: recorder.finalize(stop=stop, acknowledge=acknowledgements.append) assert acknowledgements == [stop] and recorder.stream.closed assert calls[2] == (recorder.directory.stat().st_ino, True) if fail_at != 3: assert calls[3] == (recorder.directory.parent.stat().st_ino, True) @pytest.fixture def offline_reader(): spec = importlib.util.spec_from_file_location('_cpu_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py') module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def gzip_processing_tail(kind): if kind == 'empty_members': return gzip.compress(b'') * 10000 if kind == 'oversized_header': member = gzip.compress(b'') return member[:3] + b'\x08' + member[4:10] + b'n' * (1024 * 1024) + b'\0' + member[10:] if kind == 'empty_blocks': # Valid deflate stream that consumes compressed bytes without output. return gzip.compress(b'')[:10] + b'\x00\x00\x00\xff\xff' * 200000 + b'\x03\x00' + b'\0' * 8 raise AssertionError(kind) @pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks']) def test_offline_gzip_budget_during_no_output_processing(offline_reader, tmp_path, monkeypatch, tail): path = tmp_path / 'trace.v1.jsonl.gz' terminal = b'{"kind":"terminal"}\n' path.write_bytes(gzip.compress(terminal) + gzip_processing_tail(tail)) now = 0 def monotonic(): nonlocal now now += 1 return now monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic) lines = offline_reader.trace_lines(path, compressed=True, max_bytes=2**21, max_record_bytes=2**20, deadline=12) assert next(lines) == terminal with pytest.raises(ValueError, match='budget_audit'): list(lines) assert now == 13 # Rejection inside processing, not merely after decoded output. @pytest.mark.parametrize('compressed', [False, True]) def test_offline_budget_checked_at_eof(offline_reader, tmp_path, monkeypatch, compressed): path = tmp_path / 'trace' raw = b'{"kind":"terminal"}\n' path.write_bytes(gzip.compress(raw) if compressed else raw) monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 0) lines = offline_reader.trace_lines(path, compressed=compressed, max_bytes=4096, max_record_bytes=4096, deadline=1) assert next(lines) == raw monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 2) with pytest.raises(ValueError, match='budget_audit'): next(lines) def test_offline_gzip_expansion_record_limit(offline_reader, tmp_path): path = tmp_path / 'trace.v1.jsonl.gz' path.write_bytes(gzip.compress(b'x' * (2**20 + 1))) with pytest.raises(ValueError, match='budget_audit'): list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20, max_record_bytes=2**20, deadline=math.inf)) def test_offline_gzip_decoder_no_progress_rejected(offline_reader, tmp_path, monkeypatch): path = tmp_path / 'trace.v1.jsonl.gz' path.write_bytes(gzip.compress(b'')) class StalledDecoder: eof = False def decompress(self, data, limit): self.unconsumed_tail = data return b'' monkeypatch.setattr(offline_reader.zlib, 'decompressobj', lambda **kwargs: StalledDecoder()) with pytest.raises(ValueError, match='no progress'): list(offline_reader.trace_lines(path, compressed=True, max_bytes=4096, max_record_bytes=4096, deadline=math.inf)) @pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks']) def test_offline_cli_budget_failure_after_terminal_is_reported( recording, offline_reader, tmp_path, monkeypatch, tail): recorder, stop, _ = recorder_prefix(recording, tmp_path) recorder.finalize(stop=stop) trace = tmp_path / 'legacy.jsonl.gz' trace.write_bytes(gzip.compress(lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()))) with trace.open('ab') as output: output.write(gzip_processing_tail(tail)) protocol_module, audit_module = offline_reader.cpu_modules() audit = audit_module.TrialAudit(protocol_module.load_protocol( ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), 0) monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit) monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module)) now = 0 def monotonic(): nonlocal now if audit.terminal is not None: now += 1 return now monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic) output = tmp_path / 'report.json' monkeypatch.setattr(sys, 'argv', ['audit', '--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0', '--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**21), '--wall-seconds', '4']) assert offline_reader.main() == 1 report = json.loads(output.read_text()) assert report['terminal'] is not None and not report['functional_passed'] assert 'recording_offline:budget_audit_resource_or_record_limit' in report['all_causes'] assert now == 5 @pytest.mark.parametrize('codec', ['gzip', 'xz']) def test_offline_passing_terminal_cannot_hide_empty_member_budget_overrun(offline_reader, tmp_path, monkeypatch, codec): p = protocol('P2') trace = tmp_path / ('trace.v1.jsonl.gz' if codec == 'gzip' else TRACE_FILENAME) encode = (lambda raw: gzip.compress(raw, compresslevel=1)) if codec == 'gzip' else ( lambda raw: lzma.compress(raw, preset=0)) stop = plain(asdict(Stop(physics_ticks=4500, causes=(), reason='protocol_complete', clock=plain(asdict(p.evidence_clock(0, 4500))), completed_controls=900, partial_control_ticks=0, completed_clock_cycles=1, all_sim_steps_attempted=4502, all_sim_steps_completed=4502, rollout_steps=4500, composer_buffer_replacements=4500))) with trace.open('wb') as output: def write(record): output.write(encode((json.dumps(record) + '\n').encode())) for record in records(p): write(record) if record['kind'] == 'tick' and record['state']['substeps'] % 5 == 0: g = record['state']['substeps'] write({'kind': 'step_result', 'ticks_advanced': 5, 'observed_tick': g, 'reward': [0.], 'has_next_observation': g != 4500, 'stop': stop if g == 4500 else None}) write({'kind': 'terminal', 'runtime_stop': stop, 'external_failure': None, 'recording_faults': []}) output.write(encode(b'') * 10000) protocol_module, audit_module = offline_reader.cpu_modules() audit = audit_module.TrialAudit(p, 0) monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit) monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module)) now = 0 def monotonic(): nonlocal now if audit.terminal is not None: assert audit.report()['functional_passed'] # Passing until budget expiry is recorded. now += 1 return now monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic) output = tmp_path / 'report.json' monkeypatch.setattr(sys, 'argv', ['audit', '--protocol', str(ROOT / 'configs/contact_cycle/bounded_diagnostic_p2_v1.json'), '--trial-index', '0', '--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**26), '--wall-seconds', '4']) assert offline_reader.main() == 1 report = json.loads(output.read_text()) assert not report['functional_passed'] assert report['first_failure']['causes'] == ['recording_offline:budget_audit_resource_or_record_limit'] def test_offline_gzip_bounded_chunks_are_lossless(offline_reader, tmp_path): path = tmp_path / 'trace.v1.jsonl.gz' lines = [b'a' * (2**20 - 1) + b'\n', b'b\n', b'c' * (2**17) + b'\n'] # Deliberately span record/member boundaries, including empty members. raw = b''.join(lines) path.write_bytes(gzip.compress(raw[:300000]) + gzip.compress(b'') + gzip.compress(raw[300000:])) assert list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20, max_record_bytes=2**20, deadline=math.inf)) == lines