Files
SkillFactory/tests/test_contact_cycle_sim_evaluation_audit.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

1002 lines
46 KiB
Python

"""CPU synthetic evidence only: no simulator, actor, checkpoint or optimizer execution."""
import copy
import gzip
import importlib.util
import json
import lzma
import math
import sys
from dataclasses import asdict, dataclass, field
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import (
BOOLS,
SCALARS,
TRACE_FILENAME,
VECTORS,
TrialAudit,
matrix_report,
plain,
sub,
)
from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import load_protocol
ROOT = Path(__file__).resolve().parents[1]
def protocol(stage='P1'):
return load_protocol(ROOT / 'configs/contact_cycle' / (
'continuous_eval_p1_v1.json' if stage == 'P1' else 'bounded_diagnostic_p2_v1.json'))
def state(p, trial, clock, refreshes=1):
anchors = {'anchor_a_world': [0., 0., .022], 'root_origin_world': [0., 0., .3],
'initial_root_world': [0., 0., .3]}
c = p.command(trial, clock, anchor_a_world=anchors['anchor_a_world'],
root_origin_world=anchors['root_origin_world'], weight_N=40.)
g = clock.global_tick
s = {k: 0. for k in SCALARS}
s.update({k: False for k in BOOLS})
s.update({k: [0.] * n for k, n in VECTORS.items()})
s.update(weight=40., time=g * .002, substeps=g, anchor_height=.3, anchors=anchors,
body_names=['virtual_payload', 'FL_thigh', 'FL_calf'], foot=list(c.target_world_m),
normal=[0., 0., 40. if c.phase in (0, 5) else 16. if c.phase == 1 else 0.],
root=[0., 0., .3, 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
gap=c.target_world_m[2] - .022, q_ik=[refreshes * .001] * 3, refreshes=refreshes,
contact=c.phase in (0, 1, 5), acquired=g >= 5, armed=g >= 5,
separated=c.phase in (2, 3), onset_phase=2 if c.phase in (2, 3) else -1, edge_phase=-1,
ik_target=sub(c.target_world_m, [0., 0., .3]), scenario=p.trial(trial).scenario_id,
event_program=plain(p.document['events']), external=[[0.] * 6 for _ in range(3)],
guide=[[0.] * 6 for _ in range(3)], auxiliary=[[0.] * 6 for _ in range(3)],
sensor_delay=[[0.] * 18], action_delay=[[0.] * 3], rng={'test_rng': [1, 2, 3]},
actuator_state={}, applied=None,
counters={'reset_calls': 3, 'pose_write_calls': 1, 'composer_buffer_replacements': g,
'initialization_steps': 2, 'rollout_steps': g,
'attempted_steps': g + 2, 'completed_steps': g + 2},
reward_components={k: 0. for k in ('task', 'effort', 'action_rate', 'root_position',
'root_velocity', 'residual')})
s['events'] = [{'fired': False, 'completed': False, 'start': 0., 'envelope': 0., 'integral': 0.}
for _ in s['event_program']]
return s, plain(asdict(c))
def records(p, trial=0, end=None):
total = p.trial(trial).total_ticks if end is None else end
clock = p.next_command_clock(trial, 0)
s, command = state(p, trial, clock)
refclock = plain(asdict(clock))
event = dict(kind='initial', trial_index=trial, clock=refclock, command=command,
held_control_index=None, action_origin_tick=None, reference_used_clock=None,
reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[])
yield {'kind': 'initial', 'event': event, 'state': s}
refreshes = 1
for g in range(1, total + 1):
if (g - 1) % 5 == 0:
learned = p.trial(trial).controller_kind == 'learned_policy'
yield {'kind': 'control', 'origin_tick': g - 1, 'observation': [0.] * 46, 'action': [0.] * 3,
'mean': [0.] * 3 if learned else None, 'std': [.1] * 3 if learned else None}
clock = p.evidence_clock(trial, g)
s, command = state(p, trial, clock, refreshes)
event = dict(kind='tick', trial_index=trial, clock=plain(asdict(clock)), command=command,
held_control_index=(g-1)//5, action_origin_tick=5*((g-1)//5), reference_used_clock=refclock,
reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[])
record = {'kind': 'tick', 'event': event, 'state': s}
yield record
if g % 5 == 0 and g < p.trial(trial).total_ticks:
if g % 4500 == 0:
yield {**copy.deepcopy(record), 'kind': 'seam'}
clock = p.next_command_clock(trial, g, seam_admitted=True)
_, command = state(p, trial, clock)
refreshes += 1
refclock = plain(asdict(clock))
s = copy.deepcopy(s)
s.update(refreshes=refreshes, q_ik=[refreshes * .001] * 3,
ik_target=sub(command['target_world_m'], s['root'][:3]))
yield {'kind': 'next_reference', 'state': s, 'event': {
**event, 'kind': 'next_reference', 'clock': refclock, 'command': command,
'held_control_index': None, 'action_origin_tick': None, 'reference_next_clock': refclock}}
def run(p=None, trial=0, end=None, mutate=None):
p = protocol() if p is None else p
audit = TrialAudit(p, trial)
for record in records(p, trial, end):
if mutate:
mutate(record)
causes = audit.consume(record)
if causes:
break
return audit
def test_complete_three_cycles_and_clock_prefix():
audit = run()
report = audit.report()
assert report['first_failure'] is None
assert report['functional_passed'] and report['passed_prefix_cycles'] == 3
assert report['completed_clock_cycles'] == 3
assert all(c['metrics']['phase_counts'] == [1499, 500, 300, 700, 400, 1101] for c in report['cycles'])
assert all(s['carry_pre'] == s['carry_post'] and s['post'] is not None for s in report['seams'])
assert not report['accepted_skill'] and report['optimizer_steps'] == 0
def test_incomplete_tail_not_passed_and_later_gates_not_evaluated():
report = run(end=4512).report()
assert not report['functional_passed'] and report['passed_prefix_cycles'] == 1
assert report['completed_clock_cycles'] == 1
assert report['cycles'][1]['gates']['landing_position'] == 'not_evaluated'
assert report['cycles'][2]['status'] == 'not_evaluated'
@pytest.mark.parametrize('field,value', [('root', [1.] * 13), ('q_ik', [0.] * 3),
('sensor_delay', [[1.] * 18]), ('contact', False),
('rng', {'changed': [4]}), ('action_delay', [[1.] * 3])])
def test_seam_fingerprint_rejects_teleport_and_memory_reset(field, value):
def mutate(r):
if r['kind'] == 'seam':
r['state'][field] = value
audit = run(mutate=mutate)
assert audit.first_failure['global_tick'] == 4500
assert any('recording' in c for c in audit.first_failure['causes'])
@pytest.mark.parametrize('kind', ['duplicate', 'skip', 'time_wrap', 'reset_counter'])
def test_clock_and_reset_faults(kind):
p = protocol()
audit = TrialAudit(p, 0)
stream = iter(records(p, end=3))
for _ in range(3):
r = next(stream)
assert not audit.consume(r)
r = copy.deepcopy(r) if kind == 'duplicate' else next(stream)
if kind == 'skip':
r['event']['clock']['global_tick'] = 3
elif kind == 'time_wrap':
r['state']['time'] = 0
elif kind == 'reset_counter':
r['state']['counters']['reset_calls'] += 1
assert audit.consume(r)
assert audit.first_failure
@pytest.mark.parametrize('field,value', [('normal', [0., 0., float('nan')]), ('tilt', float('inf')),
('foot', [0., 0.]), ('q', None)])
def test_nonfinite_and_malformed_first_offending_tick(field, value):
def mutate(r):
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 2:
r['state'][field] = value
audit = run(end=3, mutate=mutate)
assert audit.first_failure['global_tick'] == 2
assert not audit.report()['functional_passed']
@pytest.mark.parametrize('value,passed', [(.03, True), (.03000001, False)])
def test_exact_root_threshold(value, passed):
def mutate(r):
if r['kind'] == 'tick':
r['state']['root'][0] = value
r['state']['ik_target'][0] -= value
audit = run(end=1, mutate=mutate)
assert (audit.first_failure is None) == passed
def test_overlapping_failures_preserved():
def mutate(r):
if r['kind'] == 'tick':
r['state'].update(tilt=.7, joint_excess=.02, limit_ratio=2.)
audit = run(end=1, mutate=mutate)
assert {'tilt', 'hardware_joint_limit', 'hardware_effort_velocity'} <= set(audit.first_failure['causes'])
@pytest.mark.parametrize('valid_ticks,passed', [(4, False), (5, True)])
def test_lift_dwell_exact_deadline(valid_ticks, passed):
def mutate(r):
if r['kind'] == 'tick' and r['event']['command']['phase'] == 2:
t = r['event']['clock']['local_tick']
r['state']['gap'] = .022 if t > 2260 - valid_ticks else .021
audit = run(end=2260, mutate=mutate)
assert (audit.first_failure is None) == passed
assert audit.margins['lift_dwell'] == passed
if not passed:
assert audit.first_failure['global_tick'] == 2260
def test_lift_onset_unload_not_laundered():
def mutate(r):
if r['kind'] == 'tick' and r['event']['command']['phase'] == 2:
r['state']['onset_phase'] = 1
audit = run(end=2260, mutate=mutate)
assert 'P1_lift_dwell' in audit.first_failure['causes']
@pytest.mark.parametrize('bad_tick,passed', [(4450, True), (4451, False)])
def test_final_reload_dwell_boundary(bad_tick, passed):
def mutate(r):
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == bad_tick:
r['state']['normal'][2] = 31.9
audit = run(end=4500, mutate=mutate)
assert (audit.first_failure is None) == passed
assert audit.cycles[0]['P1_margins']['reload_dwell'] == passed
def test_support_and_unload_stop_at_window_last_sample():
def mutate(r):
if r['kind'] == 'tick' and r['event']['command']['phase'] == 1:
r['state']['normal'][2] = 29.
audit = run(end=2100, mutate=mutate)
assert audit.first_failure['global_tick'] == 1999
assert audit.gates['unloaded'] is True
assert audit.margins['unload_ratio'] is False
assert audit.gates['real_flight'] == 'not_evaluated'
def test_P2_stricter_margins_are_diagnostic_not_suppressed_functional_failures():
def mutate(r):
if r['kind'] in ('initial', 'tick', 'next_reference'):
r['state']['root'][0] = .04
r['state']['ik_target'][0] = r['event']['command']['target_world_m'][0] - .04
report = run(protocol('P2'), mutate=mutate).report()
assert report['functional_passed']
assert report['cycles'][0]['P1_margins']['root_horizontal'] is False
assert report['cycles'][0]['P1_margins_role'] == 'diagnostic_only'
def applied_zero(r, p, trial):
"""Pre-step capture with zero envelope, used before the event should start."""
s = r['state']
events = []
for i in p.trial(trial).event_indices:
program = s['event_program'][i]
events.append(dict(event_index=i, body_index=s['body_names'].index(program['body']), envelope=0.,
force_world=[0.] * 3, couple_world=[0.] * 3, point_world=program['point_m'],
com_world=[0.] * 3, frame_quat_xyzw=[0., 0., 0., 1.],
point_frame_quat_xyzw=[0., 0., 0., 1.], point_frame_origin_world=[0.] * 3))
s['applied'] = dict(format='skillfactory.contact-cycle-applied-wrench.v1', trial_index=trial,
global_tick=r['event']['clock']['global_tick'], events=events,
total_com_wrench=[[0.] * 6 for _ in s['body_names']])
@pytest.mark.parametrize('fault', ['missing', 'stale', 'frame', 'moment'])
def test_required_prephysics_applied_evidence(fault):
p, trial = protocol('P2'), 4
def mutate(r):
if r['kind'] == 'tick':
applied_zero(r, p, trial)
a = r['state']['applied']
if fault == 'missing':
r['state']['applied'] = None
elif fault == 'stale':
a['global_tick'] = 0
elif fault == 'frame':
a['events'][0]['frame_quat_xyzw'] = [0., 0., 1., 0.]
else:
a['total_com_wrench'][0][3] = .01
audit = run(p, trial, end=1, mutate=mutate)
assert any('recording_applied' in c for c in audit.first_failure['causes'])
def test_undelivered_time_event_fails_at_first_delivery_deadline():
p, trial = protocol('P2'), 4
# Use a direct event-window probe so original functional windows are irrelevant.
audit = TrialAudit(p, trial)
initial = next(records(p, trial, end=0))
assert not audit.consume(initial)
audit.event_previous['state']['time'] = 7.5
r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick')
clock = p.evidence_clock(trial, 3751)
r['event']['clock'] = plain(asdict(clock))
applied_zero(r, p, trial)
audit.last = r
causes = audit._events(r['state'], clock)
assert 'event_delivery:0' in causes
assert audit.delivery[0]['expected'] and audit.delivery[0]['missed']
assert audit.delivery[0]['expected_area_s'] == 6.
def test_matrix_includes_all_not_run_rows_and_does_not_assume_pairing():
result = matrix_report(protocol('P2'), [])
assert len(result['rows']) == 20 and len(result['pairs']) == 10
assert all(r['status'] == 'not_run' for r in result['rows'])
assert all(p['status'] == 'not_evaluated' for p in result['pairs'])
@pytest.fixture
def recording():
path = ROOT / 'source/SkillFactory/SkillFactory/tasks/manager_based/contact_cycle_sim/evaluation_recording.py'
spec = importlib.util.spec_from_file_location('_cpu_evaluation_recording', path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@dataclass
class Stop:
trial_index: int = 0
physics_ticks: int = 2
causes: tuple = ('swing_clearance',)
reason: str = 'failure'
fatal: bool = False
clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2})
completed_controls: int = 0
partial_control_ticks: int = 2
completed_clock_cycles: int = 0
all_sim_steps_attempted: int = 4
all_sim_steps_completed: int = 4
initialization_steps: int = 2
rollout_steps: int = 2
reset_calls: int = 3
pose_write_calls: int = 1
composer_buffer_replacements: int = 2
def recorder_prefix(recording, tmp_path):
budgets = []
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
budget_check=lambda *args: budgets.append(args), supplemental=lambda s: {})
for r in records(protocol(), end=2):
assert not recorder._record(r)
stop = Stop()
assert not recorder.after_step(SimpleNamespace(ticks_advanced=2, observation=None, reward=None, stop=stop))
return recorder, stop, budgets
def test_partial_control_durable_persistence_then_exact_ack(recording, tmp_path):
recorder, stop, budgets = recorder_prefix(recording, tmp_path)
acknowledgements = []
def acknowledge(value):
assert (recorder.directory / 'metrics.json').is_file()
acknowledgements.append(value)
report = recorder.finalize(stop=stop, acknowledge=acknowledge)
assert acknowledgements == [stop]
assert report['first_failure']['global_tick'] == 2
decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
raw = [json.loads(x) for x in decoded.splitlines()]
assert len([r for r in raw if r['kind'] == 'tick']) == 2
assert raw[-2]['ticks_advanced'] == 2 and raw[-2]['reward'] is None
assert raw[-1]['kind'] == 'terminal' and not report['functional_passed']
assert any(b[0] == 'finalize' for b in budgets)
def test_io_error_never_acknowledges(recording, tmp_path, monkeypatch):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
acknowledgements = []
def fail(_):
raise OSError('disk fault')
monkeypatch.setattr(recording.os, 'fsync', fail)
with pytest.raises(OSError, match='disk fault'):
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert not acknowledgements and recorder.faults
assert (recorder.directory / TRACE_FILENAME).exists()
recorder.stream.close()
def test_external_failure_terminal_no_fake_tick_and_no_ack(recording, tmp_path):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
acknowledgements = []
report = recorder.finalize(stop=stop, external_failure={
'causes': ['identity_changed'], 'fatal': True, 'context': {'stage': 'after_step'}},
acknowledge=acknowledgements.append)
assert not acknowledgements
assert report['observed_physics_ticks'] == 2
assert 'identity_changed' in report['first_failure']['causes']
def test_nonfinite_raw_and_budget_fault_are_retained(recording, tmp_path):
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
budget_check=lambda *args: None, supplemental=lambda s: {})
for r in records(protocol(), end=1):
if r['kind'] == 'tick':
r['state']['tilt'] = math.nan
causes = recorder._record(r)
assert causes
recorder.finalize(external_failure={'causes': ['recording_nonfinite'], 'fatal': True, 'context': {}})
assert '"nonfinite":"nan"' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
def test_tensor_copy_is_independent(recording):
tensor = torch.ones(1, 3)
result = recording.copied({'x': tensor})
tensor.zero_()
assert result == {'x': [[1., 1., 1.]]}
def test_incomplete_endpoint_gates_return_immediate_stop():
p = protocol()
audit = TrialAudit(p, 0)
for record in records(p, end=4500):
if record['kind'] == 'tick' and record['event']['clock']['global_tick'] == 4500:
audit.gates['real_flight'] = 'not_evaluated'
causes = audit.consume(record)
assert 'recording_incomplete_cycle_gates' in causes
break
assert not audit.consume(record)
assert audit.first_failure['global_tick'] == 4500
assert not audit.report()['cycles'][0]['passed']
def test_no_event_P1_requires_zero_wrench_not_fabricated_event_capture():
assert run(end=1).first_failure is None
def mutate(r):
if r['kind'] == 'tick':
r['state']['external'][0][0] = .01
audit = run(end=1, mutate=mutate)
assert 'identity_unexpected_external_wrench' in audit.first_failure['causes']
@pytest.mark.parametrize('fault', ['trial', 'point_frame', 'free_couple', 'moment_arm'])
def test_nonzero_applied_evidence_checks_frame_and_COM_moment(fault):
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross, rotate
p, trial = protocol('P2'), 8
audit = TrialAudit(p, trial)
r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick')
applied_zero(r, p, trial)
s = r['state']
e = s['applied']['events'][0]
program = s['event_program'][1]
q = [0., 0., math.sqrt(.5), math.sqrt(.5)]
e.update(envelope=.5, frame_quat_xyzw=q, point_frame_quat_xyzw=q,
point_frame_origin_world=[.2, .3, .4], com_world=[.1, .2, .3])
s['events'][1]['envelope'] = .5
e['point_world'] = [a+b for a, b in zip(e['point_frame_origin_world'], rotate(q, program['point_m']))]
e['force_world'] = [v*.5 for v in rotate(q, program['force_N'])]
e['couple_world'] = [v*.5 for v in rotate(q, program['couple_Nm'])]
wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'],
cross(sub(e['point_world'], e['com_world']), e['force_world']))]
s['external'][e['body_index']] = wrench
s['applied']['total_com_wrench'][e['body_index']] = wrench.copy()
clock = p.evidence_clock(trial, 1)
assert audit._applied(s, clock)[1] == e
if fault == 'trial':
s['applied']['trial_index'] = 4
elif fault == 'point_frame':
e['point_frame_quat_xyzw'] = [0., 0., 0., 1.]
elif fault == 'free_couple':
e['couple_world'][0] += .01
else:
e['com_world'][0] += .01
with pytest.raises(ValueError, match='applied'):
audit._applied(s, clock)
def test_trigger_provenance_rejects_fabricated_early_fire():
p, trial = protocol('P2'), 4
def mutate(r):
if r['kind'] == 'tick':
applied_zero(r, p, trial)
r['state']['events'][0]['fired'] = True
audit = run(p, trial, end=1, mutate=mutate)
assert 'recording_event_trigger_or_envelope:0' in audit.first_failure['causes']
def frozen_schedule_mutator(p, trial, *, edge_tick=None, inspect=None):
from SkillFactory.legs.contact_cycle.mechanics import WrenchSchedule
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross
schedule = WrenchSchedule(plain(p.document['events']), 1, 'cpu')
def mutate(r):
if r['kind'] != 'tick':
return
s = r['state']
g = r['event']['clock']['global_tick']
clock = p.next_command_clock(trial, 0) if g == 1 else p.evidence_clock(trial, g-1)
command = p.command(trial, clock, anchor_a_world=s['anchors']['anchor_a_world'],
root_origin_world=s['anchors']['root_origin_world'], weight_N=40.)
# Match the actual frozen runtime: integer substeps multiplied by dt
# in float32, NOT a Python double product subsequently cast to float32.
time = torch.tensor([g-1]) * .002
s['time'] = float((torch.tensor([g]) * .002)[0])
s['liftoff'] = g == edge_tick and s['scenario'] == 2
s['touchdown'] = g == edge_tick and s['scenario'] == 3
s['edge_phase'] = 2 if s['liftoff'] else -1
schedule.step(time, torch.tensor([command.phase]),
torch.tensor([g-1 == edge_tick and s['scenario'] == 2]),
torch.tensor([g-1 == edge_tick and s['scenario'] == 3]),
torch.tensor([s['scenario']]), .002)
if inspect:
inspect(g, time, schedule)
for i, event in enumerate(s['events']):
event.update(fired=bool(schedule.fired[0, i]), completed=bool(schedule.completed[0, i]),
start=float(schedule.start[0, i]), envelope=float(schedule.envelope[0, i]),
integral=float(schedule.integral[0, i]))
applied_zero(r, p, trial)
for e in s['applied']['events']:
i, body = e['event_index'], e['body_index']
e['envelope'] = s['events'][i]['envelope']
program = s['event_program'][i]
e['force_world'] = [v*e['envelope'] for v in program['force_N']]
e['couple_world'] = [v*e['envelope'] for v in program['couple_Nm']]
wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'],
cross(sub(e['point_world'], e['com_world']), e['force_world']))]
s['external'][body] = wrench
s['applied']['total_com_wrench'][body] = wrench.copy()
return mutate
def test_frozen_time_schedule_complete_delivery_and_impulses():
p, trial = protocol('P2'), 4
report = run(p, trial, mutate=frozen_schedule_mutator(p, trial)).report()
assert report['first_failure'] is None
assert report['functional_passed'] and report['protocol_complete']
event = report['delivery'][0]
assert event['status'] is True and event['actual_applied_evidence_available']
assert event['force_impulse_Ns'] == pytest.approx([18., -24., -12.], abs=1e-4)
assert event['couple_impulse_Nms'] == pytest.approx([1.2, .6, -.6], abs=1e-5)
@pytest.mark.parametrize('fault', ['partial_reward', 'next_observation', 'duplicate_step', 'terminal_counters'])
def test_partial_and_terminal_records_fail_closed(recording, tmp_path, fault):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
if fault == 'terminal_counters':
stop.all_sim_steps_completed += 1
else:
bad = copy.deepcopy(recorder.audit.step_results[-1])
if fault == 'partial_reward':
bad['reward'] = [1.]
elif fault == 'next_observation':
bad['has_next_observation'] = True
assert recorder.audit.consume(bad)
acknowledgements = []
report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert not acknowledgements
assert any(c.startswith('recording') for c in report['all_causes'])
def test_recording_fault_after_earlier_failure_is_not_hidden(recording, tmp_path):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
recorder.audit.fail(('recording_late_fault',), kind='terminal', tick=3)
acknowledgements = []
report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert report['first_failure']['global_tick'] == 2
assert 'recording_late_fault' in report['all_causes']
assert not acknowledgements
def test_capture_fault_reports_actual_tick_without_fabricating_sample(recording, tmp_path, monkeypatch):
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
budget_check=lambda *args: None, supplemental=lambda s: {})
for record in records(protocol(), end=1):
assert not recorder._record(record)
@dataclass
class Event:
kind: str = 'tick'
clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2})
causes: tuple = ('hardware_joint_limit',)
def broken(*args):
raise ValueError('accessor failed')
monkeypatch.setattr(recording, 'snapshot', broken)
assert recorder(Event(), None)
report = recorder.finalize(external_failure={'causes': ['recording_capture'], 'fatal': True, 'context': {}})
assert report['first_failure']['global_tick'] == 2
assert report['observed_physics_ticks'] == 1
assert 'hardware_joint_limit' in report['first_failure']['causes']
assert 'capture_fault' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
def test_offline_reaudit_matches_live_partial_report(recording, tmp_path):
import subprocess
recorder, stop, _ = recorder_prefix(recording, tmp_path)
live = recorder.finalize(stop=stop)
output = tmp_path / 'offline.json'
result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'),
'--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
'--trace', str(recorder.directory / TRACE_FILENAME), '--output', str(output),
'--max-trace-bytes', '1048576', '--wall-seconds', '20'], capture_output=True, text=True)
assert result.returncode == 1, result.stderr
offline = json.loads(output.read_text())
for key in ('first_failure', 'all_causes', 'cycles', 'functional_passed', 'protocol_complete', 'terminal'):
assert offline[key] == live[key]
assert offline['new_simulation_steps'] == offline['new_optimizer_steps'] == 0
def test_matrix_mismatched_pair_never_publishes_controller_delta():
p = protocol('P2')
rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': {'anchor': i},
'functional_passed': True} for i in (0, 1)]
pair = matrix_report(p, rows)['pairs'][0]
assert pair['status'] == 'invalid_initial_pair'
assert pair['initial_exactly_equal'] is False
assert pair['both_functional_passed'] is False
assert pair['learned_minus_reference'] == 'not_evaluated'
def test_matrix_absent_pair_snapshots_are_not_equal_evidence():
p = protocol('P2')
rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': None} for i in (0, 1)]
report = matrix_report(p, rows)
assert report['pairs'][0]['status'] == 'not_evaluated'
def test_xz_lossless_members_and_exact_inclusive_byte_accounting(recording, tmp_path):
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
budget_check=lambda *args: None, supplemental=lambda s: {})
original = list(records(protocol(), end=2))
for record in original:
assert not recorder._record(record)
assert (recorder.directory / TRACE_FILENAME).stat().st_size == recorder.written_bytes
# The initial is a checkpoint; control/ticks remain bounded and uncommitted.
assert len(recorder.pending) == len(original) - 1
assert [json.loads(x) for x in lzma.decompress(
(recorder.directory / TRACE_FILENAME).read_bytes()).splitlines()] == original[:1]
recorder._flush()
assert not recorder.pending and recorder.pending_raw_bytes == 0
decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).splitlines()
assert [json.loads(line) for line in decoded] == original
recorder.stream.close()
@pytest.mark.parametrize('stage', ['record', 'terminal_member', 'metrics', 'close', 'compression'])
def test_xz_storage_failures_preserve_complete_prefix_never_ack(recording, tmp_path, monkeypatch, stage):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
path = recorder.directory / TRACE_FILENAME
prefix = path.read_bytes()
before = [json.loads(line) for line in lzma.decompress(prefix).splitlines()]
finalize_calls = 0
def budget(where, pending, written):
nonlocal finalize_calls
assert written == path.stat().st_size
if where == 'finalize':
finalize_calls += 1
fail = ((stage == 'record' and where == 'record')
or (stage == 'terminal_member' and finalize_calls == 2)
or (stage == 'metrics' and finalize_calls == 3))
if fail:
# Deny by one byte: the pending count must INCLUDE XZ header/check/footer.
assert pending > 18
raise OSError('aggregate byte budget includes complete compressed member')
recorder.budget_check = budget
if stage == 'compression':
def broken_compression(*args, **kwargs):
raise OSError('compression fault')
monkeypatch.setattr(recording.lzma, 'compress', broken_compression)
if stage == 'close':
stream = recorder.stream
class CloseFault:
def __getattr__(self, key):
return getattr(stream, key)
def close(self):
raise OSError('close fault')
recorder.stream = CloseFault()
acknowledgements = []
if stage == 'record':
assert recorder.after_step(SimpleNamespace(ticks_advanced=0, observation=None, reward=None, stop=stop))
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
else:
with pytest.raises(OSError):
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert not acknowledgements and recorder.faults
decoded = [json.loads(line) for line in lzma.decompress(path.read_bytes()).splitlines()]
assert decoded[:len(before)] == before
if stage == 'close':
stream.close()
else:
recorder.stream.close()
@pytest.mark.parametrize('fault', ['truncated', 'corrupt', 'oversized', 'missing_terminal'])
def test_offline_xz_rejects_damaged_or_incomplete_tail(recording, tmp_path, fault):
import subprocess
recorder, stop, _ = recorder_prefix(recording, tmp_path)
trace = recorder.directory / TRACE_FILENAME
if fault == 'missing_terminal':
recorder.stream.close()
else:
recorder.finalize(stop=stop)
data = trace.read_bytes()
if fault == 'truncated':
trace.write_bytes(data[:-4])
elif fault == 'corrupt':
trace.write_bytes(data[:-8] + bytes([data[-8] ^ 1]) + data[-7:])
elif fault == 'oversized':
trace.write_bytes(data + lzma.compress(b'x' * (1024 * 1024 + 1) + b'\n'))
output = tmp_path / 'offline.json'
result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'),
'--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', '1048576',
'--wall-seconds', '20'], capture_output=True, text=True)
assert result.returncode == 1, result.stderr
report = json.loads(output.read_text())
assert not report['functional_passed']
assert any(c.startswith(('recording_offline', 'recording_missing_terminal')) for c in report['all_causes'])
assert report['observed_physics_ticks'] == 2
@pytest.mark.parametrize('value', [None, [], 'not a state'])
def test_malformed_state_container_is_a_recorded_failure(value):
def mutate(r):
if r['kind'] == 'tick':
r['state'] = value
audit = run(end=1, mutate=mutate)
assert 'recording_invalid:state_not_object' in audit.first_failure['causes']
assert audit.first_failure['global_tick'] == 1
def test_protocol_complete_is_not_a_passing_final_endpoint():
def mutate(r):
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 13500:
r['state']['tilt'] = .7
report = run(mutate=mutate).report()
assert report['protocol_complete'] and report['completed_clock_cycles'] == 3
assert report['passed_prefix_cycles'] == 2 and not report['functional_passed']
assert report['first_failure']['global_tick'] == 13500
def test_xz_short_write_retains_readable_complete_record_prefix(recording, tmp_path):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
path = recorder.directory / TRACE_FILENAME
prefix = path.read_bytes()
stream = recorder.stream
class ShortWrite:
def __getattr__(self, key):
return getattr(stream, key)
def write(self, data):
return stream.write(data[:len(data)//2])
recorder.stream = ShortWrite()
acknowledgements = []
with pytest.raises(OSError, match='Short evidence write'):
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert not acknowledgements
assert path.read_bytes().startswith(prefix)
expected = lzma.decompress(prefix).splitlines()
with lzma.open(path, 'rb') as reader:
assert [reader.readline().rstrip(b'\n') for _ in expected] == expected
with pytest.raises(EOFError):
reader.read()
stream.close()
def test_frozen_phase_schedule_float32_completion_uses_actual_start():
p, trial = protocol('P2'), 12
seen = []
def inspect(g, time, schedule):
if g == 3351:
start = float(schedule.start[0, 2])
seen.append((start, float(time[0]), float(time[0] - schedule.start[0, 2])))
assert not schedule.completed[0, 2]
audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=3399, inspect=inspect))
assert seen == [pytest.approx((6.1000003815, 6.7000002861, .5999999046), abs=1e-10)]
assert audit.first_failure is None
assert audit.report()['functional_passed']
assert all(audit.delivery[i]['status'] is True for i in (2, 3))
@pytest.mark.parametrize('fault', ['late_completion', 'area'])
def test_phase_schedule_completion_slack_does_not_relax_delivery(fault):
p, trial = protocol('P2'), 12
schedule = frozen_schedule_mutator(p, trial)
def mutate(r):
schedule(r)
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 3352:
event = r['state']['events'][2]
if fault == 'late_completion':
event['completed'] = False
else:
event['integral'] -= .003
audit = run(p, trial, mutate=mutate)
assert audit.first_failure['global_tick'] == 3352
assert 'event_delivery:2' in audit.first_failure['causes']
@pytest.mark.parametrize('trial,event,closure', [(12, 3, 3400), (8, 1, 2300)])
@pytest.mark.parametrize('edge_offset', [-1, 0, 1, None])
def test_edge_trigger_phase_closure_respects_last_pre_step(trial, event, closure, edge_offset):
p = protocol('P2')
edge_tick = None if edge_offset is None else closure + edge_offset
audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=edge_tick))
if edge_offset == -1:
assert audit.first_failure is None
assert audit.delivery[event]['status'] is True
assert audit.delivery[event]['absolute_start_s'] >= (closure - 1) * .002
else:
# Includes boundary/reload-first landing (or swing-first liftoff),
# which cannot be consumed by a scheduler still in the eligible phase.
assert audit.first_failure['global_tick'] == closure
assert audit.first_failure['causes'] == ['event_delivery:' + str(event)]
assert audit.delivery[event]['missed']
def test_delayed_liftoff_is_not_a_missing_trigger_at_phase_closure():
p, trial = protocol('P2'), 8
audit = run(p, trial, end=2300, mutate=frozen_schedule_mutator(p, trial, edge_tick=2299))
assert audit.first_failure is None
event = audit.delivery[1]
assert event['fired'] and not event['completed'] and not event['missed']
assert event['absolute_start_s'] > 4.6
assert event['status'] == 'not_evaluated'
def test_recorder_requires_existing_parent_without_creating_ancestors(recording, tmp_path):
directory = tmp_path / 'missing' / 'nested' / 'trial'
with pytest.raises(FileNotFoundError):
recording.EvaluationRecorder(protocol(), 0, directory,
budget_check=lambda *args: None, supplemental=lambda s: {})
assert not (tmp_path / 'missing').exists()
@pytest.mark.parametrize('fail_at', [None, 3, 4])
def test_recorder_directory_and_parent_fsync_before_ack(recording, tmp_path, monkeypatch, fail_at):
import os
import stat
recorder, stop, _ = recorder_prefix(recording, tmp_path)
fsync, calls, acknowledgements = os.fsync, [], []
def sync(fd):
calls.append((os.fstat(fd).st_ino, stat.S_ISDIR(os.fstat(fd).st_mode)))
assert not acknowledgements
if len(calls) == fail_at:
raise OSError('directory fsync failed')
fsync(fd)
monkeypatch.setattr(recording.os, 'fsync', sync)
if fail_at:
with pytest.raises(OSError, match='directory fsync failed'):
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert not acknowledgements and not recorder.finalized and recorder.faults
recorder.stream.close()
else:
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
assert acknowledgements == [stop] and recorder.stream.closed
assert calls[2] == (recorder.directory.stat().st_ino, True)
if fail_at != 3:
assert calls[3] == (recorder.directory.parent.stat().st_ino, True)
@pytest.fixture
def offline_reader():
spec = importlib.util.spec_from_file_location('_cpu_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def gzip_processing_tail(kind):
if kind == 'empty_members':
return gzip.compress(b'') * 10000
if kind == 'oversized_header':
member = gzip.compress(b'')
return member[:3] + b'\x08' + member[4:10] + b'n' * (1024 * 1024) + b'\0' + member[10:]
if kind == 'empty_blocks':
# Valid deflate stream that consumes compressed bytes without output.
return gzip.compress(b'')[:10] + b'\x00\x00\x00\xff\xff' * 200000 + b'\x03\x00' + b'\0' * 8
raise AssertionError(kind)
@pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks'])
def test_offline_gzip_budget_during_no_output_processing(offline_reader, tmp_path, monkeypatch, tail):
path = tmp_path / 'trace.v1.jsonl.gz'
terminal = b'{"kind":"terminal"}\n'
path.write_bytes(gzip.compress(terminal) + gzip_processing_tail(tail))
now = 0
def monotonic():
nonlocal now
now += 1
return now
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
lines = offline_reader.trace_lines(path, compressed=True, max_bytes=2**21,
max_record_bytes=2**20, deadline=12)
assert next(lines) == terminal
with pytest.raises(ValueError, match='budget_audit'):
list(lines)
assert now == 13 # Rejection inside processing, not merely after decoded output.
@pytest.mark.parametrize('compressed', [False, True])
def test_offline_budget_checked_at_eof(offline_reader, tmp_path, monkeypatch, compressed):
path = tmp_path / 'trace'
raw = b'{"kind":"terminal"}\n'
path.write_bytes(gzip.compress(raw) if compressed else raw)
monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 0)
lines = offline_reader.trace_lines(path, compressed=compressed, max_bytes=4096,
max_record_bytes=4096, deadline=1)
assert next(lines) == raw
monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 2)
with pytest.raises(ValueError, match='budget_audit'):
next(lines)
def test_offline_gzip_expansion_record_limit(offline_reader, tmp_path):
path = tmp_path / 'trace.v1.jsonl.gz'
path.write_bytes(gzip.compress(b'x' * (2**20 + 1)))
with pytest.raises(ValueError, match='budget_audit'):
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
max_record_bytes=2**20, deadline=math.inf))
def test_offline_gzip_decoder_no_progress_rejected(offline_reader, tmp_path, monkeypatch):
path = tmp_path / 'trace.v1.jsonl.gz'
path.write_bytes(gzip.compress(b''))
class StalledDecoder:
eof = False
def decompress(self, data, limit):
self.unconsumed_tail = data
return b''
monkeypatch.setattr(offline_reader.zlib, 'decompressobj', lambda **kwargs: StalledDecoder())
with pytest.raises(ValueError, match='no progress'):
list(offline_reader.trace_lines(path, compressed=True, max_bytes=4096,
max_record_bytes=4096, deadline=math.inf))
@pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks'])
def test_offline_cli_budget_failure_after_terminal_is_reported(
recording, offline_reader, tmp_path, monkeypatch, tail):
recorder, stop, _ = recorder_prefix(recording, tmp_path)
recorder.finalize(stop=stop)
trace = tmp_path / 'legacy.jsonl.gz'
trace.write_bytes(gzip.compress(lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes())))
with trace.open('ab') as output:
output.write(gzip_processing_tail(tail))
protocol_module, audit_module = offline_reader.cpu_modules()
audit = audit_module.TrialAudit(protocol_module.load_protocol(
ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), 0)
monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit)
monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module))
now = 0
def monotonic():
nonlocal now
if audit.terminal is not None:
now += 1
return now
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
output = tmp_path / 'report.json'
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**21), '--wall-seconds', '4'])
assert offline_reader.main() == 1
report = json.loads(output.read_text())
assert report['terminal'] is not None and not report['functional_passed']
assert 'recording_offline:budget_audit_resource_or_record_limit' in report['all_causes']
assert now == 5
@pytest.mark.parametrize('codec', ['gzip', 'xz'])
def test_offline_passing_terminal_cannot_hide_empty_member_budget_overrun(offline_reader, tmp_path, monkeypatch, codec):
p = protocol('P2')
trace = tmp_path / ('trace.v1.jsonl.gz' if codec == 'gzip' else TRACE_FILENAME)
encode = (lambda raw: gzip.compress(raw, compresslevel=1)) if codec == 'gzip' else (
lambda raw: lzma.compress(raw, preset=0))
stop = plain(asdict(Stop(physics_ticks=4500, causes=(), reason='protocol_complete',
clock=plain(asdict(p.evidence_clock(0, 4500))), completed_controls=900,
partial_control_ticks=0, completed_clock_cycles=1, all_sim_steps_attempted=4502,
all_sim_steps_completed=4502, rollout_steps=4500, composer_buffer_replacements=4500)))
with trace.open('wb') as output:
def write(record):
output.write(encode((json.dumps(record) + '\n').encode()))
for record in records(p):
write(record)
if record['kind'] == 'tick' and record['state']['substeps'] % 5 == 0:
g = record['state']['substeps']
write({'kind': 'step_result', 'ticks_advanced': 5, 'observed_tick': g,
'reward': [0.], 'has_next_observation': g != 4500, 'stop': stop if g == 4500 else None})
write({'kind': 'terminal', 'runtime_stop': stop, 'external_failure': None, 'recording_faults': []})
output.write(encode(b'') * 10000)
protocol_module, audit_module = offline_reader.cpu_modules()
audit = audit_module.TrialAudit(p, 0)
monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit)
monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module))
now = 0
def monotonic():
nonlocal now
if audit.terminal is not None:
assert audit.report()['functional_passed'] # Passing until budget expiry is recorded.
now += 1
return now
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
output = tmp_path / 'report.json'
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
str(ROOT / 'configs/contact_cycle/bounded_diagnostic_p2_v1.json'), '--trial-index', '0',
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**26), '--wall-seconds', '4'])
assert offline_reader.main() == 1
report = json.loads(output.read_text())
assert not report['functional_passed']
assert report['first_failure']['causes'] == ['recording_offline:budget_audit_resource_or_record_limit']
def test_offline_gzip_bounded_chunks_are_lossless(offline_reader, tmp_path):
path = tmp_path / 'trace.v1.jsonl.gz'
lines = [b'a' * (2**20 - 1) + b'\n', b'b\n', b'c' * (2**17) + b'\n']
# Deliberately span record/member boundaries, including empty members.
raw = b''.join(lines)
path.write_bytes(gzip.compress(raw[:300000]) + gzip.compress(b'') + gzip.compress(raw[300000:]))
assert list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
max_record_bytes=2**20, deadline=math.inf)) == lines