9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
1002 lines
46 KiB
Python
1002 lines
46 KiB
Python
"""CPU synthetic evidence only: no simulator, actor, checkpoint or optimizer execution."""
|
|
import copy
|
|
import gzip
|
|
import importlib.util
|
|
import json
|
|
import lzma
|
|
import math
|
|
import sys
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import (
|
|
BOOLS,
|
|
SCALARS,
|
|
TRACE_FILENAME,
|
|
VECTORS,
|
|
TrialAudit,
|
|
matrix_report,
|
|
plain,
|
|
sub,
|
|
)
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import load_protocol
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def protocol(stage='P1'):
|
|
return load_protocol(ROOT / 'configs/contact_cycle' / (
|
|
'continuous_eval_p1_v1.json' if stage == 'P1' else 'bounded_diagnostic_p2_v1.json'))
|
|
|
|
|
|
def state(p, trial, clock, refreshes=1):
|
|
anchors = {'anchor_a_world': [0., 0., .022], 'root_origin_world': [0., 0., .3],
|
|
'initial_root_world': [0., 0., .3]}
|
|
c = p.command(trial, clock, anchor_a_world=anchors['anchor_a_world'],
|
|
root_origin_world=anchors['root_origin_world'], weight_N=40.)
|
|
g = clock.global_tick
|
|
s = {k: 0. for k in SCALARS}
|
|
s.update({k: False for k in BOOLS})
|
|
s.update({k: [0.] * n for k, n in VECTORS.items()})
|
|
s.update(weight=40., time=g * .002, substeps=g, anchor_height=.3, anchors=anchors,
|
|
body_names=['virtual_payload', 'FL_thigh', 'FL_calf'], foot=list(c.target_world_m),
|
|
normal=[0., 0., 40. if c.phase in (0, 5) else 16. if c.phase == 1 else 0.],
|
|
root=[0., 0., .3, 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
|
|
gap=c.target_world_m[2] - .022, q_ik=[refreshes * .001] * 3, refreshes=refreshes,
|
|
contact=c.phase in (0, 1, 5), acquired=g >= 5, armed=g >= 5,
|
|
separated=c.phase in (2, 3), onset_phase=2 if c.phase in (2, 3) else -1, edge_phase=-1,
|
|
ik_target=sub(c.target_world_m, [0., 0., .3]), scenario=p.trial(trial).scenario_id,
|
|
event_program=plain(p.document['events']), external=[[0.] * 6 for _ in range(3)],
|
|
guide=[[0.] * 6 for _ in range(3)], auxiliary=[[0.] * 6 for _ in range(3)],
|
|
sensor_delay=[[0.] * 18], action_delay=[[0.] * 3], rng={'test_rng': [1, 2, 3]},
|
|
actuator_state={}, applied=None,
|
|
counters={'reset_calls': 3, 'pose_write_calls': 1, 'composer_buffer_replacements': g,
|
|
'initialization_steps': 2, 'rollout_steps': g,
|
|
'attempted_steps': g + 2, 'completed_steps': g + 2},
|
|
reward_components={k: 0. for k in ('task', 'effort', 'action_rate', 'root_position',
|
|
'root_velocity', 'residual')})
|
|
s['events'] = [{'fired': False, 'completed': False, 'start': 0., 'envelope': 0., 'integral': 0.}
|
|
for _ in s['event_program']]
|
|
return s, plain(asdict(c))
|
|
|
|
|
|
def records(p, trial=0, end=None):
|
|
total = p.trial(trial).total_ticks if end is None else end
|
|
clock = p.next_command_clock(trial, 0)
|
|
s, command = state(p, trial, clock)
|
|
refclock = plain(asdict(clock))
|
|
event = dict(kind='initial', trial_index=trial, clock=refclock, command=command,
|
|
held_control_index=None, action_origin_tick=None, reference_used_clock=None,
|
|
reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[])
|
|
yield {'kind': 'initial', 'event': event, 'state': s}
|
|
refreshes = 1
|
|
for g in range(1, total + 1):
|
|
if (g - 1) % 5 == 0:
|
|
learned = p.trial(trial).controller_kind == 'learned_policy'
|
|
yield {'kind': 'control', 'origin_tick': g - 1, 'observation': [0.] * 46, 'action': [0.] * 3,
|
|
'mean': [0.] * 3 if learned else None, 'std': [.1] * 3 if learned else None}
|
|
clock = p.evidence_clock(trial, g)
|
|
s, command = state(p, trial, clock, refreshes)
|
|
event = dict(kind='tick', trial_index=trial, clock=plain(asdict(clock)), command=command,
|
|
held_control_index=(g-1)//5, action_origin_tick=5*((g-1)//5), reference_used_clock=refclock,
|
|
reference_used=[0.] * 3, reference_next_clock=refclock, reference_next=[0.] * 3, causes=[])
|
|
record = {'kind': 'tick', 'event': event, 'state': s}
|
|
yield record
|
|
if g % 5 == 0 and g < p.trial(trial).total_ticks:
|
|
if g % 4500 == 0:
|
|
yield {**copy.deepcopy(record), 'kind': 'seam'}
|
|
clock = p.next_command_clock(trial, g, seam_admitted=True)
|
|
_, command = state(p, trial, clock)
|
|
refreshes += 1
|
|
refclock = plain(asdict(clock))
|
|
s = copy.deepcopy(s)
|
|
s.update(refreshes=refreshes, q_ik=[refreshes * .001] * 3,
|
|
ik_target=sub(command['target_world_m'], s['root'][:3]))
|
|
yield {'kind': 'next_reference', 'state': s, 'event': {
|
|
**event, 'kind': 'next_reference', 'clock': refclock, 'command': command,
|
|
'held_control_index': None, 'action_origin_tick': None, 'reference_next_clock': refclock}}
|
|
|
|
|
|
def run(p=None, trial=0, end=None, mutate=None):
|
|
p = protocol() if p is None else p
|
|
audit = TrialAudit(p, trial)
|
|
for record in records(p, trial, end):
|
|
if mutate:
|
|
mutate(record)
|
|
causes = audit.consume(record)
|
|
if causes:
|
|
break
|
|
return audit
|
|
|
|
|
|
def test_complete_three_cycles_and_clock_prefix():
|
|
audit = run()
|
|
report = audit.report()
|
|
assert report['first_failure'] is None
|
|
assert report['functional_passed'] and report['passed_prefix_cycles'] == 3
|
|
assert report['completed_clock_cycles'] == 3
|
|
assert all(c['metrics']['phase_counts'] == [1499, 500, 300, 700, 400, 1101] for c in report['cycles'])
|
|
assert all(s['carry_pre'] == s['carry_post'] and s['post'] is not None for s in report['seams'])
|
|
assert not report['accepted_skill'] and report['optimizer_steps'] == 0
|
|
|
|
|
|
def test_incomplete_tail_not_passed_and_later_gates_not_evaluated():
|
|
report = run(end=4512).report()
|
|
assert not report['functional_passed'] and report['passed_prefix_cycles'] == 1
|
|
assert report['completed_clock_cycles'] == 1
|
|
assert report['cycles'][1]['gates']['landing_position'] == 'not_evaluated'
|
|
assert report['cycles'][2]['status'] == 'not_evaluated'
|
|
|
|
|
|
@pytest.mark.parametrize('field,value', [('root', [1.] * 13), ('q_ik', [0.] * 3),
|
|
('sensor_delay', [[1.] * 18]), ('contact', False),
|
|
('rng', {'changed': [4]}), ('action_delay', [[1.] * 3])])
|
|
def test_seam_fingerprint_rejects_teleport_and_memory_reset(field, value):
|
|
def mutate(r):
|
|
if r['kind'] == 'seam':
|
|
r['state'][field] = value
|
|
audit = run(mutate=mutate)
|
|
assert audit.first_failure['global_tick'] == 4500
|
|
assert any('recording' in c for c in audit.first_failure['causes'])
|
|
|
|
|
|
@pytest.mark.parametrize('kind', ['duplicate', 'skip', 'time_wrap', 'reset_counter'])
|
|
def test_clock_and_reset_faults(kind):
|
|
p = protocol()
|
|
audit = TrialAudit(p, 0)
|
|
stream = iter(records(p, end=3))
|
|
for _ in range(3):
|
|
r = next(stream)
|
|
assert not audit.consume(r)
|
|
r = copy.deepcopy(r) if kind == 'duplicate' else next(stream)
|
|
if kind == 'skip':
|
|
r['event']['clock']['global_tick'] = 3
|
|
elif kind == 'time_wrap':
|
|
r['state']['time'] = 0
|
|
elif kind == 'reset_counter':
|
|
r['state']['counters']['reset_calls'] += 1
|
|
assert audit.consume(r)
|
|
assert audit.first_failure
|
|
|
|
|
|
@pytest.mark.parametrize('field,value', [('normal', [0., 0., float('nan')]), ('tilt', float('inf')),
|
|
('foot', [0., 0.]), ('q', None)])
|
|
def test_nonfinite_and_malformed_first_offending_tick(field, value):
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 2:
|
|
r['state'][field] = value
|
|
audit = run(end=3, mutate=mutate)
|
|
assert audit.first_failure['global_tick'] == 2
|
|
assert not audit.report()['functional_passed']
|
|
|
|
|
|
@pytest.mark.parametrize('value,passed', [(.03, True), (.03000001, False)])
|
|
def test_exact_root_threshold(value, passed):
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
r['state']['root'][0] = value
|
|
r['state']['ik_target'][0] -= value
|
|
audit = run(end=1, mutate=mutate)
|
|
assert (audit.first_failure is None) == passed
|
|
|
|
|
|
def test_overlapping_failures_preserved():
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
r['state'].update(tilt=.7, joint_excess=.02, limit_ratio=2.)
|
|
audit = run(end=1, mutate=mutate)
|
|
assert {'tilt', 'hardware_joint_limit', 'hardware_effort_velocity'} <= set(audit.first_failure['causes'])
|
|
|
|
|
|
@pytest.mark.parametrize('valid_ticks,passed', [(4, False), (5, True)])
|
|
def test_lift_dwell_exact_deadline(valid_ticks, passed):
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['command']['phase'] == 2:
|
|
t = r['event']['clock']['local_tick']
|
|
r['state']['gap'] = .022 if t > 2260 - valid_ticks else .021
|
|
audit = run(end=2260, mutate=mutate)
|
|
assert (audit.first_failure is None) == passed
|
|
assert audit.margins['lift_dwell'] == passed
|
|
if not passed:
|
|
assert audit.first_failure['global_tick'] == 2260
|
|
|
|
|
|
def test_lift_onset_unload_not_laundered():
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['command']['phase'] == 2:
|
|
r['state']['onset_phase'] = 1
|
|
audit = run(end=2260, mutate=mutate)
|
|
assert 'P1_lift_dwell' in audit.first_failure['causes']
|
|
|
|
|
|
@pytest.mark.parametrize('bad_tick,passed', [(4450, True), (4451, False)])
|
|
def test_final_reload_dwell_boundary(bad_tick, passed):
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == bad_tick:
|
|
r['state']['normal'][2] = 31.9
|
|
audit = run(end=4500, mutate=mutate)
|
|
assert (audit.first_failure is None) == passed
|
|
assert audit.cycles[0]['P1_margins']['reload_dwell'] == passed
|
|
|
|
|
|
def test_support_and_unload_stop_at_window_last_sample():
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['command']['phase'] == 1:
|
|
r['state']['normal'][2] = 29.
|
|
audit = run(end=2100, mutate=mutate)
|
|
assert audit.first_failure['global_tick'] == 1999
|
|
assert audit.gates['unloaded'] is True
|
|
assert audit.margins['unload_ratio'] is False
|
|
assert audit.gates['real_flight'] == 'not_evaluated'
|
|
|
|
|
|
def test_P2_stricter_margins_are_diagnostic_not_suppressed_functional_failures():
|
|
def mutate(r):
|
|
if r['kind'] in ('initial', 'tick', 'next_reference'):
|
|
r['state']['root'][0] = .04
|
|
r['state']['ik_target'][0] = r['event']['command']['target_world_m'][0] - .04
|
|
report = run(protocol('P2'), mutate=mutate).report()
|
|
assert report['functional_passed']
|
|
assert report['cycles'][0]['P1_margins']['root_horizontal'] is False
|
|
assert report['cycles'][0]['P1_margins_role'] == 'diagnostic_only'
|
|
|
|
|
|
def applied_zero(r, p, trial):
|
|
"""Pre-step capture with zero envelope, used before the event should start."""
|
|
s = r['state']
|
|
events = []
|
|
for i in p.trial(trial).event_indices:
|
|
program = s['event_program'][i]
|
|
events.append(dict(event_index=i, body_index=s['body_names'].index(program['body']), envelope=0.,
|
|
force_world=[0.] * 3, couple_world=[0.] * 3, point_world=program['point_m'],
|
|
com_world=[0.] * 3, frame_quat_xyzw=[0., 0., 0., 1.],
|
|
point_frame_quat_xyzw=[0., 0., 0., 1.], point_frame_origin_world=[0.] * 3))
|
|
s['applied'] = dict(format='skillfactory.contact-cycle-applied-wrench.v1', trial_index=trial,
|
|
global_tick=r['event']['clock']['global_tick'], events=events,
|
|
total_com_wrench=[[0.] * 6 for _ in s['body_names']])
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['missing', 'stale', 'frame', 'moment'])
|
|
def test_required_prephysics_applied_evidence(fault):
|
|
p, trial = protocol('P2'), 4
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
applied_zero(r, p, trial)
|
|
a = r['state']['applied']
|
|
if fault == 'missing':
|
|
r['state']['applied'] = None
|
|
elif fault == 'stale':
|
|
a['global_tick'] = 0
|
|
elif fault == 'frame':
|
|
a['events'][0]['frame_quat_xyzw'] = [0., 0., 1., 0.]
|
|
else:
|
|
a['total_com_wrench'][0][3] = .01
|
|
audit = run(p, trial, end=1, mutate=mutate)
|
|
assert any('recording_applied' in c for c in audit.first_failure['causes'])
|
|
|
|
|
|
def test_undelivered_time_event_fails_at_first_delivery_deadline():
|
|
p, trial = protocol('P2'), 4
|
|
# Use a direct event-window probe so original functional windows are irrelevant.
|
|
audit = TrialAudit(p, trial)
|
|
initial = next(records(p, trial, end=0))
|
|
assert not audit.consume(initial)
|
|
audit.event_previous['state']['time'] = 7.5
|
|
r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick')
|
|
clock = p.evidence_clock(trial, 3751)
|
|
r['event']['clock'] = plain(asdict(clock))
|
|
applied_zero(r, p, trial)
|
|
audit.last = r
|
|
causes = audit._events(r['state'], clock)
|
|
assert 'event_delivery:0' in causes
|
|
assert audit.delivery[0]['expected'] and audit.delivery[0]['missed']
|
|
assert audit.delivery[0]['expected_area_s'] == 6.
|
|
|
|
|
|
def test_matrix_includes_all_not_run_rows_and_does_not_assume_pairing():
|
|
result = matrix_report(protocol('P2'), [])
|
|
assert len(result['rows']) == 20 and len(result['pairs']) == 10
|
|
assert all(r['status'] == 'not_run' for r in result['rows'])
|
|
assert all(p['status'] == 'not_evaluated' for p in result['pairs'])
|
|
|
|
|
|
@pytest.fixture
|
|
def recording():
|
|
path = ROOT / 'source/SkillFactory/SkillFactory/tasks/manager_based/contact_cycle_sim/evaluation_recording.py'
|
|
spec = importlib.util.spec_from_file_location('_cpu_evaluation_recording', path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@dataclass
|
|
class Stop:
|
|
trial_index: int = 0
|
|
physics_ticks: int = 2
|
|
causes: tuple = ('swing_clearance',)
|
|
reason: str = 'failure'
|
|
fatal: bool = False
|
|
clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2})
|
|
completed_controls: int = 0
|
|
partial_control_ticks: int = 2
|
|
completed_clock_cycles: int = 0
|
|
all_sim_steps_attempted: int = 4
|
|
all_sim_steps_completed: int = 4
|
|
initialization_steps: int = 2
|
|
rollout_steps: int = 2
|
|
reset_calls: int = 3
|
|
pose_write_calls: int = 1
|
|
composer_buffer_replacements: int = 2
|
|
|
|
|
|
def recorder_prefix(recording, tmp_path):
|
|
budgets = []
|
|
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
|
|
budget_check=lambda *args: budgets.append(args), supplemental=lambda s: {})
|
|
for r in records(protocol(), end=2):
|
|
assert not recorder._record(r)
|
|
stop = Stop()
|
|
assert not recorder.after_step(SimpleNamespace(ticks_advanced=2, observation=None, reward=None, stop=stop))
|
|
return recorder, stop, budgets
|
|
|
|
|
|
def test_partial_control_durable_persistence_then_exact_ack(recording, tmp_path):
|
|
recorder, stop, budgets = recorder_prefix(recording, tmp_path)
|
|
acknowledgements = []
|
|
def acknowledge(value):
|
|
assert (recorder.directory / 'metrics.json').is_file()
|
|
acknowledgements.append(value)
|
|
report = recorder.finalize(stop=stop, acknowledge=acknowledge)
|
|
assert acknowledgements == [stop]
|
|
assert report['first_failure']['global_tick'] == 2
|
|
decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
|
|
raw = [json.loads(x) for x in decoded.splitlines()]
|
|
assert len([r for r in raw if r['kind'] == 'tick']) == 2
|
|
assert raw[-2]['ticks_advanced'] == 2 and raw[-2]['reward'] is None
|
|
assert raw[-1]['kind'] == 'terminal' and not report['functional_passed']
|
|
assert any(b[0] == 'finalize' for b in budgets)
|
|
|
|
|
|
def test_io_error_never_acknowledges(recording, tmp_path, monkeypatch):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
acknowledgements = []
|
|
def fail(_):
|
|
raise OSError('disk fault')
|
|
monkeypatch.setattr(recording.os, 'fsync', fail)
|
|
with pytest.raises(OSError, match='disk fault'):
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert not acknowledgements and recorder.faults
|
|
assert (recorder.directory / TRACE_FILENAME).exists()
|
|
recorder.stream.close()
|
|
|
|
|
|
def test_external_failure_terminal_no_fake_tick_and_no_ack(recording, tmp_path):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
acknowledgements = []
|
|
report = recorder.finalize(stop=stop, external_failure={
|
|
'causes': ['identity_changed'], 'fatal': True, 'context': {'stage': 'after_step'}},
|
|
acknowledge=acknowledgements.append)
|
|
assert not acknowledgements
|
|
assert report['observed_physics_ticks'] == 2
|
|
assert 'identity_changed' in report['first_failure']['causes']
|
|
|
|
|
|
def test_nonfinite_raw_and_budget_fault_are_retained(recording, tmp_path):
|
|
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
|
|
budget_check=lambda *args: None, supplemental=lambda s: {})
|
|
for r in records(protocol(), end=1):
|
|
if r['kind'] == 'tick':
|
|
r['state']['tilt'] = math.nan
|
|
causes = recorder._record(r)
|
|
assert causes
|
|
recorder.finalize(external_failure={'causes': ['recording_nonfinite'], 'fatal': True, 'context': {}})
|
|
assert '"nonfinite":"nan"' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
|
|
|
|
|
|
def test_tensor_copy_is_independent(recording):
|
|
tensor = torch.ones(1, 3)
|
|
result = recording.copied({'x': tensor})
|
|
tensor.zero_()
|
|
assert result == {'x': [[1., 1., 1.]]}
|
|
|
|
|
|
def test_incomplete_endpoint_gates_return_immediate_stop():
|
|
p = protocol()
|
|
audit = TrialAudit(p, 0)
|
|
for record in records(p, end=4500):
|
|
if record['kind'] == 'tick' and record['event']['clock']['global_tick'] == 4500:
|
|
audit.gates['real_flight'] = 'not_evaluated'
|
|
causes = audit.consume(record)
|
|
assert 'recording_incomplete_cycle_gates' in causes
|
|
break
|
|
assert not audit.consume(record)
|
|
assert audit.first_failure['global_tick'] == 4500
|
|
assert not audit.report()['cycles'][0]['passed']
|
|
|
|
|
|
def test_no_event_P1_requires_zero_wrench_not_fabricated_event_capture():
|
|
assert run(end=1).first_failure is None
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
r['state']['external'][0][0] = .01
|
|
audit = run(end=1, mutate=mutate)
|
|
assert 'identity_unexpected_external_wrench' in audit.first_failure['causes']
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['trial', 'point_frame', 'free_couple', 'moment_arm'])
|
|
def test_nonzero_applied_evidence_checks_frame_and_COM_moment(fault):
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross, rotate
|
|
p, trial = protocol('P2'), 8
|
|
audit = TrialAudit(p, trial)
|
|
r = next(x for x in records(p, trial, end=1) if x['kind'] == 'tick')
|
|
applied_zero(r, p, trial)
|
|
s = r['state']
|
|
e = s['applied']['events'][0]
|
|
program = s['event_program'][1]
|
|
q = [0., 0., math.sqrt(.5), math.sqrt(.5)]
|
|
e.update(envelope=.5, frame_quat_xyzw=q, point_frame_quat_xyzw=q,
|
|
point_frame_origin_world=[.2, .3, .4], com_world=[.1, .2, .3])
|
|
s['events'][1]['envelope'] = .5
|
|
e['point_world'] = [a+b for a, b in zip(e['point_frame_origin_world'], rotate(q, program['point_m']))]
|
|
e['force_world'] = [v*.5 for v in rotate(q, program['force_N'])]
|
|
e['couple_world'] = [v*.5 for v in rotate(q, program['couple_Nm'])]
|
|
wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'],
|
|
cross(sub(e['point_world'], e['com_world']), e['force_world']))]
|
|
s['external'][e['body_index']] = wrench
|
|
s['applied']['total_com_wrench'][e['body_index']] = wrench.copy()
|
|
clock = p.evidence_clock(trial, 1)
|
|
assert audit._applied(s, clock)[1] == e
|
|
if fault == 'trial':
|
|
s['applied']['trial_index'] = 4
|
|
elif fault == 'point_frame':
|
|
e['point_frame_quat_xyzw'] = [0., 0., 0., 1.]
|
|
elif fault == 'free_couple':
|
|
e['couple_world'][0] += .01
|
|
else:
|
|
e['com_world'][0] += .01
|
|
with pytest.raises(ValueError, match='applied'):
|
|
audit._applied(s, clock)
|
|
|
|
|
|
def test_trigger_provenance_rejects_fabricated_early_fire():
|
|
p, trial = protocol('P2'), 4
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
applied_zero(r, p, trial)
|
|
r['state']['events'][0]['fired'] = True
|
|
audit = run(p, trial, end=1, mutate=mutate)
|
|
assert 'recording_event_trigger_or_envelope:0' in audit.first_failure['causes']
|
|
|
|
|
|
def frozen_schedule_mutator(p, trial, *, edge_tick=None, inspect=None):
|
|
from SkillFactory.legs.contact_cycle.mechanics import WrenchSchedule
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_audit import cross
|
|
schedule = WrenchSchedule(plain(p.document['events']), 1, 'cpu')
|
|
def mutate(r):
|
|
if r['kind'] != 'tick':
|
|
return
|
|
s = r['state']
|
|
g = r['event']['clock']['global_tick']
|
|
clock = p.next_command_clock(trial, 0) if g == 1 else p.evidence_clock(trial, g-1)
|
|
command = p.command(trial, clock, anchor_a_world=s['anchors']['anchor_a_world'],
|
|
root_origin_world=s['anchors']['root_origin_world'], weight_N=40.)
|
|
# Match the actual frozen runtime: integer substeps multiplied by dt
|
|
# in float32, NOT a Python double product subsequently cast to float32.
|
|
time = torch.tensor([g-1]) * .002
|
|
s['time'] = float((torch.tensor([g]) * .002)[0])
|
|
s['liftoff'] = g == edge_tick and s['scenario'] == 2
|
|
s['touchdown'] = g == edge_tick and s['scenario'] == 3
|
|
s['edge_phase'] = 2 if s['liftoff'] else -1
|
|
schedule.step(time, torch.tensor([command.phase]),
|
|
torch.tensor([g-1 == edge_tick and s['scenario'] == 2]),
|
|
torch.tensor([g-1 == edge_tick and s['scenario'] == 3]),
|
|
torch.tensor([s['scenario']]), .002)
|
|
if inspect:
|
|
inspect(g, time, schedule)
|
|
for i, event in enumerate(s['events']):
|
|
event.update(fired=bool(schedule.fired[0, i]), completed=bool(schedule.completed[0, i]),
|
|
start=float(schedule.start[0, i]), envelope=float(schedule.envelope[0, i]),
|
|
integral=float(schedule.integral[0, i]))
|
|
applied_zero(r, p, trial)
|
|
for e in s['applied']['events']:
|
|
i, body = e['event_index'], e['body_index']
|
|
e['envelope'] = s['events'][i]['envelope']
|
|
program = s['event_program'][i]
|
|
e['force_world'] = [v*e['envelope'] for v in program['force_N']]
|
|
e['couple_world'] = [v*e['envelope'] for v in program['couple_Nm']]
|
|
wrench = e['force_world'] + [a+b for a, b in zip(e['couple_world'],
|
|
cross(sub(e['point_world'], e['com_world']), e['force_world']))]
|
|
s['external'][body] = wrench
|
|
s['applied']['total_com_wrench'][body] = wrench.copy()
|
|
return mutate
|
|
|
|
|
|
def test_frozen_time_schedule_complete_delivery_and_impulses():
|
|
p, trial = protocol('P2'), 4
|
|
report = run(p, trial, mutate=frozen_schedule_mutator(p, trial)).report()
|
|
assert report['first_failure'] is None
|
|
assert report['functional_passed'] and report['protocol_complete']
|
|
event = report['delivery'][0]
|
|
assert event['status'] is True and event['actual_applied_evidence_available']
|
|
assert event['force_impulse_Ns'] == pytest.approx([18., -24., -12.], abs=1e-4)
|
|
assert event['couple_impulse_Nms'] == pytest.approx([1.2, .6, -.6], abs=1e-5)
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['partial_reward', 'next_observation', 'duplicate_step', 'terminal_counters'])
|
|
def test_partial_and_terminal_records_fail_closed(recording, tmp_path, fault):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
if fault == 'terminal_counters':
|
|
stop.all_sim_steps_completed += 1
|
|
else:
|
|
bad = copy.deepcopy(recorder.audit.step_results[-1])
|
|
if fault == 'partial_reward':
|
|
bad['reward'] = [1.]
|
|
elif fault == 'next_observation':
|
|
bad['has_next_observation'] = True
|
|
assert recorder.audit.consume(bad)
|
|
acknowledgements = []
|
|
report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert not acknowledgements
|
|
assert any(c.startswith('recording') for c in report['all_causes'])
|
|
|
|
|
|
def test_recording_fault_after_earlier_failure_is_not_hidden(recording, tmp_path):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
recorder.audit.fail(('recording_late_fault',), kind='terminal', tick=3)
|
|
acknowledgements = []
|
|
report = recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert report['first_failure']['global_tick'] == 2
|
|
assert 'recording_late_fault' in report['all_causes']
|
|
assert not acknowledgements
|
|
|
|
|
|
def test_capture_fault_reports_actual_tick_without_fabricating_sample(recording, tmp_path, monkeypatch):
|
|
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
|
|
budget_check=lambda *args: None, supplemental=lambda s: {})
|
|
for record in records(protocol(), end=1):
|
|
assert not recorder._record(record)
|
|
@dataclass
|
|
class Event:
|
|
kind: str = 'tick'
|
|
clock: dict = field(default_factory=lambda: {'global_tick': 2, 'cycle_index': 0, 'local_tick': 2})
|
|
causes: tuple = ('hardware_joint_limit',)
|
|
def broken(*args):
|
|
raise ValueError('accessor failed')
|
|
monkeypatch.setattr(recording, 'snapshot', broken)
|
|
assert recorder(Event(), None)
|
|
report = recorder.finalize(external_failure={'causes': ['recording_capture'], 'fatal': True, 'context': {}})
|
|
assert report['first_failure']['global_tick'] == 2
|
|
assert report['observed_physics_ticks'] == 1
|
|
assert 'hardware_joint_limit' in report['first_failure']['causes']
|
|
assert 'capture_fault' in lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).decode()
|
|
|
|
|
|
def test_offline_reaudit_matches_live_partial_report(recording, tmp_path):
|
|
import subprocess
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
live = recorder.finalize(stop=stop)
|
|
output = tmp_path / 'offline.json'
|
|
result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'),
|
|
'--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
|
|
'--trace', str(recorder.directory / TRACE_FILENAME), '--output', str(output),
|
|
'--max-trace-bytes', '1048576', '--wall-seconds', '20'], capture_output=True, text=True)
|
|
assert result.returncode == 1, result.stderr
|
|
offline = json.loads(output.read_text())
|
|
for key in ('first_failure', 'all_causes', 'cycles', 'functional_passed', 'protocol_complete', 'terminal'):
|
|
assert offline[key] == live[key]
|
|
assert offline['new_simulation_steps'] == offline['new_optimizer_steps'] == 0
|
|
|
|
|
|
def test_matrix_mismatched_pair_never_publishes_controller_delta():
|
|
p = protocol('P2')
|
|
rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': {'anchor': i},
|
|
'functional_passed': True} for i in (0, 1)]
|
|
pair = matrix_report(p, rows)['pairs'][0]
|
|
assert pair['status'] == 'invalid_initial_pair'
|
|
assert pair['initial_exactly_equal'] is False
|
|
assert pair['both_functional_passed'] is False
|
|
assert pair['learned_minus_reference'] == 'not_evaluated'
|
|
|
|
|
|
def test_matrix_absent_pair_snapshots_are_not_equal_evidence():
|
|
p = protocol('P2')
|
|
rows = [{'trial': plain(asdict(p.trial(i))), 'initial_pair_state': None} for i in (0, 1)]
|
|
report = matrix_report(p, rows)
|
|
assert report['pairs'][0]['status'] == 'not_evaluated'
|
|
|
|
|
|
def test_xz_lossless_members_and_exact_inclusive_byte_accounting(recording, tmp_path):
|
|
recorder = recording.EvaluationRecorder(protocol(), 0, tmp_path / 'evidence',
|
|
budget_check=lambda *args: None, supplemental=lambda s: {})
|
|
original = list(records(protocol(), end=2))
|
|
for record in original:
|
|
assert not recorder._record(record)
|
|
assert (recorder.directory / TRACE_FILENAME).stat().st_size == recorder.written_bytes
|
|
# The initial is a checkpoint; control/ticks remain bounded and uncommitted.
|
|
assert len(recorder.pending) == len(original) - 1
|
|
assert [json.loads(x) for x in lzma.decompress(
|
|
(recorder.directory / TRACE_FILENAME).read_bytes()).splitlines()] == original[:1]
|
|
recorder._flush()
|
|
assert not recorder.pending and recorder.pending_raw_bytes == 0
|
|
decoded = lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes()).splitlines()
|
|
assert [json.loads(line) for line in decoded] == original
|
|
recorder.stream.close()
|
|
|
|
|
|
@pytest.mark.parametrize('stage', ['record', 'terminal_member', 'metrics', 'close', 'compression'])
|
|
def test_xz_storage_failures_preserve_complete_prefix_never_ack(recording, tmp_path, monkeypatch, stage):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
path = recorder.directory / TRACE_FILENAME
|
|
prefix = path.read_bytes()
|
|
before = [json.loads(line) for line in lzma.decompress(prefix).splitlines()]
|
|
finalize_calls = 0
|
|
def budget(where, pending, written):
|
|
nonlocal finalize_calls
|
|
assert written == path.stat().st_size
|
|
if where == 'finalize':
|
|
finalize_calls += 1
|
|
fail = ((stage == 'record' and where == 'record')
|
|
or (stage == 'terminal_member' and finalize_calls == 2)
|
|
or (stage == 'metrics' and finalize_calls == 3))
|
|
if fail:
|
|
# Deny by one byte: the pending count must INCLUDE XZ header/check/footer.
|
|
assert pending > 18
|
|
raise OSError('aggregate byte budget includes complete compressed member')
|
|
recorder.budget_check = budget
|
|
if stage == 'compression':
|
|
def broken_compression(*args, **kwargs):
|
|
raise OSError('compression fault')
|
|
monkeypatch.setattr(recording.lzma, 'compress', broken_compression)
|
|
if stage == 'close':
|
|
stream = recorder.stream
|
|
class CloseFault:
|
|
def __getattr__(self, key):
|
|
return getattr(stream, key)
|
|
def close(self):
|
|
raise OSError('close fault')
|
|
recorder.stream = CloseFault()
|
|
acknowledgements = []
|
|
if stage == 'record':
|
|
assert recorder.after_step(SimpleNamespace(ticks_advanced=0, observation=None, reward=None, stop=stop))
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
else:
|
|
with pytest.raises(OSError):
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert not acknowledgements and recorder.faults
|
|
decoded = [json.loads(line) for line in lzma.decompress(path.read_bytes()).splitlines()]
|
|
assert decoded[:len(before)] == before
|
|
if stage == 'close':
|
|
stream.close()
|
|
else:
|
|
recorder.stream.close()
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['truncated', 'corrupt', 'oversized', 'missing_terminal'])
|
|
def test_offline_xz_rejects_damaged_or_incomplete_tail(recording, tmp_path, fault):
|
|
import subprocess
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
trace = recorder.directory / TRACE_FILENAME
|
|
if fault == 'missing_terminal':
|
|
recorder.stream.close()
|
|
else:
|
|
recorder.finalize(stop=stop)
|
|
data = trace.read_bytes()
|
|
if fault == 'truncated':
|
|
trace.write_bytes(data[:-4])
|
|
elif fault == 'corrupt':
|
|
trace.write_bytes(data[:-8] + bytes([data[-8] ^ 1]) + data[-7:])
|
|
elif fault == 'oversized':
|
|
trace.write_bytes(data + lzma.compress(b'x' * (1024 * 1024 + 1) + b'\n'))
|
|
output = tmp_path / 'offline.json'
|
|
result = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_cycle_sim_evaluation.py'),
|
|
'--protocol', str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
|
|
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', '1048576',
|
|
'--wall-seconds', '20'], capture_output=True, text=True)
|
|
assert result.returncode == 1, result.stderr
|
|
report = json.loads(output.read_text())
|
|
assert not report['functional_passed']
|
|
assert any(c.startswith(('recording_offline', 'recording_missing_terminal')) for c in report['all_causes'])
|
|
assert report['observed_physics_ticks'] == 2
|
|
|
|
|
|
@pytest.mark.parametrize('value', [None, [], 'not a state'])
|
|
def test_malformed_state_container_is_a_recorded_failure(value):
|
|
def mutate(r):
|
|
if r['kind'] == 'tick':
|
|
r['state'] = value
|
|
audit = run(end=1, mutate=mutate)
|
|
assert 'recording_invalid:state_not_object' in audit.first_failure['causes']
|
|
assert audit.first_failure['global_tick'] == 1
|
|
|
|
|
|
def test_protocol_complete_is_not_a_passing_final_endpoint():
|
|
def mutate(r):
|
|
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 13500:
|
|
r['state']['tilt'] = .7
|
|
report = run(mutate=mutate).report()
|
|
assert report['protocol_complete'] and report['completed_clock_cycles'] == 3
|
|
assert report['passed_prefix_cycles'] == 2 and not report['functional_passed']
|
|
assert report['first_failure']['global_tick'] == 13500
|
|
|
|
|
|
def test_xz_short_write_retains_readable_complete_record_prefix(recording, tmp_path):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
path = recorder.directory / TRACE_FILENAME
|
|
prefix = path.read_bytes()
|
|
stream = recorder.stream
|
|
class ShortWrite:
|
|
def __getattr__(self, key):
|
|
return getattr(stream, key)
|
|
def write(self, data):
|
|
return stream.write(data[:len(data)//2])
|
|
recorder.stream = ShortWrite()
|
|
acknowledgements = []
|
|
with pytest.raises(OSError, match='Short evidence write'):
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert not acknowledgements
|
|
assert path.read_bytes().startswith(prefix)
|
|
expected = lzma.decompress(prefix).splitlines()
|
|
with lzma.open(path, 'rb') as reader:
|
|
assert [reader.readline().rstrip(b'\n') for _ in expected] == expected
|
|
with pytest.raises(EOFError):
|
|
reader.read()
|
|
stream.close()
|
|
|
|
|
|
def test_frozen_phase_schedule_float32_completion_uses_actual_start():
|
|
p, trial = protocol('P2'), 12
|
|
seen = []
|
|
def inspect(g, time, schedule):
|
|
if g == 3351:
|
|
start = float(schedule.start[0, 2])
|
|
seen.append((start, float(time[0]), float(time[0] - schedule.start[0, 2])))
|
|
assert not schedule.completed[0, 2]
|
|
audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=3399, inspect=inspect))
|
|
assert seen == [pytest.approx((6.1000003815, 6.7000002861, .5999999046), abs=1e-10)]
|
|
assert audit.first_failure is None
|
|
assert audit.report()['functional_passed']
|
|
assert all(audit.delivery[i]['status'] is True for i in (2, 3))
|
|
|
|
|
|
@pytest.mark.parametrize('fault', ['late_completion', 'area'])
|
|
def test_phase_schedule_completion_slack_does_not_relax_delivery(fault):
|
|
p, trial = protocol('P2'), 12
|
|
schedule = frozen_schedule_mutator(p, trial)
|
|
def mutate(r):
|
|
schedule(r)
|
|
if r['kind'] == 'tick' and r['event']['clock']['global_tick'] == 3352:
|
|
event = r['state']['events'][2]
|
|
if fault == 'late_completion':
|
|
event['completed'] = False
|
|
else:
|
|
event['integral'] -= .003
|
|
audit = run(p, trial, mutate=mutate)
|
|
assert audit.first_failure['global_tick'] == 3352
|
|
assert 'event_delivery:2' in audit.first_failure['causes']
|
|
|
|
|
|
@pytest.mark.parametrize('trial,event,closure', [(12, 3, 3400), (8, 1, 2300)])
|
|
@pytest.mark.parametrize('edge_offset', [-1, 0, 1, None])
|
|
def test_edge_trigger_phase_closure_respects_last_pre_step(trial, event, closure, edge_offset):
|
|
p = protocol('P2')
|
|
edge_tick = None if edge_offset is None else closure + edge_offset
|
|
audit = run(p, trial, mutate=frozen_schedule_mutator(p, trial, edge_tick=edge_tick))
|
|
if edge_offset == -1:
|
|
assert audit.first_failure is None
|
|
assert audit.delivery[event]['status'] is True
|
|
assert audit.delivery[event]['absolute_start_s'] >= (closure - 1) * .002
|
|
else:
|
|
# Includes boundary/reload-first landing (or swing-first liftoff),
|
|
# which cannot be consumed by a scheduler still in the eligible phase.
|
|
assert audit.first_failure['global_tick'] == closure
|
|
assert audit.first_failure['causes'] == ['event_delivery:' + str(event)]
|
|
assert audit.delivery[event]['missed']
|
|
|
|
|
|
def test_delayed_liftoff_is_not_a_missing_trigger_at_phase_closure():
|
|
p, trial = protocol('P2'), 8
|
|
audit = run(p, trial, end=2300, mutate=frozen_schedule_mutator(p, trial, edge_tick=2299))
|
|
assert audit.first_failure is None
|
|
event = audit.delivery[1]
|
|
assert event['fired'] and not event['completed'] and not event['missed']
|
|
assert event['absolute_start_s'] > 4.6
|
|
assert event['status'] == 'not_evaluated'
|
|
|
|
|
|
def test_recorder_requires_existing_parent_without_creating_ancestors(recording, tmp_path):
|
|
directory = tmp_path / 'missing' / 'nested' / 'trial'
|
|
with pytest.raises(FileNotFoundError):
|
|
recording.EvaluationRecorder(protocol(), 0, directory,
|
|
budget_check=lambda *args: None, supplemental=lambda s: {})
|
|
assert not (tmp_path / 'missing').exists()
|
|
|
|
|
|
@pytest.mark.parametrize('fail_at', [None, 3, 4])
|
|
def test_recorder_directory_and_parent_fsync_before_ack(recording, tmp_path, monkeypatch, fail_at):
|
|
import os
|
|
import stat
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
fsync, calls, acknowledgements = os.fsync, [], []
|
|
def sync(fd):
|
|
calls.append((os.fstat(fd).st_ino, stat.S_ISDIR(os.fstat(fd).st_mode)))
|
|
assert not acknowledgements
|
|
if len(calls) == fail_at:
|
|
raise OSError('directory fsync failed')
|
|
fsync(fd)
|
|
monkeypatch.setattr(recording.os, 'fsync', sync)
|
|
if fail_at:
|
|
with pytest.raises(OSError, match='directory fsync failed'):
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert not acknowledgements and not recorder.finalized and recorder.faults
|
|
recorder.stream.close()
|
|
else:
|
|
recorder.finalize(stop=stop, acknowledge=acknowledgements.append)
|
|
assert acknowledgements == [stop] and recorder.stream.closed
|
|
assert calls[2] == (recorder.directory.stat().st_ino, True)
|
|
if fail_at != 3:
|
|
assert calls[3] == (recorder.directory.parent.stat().st_ino, True)
|
|
|
|
|
|
@pytest.fixture
|
|
def offline_reader():
|
|
spec = importlib.util.spec_from_file_location('_cpu_offline_reader', ROOT / 'scripts/audit_cycle_sim_evaluation.py')
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def gzip_processing_tail(kind):
|
|
if kind == 'empty_members':
|
|
return gzip.compress(b'') * 10000
|
|
if kind == 'oversized_header':
|
|
member = gzip.compress(b'')
|
|
return member[:3] + b'\x08' + member[4:10] + b'n' * (1024 * 1024) + b'\0' + member[10:]
|
|
if kind == 'empty_blocks':
|
|
# Valid deflate stream that consumes compressed bytes without output.
|
|
return gzip.compress(b'')[:10] + b'\x00\x00\x00\xff\xff' * 200000 + b'\x03\x00' + b'\0' * 8
|
|
raise AssertionError(kind)
|
|
|
|
|
|
@pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks'])
|
|
def test_offline_gzip_budget_during_no_output_processing(offline_reader, tmp_path, monkeypatch, tail):
|
|
path = tmp_path / 'trace.v1.jsonl.gz'
|
|
terminal = b'{"kind":"terminal"}\n'
|
|
path.write_bytes(gzip.compress(terminal) + gzip_processing_tail(tail))
|
|
now = 0
|
|
def monotonic():
|
|
nonlocal now
|
|
now += 1
|
|
return now
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
|
|
lines = offline_reader.trace_lines(path, compressed=True, max_bytes=2**21,
|
|
max_record_bytes=2**20, deadline=12)
|
|
assert next(lines) == terminal
|
|
with pytest.raises(ValueError, match='budget_audit'):
|
|
list(lines)
|
|
assert now == 13 # Rejection inside processing, not merely after decoded output.
|
|
|
|
|
|
@pytest.mark.parametrize('compressed', [False, True])
|
|
def test_offline_budget_checked_at_eof(offline_reader, tmp_path, monkeypatch, compressed):
|
|
path = tmp_path / 'trace'
|
|
raw = b'{"kind":"terminal"}\n'
|
|
path.write_bytes(gzip.compress(raw) if compressed else raw)
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 0)
|
|
lines = offline_reader.trace_lines(path, compressed=compressed, max_bytes=4096,
|
|
max_record_bytes=4096, deadline=1)
|
|
assert next(lines) == raw
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', lambda: 2)
|
|
with pytest.raises(ValueError, match='budget_audit'):
|
|
next(lines)
|
|
|
|
|
|
def test_offline_gzip_expansion_record_limit(offline_reader, tmp_path):
|
|
path = tmp_path / 'trace.v1.jsonl.gz'
|
|
path.write_bytes(gzip.compress(b'x' * (2**20 + 1)))
|
|
with pytest.raises(ValueError, match='budget_audit'):
|
|
list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=2**20, deadline=math.inf))
|
|
|
|
|
|
def test_offline_gzip_decoder_no_progress_rejected(offline_reader, tmp_path, monkeypatch):
|
|
path = tmp_path / 'trace.v1.jsonl.gz'
|
|
path.write_bytes(gzip.compress(b''))
|
|
class StalledDecoder:
|
|
eof = False
|
|
def decompress(self, data, limit):
|
|
self.unconsumed_tail = data
|
|
return b''
|
|
monkeypatch.setattr(offline_reader.zlib, 'decompressobj', lambda **kwargs: StalledDecoder())
|
|
with pytest.raises(ValueError, match='no progress'):
|
|
list(offline_reader.trace_lines(path, compressed=True, max_bytes=4096,
|
|
max_record_bytes=4096, deadline=math.inf))
|
|
|
|
|
|
@pytest.mark.parametrize('tail', ['empty_members', 'oversized_header', 'empty_blocks'])
|
|
def test_offline_cli_budget_failure_after_terminal_is_reported(
|
|
recording, offline_reader, tmp_path, monkeypatch, tail):
|
|
recorder, stop, _ = recorder_prefix(recording, tmp_path)
|
|
recorder.finalize(stop=stop)
|
|
trace = tmp_path / 'legacy.jsonl.gz'
|
|
trace.write_bytes(gzip.compress(lzma.decompress((recorder.directory / TRACE_FILENAME).read_bytes())))
|
|
with trace.open('ab') as output:
|
|
output.write(gzip_processing_tail(tail))
|
|
protocol_module, audit_module = offline_reader.cpu_modules()
|
|
audit = audit_module.TrialAudit(protocol_module.load_protocol(
|
|
ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), 0)
|
|
monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit)
|
|
monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module))
|
|
now = 0
|
|
def monotonic():
|
|
nonlocal now
|
|
if audit.terminal is not None:
|
|
now += 1
|
|
return now
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
|
|
output = tmp_path / 'report.json'
|
|
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
|
|
str(ROOT / 'configs/contact_cycle/continuous_eval_p1_v1.json'), '--trial-index', '0',
|
|
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**21), '--wall-seconds', '4'])
|
|
assert offline_reader.main() == 1
|
|
report = json.loads(output.read_text())
|
|
assert report['terminal'] is not None and not report['functional_passed']
|
|
assert 'recording_offline:budget_audit_resource_or_record_limit' in report['all_causes']
|
|
assert now == 5
|
|
|
|
|
|
@pytest.mark.parametrize('codec', ['gzip', 'xz'])
|
|
def test_offline_passing_terminal_cannot_hide_empty_member_budget_overrun(offline_reader, tmp_path, monkeypatch, codec):
|
|
p = protocol('P2')
|
|
trace = tmp_path / ('trace.v1.jsonl.gz' if codec == 'gzip' else TRACE_FILENAME)
|
|
encode = (lambda raw: gzip.compress(raw, compresslevel=1)) if codec == 'gzip' else (
|
|
lambda raw: lzma.compress(raw, preset=0))
|
|
stop = plain(asdict(Stop(physics_ticks=4500, causes=(), reason='protocol_complete',
|
|
clock=plain(asdict(p.evidence_clock(0, 4500))), completed_controls=900,
|
|
partial_control_ticks=0, completed_clock_cycles=1, all_sim_steps_attempted=4502,
|
|
all_sim_steps_completed=4502, rollout_steps=4500, composer_buffer_replacements=4500)))
|
|
with trace.open('wb') as output:
|
|
def write(record):
|
|
output.write(encode((json.dumps(record) + '\n').encode()))
|
|
for record in records(p):
|
|
write(record)
|
|
if record['kind'] == 'tick' and record['state']['substeps'] % 5 == 0:
|
|
g = record['state']['substeps']
|
|
write({'kind': 'step_result', 'ticks_advanced': 5, 'observed_tick': g,
|
|
'reward': [0.], 'has_next_observation': g != 4500, 'stop': stop if g == 4500 else None})
|
|
write({'kind': 'terminal', 'runtime_stop': stop, 'external_failure': None, 'recording_faults': []})
|
|
output.write(encode(b'') * 10000)
|
|
protocol_module, audit_module = offline_reader.cpu_modules()
|
|
audit = audit_module.TrialAudit(p, 0)
|
|
monkeypatch.setattr(audit_module, 'TrialAudit', lambda *args: audit)
|
|
monkeypatch.setattr(offline_reader, 'cpu_modules', lambda: (protocol_module, audit_module))
|
|
now = 0
|
|
def monotonic():
|
|
nonlocal now
|
|
if audit.terminal is not None:
|
|
assert audit.report()['functional_passed'] # Passing until budget expiry is recorded.
|
|
now += 1
|
|
return now
|
|
monkeypatch.setattr(offline_reader.time, 'monotonic', monotonic)
|
|
output = tmp_path / 'report.json'
|
|
monkeypatch.setattr(sys, 'argv', ['audit', '--protocol',
|
|
str(ROOT / 'configs/contact_cycle/bounded_diagnostic_p2_v1.json'), '--trial-index', '0',
|
|
'--trace', str(trace), '--output', str(output), '--max-trace-bytes', str(2**26), '--wall-seconds', '4'])
|
|
assert offline_reader.main() == 1
|
|
report = json.loads(output.read_text())
|
|
assert not report['functional_passed']
|
|
assert report['first_failure']['causes'] == ['recording_offline:budget_audit_resource_or_record_limit']
|
|
|
|
|
|
def test_offline_gzip_bounded_chunks_are_lossless(offline_reader, tmp_path):
|
|
path = tmp_path / 'trace.v1.jsonl.gz'
|
|
lines = [b'a' * (2**20 - 1) + b'\n', b'b\n', b'c' * (2**17) + b'\n']
|
|
# Deliberately span record/member boundaries, including empty members.
|
|
raw = b''.join(lines)
|
|
path.write_bytes(gzip.compress(raw[:300000]) + gzip.compress(b'') + gzip.compress(raw[300000:]))
|
|
assert list(offline_reader.trace_lines(path, compressed=True, max_bytes=2**20,
|
|
max_record_bytes=2**20, deadline=math.inf)) == lines
|