"""CPU-only protocol tests: these are not simulator or development-gate evidence.""" import copy import dataclasses import json import math import subprocess import sys from collections import Counter from pathlib import Path import pytest import torch from SkillFactory.legs.contact_cycle.mechanics import inverse_rotate from SkillFactory.legs.contact_cycle.task import command as frozen_command from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import ( CYCLE_TICKS, EVIDENCE_PHASE_COUNTS, PHYSICS_DT_S, Clock, load_protocol, validate_protocol, ) ROOT = Path(__file__).resolve().parents[1] PATHS = {"P1": ROOT / "configs/contact_cycle/continuous_eval_p1_v1.json", "P2": ROOT / "configs/contact_cycle/bounded_diagnostic_p2_v1.json"} ORIGINAL = json.loads((ROOT / "configs/contact_cycle/go2_fl_v1.json").read_text()) A = (0.12, -0.07, 0.022) PORT = (0.02, 0.03, 0.30) def sample(protocol, clock, trial=0, **kwargs): return protocol.command(trial, clock, anchor_a_world=A, root_origin_world=PORT, weight_N=40.0, **kwargs) @pytest.mark.parametrize("stage,rows,ticks", [("P1", 2, 13500), ("P2", 20, 4500)]) def test_evaluation_protocol_schema_and_immutable_trials(stage, rows, ticks): raw = json.loads(PATHS[stage].read_text()) protocol = validate_protocol(raw) assert len(protocol.trials) == rows assert all(row.total_ticks == ticks for row in protocol.trials) assert protocol.document["execution_authorized"] is False assert protocol.document["accepted_skill"] is False assert protocol.document["optimizer_steps"] == 0 raw["clock"]["cycle_ticks"] = 7 assert protocol.document["clock"]["cycle_ticks"] == 4500 with pytest.raises(TypeError): protocol.document["clock"]["cycle_ticks"] = 7 with pytest.raises(dataclasses.FrozenInstanceError): protocol.trials[0].seed = 0 with pytest.raises(dataclasses.FrozenInstanceError): protocol.trials = () def test_evaluation_protocol_exact_preregistered_order_and_pairing(): p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"]) controllers = ("learned_policy", "analytic_reference") conditions = ("nominal", "oblique_sustained", "eccentric_liftoff", "landing_combination", "target_only") assert [(r.seed, r.controller_kind) for r in p1.trials] == [(93501, c) for c in controllers] assert [(r.condition_id, r.seed, r.controller_kind) for r in p2.trials] == [ (condition, seed, controller) for condition in conditions for seed in (94501, 94502) for controller in controllers ] assert len({r.trial_id for r in p2.trials}) == 20 for learned, reference in zip(p2.trials[::2], p2.trials[1::2]): assert learned.pair_id == reference.pair_id assert dataclasses.replace(learned, index=reference.index, trial_id=reference.trial_id, controller_kind=reference.controller_kind) == reference assert [p2.trials[i].event_indices for i in (0, 4, 8, 12, 16)] == [(), (0,), (1,), (2, 3), ()] assert p2.trials[16].swing_displacement_m == (0.030, 0.025, 0.0) assert p1.document["events"] == () assert json.loads(json.dumps(ORIGINAL["events"])) == json.loads(PATHS["P2"].read_text())["events"] def test_evaluation_protocol_frozen_threshold_provenance_and_budget(): p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"]) functional = json.loads((ROOT / "configs/contact_cycle/simulation_functional_v1.json").read_text()) assert p1.document["thresholds"]["functional"] == functional["gates"] assert p2.document["thresholds"]["P1_margins_role"] == "diagnostic_only" assert p1.document["thresholds"]["P1_margins_role"] == "gate" assert p1.document["thresholds"]["P1_margins"] == p2.document["thresholds"]["P1_margins"] assert p2.document["requires_P1_evidence_review"] is True limits = p1.document["resources"] assert limits == p2.document["resources"] assert sum(r.total_ticks for r in (*p1.trials, *p2.trials)) == limits["max_rollout_ticks"] == 117000 assert limits["max_rollout_ticks"] + limits["max_initialization_reset_ticks"] == 120000 assert limits["max_sim_step_ticks_including_initialization"] == 120000 assert limits["max_launches"] == 3 assert limits["aggregate_same_boot_window_s"] == 3600 assert limits["max_artifact_bytes"] == 2**30 assert limits["cleanup_reserve_s"] == 120 assert limits["max_control_steps"] == 117000 // 5 def _leaves(value, prefix=()): if isinstance(value, dict): for key, item in value.items(): yield from _leaves(item, (*prefix, key)) elif isinstance(value, list): for index, item in enumerate(value): yield from _leaves(item, (*prefix, index)) else: yield prefix, value @pytest.mark.parametrize("stage", ["P1", "P2"]) def test_evaluation_protocol_every_leaf_is_locked_and_nonfinite_fails_closed(stage): raw = json.loads(PATHS[stage].read_text()) for path, value in _leaves(raw): bad_values = [None] if type(value) in (int, float): bad_values += [float("nan"), float("inf"), -float("inf"), True, -1, value + 0.001] for bad in bad_values: changed = copy.deepcopy(raw) parent = changed for key in path[:-1]: parent = parent[key] parent[path[-1]] = bad with pytest.raises(ValueError): validate_protocol(changed) @pytest.mark.parametrize("bad", [None, [], {}, {"stage": "P3"}, {"stage": []}]) def test_evaluation_protocol_invalid_root(bad): with pytest.raises(ValueError): validate_protocol(bad) def test_evaluation_protocol_unknown_missing_and_reordered_values_fail(tmp_path): for mutate in ( lambda d: d.update(unknown=True), lambda d: d.pop("resources"), lambda d: d["resources"].update(unknown=1), lambda d: d["seeds"].reverse(), lambda d: d["conditions"].pop(), lambda d: d["clock"]["phase_ticks"].append(1), lambda d: d["events"][0].update(force_N=[30.0, -4.0, -2.0]), ): raw = json.loads(PATHS["P2"].read_text()) mutate(raw) with pytest.raises(ValueError): validate_protocol(raw) path = tmp_path / "duplicate.json" path.write_text('{"stage": "P1", "stage": "P2"}') with pytest.raises(ValueError, match="Duplicate"): load_protocol(path) path.write_text(PATHS["P1"].read_text().replace('"physics_dt_s": 0.002', '"physics_dt_s": NaN')) with pytest.raises(ValueError): load_protocol(path) @pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) def test_evaluation_protocol_first_cycle_matches_frozen_command_every_tick(dtype): protocol = load_protocol(PATHS["P1"]) time = torch.arange(4501, dtype=dtype) * PHYSICS_DT_S origin = torch.tensor(A, dtype=dtype).expand(4501, -1) expected = frozen_command(time, ORIGINAL, origin, torch.full_like(time, 40.0)) actual = [sample(protocol, Clock(t, 0, t)) for t in range(4501)] assert torch.equal(torch.tensor([c.phase for c in actual]), expected[0]) for field, reference in zip(("progress", "target_world_m", "desired_normal_N", "auxiliary_fraction"), expected[1:]): torch.testing.assert_close(torch.tensor([getattr(c, field) for c in actual], dtype=dtype), reference, rtol=1e-5, atol=3e-5 if dtype == torch.float32 else 1e-12) def test_evaluation_protocol_exact_phase_counts_right_closed_endpoints(): protocol = load_protocol(PATHS["P1"]) counts = [Counter() for _ in range(3)] for tick in range(1, 13501): clock = protocol.evidence_clock(0, tick) assert clock.global_tick == clock.cycle_index * CYCLE_TICKS + clock.local_tick counts[clock.cycle_index][sample(protocol, clock).phase] += 1 assert [tuple(c[p] for p in range(6)) for c in counts] == [EVIDENCE_PHASE_COUNTS] * 3 for tick, cycle, local in ((4499, 0, 4499), (4500, 0, 4500), (4501, 1, 1), (9000, 1, 4500), (9001, 2, 1), (13500, 2, 4500)): assert protocol.evidence_clock(0, tick) == Clock(tick, cycle, local) for tick, phase in ((1499, 0), (1500, 1), (1999, 1), (2000, 2), (2299, 2), (2300, 3), (2999, 3), (3000, 4), (3399, 4), (3400, 5), (4500, 5)): c = sample(protocol, Clock(tick, 0, tick)) assert c.phase == phase assert protocol.evidence_clock(0, 4500).local_time_s == 9.0 assert protocol.evidence_clock(0, 13500).global_time_s == 27.0 @pytest.mark.parametrize("tick", [4500, 9000]) def test_evaluation_protocol_seam_requires_admission_and_preserves_command_continuity(tick): protocol = load_protocol(PATHS["P1"]) old = sample(protocol, protocol.evidence_clock(0, tick)) with pytest.raises(ValueError, match="endpoint must pass"): protocol.next_command_clock(0, tick) clock = protocol.next_command_clock(0, tick, seam_admitted=True) new = sample(protocol, clock) assert old.phase == 5 and old.progress == 1.0 assert new.phase == 0 and new.progress == 0.0 assert old.end_anchor == new.start_anchor assert old.target_world_m == new.target_world_m assert old.desired_normal_N == new.desired_normal_N == 40.0 assert old.auxiliary_fraction == new.auxiliary_fraction == 0.0 assert old.target_velocity_world_m_s == new.target_velocity_world_m_s == (0.0, 0.0, 0.0) assert old.root_target_world_m == new.root_target_world_m == PORT assert old.root_velocity_world_m_s == new.root_velocity_world_m_s == (0.0, 0.0, 0.0) assert protocol.evidence_clock(0, tick + 1).local_tick == 1 def test_evaluation_protocol_fixed_anchors_reverse_and_repeat_without_state_or_rng_writes(): protocol = load_protocol(PATHS["P1"]) rng = torch.get_rng_state().clone() anchors, root = list(A), list(PORT) args = {"anchor_a_world": anchors, "root_origin_world": root, "weight_N": 40.0} endpoints = [protocol.command(0, protocol.evidence_clock(0, t), **args) for t in (4500, 9000, 13500)] assert endpoints[0].target_world_m == endpoints[2].target_world_m assert endpoints[1].target_world_m == A assert [c.start_anchor + c.end_anchor for c in endpoints] == ["AB", "BA", "AB"] assert anchors == list(A) and root == list(PORT) assert torch.equal(rng, torch.get_rng_state()) assert protocol.next_command_clock(0, 0) == Clock(0, 0, 0) assert protocol.next_command_clock(0, 13500) is None assert protocol.next_command_clock(1, 13500) is None assert load_protocol(PATHS["P2"]).next_command_clock(0, 4500) is None # Mid-swing reverses direction; it does not accumulate a third positive displacement. first = sample(protocol, Clock(2650, 0, 2650)) reverse = sample(protocol, Clock(7150, 1, 2650)) assert reverse.target_world_m == pytest.approx(first.target_world_m) assert reverse.target_velocity_world_m_s[:2] == pytest.approx( tuple(-v for v in first.target_velocity_world_m_s[:2])) def test_evaluation_protocol_target_only_is_smooth_and_predeclared(): protocol = load_protocol(PATHS["P2"]) start = sample(protocol, Clock(0, 0, 0), trial=16) endpoint = sample(protocol, Clock(4500, 0, 4500), trial=16) assert start.target_world_m == A assert endpoint.target_world_m == pytest.approx(tuple(x + d for x, d in zip(A, (0.030, 0.025, 0.0)))) assert start.target_velocity_world_m_s == endpoint.target_velocity_world_m_s == (0.0, 0.0, 0.0) for tick in (2100, 2600, 3200): before, center, after = [sample(protocol, Clock(t, 0, t), trial=16) for t in (tick - 1, tick, tick + 1)] derivative = tuple((y - x) / (2 * PHYSICS_DT_S) for x, y in zip(before.target_world_m, after.target_world_m)) assert center.target_velocity_world_m_s == pytest.approx(derivative, abs=2e-6) def test_evaluation_protocol_world_axis_actor_displacement_is_not_actual_body_ik(): protocol = load_protocol(PATHS["P1"]) cmd = sample(protocol, Clock(2600, 0, 2600)) target = torch.tensor([cmd.target_world_m], dtype=torch.float64) commanded_port = torch.tensor([PORT], dtype=torch.float64) actual_root = commanded_port + torch.tensor([[0.01, -0.02, 0.03]], dtype=torch.float64) q = torch.tensor([[0.0, 0.0, math.sqrt(0.5), math.sqrt(0.5)]], dtype=torch.float64) actor = torch.tensor([cmd.actor_target_displacement_world_m], dtype=torch.float64) ik = inverse_rotate(q, target - actual_root) torch.testing.assert_close(actor, target - commanded_port) assert not torch.allclose(actor, ik) torch.testing.assert_close(ik[:, 0], (target - actual_root)[:, 1]) torch.testing.assert_close(ik[:, 1], -(target - actual_root)[:, 0]) @pytest.mark.parametrize("bad", [-1, 0, 13501, True, 1.0, float("nan"), float("inf")]) def test_evaluation_protocol_bad_evidence_ticks(bad): with pytest.raises(ValueError): load_protocol(PATHS["P1"]).evidence_clock(0, bad) @pytest.mark.parametrize("bad", [-1, 1, 4499, 13501, True, 5.0, float("nan")]) def test_evaluation_protocol_bad_control_ticks(bad): with pytest.raises(ValueError): load_protocol(PATHS["P1"]).next_command_clock(0, bad) @pytest.mark.parametrize("clock", [Clock(1, 0, 0), Clock(0, -1, 4500), Clock(13500, 3, 0), Clock(4501, 0, 4501), Clock(True, 0, 1), Clock(0, 0, 0.0), None]) def test_evaluation_protocol_bad_command_clock(clock): with pytest.raises(ValueError): sample(load_protocol(PATHS["P1"]), clock) @pytest.mark.parametrize("key,value", [("weight_N", 0), ("weight_N", True), ("weight_N", float("inf")), ("weight_N", float("nan")), ("anchor_a_world", [0, 0]), ("anchor_a_world", [0, float("nan"), 0]), ("root_origin_world", [0, float("inf"), 0]), ("root_origin_world", [0, True, 0])]) def test_evaluation_protocol_bad_command_parameters(key, value): args = {"weight_N": 40.0, "anchor_a_world": A, "root_origin_world": PORT} args[key] = value with pytest.raises(ValueError): load_protocol(PATHS["P1"]).command(0, Clock(0, 0, 0), **args) @pytest.mark.parametrize("index", [-1, 2, True, 0.0]) def test_evaluation_protocol_bad_trial_index(index): with pytest.raises(ValueError): load_protocol(PATHS["P1"]).trial(index) def test_evaluation_protocol_import_and_load_need_only_stdlib_no_runtime(): path = ROOT / "source/SkillFactory/SkillFactory/legs/contact_cycle_sim/evaluation_protocol.py" script = f""" import importlib.util, sys spec = importlib.util.spec_from_file_location('pure_protocol', {str(path)!r}) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) protocol = module.load_protocol({str(PATHS['P1'])!r}) assert len(protocol.trials) == 2 assert not any(name.split('.')[0] in ('torch', 'isaaclab', 'isaacsim', 'omni') for name in sys.modules) """ subprocess.run([sys.executable, "-I", "-S", "-c", script], check=True)