Files
SkillFactory/tests/test_contact_cycle_sim_evaluation_protocol.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

303 lines
15 KiB
Python

"""CPU-only protocol tests: these are not simulator or development-gate evidence."""
import copy
import dataclasses
import json
import math
import subprocess
import sys
from collections import Counter
from pathlib import Path
import pytest
import torch
from SkillFactory.legs.contact_cycle.mechanics import inverse_rotate
from SkillFactory.legs.contact_cycle.task import command as frozen_command
from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import (
CYCLE_TICKS,
EVIDENCE_PHASE_COUNTS,
PHYSICS_DT_S,
Clock,
load_protocol,
validate_protocol,
)
ROOT = Path(__file__).resolve().parents[1]
PATHS = {"P1": ROOT / "configs/contact_cycle/continuous_eval_p1_v1.json",
"P2": ROOT / "configs/contact_cycle/bounded_diagnostic_p2_v1.json"}
ORIGINAL = json.loads((ROOT / "configs/contact_cycle/go2_fl_v1.json").read_text())
A = (0.12, -0.07, 0.022)
PORT = (0.02, 0.03, 0.30)
def sample(protocol, clock, trial=0, **kwargs):
return protocol.command(trial, clock, anchor_a_world=A, root_origin_world=PORT, weight_N=40.0, **kwargs)
@pytest.mark.parametrize("stage,rows,ticks", [("P1", 2, 13500), ("P2", 20, 4500)])
def test_evaluation_protocol_schema_and_immutable_trials(stage, rows, ticks):
raw = json.loads(PATHS[stage].read_text())
protocol = validate_protocol(raw)
assert len(protocol.trials) == rows
assert all(row.total_ticks == ticks for row in protocol.trials)
assert protocol.document["execution_authorized"] is False
assert protocol.document["accepted_skill"] is False
assert protocol.document["optimizer_steps"] == 0
raw["clock"]["cycle_ticks"] = 7
assert protocol.document["clock"]["cycle_ticks"] == 4500
with pytest.raises(TypeError):
protocol.document["clock"]["cycle_ticks"] = 7
with pytest.raises(dataclasses.FrozenInstanceError):
protocol.trials[0].seed = 0
with pytest.raises(dataclasses.FrozenInstanceError):
protocol.trials = ()
def test_evaluation_protocol_exact_preregistered_order_and_pairing():
p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"])
controllers = ("learned_policy", "analytic_reference")
conditions = ("nominal", "oblique_sustained", "eccentric_liftoff", "landing_combination", "target_only")
assert [(r.seed, r.controller_kind) for r in p1.trials] == [(93501, c) for c in controllers]
assert [(r.condition_id, r.seed, r.controller_kind) for r in p2.trials] == [
(condition, seed, controller)
for condition in conditions for seed in (94501, 94502) for controller in controllers
]
assert len({r.trial_id for r in p2.trials}) == 20
for learned, reference in zip(p2.trials[::2], p2.trials[1::2]):
assert learned.pair_id == reference.pair_id
assert dataclasses.replace(learned, index=reference.index, trial_id=reference.trial_id,
controller_kind=reference.controller_kind) == reference
assert [p2.trials[i].event_indices for i in (0, 4, 8, 12, 16)] == [(), (0,), (1,), (2, 3), ()]
assert p2.trials[16].swing_displacement_m == (0.030, 0.025, 0.0)
assert p1.document["events"] == ()
assert json.loads(json.dumps(ORIGINAL["events"])) == json.loads(PATHS["P2"].read_text())["events"]
def test_evaluation_protocol_frozen_threshold_provenance_and_budget():
p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"])
functional = json.loads((ROOT / "configs/contact_cycle/simulation_functional_v1.json").read_text())
assert p1.document["thresholds"]["functional"] == functional["gates"]
assert p2.document["thresholds"]["P1_margins_role"] == "diagnostic_only"
assert p1.document["thresholds"]["P1_margins_role"] == "gate"
assert p1.document["thresholds"]["P1_margins"] == p2.document["thresholds"]["P1_margins"]
assert p2.document["requires_P1_evidence_review"] is True
limits = p1.document["resources"]
assert limits == p2.document["resources"]
assert sum(r.total_ticks for r in (*p1.trials, *p2.trials)) == limits["max_rollout_ticks"] == 117000
assert limits["max_rollout_ticks"] + limits["max_initialization_reset_ticks"] == 120000
assert limits["max_sim_step_ticks_including_initialization"] == 120000
assert limits["max_launches"] == 3
assert limits["aggregate_same_boot_window_s"] == 3600
assert limits["max_artifact_bytes"] == 2**30
assert limits["cleanup_reserve_s"] == 120
assert limits["max_control_steps"] == 117000 // 5
def _leaves(value, prefix=()):
if isinstance(value, dict):
for key, item in value.items():
yield from _leaves(item, (*prefix, key))
elif isinstance(value, list):
for index, item in enumerate(value):
yield from _leaves(item, (*prefix, index))
else:
yield prefix, value
@pytest.mark.parametrize("stage", ["P1", "P2"])
def test_evaluation_protocol_every_leaf_is_locked_and_nonfinite_fails_closed(stage):
raw = json.loads(PATHS[stage].read_text())
for path, value in _leaves(raw):
bad_values = [None]
if type(value) in (int, float):
bad_values += [float("nan"), float("inf"), -float("inf"), True, -1, value + 0.001]
for bad in bad_values:
changed = copy.deepcopy(raw)
parent = changed
for key in path[:-1]:
parent = parent[key]
parent[path[-1]] = bad
with pytest.raises(ValueError):
validate_protocol(changed)
@pytest.mark.parametrize("bad", [None, [], {}, {"stage": "P3"}, {"stage": []}])
def test_evaluation_protocol_invalid_root(bad):
with pytest.raises(ValueError):
validate_protocol(bad)
def test_evaluation_protocol_unknown_missing_and_reordered_values_fail(tmp_path):
for mutate in (
lambda d: d.update(unknown=True), lambda d: d.pop("resources"),
lambda d: d["resources"].update(unknown=1), lambda d: d["seeds"].reverse(),
lambda d: d["conditions"].pop(), lambda d: d["clock"]["phase_ticks"].append(1),
lambda d: d["events"][0].update(force_N=[30.0, -4.0, -2.0]),
):
raw = json.loads(PATHS["P2"].read_text())
mutate(raw)
with pytest.raises(ValueError):
validate_protocol(raw)
path = tmp_path / "duplicate.json"
path.write_text('{"stage": "P1", "stage": "P2"}')
with pytest.raises(ValueError, match="Duplicate"):
load_protocol(path)
path.write_text(PATHS["P1"].read_text().replace('"physics_dt_s": 0.002', '"physics_dt_s": NaN'))
with pytest.raises(ValueError):
load_protocol(path)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float64])
def test_evaluation_protocol_first_cycle_matches_frozen_command_every_tick(dtype):
protocol = load_protocol(PATHS["P1"])
time = torch.arange(4501, dtype=dtype) * PHYSICS_DT_S
origin = torch.tensor(A, dtype=dtype).expand(4501, -1)
expected = frozen_command(time, ORIGINAL, origin, torch.full_like(time, 40.0))
actual = [sample(protocol, Clock(t, 0, t)) for t in range(4501)]
assert torch.equal(torch.tensor([c.phase for c in actual]), expected[0])
for field, reference in zip(("progress", "target_world_m", "desired_normal_N", "auxiliary_fraction"), expected[1:]):
torch.testing.assert_close(torch.tensor([getattr(c, field) for c in actual], dtype=dtype), reference,
rtol=1e-5, atol=3e-5 if dtype == torch.float32 else 1e-12)
def test_evaluation_protocol_exact_phase_counts_right_closed_endpoints():
protocol = load_protocol(PATHS["P1"])
counts = [Counter() for _ in range(3)]
for tick in range(1, 13501):
clock = protocol.evidence_clock(0, tick)
assert clock.global_tick == clock.cycle_index * CYCLE_TICKS + clock.local_tick
counts[clock.cycle_index][sample(protocol, clock).phase] += 1
assert [tuple(c[p] for p in range(6)) for c in counts] == [EVIDENCE_PHASE_COUNTS] * 3
for tick, cycle, local in ((4499, 0, 4499), (4500, 0, 4500), (4501, 1, 1),
(9000, 1, 4500), (9001, 2, 1), (13500, 2, 4500)):
assert protocol.evidence_clock(0, tick) == Clock(tick, cycle, local)
for tick, phase in ((1499, 0), (1500, 1), (1999, 1), (2000, 2), (2299, 2), (2300, 3),
(2999, 3), (3000, 4), (3399, 4), (3400, 5), (4500, 5)):
c = sample(protocol, Clock(tick, 0, tick))
assert c.phase == phase
assert protocol.evidence_clock(0, 4500).local_time_s == 9.0
assert protocol.evidence_clock(0, 13500).global_time_s == 27.0
@pytest.mark.parametrize("tick", [4500, 9000])
def test_evaluation_protocol_seam_requires_admission_and_preserves_command_continuity(tick):
protocol = load_protocol(PATHS["P1"])
old = sample(protocol, protocol.evidence_clock(0, tick))
with pytest.raises(ValueError, match="endpoint must pass"):
protocol.next_command_clock(0, tick)
clock = protocol.next_command_clock(0, tick, seam_admitted=True)
new = sample(protocol, clock)
assert old.phase == 5 and old.progress == 1.0
assert new.phase == 0 and new.progress == 0.0
assert old.end_anchor == new.start_anchor
assert old.target_world_m == new.target_world_m
assert old.desired_normal_N == new.desired_normal_N == 40.0
assert old.auxiliary_fraction == new.auxiliary_fraction == 0.0
assert old.target_velocity_world_m_s == new.target_velocity_world_m_s == (0.0, 0.0, 0.0)
assert old.root_target_world_m == new.root_target_world_m == PORT
assert old.root_velocity_world_m_s == new.root_velocity_world_m_s == (0.0, 0.0, 0.0)
assert protocol.evidence_clock(0, tick + 1).local_tick == 1
def test_evaluation_protocol_fixed_anchors_reverse_and_repeat_without_state_or_rng_writes():
protocol = load_protocol(PATHS["P1"])
rng = torch.get_rng_state().clone()
anchors, root = list(A), list(PORT)
args = {"anchor_a_world": anchors, "root_origin_world": root, "weight_N": 40.0}
endpoints = [protocol.command(0, protocol.evidence_clock(0, t), **args) for t in (4500, 9000, 13500)]
assert endpoints[0].target_world_m == endpoints[2].target_world_m
assert endpoints[1].target_world_m == A
assert [c.start_anchor + c.end_anchor for c in endpoints] == ["AB", "BA", "AB"]
assert anchors == list(A) and root == list(PORT)
assert torch.equal(rng, torch.get_rng_state())
assert protocol.next_command_clock(0, 0) == Clock(0, 0, 0)
assert protocol.next_command_clock(0, 13500) is None
assert protocol.next_command_clock(1, 13500) is None
assert load_protocol(PATHS["P2"]).next_command_clock(0, 4500) is None
# Mid-swing reverses direction; it does not accumulate a third positive displacement.
first = sample(protocol, Clock(2650, 0, 2650))
reverse = sample(protocol, Clock(7150, 1, 2650))
assert reverse.target_world_m == pytest.approx(first.target_world_m)
assert reverse.target_velocity_world_m_s[:2] == pytest.approx(
tuple(-v for v in first.target_velocity_world_m_s[:2]))
def test_evaluation_protocol_target_only_is_smooth_and_predeclared():
protocol = load_protocol(PATHS["P2"])
start = sample(protocol, Clock(0, 0, 0), trial=16)
endpoint = sample(protocol, Clock(4500, 0, 4500), trial=16)
assert start.target_world_m == A
assert endpoint.target_world_m == pytest.approx(tuple(x + d for x, d in zip(A, (0.030, 0.025, 0.0))))
assert start.target_velocity_world_m_s == endpoint.target_velocity_world_m_s == (0.0, 0.0, 0.0)
for tick in (2100, 2600, 3200):
before, center, after = [sample(protocol, Clock(t, 0, t), trial=16) for t in (tick - 1, tick, tick + 1)]
derivative = tuple((y - x) / (2 * PHYSICS_DT_S) for x, y in zip(before.target_world_m, after.target_world_m))
assert center.target_velocity_world_m_s == pytest.approx(derivative, abs=2e-6)
def test_evaluation_protocol_world_axis_actor_displacement_is_not_actual_body_ik():
protocol = load_protocol(PATHS["P1"])
cmd = sample(protocol, Clock(2600, 0, 2600))
target = torch.tensor([cmd.target_world_m], dtype=torch.float64)
commanded_port = torch.tensor([PORT], dtype=torch.float64)
actual_root = commanded_port + torch.tensor([[0.01, -0.02, 0.03]], dtype=torch.float64)
q = torch.tensor([[0.0, 0.0, math.sqrt(0.5), math.sqrt(0.5)]], dtype=torch.float64)
actor = torch.tensor([cmd.actor_target_displacement_world_m], dtype=torch.float64)
ik = inverse_rotate(q, target - actual_root)
torch.testing.assert_close(actor, target - commanded_port)
assert not torch.allclose(actor, ik)
torch.testing.assert_close(ik[:, 0], (target - actual_root)[:, 1])
torch.testing.assert_close(ik[:, 1], -(target - actual_root)[:, 0])
@pytest.mark.parametrize("bad", [-1, 0, 13501, True, 1.0, float("nan"), float("inf")])
def test_evaluation_protocol_bad_evidence_ticks(bad):
with pytest.raises(ValueError):
load_protocol(PATHS["P1"]).evidence_clock(0, bad)
@pytest.mark.parametrize("bad", [-1, 1, 4499, 13501, True, 5.0, float("nan")])
def test_evaluation_protocol_bad_control_ticks(bad):
with pytest.raises(ValueError):
load_protocol(PATHS["P1"]).next_command_clock(0, bad)
@pytest.mark.parametrize("clock", [Clock(1, 0, 0), Clock(0, -1, 4500), Clock(13500, 3, 0),
Clock(4501, 0, 4501), Clock(True, 0, 1), Clock(0, 0, 0.0), None])
def test_evaluation_protocol_bad_command_clock(clock):
with pytest.raises(ValueError):
sample(load_protocol(PATHS["P1"]), clock)
@pytest.mark.parametrize("key,value", [("weight_N", 0), ("weight_N", True), ("weight_N", float("inf")),
("weight_N", float("nan")), ("anchor_a_world", [0, 0]),
("anchor_a_world", [0, float("nan"), 0]),
("root_origin_world", [0, float("inf"), 0]),
("root_origin_world", [0, True, 0])])
def test_evaluation_protocol_bad_command_parameters(key, value):
args = {"weight_N": 40.0, "anchor_a_world": A, "root_origin_world": PORT}
args[key] = value
with pytest.raises(ValueError):
load_protocol(PATHS["P1"]).command(0, Clock(0, 0, 0), **args)
@pytest.mark.parametrize("index", [-1, 2, True, 0.0])
def test_evaluation_protocol_bad_trial_index(index):
with pytest.raises(ValueError):
load_protocol(PATHS["P1"]).trial(index)
def test_evaluation_protocol_import_and_load_need_only_stdlib_no_runtime():
path = ROOT / "source/SkillFactory/SkillFactory/legs/contact_cycle_sim/evaluation_protocol.py"
script = f"""
import importlib.util, sys
spec = importlib.util.spec_from_file_location('pure_protocol', {str(path)!r})
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
protocol = module.load_protocol({str(PATHS['P1'])!r})
assert len(protocol.trials) == 2
assert not any(name.split('.')[0] in ('torch', 'isaaclab', 'isaacsim', 'omni') for name in sys.modules)
"""
subprocess.run([sys.executable, "-I", "-S", "-c", script], check=True)