9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
303 lines
15 KiB
Python
303 lines
15 KiB
Python
"""CPU-only protocol tests: these are not simulator or development-gate evidence."""
|
|
|
|
import copy
|
|
import dataclasses
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import torch
|
|
from SkillFactory.legs.contact_cycle.mechanics import inverse_rotate
|
|
from SkillFactory.legs.contact_cycle.task import command as frozen_command
|
|
from SkillFactory.legs.contact_cycle_sim.evaluation_protocol import (
|
|
CYCLE_TICKS,
|
|
EVIDENCE_PHASE_COUNTS,
|
|
PHYSICS_DT_S,
|
|
Clock,
|
|
load_protocol,
|
|
validate_protocol,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PATHS = {"P1": ROOT / "configs/contact_cycle/continuous_eval_p1_v1.json",
|
|
"P2": ROOT / "configs/contact_cycle/bounded_diagnostic_p2_v1.json"}
|
|
ORIGINAL = json.loads((ROOT / "configs/contact_cycle/go2_fl_v1.json").read_text())
|
|
A = (0.12, -0.07, 0.022)
|
|
PORT = (0.02, 0.03, 0.30)
|
|
|
|
|
|
def sample(protocol, clock, trial=0, **kwargs):
|
|
return protocol.command(trial, clock, anchor_a_world=A, root_origin_world=PORT, weight_N=40.0, **kwargs)
|
|
|
|
|
|
@pytest.mark.parametrize("stage,rows,ticks", [("P1", 2, 13500), ("P2", 20, 4500)])
|
|
def test_evaluation_protocol_schema_and_immutable_trials(stage, rows, ticks):
|
|
raw = json.loads(PATHS[stage].read_text())
|
|
protocol = validate_protocol(raw)
|
|
assert len(protocol.trials) == rows
|
|
assert all(row.total_ticks == ticks for row in protocol.trials)
|
|
assert protocol.document["execution_authorized"] is False
|
|
assert protocol.document["accepted_skill"] is False
|
|
assert protocol.document["optimizer_steps"] == 0
|
|
raw["clock"]["cycle_ticks"] = 7
|
|
assert protocol.document["clock"]["cycle_ticks"] == 4500
|
|
with pytest.raises(TypeError):
|
|
protocol.document["clock"]["cycle_ticks"] = 7
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
protocol.trials[0].seed = 0
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
protocol.trials = ()
|
|
|
|
|
|
def test_evaluation_protocol_exact_preregistered_order_and_pairing():
|
|
p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"])
|
|
controllers = ("learned_policy", "analytic_reference")
|
|
conditions = ("nominal", "oblique_sustained", "eccentric_liftoff", "landing_combination", "target_only")
|
|
assert [(r.seed, r.controller_kind) for r in p1.trials] == [(93501, c) for c in controllers]
|
|
assert [(r.condition_id, r.seed, r.controller_kind) for r in p2.trials] == [
|
|
(condition, seed, controller)
|
|
for condition in conditions for seed in (94501, 94502) for controller in controllers
|
|
]
|
|
assert len({r.trial_id for r in p2.trials}) == 20
|
|
for learned, reference in zip(p2.trials[::2], p2.trials[1::2]):
|
|
assert learned.pair_id == reference.pair_id
|
|
assert dataclasses.replace(learned, index=reference.index, trial_id=reference.trial_id,
|
|
controller_kind=reference.controller_kind) == reference
|
|
assert [p2.trials[i].event_indices for i in (0, 4, 8, 12, 16)] == [(), (0,), (1,), (2, 3), ()]
|
|
assert p2.trials[16].swing_displacement_m == (0.030, 0.025, 0.0)
|
|
assert p1.document["events"] == ()
|
|
assert json.loads(json.dumps(ORIGINAL["events"])) == json.loads(PATHS["P2"].read_text())["events"]
|
|
|
|
|
|
def test_evaluation_protocol_frozen_threshold_provenance_and_budget():
|
|
p1, p2 = load_protocol(PATHS["P1"]), load_protocol(PATHS["P2"])
|
|
functional = json.loads((ROOT / "configs/contact_cycle/simulation_functional_v1.json").read_text())
|
|
assert p1.document["thresholds"]["functional"] == functional["gates"]
|
|
assert p2.document["thresholds"]["P1_margins_role"] == "diagnostic_only"
|
|
assert p1.document["thresholds"]["P1_margins_role"] == "gate"
|
|
assert p1.document["thresholds"]["P1_margins"] == p2.document["thresholds"]["P1_margins"]
|
|
assert p2.document["requires_P1_evidence_review"] is True
|
|
limits = p1.document["resources"]
|
|
assert limits == p2.document["resources"]
|
|
assert sum(r.total_ticks for r in (*p1.trials, *p2.trials)) == limits["max_rollout_ticks"] == 117000
|
|
assert limits["max_rollout_ticks"] + limits["max_initialization_reset_ticks"] == 120000
|
|
assert limits["max_sim_step_ticks_including_initialization"] == 120000
|
|
assert limits["max_launches"] == 3
|
|
assert limits["aggregate_same_boot_window_s"] == 3600
|
|
assert limits["max_artifact_bytes"] == 2**30
|
|
assert limits["cleanup_reserve_s"] == 120
|
|
assert limits["max_control_steps"] == 117000 // 5
|
|
|
|
|
|
def _leaves(value, prefix=()):
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
yield from _leaves(item, (*prefix, key))
|
|
elif isinstance(value, list):
|
|
for index, item in enumerate(value):
|
|
yield from _leaves(item, (*prefix, index))
|
|
else:
|
|
yield prefix, value
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["P1", "P2"])
|
|
def test_evaluation_protocol_every_leaf_is_locked_and_nonfinite_fails_closed(stage):
|
|
raw = json.loads(PATHS[stage].read_text())
|
|
for path, value in _leaves(raw):
|
|
bad_values = [None]
|
|
if type(value) in (int, float):
|
|
bad_values += [float("nan"), float("inf"), -float("inf"), True, -1, value + 0.001]
|
|
for bad in bad_values:
|
|
changed = copy.deepcopy(raw)
|
|
parent = changed
|
|
for key in path[:-1]:
|
|
parent = parent[key]
|
|
parent[path[-1]] = bad
|
|
with pytest.raises(ValueError):
|
|
validate_protocol(changed)
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [None, [], {}, {"stage": "P3"}, {"stage": []}])
|
|
def test_evaluation_protocol_invalid_root(bad):
|
|
with pytest.raises(ValueError):
|
|
validate_protocol(bad)
|
|
|
|
|
|
def test_evaluation_protocol_unknown_missing_and_reordered_values_fail(tmp_path):
|
|
for mutate in (
|
|
lambda d: d.update(unknown=True), lambda d: d.pop("resources"),
|
|
lambda d: d["resources"].update(unknown=1), lambda d: d["seeds"].reverse(),
|
|
lambda d: d["conditions"].pop(), lambda d: d["clock"]["phase_ticks"].append(1),
|
|
lambda d: d["events"][0].update(force_N=[30.0, -4.0, -2.0]),
|
|
):
|
|
raw = json.loads(PATHS["P2"].read_text())
|
|
mutate(raw)
|
|
with pytest.raises(ValueError):
|
|
validate_protocol(raw)
|
|
path = tmp_path / "duplicate.json"
|
|
path.write_text('{"stage": "P1", "stage": "P2"}')
|
|
with pytest.raises(ValueError, match="Duplicate"):
|
|
load_protocol(path)
|
|
path.write_text(PATHS["P1"].read_text().replace('"physics_dt_s": 0.002', '"physics_dt_s": NaN'))
|
|
with pytest.raises(ValueError):
|
|
load_protocol(path)
|
|
|
|
|
|
@pytest.mark.parametrize("dtype", [torch.float32, torch.float64])
|
|
def test_evaluation_protocol_first_cycle_matches_frozen_command_every_tick(dtype):
|
|
protocol = load_protocol(PATHS["P1"])
|
|
time = torch.arange(4501, dtype=dtype) * PHYSICS_DT_S
|
|
origin = torch.tensor(A, dtype=dtype).expand(4501, -1)
|
|
expected = frozen_command(time, ORIGINAL, origin, torch.full_like(time, 40.0))
|
|
actual = [sample(protocol, Clock(t, 0, t)) for t in range(4501)]
|
|
assert torch.equal(torch.tensor([c.phase for c in actual]), expected[0])
|
|
for field, reference in zip(("progress", "target_world_m", "desired_normal_N", "auxiliary_fraction"), expected[1:]):
|
|
torch.testing.assert_close(torch.tensor([getattr(c, field) for c in actual], dtype=dtype), reference,
|
|
rtol=1e-5, atol=3e-5 if dtype == torch.float32 else 1e-12)
|
|
|
|
|
|
def test_evaluation_protocol_exact_phase_counts_right_closed_endpoints():
|
|
protocol = load_protocol(PATHS["P1"])
|
|
counts = [Counter() for _ in range(3)]
|
|
for tick in range(1, 13501):
|
|
clock = protocol.evidence_clock(0, tick)
|
|
assert clock.global_tick == clock.cycle_index * CYCLE_TICKS + clock.local_tick
|
|
counts[clock.cycle_index][sample(protocol, clock).phase] += 1
|
|
assert [tuple(c[p] for p in range(6)) for c in counts] == [EVIDENCE_PHASE_COUNTS] * 3
|
|
for tick, cycle, local in ((4499, 0, 4499), (4500, 0, 4500), (4501, 1, 1),
|
|
(9000, 1, 4500), (9001, 2, 1), (13500, 2, 4500)):
|
|
assert protocol.evidence_clock(0, tick) == Clock(tick, cycle, local)
|
|
for tick, phase in ((1499, 0), (1500, 1), (1999, 1), (2000, 2), (2299, 2), (2300, 3),
|
|
(2999, 3), (3000, 4), (3399, 4), (3400, 5), (4500, 5)):
|
|
c = sample(protocol, Clock(tick, 0, tick))
|
|
assert c.phase == phase
|
|
assert protocol.evidence_clock(0, 4500).local_time_s == 9.0
|
|
assert protocol.evidence_clock(0, 13500).global_time_s == 27.0
|
|
|
|
|
|
@pytest.mark.parametrize("tick", [4500, 9000])
|
|
def test_evaluation_protocol_seam_requires_admission_and_preserves_command_continuity(tick):
|
|
protocol = load_protocol(PATHS["P1"])
|
|
old = sample(protocol, protocol.evidence_clock(0, tick))
|
|
with pytest.raises(ValueError, match="endpoint must pass"):
|
|
protocol.next_command_clock(0, tick)
|
|
clock = protocol.next_command_clock(0, tick, seam_admitted=True)
|
|
new = sample(protocol, clock)
|
|
assert old.phase == 5 and old.progress == 1.0
|
|
assert new.phase == 0 and new.progress == 0.0
|
|
assert old.end_anchor == new.start_anchor
|
|
assert old.target_world_m == new.target_world_m
|
|
assert old.desired_normal_N == new.desired_normal_N == 40.0
|
|
assert old.auxiliary_fraction == new.auxiliary_fraction == 0.0
|
|
assert old.target_velocity_world_m_s == new.target_velocity_world_m_s == (0.0, 0.0, 0.0)
|
|
assert old.root_target_world_m == new.root_target_world_m == PORT
|
|
assert old.root_velocity_world_m_s == new.root_velocity_world_m_s == (0.0, 0.0, 0.0)
|
|
assert protocol.evidence_clock(0, tick + 1).local_tick == 1
|
|
|
|
|
|
def test_evaluation_protocol_fixed_anchors_reverse_and_repeat_without_state_or_rng_writes():
|
|
protocol = load_protocol(PATHS["P1"])
|
|
rng = torch.get_rng_state().clone()
|
|
anchors, root = list(A), list(PORT)
|
|
args = {"anchor_a_world": anchors, "root_origin_world": root, "weight_N": 40.0}
|
|
endpoints = [protocol.command(0, protocol.evidence_clock(0, t), **args) for t in (4500, 9000, 13500)]
|
|
assert endpoints[0].target_world_m == endpoints[2].target_world_m
|
|
assert endpoints[1].target_world_m == A
|
|
assert [c.start_anchor + c.end_anchor for c in endpoints] == ["AB", "BA", "AB"]
|
|
assert anchors == list(A) and root == list(PORT)
|
|
assert torch.equal(rng, torch.get_rng_state())
|
|
assert protocol.next_command_clock(0, 0) == Clock(0, 0, 0)
|
|
assert protocol.next_command_clock(0, 13500) is None
|
|
assert protocol.next_command_clock(1, 13500) is None
|
|
assert load_protocol(PATHS["P2"]).next_command_clock(0, 4500) is None
|
|
# Mid-swing reverses direction; it does not accumulate a third positive displacement.
|
|
first = sample(protocol, Clock(2650, 0, 2650))
|
|
reverse = sample(protocol, Clock(7150, 1, 2650))
|
|
assert reverse.target_world_m == pytest.approx(first.target_world_m)
|
|
assert reverse.target_velocity_world_m_s[:2] == pytest.approx(
|
|
tuple(-v for v in first.target_velocity_world_m_s[:2]))
|
|
|
|
|
|
def test_evaluation_protocol_target_only_is_smooth_and_predeclared():
|
|
protocol = load_protocol(PATHS["P2"])
|
|
start = sample(protocol, Clock(0, 0, 0), trial=16)
|
|
endpoint = sample(protocol, Clock(4500, 0, 4500), trial=16)
|
|
assert start.target_world_m == A
|
|
assert endpoint.target_world_m == pytest.approx(tuple(x + d for x, d in zip(A, (0.030, 0.025, 0.0))))
|
|
assert start.target_velocity_world_m_s == endpoint.target_velocity_world_m_s == (0.0, 0.0, 0.0)
|
|
for tick in (2100, 2600, 3200):
|
|
before, center, after = [sample(protocol, Clock(t, 0, t), trial=16) for t in (tick - 1, tick, tick + 1)]
|
|
derivative = tuple((y - x) / (2 * PHYSICS_DT_S) for x, y in zip(before.target_world_m, after.target_world_m))
|
|
assert center.target_velocity_world_m_s == pytest.approx(derivative, abs=2e-6)
|
|
|
|
|
|
def test_evaluation_protocol_world_axis_actor_displacement_is_not_actual_body_ik():
|
|
protocol = load_protocol(PATHS["P1"])
|
|
cmd = sample(protocol, Clock(2600, 0, 2600))
|
|
target = torch.tensor([cmd.target_world_m], dtype=torch.float64)
|
|
commanded_port = torch.tensor([PORT], dtype=torch.float64)
|
|
actual_root = commanded_port + torch.tensor([[0.01, -0.02, 0.03]], dtype=torch.float64)
|
|
q = torch.tensor([[0.0, 0.0, math.sqrt(0.5), math.sqrt(0.5)]], dtype=torch.float64)
|
|
actor = torch.tensor([cmd.actor_target_displacement_world_m], dtype=torch.float64)
|
|
ik = inverse_rotate(q, target - actual_root)
|
|
torch.testing.assert_close(actor, target - commanded_port)
|
|
assert not torch.allclose(actor, ik)
|
|
torch.testing.assert_close(ik[:, 0], (target - actual_root)[:, 1])
|
|
torch.testing.assert_close(ik[:, 1], -(target - actual_root)[:, 0])
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [-1, 0, 13501, True, 1.0, float("nan"), float("inf")])
|
|
def test_evaluation_protocol_bad_evidence_ticks(bad):
|
|
with pytest.raises(ValueError):
|
|
load_protocol(PATHS["P1"]).evidence_clock(0, bad)
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [-1, 1, 4499, 13501, True, 5.0, float("nan")])
|
|
def test_evaluation_protocol_bad_control_ticks(bad):
|
|
with pytest.raises(ValueError):
|
|
load_protocol(PATHS["P1"]).next_command_clock(0, bad)
|
|
|
|
|
|
@pytest.mark.parametrize("clock", [Clock(1, 0, 0), Clock(0, -1, 4500), Clock(13500, 3, 0),
|
|
Clock(4501, 0, 4501), Clock(True, 0, 1), Clock(0, 0, 0.0), None])
|
|
def test_evaluation_protocol_bad_command_clock(clock):
|
|
with pytest.raises(ValueError):
|
|
sample(load_protocol(PATHS["P1"]), clock)
|
|
|
|
|
|
@pytest.mark.parametrize("key,value", [("weight_N", 0), ("weight_N", True), ("weight_N", float("inf")),
|
|
("weight_N", float("nan")), ("anchor_a_world", [0, 0]),
|
|
("anchor_a_world", [0, float("nan"), 0]),
|
|
("root_origin_world", [0, float("inf"), 0]),
|
|
("root_origin_world", [0, True, 0])])
|
|
def test_evaluation_protocol_bad_command_parameters(key, value):
|
|
args = {"weight_N": 40.0, "anchor_a_world": A, "root_origin_world": PORT}
|
|
args[key] = value
|
|
with pytest.raises(ValueError):
|
|
load_protocol(PATHS["P1"]).command(0, Clock(0, 0, 0), **args)
|
|
|
|
|
|
@pytest.mark.parametrize("index", [-1, 2, True, 0.0])
|
|
def test_evaluation_protocol_bad_trial_index(index):
|
|
with pytest.raises(ValueError):
|
|
load_protocol(PATHS["P1"]).trial(index)
|
|
|
|
|
|
def test_evaluation_protocol_import_and_load_need_only_stdlib_no_runtime():
|
|
path = ROOT / "source/SkillFactory/SkillFactory/legs/contact_cycle_sim/evaluation_protocol.py"
|
|
script = f"""
|
|
import importlib.util, sys
|
|
spec = importlib.util.spec_from_file_location('pure_protocol', {str(path)!r})
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
protocol = module.load_protocol({str(PATHS['P1'])!r})
|
|
assert len(protocol.trials) == 2
|
|
assert not any(name.split('.')[0] in ('torch', 'isaaclab', 'isaacsim', 'omni') for name in sys.modules)
|
|
"""
|
|
subprocess.run([sys.executable, "-I", "-S", "-c", script], check=True)
|