Files
SkillFactory/tests/test_contact_cycle_sim_learning.py
chenlin 9fe9d27865 feat: checkpoint contact-cycle evaluation and RootXY PPO progress
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations.

Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration.

Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
2026-10-15 14:18:17 +08:00

370 lines
17 KiB
Python

"""Synthetic CPU-only 46D learner API tests; never robot-data optimization or Kit."""
import copy
import pytest
import torch
from SkillFactory.legs import standing_ppo
from SkillFactory.legs.contact_cycle.ppo import CycleActorCritic
from SkillFactory.legs.contact_cycle.task import FAILURE_NAMES
from SkillFactory.legs.contact_cycle_sim import ppo
from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic, features
from SkillFactory.legs.policy import compute_gae
MODEL_CONFIG = {"hidden": 16, "initial_std": .1}
CFG = {"horizon": 4, "failure_penalty": 10., "gamma": .99, "gae_lambda": .95,
"epochs": 2, "minibatch_size": 4, "clip_ratio": .2, "value_coefficient": .5,
"max_gradient_norm": 1., "target_kl": .02}
@pytest.fixture(autouse=True)
def cpu_synthetic_seed():
torch.set_num_threads(1)
torch.manual_seed(71)
def learner():
model = SimResidualActorCritic(**MODEL_CONFIG)
return model, torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5)
class ValueModel:
def value(self, x):
# Return a view deliberately: bootstrap must not modify any input.
return x[:, 0]
def terminal_case(dtype=torch.int32):
x = torch.full((4, 46), -100.) # reset observations never supply terminal values
x[3] = 7.
terminal = torch.zeros(3, 46)
terminal[:, 0] = torch.tensor([30., 99., 20.])
terminated = torch.tensor([True, False, False, False])
truncated = torch.tensor([True, True, True, False])
info = {"terminal_ids": torch.tensor([2, 0, 1], dtype=dtype),
"terminal_observation": {"policy": terminal}}
return x, terminated, truncated, info
def test_reexports_are_unchanged_generic_math_and_46d_features():
assert ppo.features is features
assert ppo.ppo_step is standing_ppo.ppo_step
assert ppo.rng_state is standing_ppo.rng_state
assert ppo.restore_rng is standing_ppo.restore_rng
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
def test_bootstrap_timeout_prereset_failure_zero_and_no_alias(dtype):
x, terminated, truncated, info = terminal_case(dtype)
before = x.clone()
actual = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
torch.testing.assert_close(actual, torch.tensor([0., 20., 30., 7.]))
torch.testing.assert_close(x, before)
torch.testing.assert_close(info["terminal_observation"]["policy"][:, 0], torch.tensor([30., 99., 20.]))
# True failures need no terminal observations, even when also truncated.
torch.testing.assert_close(ppo.bootstrap_values(ValueModel(), x, torch.ones(4, dtype=torch.bool),
torch.ones(4, dtype=torch.bool), {}), torch.zeros(4))
@pytest.mark.parametrize("fault", ["missing", "missing_policy", "32d", "count", "duplicate", "float_ids",
"negative_id", "out_of_range", "coverage", "nonterminal_id", "nan"])
def test_bootstrap_rejects_bad_terminal_contract(fault):
x, terminated, truncated, info = terminal_case()
if fault == "missing":
info = {}
elif fault == "missing_policy":
info["terminal_observation"] = {}
elif fault == "32d":
info["terminal_observation"]["policy"] = torch.zeros(3, 32)
elif fault == "count":
info["terminal_observation"]["policy"] = torch.zeros(2, 46)
elif fault == "nan":
info["terminal_observation"]["policy"][0, 0] = float("nan")
else:
info["terminal_ids"] = {
"duplicate": torch.tensor([2, 2, 1]), "float_ids": torch.tensor([2., 0., 1.]),
"negative_id": torch.tensor([2, -1, 1]), "out_of_range": torch.tensor([2, 4, 1]),
"coverage": torch.tensor([2, 0]), "nonterminal_id": torch.tensor([2, 3, 1]),
}[fault]
with pytest.raises((ValueError, RuntimeError)):
ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
@pytest.mark.parametrize("bad", [torch.zeros(2, 32), torch.zeros(2, 46, dtype=torch.float64),
torch.full((2, 46), float("nan")), torch.zeros(0, 46)])
def test_features_reject_old_or_malformed_data(bad):
with pytest.raises(ValueError):
ppo.features(bad)
def test_bootstrap_shapes_types_and_nonfinite_value():
x, terminated, truncated, info = terminal_case()
for bad in (terminated.float(), terminated[:, None], terminated[:1]):
with pytest.raises(ValueError):
ppo.bootstrap_values(ValueModel(), x, bad, truncated, info)
class BadValue:
def value(self, x):
return torch.full((len(x),), float("nan"))
with pytest.raises(ValueError):
ppo.bootstrap_values(BadValue(), x, terminated, truncated, info)
def test_gae_failure_and_timeout_end_recursion_but_timeout_bootstraps():
x, terminated, truncated, info = terminal_case()
nv = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
rewards = torch.tensor([[1., 1., 1., 1.], [100., 100., 100., 100.]])
values = torch.zeros_like(rewards)
next_values = torch.stack([nv, torch.zeros(4)])
done = torch.stack([terminated | truncated, torch.ones(4, dtype=torch.bool)])
adv, returns = compute_gae(rewards, values, next_values, done, .9, .5)
torch.testing.assert_close(adv[0], torch.tensor([1., 19., 28., 52.3]))
torch.testing.assert_close(returns, adv)
class ReusedBufferEnv:
num_envs, device, step_dt = 2, "cpu", .01
def __init__(self, fault=None):
self.x = torch.zeros(2, 46)
self.phase = torch.zeros(2, dtype=torch.int32)
self.flags = torch.zeros(2, len(FAILURE_NAMES), dtype=torch.bool)
self.steps, self.fault = 0, fault
def step(self, action):
assert not torch.is_grad_enabled()
assert action.shape == (2, 3) and (action.abs() <= 1).all()
self.x.fill_(self.steps + 1.)
self.phase.fill_(self.steps % 6) # phase of pre-step state, not next observation
self.flags.zero_()
self.flags[0, 0] = True
terminal = torch.full((2, 46), 10. + self.steps)
self.steps += 1
info = {"terminal_ids": torch.tensor([0, 1], dtype=torch.int32),
"terminal_observation": {"policy": terminal},
"cycle_step": {"phase": self.phase, "failure_flags": self.flags}}
reward = torch.full((2,), .01)
if self.fault == "missing":
del info["cycle_step"]
elif self.fault == "phase":
info["cycle_step"]["phase"] = torch.full((2,), -1)
elif self.fault == "flags":
info["cycle_step"]["failure_flags"] = torch.zeros(2, 7, dtype=torch.bool)
elif self.fault == "reward":
reward[0] = float("inf")
elif self.fault == "reward_shape":
reward = reward[:, None]
return {"policy": self.x}, reward, torch.tensor([True, False]), torch.tensor([True, True]), info
def test_collect_preserves_prestep_buffers_diagnostics_logp_and_reward():
model, _ = learner()
env = ReusedBufferEnv()
calls = []
data, nx, failures, timeouts = ppo.collect_rollout(env, model, env.x, CFG, lambda: calls.append(True))
assert len(calls) == CFG["horizon"] and failures == timeouts == 4
assert data["features"].shape == (4, 2, 46) and data["latent"].shape == (4, 2, 3)
for step in range(4):
torch.testing.assert_close(data["features"][step], torch.full((2, 46), float(step)))
assert (data["phase"][step] == step).all()
logp, values = model.evaluate(data["features"].flatten(0, 1), data["latent"].flatten(0, 1))
torch.testing.assert_close(logp, data["log_prob"].flatten())
torch.testing.assert_close(values, data["value"].flatten())
torch.testing.assert_close(data["reward"], torch.tensor([[-9., 1.]]).expand(4, 2))
assert data["done"].all() and not data["next_value"][:, 0].any()
for step in range(4):
torch.testing.assert_close(data["next_value"][step, 1], model.value(torch.full((1, 46), 10. + step))[0])
assert all(not v.requires_grad for v in data.values())
env.x.fill_(100.)
env.phase.fill_(5)
env.flags.zero_()
assert (nx == 4).all() and (data["phase"][0] == 0).all()
assert data["failure_flags"][:, 0, 0].all()
@pytest.mark.parametrize("fault", ["missing", "phase", "flags", "reward", "reward_shape"])
def test_collect_missing_or_malformed_diagnostics_fails_before_optimizer(fault):
model, optimizer = learner()
before = copy.deepcopy(model.state_dict())
env = ReusedBufferEnv(fault)
with pytest.raises((ValueError, RuntimeError)):
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
ppo.ppo_step(model, optimizer, data, CFG)
assert not optimizer.state
for key, value in before.items():
torch.testing.assert_close(value, model.state_dict()[key], rtol=0, atol=0)
def test_synthetic_generic_ppo_and_immutable_pre_normalization_diagnostics():
model, optimizer = learner()
env = ReusedBufferEnv()
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
before_data = copy.deepcopy(data)
before_model = copy.deepcopy(model.state_dict())
diagnostics = ppo.rollout_diagnostics(data, CFG)
assert diagnostics["missing_diagnostic_samples"] == 0
assert diagnostics["phase_control_samples"] == [2, 2, 2, 2, 0, 0]
assert diagnostics["terminal_causes"][FAILURE_NAMES[0]] == 4
adv, _ = compute_gae(data["reward"], data["value"], data["next_value"], data["done"], .99, .95)
assert diagnostics["advantage_before_normalization"]["mean"] == pytest.approx(float(adv.mean()))
metrics = ppo.ppo_step(model, optimizer, data, CFG)
assert metrics["sampling_log_prob_error"] < 2e-4 and metrics["minibatches"] > 0
assert any(not torch.equal(v, model.state_dict()[k]) for k, v in before_model.items())
assert ppo.rollout_diagnostics(data, CFG) == diagnostics
for key in data:
torch.testing.assert_close(data[key], before_data[key], rtol=0, atol=0)
# The generic learner itself rejects stale/corrupted sampling log probabilities.
data["log_prob"].add_(10.)
with pytest.raises(RuntimeError, match="mismatch"):
ppo.ppo_step(model, optimizer, data, CFG)
def test_selection_key_lexicographic_all_rows_and_failure_cost():
good = {"success": True, "height_rms_m": .06, "swing_rms_m": .025, "landing_error_m": .05}
assert ppo.selection_key([good]) == (0., 3.)
assert ppo.selection_key([good, {"success": False}]) == (.5, 9.5)
assert ppo.selection_key([{}]) == (1., 16.)
worst_success = {**good, "height_rms_m": 100., "swing_rms_m": 100., "landing_error_m": 100.}
assert ppo.selection_key([worst_success]) < ppo.selection_key([good, {}])
for bad in ([], [{"success": True}], [{**good, "height_rms_m": float("nan")}],
[{**good, "landing_error_m": -1.}], [{**good, "success": "false"}]):
with pytest.raises(ValueError):
ppo.selection_key(bad)
def saved_learner(tmp_path):
model, optimizer = learner()
env = ReusedBufferEnv()
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
ppo.ppo_step(model, optimizer, data, CFG)
path = tmp_path / "residual.pt"
identity = {"source_sha256": "synthetic-source", "task": ppo.CONTRACT}
size = ppo.save_checkpoint(path, model, optimizer, identity, 1, {"update": 1, "key": [0., 3.]},
{"reserved_transitions": 8}, MODEL_CONFIG)
return path, model, optimizer, identity, size
def assert_tree_equal(a, b):
if isinstance(a, torch.Tensor):
torch.testing.assert_close(a, b, rtol=0, atol=0)
elif isinstance(a, dict):
assert a.keys() == b.keys()
for key in a:
assert_tree_equal(a[key], b[key])
elif isinstance(a, (tuple, list)):
assert type(a) is type(b) and len(a) == len(b)
for av, bv in zip(a, b):
assert_tree_equal(av, bv)
else:
assert a == b
def test_checkpoint_distinct_atomic_exact_bytes_rng_and_optimizer_roundtrip(tmp_path):
path, model, optimizer, identity, size = saved_learner(tmp_path)
assert size == path.stat().st_size and list(tmp_path.iterdir()) == [path]
payload = torch.load(path, weights_only=True)
assert payload["format"] == ppo.CHECKPOINT_FORMAT
assert payload["contract"] == ppo.CONTRACT and payload["features"] == model.feature_version
assert payload["accepted_skill"] is False and "sampler" not in payload
assert payload["model_config"] == MODEL_CONFIG and payload["budget_snapshot"] == {"reserved_transitions": 8}
expected_random = torch.rand(12)
with torch.no_grad():
model.actor[-1].bias.add_(1.)
optimizer.param_groups[0]["lr"] = .005
restored = ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG)
assert restored["update"] == 1
assert_tree_equal(model.state_dict(), payload["model"])
assert_tree_equal(optimizer.state_dict(), payload["optimizer"])
torch.testing.assert_close(torch.rand(12), expected_random, rtol=0, atol=0)
original = path.read_bytes()
with pytest.raises(RuntimeError, match="budget"):
ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG, max_bytes=1)
assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path]
@pytest.mark.parametrize("fault", ["format", "contract", "features", "identity", "hidden", "config_type",
"update", "accepted", "model_shape", "model_dtype", "model_nan", "scale",
"optimizer_nan", "optimizer_shape", "optimizer_type", "optimizer_lr_type",
"optimizer_slot_type", "optimizer_missing_slot", "optimizer_step", "rng", "budget", "sampler"])
def test_checkpoint_rejects_before_mutating_any_learner_or_rng(tmp_path, fault):
path, model, optimizer, identity, _ = saved_learner(tmp_path)
payload = torch.load(path, weights_only=True)
if fault in ("format", "contract", "features"):
payload[fault] = "legacy-32D-standing-or-reach"
elif fault == "identity":
payload["identity"]["source_sha256"] = "different-source"
elif fault == "hidden":
payload["model_config"]["hidden"] = 32
elif fault == "config_type":
payload["model_config"]["hidden"] = 16.
elif fault == "update":
payload["update"] = True
elif fault == "accepted":
payload["accepted_skill"] = True
elif fault == "model_shape":
payload["model"]["actor.0.weight"] = torch.zeros(16, 32)
elif fault == "model_dtype":
payload["model"]["log_std"] = payload["model"]["log_std"].double()
elif fault == "model_nan":
payload["model"]["log_std"][0] = float("nan")
elif fault == "scale":
payload["model"]["scale"][0] = 0.
elif fault == "optimizer_nan":
payload["optimizer"]["param_groups"][0]["lr"] = float("nan")
elif fault == "optimizer_shape":
payload["optimizer"]["state"][0]["exp_avg"] = torch.zeros(32)
elif fault == "optimizer_type":
payload["optimizer_type"] = "SGD"
elif fault == "optimizer_lr_type":
payload["optimizer"]["param_groups"][0]["lr"] = "invalid"
elif fault == "optimizer_slot_type":
payload["optimizer"]["state"][0]["exp_avg"] = 0.
elif fault == "optimizer_missing_slot":
del payload["optimizer"]["state"][0]["exp_avg"]
elif fault == "optimizer_step":
payload["optimizer"]["state"][0]["step"] = torch.tensor(.5)
elif fault == "rng":
payload["rng"]["cpu"] = torch.zeros(2, dtype=torch.uint8)
elif fault == "budget":
payload["budget_snapshot"] = None
elif fault == "sampler":
payload["sampler"] = {}
torch.save(payload, path)
before = copy.deepcopy((model.state_dict(), optimizer.state_dict(), ppo.rng_state()))
with pytest.raises(ValueError):
ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG)
assert_tree_equal(before, (model.state_dict(), optimizer.state_dict(), ppo.rng_state()))
def test_save_rejects_wrong_model_configuration_type_and_nonfinite_state(tmp_path):
model, optimizer = learner()
old = CycleActorCritic(hidden=16)
cases = [(old, torch.optim.Adam(old.parameters()), MODEL_CONFIG),
(model, optimizer, {"hidden": 32, "initial_std": .1}),
(model, optimizer, {"hidden": 16, "initial_std": True}),
(model, torch.optim.Adam(old.parameters()), MODEL_CONFIG)]
for candidate, opt, cfg in cases:
with pytest.raises(ValueError):
ppo.save_checkpoint(tmp_path / "bad.pt", candidate, opt, {}, 0, None, {}, cfg)
with torch.no_grad():
model.log_std[0] = float("inf")
with pytest.raises(ValueError):
ppo.save_checkpoint(tmp_path / "bad.pt", model, optimizer, {}, 0, None, {}, MODEL_CONFIG)
assert not list(tmp_path.iterdir())
def test_atomic_replace_failure_keeps_destination_and_removes_temporary(tmp_path, monkeypatch):
path, model, optimizer, identity, _ = saved_learner(tmp_path)
original = path.read_bytes()
def refuse(source, destination):
assert source.exists() and destination == path
raise OSError("synthetic replace failure")
monkeypatch.setattr(ppo.os, "replace", refuse)
with pytest.raises(OSError, match="synthetic"):
ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG)
assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path]