"""Synthetic CPU-only 46D learner API tests; never robot-data optimization or Kit.""" import copy import pytest import torch from SkillFactory.legs import standing_ppo from SkillFactory.legs.contact_cycle.ppo import CycleActorCritic from SkillFactory.legs.contact_cycle.task import FAILURE_NAMES from SkillFactory.legs.contact_cycle_sim import ppo from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic, features from SkillFactory.legs.policy import compute_gae MODEL_CONFIG = {"hidden": 16, "initial_std": .1} CFG = {"horizon": 4, "failure_penalty": 10., "gamma": .99, "gae_lambda": .95, "epochs": 2, "minibatch_size": 4, "clip_ratio": .2, "value_coefficient": .5, "max_gradient_norm": 1., "target_kl": .02} @pytest.fixture(autouse=True) def cpu_synthetic_seed(): torch.set_num_threads(1) torch.manual_seed(71) def learner(): model = SimResidualActorCritic(**MODEL_CONFIG) return model, torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5) class ValueModel: def value(self, x): # Return a view deliberately: bootstrap must not modify any input. return x[:, 0] def terminal_case(dtype=torch.int32): x = torch.full((4, 46), -100.) # reset observations never supply terminal values x[3] = 7. terminal = torch.zeros(3, 46) terminal[:, 0] = torch.tensor([30., 99., 20.]) terminated = torch.tensor([True, False, False, False]) truncated = torch.tensor([True, True, True, False]) info = {"terminal_ids": torch.tensor([2, 0, 1], dtype=dtype), "terminal_observation": {"policy": terminal}} return x, terminated, truncated, info def test_reexports_are_unchanged_generic_math_and_46d_features(): assert ppo.features is features assert ppo.ppo_step is standing_ppo.ppo_step assert ppo.rng_state is standing_ppo.rng_state assert ppo.restore_rng is standing_ppo.restore_rng @pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) def test_bootstrap_timeout_prereset_failure_zero_and_no_alias(dtype): x, terminated, truncated, info = terminal_case(dtype) before = x.clone() actual = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info) torch.testing.assert_close(actual, torch.tensor([0., 20., 30., 7.])) torch.testing.assert_close(x, before) torch.testing.assert_close(info["terminal_observation"]["policy"][:, 0], torch.tensor([30., 99., 20.])) # True failures need no terminal observations, even when also truncated. torch.testing.assert_close(ppo.bootstrap_values(ValueModel(), x, torch.ones(4, dtype=torch.bool), torch.ones(4, dtype=torch.bool), {}), torch.zeros(4)) @pytest.mark.parametrize("fault", ["missing", "missing_policy", "32d", "count", "duplicate", "float_ids", "negative_id", "out_of_range", "coverage", "nonterminal_id", "nan"]) def test_bootstrap_rejects_bad_terminal_contract(fault): x, terminated, truncated, info = terminal_case() if fault == "missing": info = {} elif fault == "missing_policy": info["terminal_observation"] = {} elif fault == "32d": info["terminal_observation"]["policy"] = torch.zeros(3, 32) elif fault == "count": info["terminal_observation"]["policy"] = torch.zeros(2, 46) elif fault == "nan": info["terminal_observation"]["policy"][0, 0] = float("nan") else: info["terminal_ids"] = { "duplicate": torch.tensor([2, 2, 1]), "float_ids": torch.tensor([2., 0., 1.]), "negative_id": torch.tensor([2, -1, 1]), "out_of_range": torch.tensor([2, 4, 1]), "coverage": torch.tensor([2, 0]), "nonterminal_id": torch.tensor([2, 3, 1]), }[fault] with pytest.raises((ValueError, RuntimeError)): ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info) @pytest.mark.parametrize("bad", [torch.zeros(2, 32), torch.zeros(2, 46, dtype=torch.float64), torch.full((2, 46), float("nan")), torch.zeros(0, 46)]) def test_features_reject_old_or_malformed_data(bad): with pytest.raises(ValueError): ppo.features(bad) def test_bootstrap_shapes_types_and_nonfinite_value(): x, terminated, truncated, info = terminal_case() for bad in (terminated.float(), terminated[:, None], terminated[:1]): with pytest.raises(ValueError): ppo.bootstrap_values(ValueModel(), x, bad, truncated, info) class BadValue: def value(self, x): return torch.full((len(x),), float("nan")) with pytest.raises(ValueError): ppo.bootstrap_values(BadValue(), x, terminated, truncated, info) def test_gae_failure_and_timeout_end_recursion_but_timeout_bootstraps(): x, terminated, truncated, info = terminal_case() nv = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info) rewards = torch.tensor([[1., 1., 1., 1.], [100., 100., 100., 100.]]) values = torch.zeros_like(rewards) next_values = torch.stack([nv, torch.zeros(4)]) done = torch.stack([terminated | truncated, torch.ones(4, dtype=torch.bool)]) adv, returns = compute_gae(rewards, values, next_values, done, .9, .5) torch.testing.assert_close(adv[0], torch.tensor([1., 19., 28., 52.3])) torch.testing.assert_close(returns, adv) class ReusedBufferEnv: num_envs, device, step_dt = 2, "cpu", .01 def __init__(self, fault=None): self.x = torch.zeros(2, 46) self.phase = torch.zeros(2, dtype=torch.int32) self.flags = torch.zeros(2, len(FAILURE_NAMES), dtype=torch.bool) self.steps, self.fault = 0, fault def step(self, action): assert not torch.is_grad_enabled() assert action.shape == (2, 3) and (action.abs() <= 1).all() self.x.fill_(self.steps + 1.) self.phase.fill_(self.steps % 6) # phase of pre-step state, not next observation self.flags.zero_() self.flags[0, 0] = True terminal = torch.full((2, 46), 10. + self.steps) self.steps += 1 info = {"terminal_ids": torch.tensor([0, 1], dtype=torch.int32), "terminal_observation": {"policy": terminal}, "cycle_step": {"phase": self.phase, "failure_flags": self.flags}} reward = torch.full((2,), .01) if self.fault == "missing": del info["cycle_step"] elif self.fault == "phase": info["cycle_step"]["phase"] = torch.full((2,), -1) elif self.fault == "flags": info["cycle_step"]["failure_flags"] = torch.zeros(2, 7, dtype=torch.bool) elif self.fault == "reward": reward[0] = float("inf") elif self.fault == "reward_shape": reward = reward[:, None] return {"policy": self.x}, reward, torch.tensor([True, False]), torch.tensor([True, True]), info def test_collect_preserves_prestep_buffers_diagnostics_logp_and_reward(): model, _ = learner() env = ReusedBufferEnv() calls = [] data, nx, failures, timeouts = ppo.collect_rollout(env, model, env.x, CFG, lambda: calls.append(True)) assert len(calls) == CFG["horizon"] and failures == timeouts == 4 assert data["features"].shape == (4, 2, 46) and data["latent"].shape == (4, 2, 3) for step in range(4): torch.testing.assert_close(data["features"][step], torch.full((2, 46), float(step))) assert (data["phase"][step] == step).all() logp, values = model.evaluate(data["features"].flatten(0, 1), data["latent"].flatten(0, 1)) torch.testing.assert_close(logp, data["log_prob"].flatten()) torch.testing.assert_close(values, data["value"].flatten()) torch.testing.assert_close(data["reward"], torch.tensor([[-9., 1.]]).expand(4, 2)) assert data["done"].all() and not data["next_value"][:, 0].any() for step in range(4): torch.testing.assert_close(data["next_value"][step, 1], model.value(torch.full((1, 46), 10. + step))[0]) assert all(not v.requires_grad for v in data.values()) env.x.fill_(100.) env.phase.fill_(5) env.flags.zero_() assert (nx == 4).all() and (data["phase"][0] == 0).all() assert data["failure_flags"][:, 0, 0].all() @pytest.mark.parametrize("fault", ["missing", "phase", "flags", "reward", "reward_shape"]) def test_collect_missing_or_malformed_diagnostics_fails_before_optimizer(fault): model, optimizer = learner() before = copy.deepcopy(model.state_dict()) env = ReusedBufferEnv(fault) with pytest.raises((ValueError, RuntimeError)): data, *_ = ppo.collect_rollout(env, model, env.x, CFG) ppo.ppo_step(model, optimizer, data, CFG) assert not optimizer.state for key, value in before.items(): torch.testing.assert_close(value, model.state_dict()[key], rtol=0, atol=0) def test_synthetic_generic_ppo_and_immutable_pre_normalization_diagnostics(): model, optimizer = learner() env = ReusedBufferEnv() data, *_ = ppo.collect_rollout(env, model, env.x, CFG) before_data = copy.deepcopy(data) before_model = copy.deepcopy(model.state_dict()) diagnostics = ppo.rollout_diagnostics(data, CFG) assert diagnostics["missing_diagnostic_samples"] == 0 assert diagnostics["phase_control_samples"] == [2, 2, 2, 2, 0, 0] assert diagnostics["terminal_causes"][FAILURE_NAMES[0]] == 4 adv, _ = compute_gae(data["reward"], data["value"], data["next_value"], data["done"], .99, .95) assert diagnostics["advantage_before_normalization"]["mean"] == pytest.approx(float(adv.mean())) metrics = ppo.ppo_step(model, optimizer, data, CFG) assert metrics["sampling_log_prob_error"] < 2e-4 and metrics["minibatches"] > 0 assert any(not torch.equal(v, model.state_dict()[k]) for k, v in before_model.items()) assert ppo.rollout_diagnostics(data, CFG) == diagnostics for key in data: torch.testing.assert_close(data[key], before_data[key], rtol=0, atol=0) # The generic learner itself rejects stale/corrupted sampling log probabilities. data["log_prob"].add_(10.) with pytest.raises(RuntimeError, match="mismatch"): ppo.ppo_step(model, optimizer, data, CFG) def test_selection_key_lexicographic_all_rows_and_failure_cost(): good = {"success": True, "height_rms_m": .06, "swing_rms_m": .025, "landing_error_m": .05} assert ppo.selection_key([good]) == (0., 3.) assert ppo.selection_key([good, {"success": False}]) == (.5, 9.5) assert ppo.selection_key([{}]) == (1., 16.) worst_success = {**good, "height_rms_m": 100., "swing_rms_m": 100., "landing_error_m": 100.} assert ppo.selection_key([worst_success]) < ppo.selection_key([good, {}]) for bad in ([], [{"success": True}], [{**good, "height_rms_m": float("nan")}], [{**good, "landing_error_m": -1.}], [{**good, "success": "false"}]): with pytest.raises(ValueError): ppo.selection_key(bad) def saved_learner(tmp_path): model, optimizer = learner() env = ReusedBufferEnv() data, *_ = ppo.collect_rollout(env, model, env.x, CFG) ppo.ppo_step(model, optimizer, data, CFG) path = tmp_path / "residual.pt" identity = {"source_sha256": "synthetic-source", "task": ppo.CONTRACT} size = ppo.save_checkpoint(path, model, optimizer, identity, 1, {"update": 1, "key": [0., 3.]}, {"reserved_transitions": 8}, MODEL_CONFIG) return path, model, optimizer, identity, size def assert_tree_equal(a, b): if isinstance(a, torch.Tensor): torch.testing.assert_close(a, b, rtol=0, atol=0) elif isinstance(a, dict): assert a.keys() == b.keys() for key in a: assert_tree_equal(a[key], b[key]) elif isinstance(a, (tuple, list)): assert type(a) is type(b) and len(a) == len(b) for av, bv in zip(a, b): assert_tree_equal(av, bv) else: assert a == b def test_checkpoint_distinct_atomic_exact_bytes_rng_and_optimizer_roundtrip(tmp_path): path, model, optimizer, identity, size = saved_learner(tmp_path) assert size == path.stat().st_size and list(tmp_path.iterdir()) == [path] payload = torch.load(path, weights_only=True) assert payload["format"] == ppo.CHECKPOINT_FORMAT assert payload["contract"] == ppo.CONTRACT and payload["features"] == model.feature_version assert payload["accepted_skill"] is False and "sampler" not in payload assert payload["model_config"] == MODEL_CONFIG and payload["budget_snapshot"] == {"reserved_transitions": 8} expected_random = torch.rand(12) with torch.no_grad(): model.actor[-1].bias.add_(1.) optimizer.param_groups[0]["lr"] = .005 restored = ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG) assert restored["update"] == 1 assert_tree_equal(model.state_dict(), payload["model"]) assert_tree_equal(optimizer.state_dict(), payload["optimizer"]) torch.testing.assert_close(torch.rand(12), expected_random, rtol=0, atol=0) original = path.read_bytes() with pytest.raises(RuntimeError, match="budget"): ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG, max_bytes=1) assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path] @pytest.mark.parametrize("fault", ["format", "contract", "features", "identity", "hidden", "config_type", "update", "accepted", "model_shape", "model_dtype", "model_nan", "scale", "optimizer_nan", "optimizer_shape", "optimizer_type", "optimizer_lr_type", "optimizer_slot_type", "optimizer_missing_slot", "optimizer_step", "rng", "budget", "sampler"]) def test_checkpoint_rejects_before_mutating_any_learner_or_rng(tmp_path, fault): path, model, optimizer, identity, _ = saved_learner(tmp_path) payload = torch.load(path, weights_only=True) if fault in ("format", "contract", "features"): payload[fault] = "legacy-32D-standing-or-reach" elif fault == "identity": payload["identity"]["source_sha256"] = "different-source" elif fault == "hidden": payload["model_config"]["hidden"] = 32 elif fault == "config_type": payload["model_config"]["hidden"] = 16. elif fault == "update": payload["update"] = True elif fault == "accepted": payload["accepted_skill"] = True elif fault == "model_shape": payload["model"]["actor.0.weight"] = torch.zeros(16, 32) elif fault == "model_dtype": payload["model"]["log_std"] = payload["model"]["log_std"].double() elif fault == "model_nan": payload["model"]["log_std"][0] = float("nan") elif fault == "scale": payload["model"]["scale"][0] = 0. elif fault == "optimizer_nan": payload["optimizer"]["param_groups"][0]["lr"] = float("nan") elif fault == "optimizer_shape": payload["optimizer"]["state"][0]["exp_avg"] = torch.zeros(32) elif fault == "optimizer_type": payload["optimizer_type"] = "SGD" elif fault == "optimizer_lr_type": payload["optimizer"]["param_groups"][0]["lr"] = "invalid" elif fault == "optimizer_slot_type": payload["optimizer"]["state"][0]["exp_avg"] = 0. elif fault == "optimizer_missing_slot": del payload["optimizer"]["state"][0]["exp_avg"] elif fault == "optimizer_step": payload["optimizer"]["state"][0]["step"] = torch.tensor(.5) elif fault == "rng": payload["rng"]["cpu"] = torch.zeros(2, dtype=torch.uint8) elif fault == "budget": payload["budget_snapshot"] = None elif fault == "sampler": payload["sampler"] = {} torch.save(payload, path) before = copy.deepcopy((model.state_dict(), optimizer.state_dict(), ppo.rng_state())) with pytest.raises(ValueError): ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG) assert_tree_equal(before, (model.state_dict(), optimizer.state_dict(), ppo.rng_state())) def test_save_rejects_wrong_model_configuration_type_and_nonfinite_state(tmp_path): model, optimizer = learner() old = CycleActorCritic(hidden=16) cases = [(old, torch.optim.Adam(old.parameters()), MODEL_CONFIG), (model, optimizer, {"hidden": 32, "initial_std": .1}), (model, optimizer, {"hidden": 16, "initial_std": True}), (model, torch.optim.Adam(old.parameters()), MODEL_CONFIG)] for candidate, opt, cfg in cases: with pytest.raises(ValueError): ppo.save_checkpoint(tmp_path / "bad.pt", candidate, opt, {}, 0, None, {}, cfg) with torch.no_grad(): model.log_std[0] = float("inf") with pytest.raises(ValueError): ppo.save_checkpoint(tmp_path / "bad.pt", model, optimizer, {}, 0, None, {}, MODEL_CONFIG) assert not list(tmp_path.iterdir()) def test_atomic_replace_failure_keeps_destination_and_removes_temporary(tmp_path, monkeypatch): path, model, optimizer, identity, _ = saved_learner(tmp_path) original = path.read_bytes() def refuse(source, destination): assert source.exists() and destination == path raise OSError("synthetic replace failure") monkeypatch.setattr(ppo.os, "replace", refuse) with pytest.raises(OSError, match="synthetic"): ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG) assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path]