9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
370 lines
17 KiB
Python
370 lines
17 KiB
Python
"""Synthetic CPU-only 46D learner API tests; never robot-data optimization or Kit."""
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from SkillFactory.legs import standing_ppo
|
|
from SkillFactory.legs.contact_cycle.ppo import CycleActorCritic
|
|
from SkillFactory.legs.contact_cycle.task import FAILURE_NAMES
|
|
from SkillFactory.legs.contact_cycle_sim import ppo
|
|
from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic, features
|
|
from SkillFactory.legs.policy import compute_gae
|
|
|
|
MODEL_CONFIG = {"hidden": 16, "initial_std": .1}
|
|
CFG = {"horizon": 4, "failure_penalty": 10., "gamma": .99, "gae_lambda": .95,
|
|
"epochs": 2, "minibatch_size": 4, "clip_ratio": .2, "value_coefficient": .5,
|
|
"max_gradient_norm": 1., "target_kl": .02}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def cpu_synthetic_seed():
|
|
torch.set_num_threads(1)
|
|
torch.manual_seed(71)
|
|
|
|
|
|
def learner():
|
|
model = SimResidualActorCritic(**MODEL_CONFIG)
|
|
return model, torch.optim.Adam(model.parameters(), lr=3e-4, eps=1e-5)
|
|
|
|
|
|
class ValueModel:
|
|
def value(self, x):
|
|
# Return a view deliberately: bootstrap must not modify any input.
|
|
return x[:, 0]
|
|
|
|
|
|
def terminal_case(dtype=torch.int32):
|
|
x = torch.full((4, 46), -100.) # reset observations never supply terminal values
|
|
x[3] = 7.
|
|
terminal = torch.zeros(3, 46)
|
|
terminal[:, 0] = torch.tensor([30., 99., 20.])
|
|
terminated = torch.tensor([True, False, False, False])
|
|
truncated = torch.tensor([True, True, True, False])
|
|
info = {"terminal_ids": torch.tensor([2, 0, 1], dtype=dtype),
|
|
"terminal_observation": {"policy": terminal}}
|
|
return x, terminated, truncated, info
|
|
|
|
|
|
def test_reexports_are_unchanged_generic_math_and_46d_features():
|
|
assert ppo.features is features
|
|
assert ppo.ppo_step is standing_ppo.ppo_step
|
|
assert ppo.rng_state is standing_ppo.rng_state
|
|
assert ppo.restore_rng is standing_ppo.restore_rng
|
|
|
|
|
|
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
|
|
def test_bootstrap_timeout_prereset_failure_zero_and_no_alias(dtype):
|
|
x, terminated, truncated, info = terminal_case(dtype)
|
|
before = x.clone()
|
|
actual = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
|
|
torch.testing.assert_close(actual, torch.tensor([0., 20., 30., 7.]))
|
|
torch.testing.assert_close(x, before)
|
|
torch.testing.assert_close(info["terminal_observation"]["policy"][:, 0], torch.tensor([30., 99., 20.]))
|
|
# True failures need no terminal observations, even when also truncated.
|
|
torch.testing.assert_close(ppo.bootstrap_values(ValueModel(), x, torch.ones(4, dtype=torch.bool),
|
|
torch.ones(4, dtype=torch.bool), {}), torch.zeros(4))
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["missing", "missing_policy", "32d", "count", "duplicate", "float_ids",
|
|
"negative_id", "out_of_range", "coverage", "nonterminal_id", "nan"])
|
|
def test_bootstrap_rejects_bad_terminal_contract(fault):
|
|
x, terminated, truncated, info = terminal_case()
|
|
if fault == "missing":
|
|
info = {}
|
|
elif fault == "missing_policy":
|
|
info["terminal_observation"] = {}
|
|
elif fault == "32d":
|
|
info["terminal_observation"]["policy"] = torch.zeros(3, 32)
|
|
elif fault == "count":
|
|
info["terminal_observation"]["policy"] = torch.zeros(2, 46)
|
|
elif fault == "nan":
|
|
info["terminal_observation"]["policy"][0, 0] = float("nan")
|
|
else:
|
|
info["terminal_ids"] = {
|
|
"duplicate": torch.tensor([2, 2, 1]), "float_ids": torch.tensor([2., 0., 1.]),
|
|
"negative_id": torch.tensor([2, -1, 1]), "out_of_range": torch.tensor([2, 4, 1]),
|
|
"coverage": torch.tensor([2, 0]), "nonterminal_id": torch.tensor([2, 3, 1]),
|
|
}[fault]
|
|
with pytest.raises((ValueError, RuntimeError)):
|
|
ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [torch.zeros(2, 32), torch.zeros(2, 46, dtype=torch.float64),
|
|
torch.full((2, 46), float("nan")), torch.zeros(0, 46)])
|
|
def test_features_reject_old_or_malformed_data(bad):
|
|
with pytest.raises(ValueError):
|
|
ppo.features(bad)
|
|
|
|
|
|
def test_bootstrap_shapes_types_and_nonfinite_value():
|
|
x, terminated, truncated, info = terminal_case()
|
|
for bad in (terminated.float(), terminated[:, None], terminated[:1]):
|
|
with pytest.raises(ValueError):
|
|
ppo.bootstrap_values(ValueModel(), x, bad, truncated, info)
|
|
|
|
class BadValue:
|
|
def value(self, x):
|
|
return torch.full((len(x),), float("nan"))
|
|
|
|
with pytest.raises(ValueError):
|
|
ppo.bootstrap_values(BadValue(), x, terminated, truncated, info)
|
|
|
|
|
|
def test_gae_failure_and_timeout_end_recursion_but_timeout_bootstraps():
|
|
x, terminated, truncated, info = terminal_case()
|
|
nv = ppo.bootstrap_values(ValueModel(), x, terminated, truncated, info)
|
|
rewards = torch.tensor([[1., 1., 1., 1.], [100., 100., 100., 100.]])
|
|
values = torch.zeros_like(rewards)
|
|
next_values = torch.stack([nv, torch.zeros(4)])
|
|
done = torch.stack([terminated | truncated, torch.ones(4, dtype=torch.bool)])
|
|
adv, returns = compute_gae(rewards, values, next_values, done, .9, .5)
|
|
torch.testing.assert_close(adv[0], torch.tensor([1., 19., 28., 52.3]))
|
|
torch.testing.assert_close(returns, adv)
|
|
|
|
|
|
class ReusedBufferEnv:
|
|
num_envs, device, step_dt = 2, "cpu", .01
|
|
|
|
def __init__(self, fault=None):
|
|
self.x = torch.zeros(2, 46)
|
|
self.phase = torch.zeros(2, dtype=torch.int32)
|
|
self.flags = torch.zeros(2, len(FAILURE_NAMES), dtype=torch.bool)
|
|
self.steps, self.fault = 0, fault
|
|
|
|
def step(self, action):
|
|
assert not torch.is_grad_enabled()
|
|
assert action.shape == (2, 3) and (action.abs() <= 1).all()
|
|
self.x.fill_(self.steps + 1.)
|
|
self.phase.fill_(self.steps % 6) # phase of pre-step state, not next observation
|
|
self.flags.zero_()
|
|
self.flags[0, 0] = True
|
|
terminal = torch.full((2, 46), 10. + self.steps)
|
|
self.steps += 1
|
|
info = {"terminal_ids": torch.tensor([0, 1], dtype=torch.int32),
|
|
"terminal_observation": {"policy": terminal},
|
|
"cycle_step": {"phase": self.phase, "failure_flags": self.flags}}
|
|
reward = torch.full((2,), .01)
|
|
if self.fault == "missing":
|
|
del info["cycle_step"]
|
|
elif self.fault == "phase":
|
|
info["cycle_step"]["phase"] = torch.full((2,), -1)
|
|
elif self.fault == "flags":
|
|
info["cycle_step"]["failure_flags"] = torch.zeros(2, 7, dtype=torch.bool)
|
|
elif self.fault == "reward":
|
|
reward[0] = float("inf")
|
|
elif self.fault == "reward_shape":
|
|
reward = reward[:, None]
|
|
return {"policy": self.x}, reward, torch.tensor([True, False]), torch.tensor([True, True]), info
|
|
|
|
|
|
def test_collect_preserves_prestep_buffers_diagnostics_logp_and_reward():
|
|
model, _ = learner()
|
|
env = ReusedBufferEnv()
|
|
calls = []
|
|
data, nx, failures, timeouts = ppo.collect_rollout(env, model, env.x, CFG, lambda: calls.append(True))
|
|
assert len(calls) == CFG["horizon"] and failures == timeouts == 4
|
|
assert data["features"].shape == (4, 2, 46) and data["latent"].shape == (4, 2, 3)
|
|
for step in range(4):
|
|
torch.testing.assert_close(data["features"][step], torch.full((2, 46), float(step)))
|
|
assert (data["phase"][step] == step).all()
|
|
logp, values = model.evaluate(data["features"].flatten(0, 1), data["latent"].flatten(0, 1))
|
|
torch.testing.assert_close(logp, data["log_prob"].flatten())
|
|
torch.testing.assert_close(values, data["value"].flatten())
|
|
torch.testing.assert_close(data["reward"], torch.tensor([[-9., 1.]]).expand(4, 2))
|
|
assert data["done"].all() and not data["next_value"][:, 0].any()
|
|
for step in range(4):
|
|
torch.testing.assert_close(data["next_value"][step, 1], model.value(torch.full((1, 46), 10. + step))[0])
|
|
assert all(not v.requires_grad for v in data.values())
|
|
env.x.fill_(100.)
|
|
env.phase.fill_(5)
|
|
env.flags.zero_()
|
|
assert (nx == 4).all() and (data["phase"][0] == 0).all()
|
|
assert data["failure_flags"][:, 0, 0].all()
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["missing", "phase", "flags", "reward", "reward_shape"])
|
|
def test_collect_missing_or_malformed_diagnostics_fails_before_optimizer(fault):
|
|
model, optimizer = learner()
|
|
before = copy.deepcopy(model.state_dict())
|
|
env = ReusedBufferEnv(fault)
|
|
with pytest.raises((ValueError, RuntimeError)):
|
|
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
|
|
ppo.ppo_step(model, optimizer, data, CFG)
|
|
assert not optimizer.state
|
|
for key, value in before.items():
|
|
torch.testing.assert_close(value, model.state_dict()[key], rtol=0, atol=0)
|
|
|
|
|
|
def test_synthetic_generic_ppo_and_immutable_pre_normalization_diagnostics():
|
|
model, optimizer = learner()
|
|
env = ReusedBufferEnv()
|
|
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
|
|
before_data = copy.deepcopy(data)
|
|
before_model = copy.deepcopy(model.state_dict())
|
|
diagnostics = ppo.rollout_diagnostics(data, CFG)
|
|
assert diagnostics["missing_diagnostic_samples"] == 0
|
|
assert diagnostics["phase_control_samples"] == [2, 2, 2, 2, 0, 0]
|
|
assert diagnostics["terminal_causes"][FAILURE_NAMES[0]] == 4
|
|
adv, _ = compute_gae(data["reward"], data["value"], data["next_value"], data["done"], .99, .95)
|
|
assert diagnostics["advantage_before_normalization"]["mean"] == pytest.approx(float(adv.mean()))
|
|
metrics = ppo.ppo_step(model, optimizer, data, CFG)
|
|
assert metrics["sampling_log_prob_error"] < 2e-4 and metrics["minibatches"] > 0
|
|
assert any(not torch.equal(v, model.state_dict()[k]) for k, v in before_model.items())
|
|
assert ppo.rollout_diagnostics(data, CFG) == diagnostics
|
|
for key in data:
|
|
torch.testing.assert_close(data[key], before_data[key], rtol=0, atol=0)
|
|
# The generic learner itself rejects stale/corrupted sampling log probabilities.
|
|
data["log_prob"].add_(10.)
|
|
with pytest.raises(RuntimeError, match="mismatch"):
|
|
ppo.ppo_step(model, optimizer, data, CFG)
|
|
|
|
|
|
def test_selection_key_lexicographic_all_rows_and_failure_cost():
|
|
good = {"success": True, "height_rms_m": .06, "swing_rms_m": .025, "landing_error_m": .05}
|
|
assert ppo.selection_key([good]) == (0., 3.)
|
|
assert ppo.selection_key([good, {"success": False}]) == (.5, 9.5)
|
|
assert ppo.selection_key([{}]) == (1., 16.)
|
|
worst_success = {**good, "height_rms_m": 100., "swing_rms_m": 100., "landing_error_m": 100.}
|
|
assert ppo.selection_key([worst_success]) < ppo.selection_key([good, {}])
|
|
for bad in ([], [{"success": True}], [{**good, "height_rms_m": float("nan")}],
|
|
[{**good, "landing_error_m": -1.}], [{**good, "success": "false"}]):
|
|
with pytest.raises(ValueError):
|
|
ppo.selection_key(bad)
|
|
|
|
|
|
def saved_learner(tmp_path):
|
|
model, optimizer = learner()
|
|
env = ReusedBufferEnv()
|
|
data, *_ = ppo.collect_rollout(env, model, env.x, CFG)
|
|
ppo.ppo_step(model, optimizer, data, CFG)
|
|
path = tmp_path / "residual.pt"
|
|
identity = {"source_sha256": "synthetic-source", "task": ppo.CONTRACT}
|
|
size = ppo.save_checkpoint(path, model, optimizer, identity, 1, {"update": 1, "key": [0., 3.]},
|
|
{"reserved_transitions": 8}, MODEL_CONFIG)
|
|
return path, model, optimizer, identity, size
|
|
|
|
|
|
def assert_tree_equal(a, b):
|
|
if isinstance(a, torch.Tensor):
|
|
torch.testing.assert_close(a, b, rtol=0, atol=0)
|
|
elif isinstance(a, dict):
|
|
assert a.keys() == b.keys()
|
|
for key in a:
|
|
assert_tree_equal(a[key], b[key])
|
|
elif isinstance(a, (tuple, list)):
|
|
assert type(a) is type(b) and len(a) == len(b)
|
|
for av, bv in zip(a, b):
|
|
assert_tree_equal(av, bv)
|
|
else:
|
|
assert a == b
|
|
|
|
|
|
def test_checkpoint_distinct_atomic_exact_bytes_rng_and_optimizer_roundtrip(tmp_path):
|
|
path, model, optimizer, identity, size = saved_learner(tmp_path)
|
|
assert size == path.stat().st_size and list(tmp_path.iterdir()) == [path]
|
|
payload = torch.load(path, weights_only=True)
|
|
assert payload["format"] == ppo.CHECKPOINT_FORMAT
|
|
assert payload["contract"] == ppo.CONTRACT and payload["features"] == model.feature_version
|
|
assert payload["accepted_skill"] is False and "sampler" not in payload
|
|
assert payload["model_config"] == MODEL_CONFIG and payload["budget_snapshot"] == {"reserved_transitions": 8}
|
|
expected_random = torch.rand(12)
|
|
with torch.no_grad():
|
|
model.actor[-1].bias.add_(1.)
|
|
optimizer.param_groups[0]["lr"] = .005
|
|
restored = ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG)
|
|
assert restored["update"] == 1
|
|
assert_tree_equal(model.state_dict(), payload["model"])
|
|
assert_tree_equal(optimizer.state_dict(), payload["optimizer"])
|
|
torch.testing.assert_close(torch.rand(12), expected_random, rtol=0, atol=0)
|
|
original = path.read_bytes()
|
|
with pytest.raises(RuntimeError, match="budget"):
|
|
ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG, max_bytes=1)
|
|
assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path]
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["format", "contract", "features", "identity", "hidden", "config_type",
|
|
"update", "accepted", "model_shape", "model_dtype", "model_nan", "scale",
|
|
"optimizer_nan", "optimizer_shape", "optimizer_type", "optimizer_lr_type",
|
|
"optimizer_slot_type", "optimizer_missing_slot", "optimizer_step", "rng", "budget", "sampler"])
|
|
def test_checkpoint_rejects_before_mutating_any_learner_or_rng(tmp_path, fault):
|
|
path, model, optimizer, identity, _ = saved_learner(tmp_path)
|
|
payload = torch.load(path, weights_only=True)
|
|
if fault in ("format", "contract", "features"):
|
|
payload[fault] = "legacy-32D-standing-or-reach"
|
|
elif fault == "identity":
|
|
payload["identity"]["source_sha256"] = "different-source"
|
|
elif fault == "hidden":
|
|
payload["model_config"]["hidden"] = 32
|
|
elif fault == "config_type":
|
|
payload["model_config"]["hidden"] = 16.
|
|
elif fault == "update":
|
|
payload["update"] = True
|
|
elif fault == "accepted":
|
|
payload["accepted_skill"] = True
|
|
elif fault == "model_shape":
|
|
payload["model"]["actor.0.weight"] = torch.zeros(16, 32)
|
|
elif fault == "model_dtype":
|
|
payload["model"]["log_std"] = payload["model"]["log_std"].double()
|
|
elif fault == "model_nan":
|
|
payload["model"]["log_std"][0] = float("nan")
|
|
elif fault == "scale":
|
|
payload["model"]["scale"][0] = 0.
|
|
elif fault == "optimizer_nan":
|
|
payload["optimizer"]["param_groups"][0]["lr"] = float("nan")
|
|
elif fault == "optimizer_shape":
|
|
payload["optimizer"]["state"][0]["exp_avg"] = torch.zeros(32)
|
|
elif fault == "optimizer_type":
|
|
payload["optimizer_type"] = "SGD"
|
|
elif fault == "optimizer_lr_type":
|
|
payload["optimizer"]["param_groups"][0]["lr"] = "invalid"
|
|
elif fault == "optimizer_slot_type":
|
|
payload["optimizer"]["state"][0]["exp_avg"] = 0.
|
|
elif fault == "optimizer_missing_slot":
|
|
del payload["optimizer"]["state"][0]["exp_avg"]
|
|
elif fault == "optimizer_step":
|
|
payload["optimizer"]["state"][0]["step"] = torch.tensor(.5)
|
|
elif fault == "rng":
|
|
payload["rng"]["cpu"] = torch.zeros(2, dtype=torch.uint8)
|
|
elif fault == "budget":
|
|
payload["budget_snapshot"] = None
|
|
elif fault == "sampler":
|
|
payload["sampler"] = {}
|
|
torch.save(payload, path)
|
|
before = copy.deepcopy((model.state_dict(), optimizer.state_dict(), ppo.rng_state()))
|
|
with pytest.raises(ValueError):
|
|
ppo.restore_checkpoint(path, model, optimizer, identity, MODEL_CONFIG)
|
|
assert_tree_equal(before, (model.state_dict(), optimizer.state_dict(), ppo.rng_state()))
|
|
|
|
|
|
def test_save_rejects_wrong_model_configuration_type_and_nonfinite_state(tmp_path):
|
|
model, optimizer = learner()
|
|
old = CycleActorCritic(hidden=16)
|
|
cases = [(old, torch.optim.Adam(old.parameters()), MODEL_CONFIG),
|
|
(model, optimizer, {"hidden": 32, "initial_std": .1}),
|
|
(model, optimizer, {"hidden": 16, "initial_std": True}),
|
|
(model, torch.optim.Adam(old.parameters()), MODEL_CONFIG)]
|
|
for candidate, opt, cfg in cases:
|
|
with pytest.raises(ValueError):
|
|
ppo.save_checkpoint(tmp_path / "bad.pt", candidate, opt, {}, 0, None, {}, cfg)
|
|
with torch.no_grad():
|
|
model.log_std[0] = float("inf")
|
|
with pytest.raises(ValueError):
|
|
ppo.save_checkpoint(tmp_path / "bad.pt", model, optimizer, {}, 0, None, {}, MODEL_CONFIG)
|
|
assert not list(tmp_path.iterdir())
|
|
|
|
|
|
def test_atomic_replace_failure_keeps_destination_and_removes_temporary(tmp_path, monkeypatch):
|
|
path, model, optimizer, identity, _ = saved_learner(tmp_path)
|
|
original = path.read_bytes()
|
|
|
|
def refuse(source, destination):
|
|
assert source.exists() and destination == path
|
|
raise OSError("synthetic replace failure")
|
|
|
|
monkeypatch.setattr(ppo.os, "replace", refuse)
|
|
with pytest.raises(OSError, match="synthetic"):
|
|
ppo.save_checkpoint(path, model, optimizer, identity, 2, None, {}, MODEL_CONFIG)
|
|
assert path.read_bytes() == original and list(tmp_path.iterdir()) == [path]
|