9fe9d27865
Consolidate next-stage docs, archive superseded narratives, and preserve frozen document inputs. Retain invalid P2 pairs, partial RootXY training, failed attempts and nonselectable checkpoints as explicit limitations. Include supervised evaluation/training, desktop GPU admission, bounded evidence and atomic-publication monitor repairs with CPU regressions. Ignore local dotenv configuration. Validation: 1403 passed, 6 skipped for contact-cycle and related contracts/CLI tests. Full-suite legacy ABI and registration tests require Python 3.12/Torch 2.10/pxr; no runtime checks relaxed. No new physical run or accepted skill.
573 lines
26 KiB
Python
573 lines
26 KiB
Python
"""CPU/synthetic protocol tests. No saved/physical robot data is optimized here."""
|
|
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import torch
|
|
from SkillFactory.legs.contact_cycle_sim import training as protocol
|
|
from SkillFactory.legs.contact_cycle_sim.model import SimResidualActorCritic
|
|
from SkillFactory.legs.contact_cycle_sim.ppo import restore_checkpoint, save_checkpoint, selection_key
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CFG = ROOT / protocol.CONFIG_PATH
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def threads():
|
|
old = torch.get_num_threads()
|
|
torch.set_num_threads(1)
|
|
yield
|
|
torch.set_num_threads(old)
|
|
|
|
|
|
def write(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value))
|
|
return path
|
|
|
|
|
|
def test_exact_pilot_config_and_proposal():
|
|
cfg = protocol.load_training_config(CFG, ROOT)
|
|
assert cfg == protocol.PILOT
|
|
assert cfg["num_envs"] * cfg["horizon"] * cfg["updates_per_seed"] == 1048576
|
|
assert cfg["validation_updates"] == [0, 64, 128, 192, 256]
|
|
assert protocol.PROPOSAL["actual_training_launches"] == 1
|
|
|
|
|
|
@pytest.mark.parametrize("key,value", [
|
|
("seeds", [91500]), ("num_envs", 8), ("horizon", 64), ("max_total_transitions", 1048577),
|
|
("max_total_wall_s", 3601), ("max_artifact_bytes", 1073741825), ("validation_seed", 91501),
|
|
("validation_updates", [0, 32, 256]), ("features", "policy32"), ("resume", False),
|
|
("updates_per_seed", True), ("task_id", "SkillFactory-Go2FL-ContactCycle-v0"),
|
|
])
|
|
def test_config_rejects_scope_changes(tmp_path, key, value):
|
|
cfg = copy.deepcopy(protocol.PILOT)
|
|
cfg[key] = value
|
|
with pytest.raises(ValueError):
|
|
protocol.load_training_config(write(tmp_path / "bad.json", cfg), ROOT)
|
|
|
|
|
|
@pytest.mark.parametrize("key,value", [("hidden", 32), ("initial_std", .2), ("adam_eps", 1e-8),
|
|
("gamma", True), ("learning_rate", float("nan"))])
|
|
def test_algorithm_is_fixed(tmp_path, key, value):
|
|
cfg = copy.deepcopy(protocol.PILOT)
|
|
cfg["algorithm"][key] = value
|
|
with pytest.raises(ValueError):
|
|
protocol.load_training_config(write(tmp_path / "bad.json", cfg), ROOT)
|
|
|
|
|
|
def test_changed_proposal_rejected(tmp_path):
|
|
proposal = {**protocol.PROPOSAL, "resume": True}
|
|
write(tmp_path / protocol.PROPOSAL_PATH, proposal)
|
|
with pytest.raises(ValueError, match="proposal"):
|
|
protocol.load_training_config(CFG, tmp_path)
|
|
|
|
|
|
def test_manifest_binds_new_seam_math_assets_auditor_and_parent_plan():
|
|
m = protocol.make_manifest(ROOT, CFG)
|
|
assert m["features"] == "contact-cycle-sim-truth-residual-46-v1"
|
|
assert m["sampler"] is None and m["resume"] is False and m["accepted_skill"] is False
|
|
for name in ("source/SkillFactory/SkillFactory/legs/standing_ppo.py",
|
|
"source/SkillFactory/SkillFactory/legs/policy.py",
|
|
"source/SkillFactory/SkillFactory/legs/contact_cycle_sim/ppo.py",
|
|
"source/SkillFactory/SkillFactory/legs/contact_cycle_sim/training.py",
|
|
"scripts/train_cycle_sim_residual.py", "scripts/audit_contact_cycle_functional.py",
|
|
"scripts/verify_cycle_sim_learning_runtime.py", "scripts/supervise_sim.py",
|
|
"docs/contact_cycle_sim_training_plan.md", protocol.CONFIG_PATH, protocol.PROPOSAL_PATH):
|
|
assert name in m["sources"]
|
|
assert any(p.startswith("assets/generated/support/") for p in m["sources"])
|
|
assert any(p.startswith("assets/generated/legs/") for p in m["sources"])
|
|
|
|
|
|
def synthetic_grant(manifest):
|
|
# NOT a grant for any actual pilot: the bound manifest below is a synthetic CPU object.
|
|
return {"format": protocol.AUTHORIZATION_FORMAT, "approved": True,
|
|
"manifest_sha256": protocol.digest(manifest), "operator": "synthetic_cpu_test",
|
|
"authorization_reference": "synthetic_only_never_execute"}
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["missing", "old", "stale", "unapproved", "operator", "extra"])
|
|
def test_grants_fail_closed(tmp_path, fault):
|
|
manifest = {"synthetic_cpu_only": True}
|
|
grant = synthetic_grant(manifest)
|
|
if fault == "old":
|
|
grant["format"] = "skillfactory.contact-cycle-authorization.v1"
|
|
elif fault == "stale":
|
|
grant["manifest_sha256"] = "0" * 64
|
|
elif fault == "unapproved":
|
|
grant["approved"] = 1
|
|
elif fault == "operator":
|
|
grant["operator"] = " "
|
|
elif fault == "extra":
|
|
grant["resume"] = True
|
|
path = None if fault == "missing" else write(tmp_path / "synthetic.json", grant)
|
|
with pytest.raises(PermissionError):
|
|
protocol.require_authorization(path, manifest)
|
|
|
|
|
|
def test_synthetic_bound_grant_only(tmp_path):
|
|
manifest = {"synthetic_cpu_only": True}
|
|
grant = synthetic_grant(manifest)
|
|
assert protocol.require_authorization(write(tmp_path / "synthetic.json", grant), manifest) == grant
|
|
|
|
|
|
def test_frozen_and_symlink_output_rejected(tmp_path):
|
|
with pytest.raises(ValueError, match="Frozen"):
|
|
protocol.writable_output(ROOT, ROOT / "releases/child")
|
|
(tmp_path / "link").symlink_to(ROOT / "releases", target_is_directory=True)
|
|
with pytest.raises(ValueError, match="Frozen"):
|
|
protocol.writable_output(ROOT, tmp_path / "link/child")
|
|
|
|
|
|
@pytest.fixture
|
|
def prepared(tmp_path, monkeypatch):
|
|
# Small source snapshot fixture; never prepare a live training manifest or grant.
|
|
source = tmp_path / "source_root"
|
|
source.mkdir()
|
|
(source / "input.txt").write_text("synthetic input")
|
|
import hashlib
|
|
|
|
manifest = {"synthetic_cpu_only": True, "training": copy.deepcopy(protocol.PILOT),
|
|
"sources": {"input.txt": hashlib.sha256(b"synthetic input").hexdigest()}}
|
|
monkeypatch.setattr(protocol, "make_manifest", lambda *_: manifest)
|
|
output = tmp_path / "prepared"
|
|
protocol.prepare(source, source / "config", output)
|
|
return source, output, manifest
|
|
|
|
|
|
def test_preparation_not_training(prepared):
|
|
root, out, manifest = prepared
|
|
protocol.validate_prepared(root, out, manifest)
|
|
assert not (out / "budget.json").exists()
|
|
assert not (out / "launch_claim.json").exists()
|
|
assert not list(out.glob("*.pt"))
|
|
with pytest.raises(FileExistsError):
|
|
protocol.prepare(root, root / "config", out)
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["snapshot", "manifest", "budget", "lock", "claim", "old_checkpoint", "cap"])
|
|
def test_prepared_must_be_fresh_immutable_and_bounded(prepared, fault):
|
|
root, out, manifest = prepared
|
|
if fault == "snapshot":
|
|
(out / "source_snapshot/input.txt").write_text("changed")
|
|
elif fault == "manifest":
|
|
write(out / "manifest.json", {})
|
|
elif fault == "cap":
|
|
manifest["training"]["max_artifact_bytes"] = 1
|
|
write(out / "manifest.json", manifest)
|
|
write(out / "preparation.json", {"prepared": True, "authorized": False, "trained": False,
|
|
"manifest_sha256": protocol.digest(manifest)})
|
|
else:
|
|
name = {"budget": "budget.json", "lock": "budget.lock", "claim": "launch_claim.json",
|
|
"old_checkpoint": "old32.pt"}[fault]
|
|
(out / name).write_text("old")
|
|
with pytest.raises((ValueError, FileExistsError, protocol.BudgetExceeded)):
|
|
protocol.validate_prepared(root, out, manifest)
|
|
|
|
|
|
def small_manifest():
|
|
# Fixed pilot batch/cap, simulated BOOTTIME; no actual sources/grant.
|
|
return {"synthetic_cpu_only": True, "training": copy.deepcopy(protocol.PILOT)}
|
|
|
|
|
|
def ledger_at(path, clock=None):
|
|
clock = clock if clock is not None else [10.]
|
|
path.mkdir(exist_ok=True)
|
|
return protocol.FreshBudgetLedger(path, small_manifest(), now=lambda: 100., boot_now=lambda: clock[0],
|
|
boot_id="synthetic-boot")
|
|
|
|
|
|
def test_reservation_is_durable_and_never_refunded(tmp_path):
|
|
with ledger_at(tmp_path) as ledger:
|
|
ledger.reserve_rollout(91501, 1)
|
|
disk = json.loads((tmp_path / "budget.json").read_text())
|
|
assert disk["reserved_transitions"] == 4096
|
|
assert disk["pending"] == {"seed": 91501, "update": 1}
|
|
with pytest.raises(PermissionError):
|
|
ledger.reconcile_checkpoint(91501, 0)
|
|
with pytest.raises(FileExistsError):
|
|
with ledger_at(tmp_path):
|
|
pass
|
|
assert json.loads((tmp_path / "budget.json").read_text())["reserved_transitions"] == 4096
|
|
|
|
|
|
def test_checkpoint_before_commit_and_failure_keeps_pending(tmp_path):
|
|
with ledger_at(tmp_path) as ledger:
|
|
ledger.reserve_rollout(91501, 1)
|
|
def failed_save():
|
|
raise OSError("synthetic full disk")
|
|
with pytest.raises(OSError):
|
|
protocol.checkpoint_then_commit(ledger, 91501, 1, failed_save)
|
|
assert ledger.data["pending"] == {"seed": 91501, "update": 1}
|
|
assert ledger.data["updates"]["91501"] == 0
|
|
def save():
|
|
assert ledger.data["updates"]["91501"] == 0
|
|
(tmp_path / "synthetic_latest").write_text("atomic checkpoint proof")
|
|
return 23
|
|
assert protocol.checkpoint_then_commit(ledger, 91501, 1, save) == 23
|
|
assert ledger.data["pending"] is None and ledger.data["updates"]["91501"] == 1
|
|
|
|
|
|
def test_boot_deadline_includes_validation_suspend_and_startup(tmp_path):
|
|
clock = [10.]
|
|
with ledger_at(tmp_path, clock) as ledger:
|
|
assert ledger.data["clock"]["deadline_s"] == 3610.
|
|
clock[0] = 3610.
|
|
with pytest.raises(protocol.BudgetExceeded):
|
|
ledger.reserve_rollout(91501, 1)
|
|
assert ledger.data["reserved_transitions"] == 0
|
|
|
|
|
|
def test_all_256_reservations_reach_exact_cap(tmp_path):
|
|
with ledger_at(tmp_path) as ledger:
|
|
for update in range(1, 257):
|
|
ledger.reserve_rollout(91501, update)
|
|
ledger.commit_update(91501, update)
|
|
assert ledger.data["reserved_transitions"] == 1048576
|
|
with pytest.raises(ValueError):
|
|
ledger.reserve_rollout(91501, 257)
|
|
|
|
|
|
def test_artifact_cap_counts_console_supervisor_and_atomic_temporary(tmp_path):
|
|
with ledger_at(tmp_path) as ledger:
|
|
(tmp_path / "console.log").write_bytes(b"x" * 11)
|
|
(tmp_path / "supervisor.json").write_bytes(b"x" * 17)
|
|
used = protocol.artifact_bytes(tmp_path)
|
|
ledger.cfg["max_artifact_bytes"] = used + 20
|
|
ledger.guard(reserve_bytes=20)
|
|
with pytest.raises(protocol.BudgetExceeded):
|
|
ledger.guard(reserve_bytes=21)
|
|
|
|
|
|
def success(height=0.):
|
|
return {"success": True, "height_rms_m": height, "swing_rms_m": 0., "landing_error_m": 0.}
|
|
|
|
|
|
def test_selection_lexicographic_earliest_tie_and_never_u0():
|
|
with pytest.raises(ValueError, match="never u0"):
|
|
protocol.choose_candidate(None, 0, [success()])
|
|
best = {"update": 64, "selection_key": selection_key([success(.06)])}
|
|
assert not protocol.choose_candidate(best, 128, [success(.06)])
|
|
assert protocol.choose_candidate(best, 128, [success(.03)])
|
|
assert not protocol.choose_candidate(best, 128, [{"success": False}, success()])
|
|
assert protocol.choose_candidate(None, 64, [{"success": False}]) # failed trained candidate still lockable
|
|
assert selection_key([success(), {"success": False}]) == (.5, 8.)
|
|
|
|
|
|
class SyntheticEnv:
|
|
num_envs = 128
|
|
device = "cpu"
|
|
step_dt = .01
|
|
|
|
def __init__(self, ledger, malformed=False):
|
|
self.ledger, self.malformed, self.steps = ledger, malformed, 0
|
|
|
|
def step(self, action):
|
|
assert self.ledger.data["pending"] == {"seed": 91501, "update": 1}
|
|
self.steps += 1
|
|
x = torch.zeros(128, 46)
|
|
x[:, 32] = self.steps * .001
|
|
info = {} if self.malformed else {"cycle_step": {
|
|
"phase": torch.zeros(128, dtype=torch.long), "failure_flags": torch.zeros(128, 8, dtype=torch.bool)}}
|
|
return {"policy": x}, -action.square().sum(-1) * .01, torch.zeros(128, dtype=torch.bool), \
|
|
torch.zeros(128, dtype=torch.bool), info
|
|
|
|
|
|
def test_real_optimizer_on_fresh_synthetic_data_only_after_diagnostics(tmp_path, monkeypatch):
|
|
model = SimResidualActorCritic(hidden=16)
|
|
initial = {k: v.detach().clone() for k, v in model.named_parameters()}
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=.0003, eps=1e-5)
|
|
original = protocol.ppo_step
|
|
calls = []
|
|
def checked_step(*args):
|
|
assert (tmp_path / "rollout_diagnostics.jsonl").is_file()
|
|
record = json.loads((tmp_path / "rollout_diagnostics.jsonl").read_text())
|
|
assert record["missing_diagnostic_samples"] == 0
|
|
assert sum(record["phase_control_samples"]) == 4096
|
|
calls.append("optimizer")
|
|
return original(*args)
|
|
monkeypatch.setattr(protocol, "ppo_step", checked_step)
|
|
algo = {**protocol.PILOT["algorithm"], "horizon": 32}
|
|
with ledger_at(tmp_path) as ledger:
|
|
env = SyntheticEnv(ledger)
|
|
_, failures, timeouts, learning = protocol.optimize_reserved_rollout(
|
|
env, model, optimizer, torch.zeros(128, 46), algo, 91501, 1, ledger, ledger.guard, ledger.guard)
|
|
assert env.steps == 32 and failures == timeouts == 0 and learning["minibatches"] > 0
|
|
assert calls == ["optimizer"]
|
|
assert ledger.data["pending"] is not None # no premature commit
|
|
delta = protocol.parameter_deltas(model, initial)
|
|
assert delta["actor"]["l2"] > 0 and delta["critic"]["l2"] > 0 and delta["log_std"]["l2"] > 0
|
|
|
|
|
|
def test_missing_diagnostics_never_reach_optimizer(tmp_path, monkeypatch):
|
|
model = SimResidualActorCritic(hidden=16)
|
|
before = {k: v.clone() for k, v in model.state_dict().items()}
|
|
def forbidden(*_):
|
|
pytest.fail("optimizer must never be called")
|
|
monkeypatch.setattr(protocol, "ppo_step", forbidden)
|
|
with ledger_at(tmp_path) as ledger:
|
|
with pytest.raises(RuntimeError, match="diagnostics"):
|
|
protocol.optimize_reserved_rollout(SyntheticEnv(ledger, True), model, None, torch.zeros(128, 46),
|
|
{**protocol.PILOT["algorithm"], "horizon": 32}, 91501, 1, ledger, ledger.guard, ledger.guard)
|
|
assert ledger.data["reserved_transitions"] == 4096 and ledger.data["pending"] is not None
|
|
assert all(torch.equal(v, before[k]) for k, v in model.state_dict().items())
|
|
|
|
|
|
def test_old32_checkpoint_rejected_without_mutation(tmp_path):
|
|
model = SimResidualActorCritic(hidden=16)
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=.0003)
|
|
path = tmp_path / "synthetic.pt"
|
|
config = {"hidden": 16, "initial_std": .1}
|
|
save_checkpoint(path, model, optimizer, {"synthetic": True}, 0, None, {}, config)
|
|
raw = torch.load(path, weights_only=True)
|
|
raw["format"] = "skillfactory.contact-cycle-ppo-checkpoint.v1"
|
|
raw["features"] = "policy32"
|
|
torch.save(raw, path)
|
|
initial = {k: v.clone() for k, v in model.state_dict().items()}
|
|
with pytest.raises(ValueError):
|
|
restore_checkpoint(path, model, optimizer, {"synthetic": True}, config)
|
|
assert all(torch.equal(v, initial[k]) for k, v in model.state_dict().items())
|
|
|
|
|
|
def synthetic_trace():
|
|
n, b = 4, 2
|
|
z = torch.zeros(n, b)
|
|
return {"actor_input": torch.zeros(n, b, 46), "actor_metadata_available": torch.ones(n, b, dtype=torch.bool),
|
|
"control_action_available": torch.ones(n, b, dtype=torch.bool), "actor_mean": torch.zeros(n, b, 3),
|
|
"requested_action": torch.zeros(n, b, 3), "reference_used_normalized": torch.zeros(n, b, 3),
|
|
"processed_target": torch.zeros(n, b, 3), "effective_residual_rad": torch.zeros(n, b, 3),
|
|
"scenario": torch.zeros(n, b, dtype=torch.long), "external": torch.zeros(n, b, 2, 6),
|
|
"phase": torch.full((n, b), 3, dtype=torch.long), "signed_height_error": z,
|
|
"foot": torch.zeros(n, b, 3), "target": torch.zeros(n, b, 3),
|
|
"would_fail": torch.zeros(n, b, dtype=torch.bool), "failure_flags": torch.zeros(n, b, 8, dtype=torch.bool)}
|
|
|
|
|
|
def synthetic_audit(episode, _, kind):
|
|
assert kind == "learned_policy"
|
|
return {"sim_functional_passed": True, "metrics": {"landing_error_m": 0.}}
|
|
|
|
|
|
def test_first_episode_slicing_no_autoreset_stitch_and_failure_cost():
|
|
trace = synthetic_trace()
|
|
trace["signed_height_error"][2:, 0] = 100 # autoreset evidence must not improve/contaminate first episode
|
|
rows = protocol.validation_rows(trace, [2, 4], [True, False], torch.zeros(3), {}, synthetic_audit)
|
|
assert rows[0]["height_rms_m"] == 0 and not rows[0]["success"]
|
|
assert rows[0]["first_episode_samples"] == 2 and rows[1]["success"]
|
|
assert selection_key(rows) == (.5, 8.)
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["width32", "nan", "missing_metadata", "delta", "template", "reference",
|
|
"stochastic", "scenario", "external", "missing_stop"])
|
|
def test_validation_required_invariants_fail_closed(fault):
|
|
trace = synthetic_trace()
|
|
stops = [4, 4]
|
|
if fault == "width32":
|
|
trace["actor_input"] = torch.zeros(4, 2, 32)
|
|
elif fault == "nan":
|
|
trace["actor_input"][0, 0, 0] = float("nan")
|
|
elif fault == "missing_metadata":
|
|
trace["actor_metadata_available"][0, 0] = False
|
|
elif fault == "delta":
|
|
trace["effective_residual_rad"][0, 0, 0] = .1
|
|
elif fault == "template":
|
|
trace["processed_target"][0, 0, 0] = .6
|
|
elif fault == "reference":
|
|
trace["reference_used_normalized"][0, 0, 0] = 1.1
|
|
elif fault == "stochastic":
|
|
trace["requested_action"][0, 0, 0] = .1
|
|
elif fault == "scenario":
|
|
trace["scenario"][0, 0] = 1
|
|
elif fault == "external":
|
|
trace["external"][0, 0, 0, 0] = 1
|
|
else:
|
|
stops[0] = None
|
|
with pytest.raises(RuntimeError):
|
|
protocol.validation_rows(trace, stops, [False, False], torch.zeros(3), {}, synthetic_audit)
|
|
|
|
|
|
def test_compact_trace_subset_clones_full_width_and_exact_guard(tmp_path):
|
|
trace = {"actor_input": torch.zeros(100, 128, 46)}
|
|
reserved = []
|
|
size = protocol.save_trace_subset(tmp_path / "trace.pt", trace, [2], [3] * 128, reserved.append)
|
|
assert reserved == [size] and (tmp_path / "trace.pt").stat().st_size == size
|
|
loaded = torch.load(tmp_path / "trace.pt", weights_only=True)["2"]["actor_input"]
|
|
assert loaded.shape == (3, 1, 46) and loaded.untyped_storage().nbytes() == 3 * 46 * 4
|
|
def reject(_):
|
|
raise protocol.BudgetExceeded("no capacity")
|
|
with pytest.raises(protocol.BudgetExceeded):
|
|
protocol.save_trace_subset(tmp_path / "rejected.pt", trace, [2], [3] * 128, reject)
|
|
assert not (tmp_path / "rejected.pt").exists()
|
|
|
|
|
|
def test_conflicting_sim_processes_and_ignore_shell_command_text(tmp_path):
|
|
for pid, command in ((123, b"/bin/bash\0-c\0kit scripts/train_cycle_sim_residual.py\0"),
|
|
(124, b"/usr/bin/python3\0scripts/supervise_sim.py\0--\0/isaaclab.sh\0-p\0"
|
|
b"scripts/verify_cycle_sim_learning_runtime.py\0"),
|
|
(126, b"/kit/python.sh\0scripts/verify_cycle_sim_learning_runtime.py\0"),
|
|
(127, b"/kit/python3\0-c\0from isaaclab.cli import cli; cli()\0-p\0"
|
|
b"scripts/verify_cycle_sim_learning_runtime.py\0")):
|
|
p = tmp_path / str(pid)
|
|
p.mkdir()
|
|
(p / "cmdline").write_bytes(command)
|
|
protocol.assert_no_conflicting_processes(tmp_path)
|
|
p = tmp_path / "125"
|
|
p.mkdir()
|
|
(p / "cmdline").write_bytes(b"/opt/kit/kit\0--headless\0")
|
|
with pytest.raises(RuntimeError, match="125"):
|
|
protocol.assert_no_conflicting_processes(tmp_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def cli(monkeypatch):
|
|
monkeypatch.syspath_prepend(str(ROOT / "scripts"))
|
|
spec = importlib.util.spec_from_file_location("residual_training_cli", ROOT / "scripts/train_cycle_sim_residual.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
# A pre-Kit rejection must not silently import the simulator.
|
|
import builtins
|
|
original = builtins.__import__
|
|
def no_kit(name, *args, **kwargs):
|
|
if name.startswith(("isaaclab", "omni", "isaacsim")):
|
|
pytest.fail("Kit imported before authorization/preparation gates")
|
|
return original(name, *args, **kwargs)
|
|
monkeypatch.setattr(builtins, "__import__", no_kit)
|
|
return module
|
|
|
|
|
|
@pytest.mark.parametrize("flag", ["--resume", "--checkpoint", "--warm-start"])
|
|
def test_cli_has_no_resume_checkpoint_or_warmstart(cli, tmp_path, flag):
|
|
with pytest.raises(SystemExit) as error:
|
|
cli.main(["--train", "--output", str(tmp_path / "no-output"), flag])
|
|
assert error.value.code == 2 and not (tmp_path / "no-output").exists()
|
|
|
|
|
|
def test_cli_old_grant_before_kit_budget(cli, tmp_path):
|
|
grant = synthetic_grant({"synthetic_cpu_only": True})
|
|
grant["format"] = "skillfactory.contact-cycle-authorization.v1"
|
|
path = write(tmp_path / "old_synthetic_grant.json", grant)
|
|
out = tmp_path / "no-output"
|
|
with pytest.raises(PermissionError):
|
|
cli.main(["--train", "--output", str(out), "--authorization", str(path), "--headless"])
|
|
assert not out.exists()
|
|
|
|
|
|
def test_cli_frozen_and_wrong_config_before_kit_budget(cli, tmp_path):
|
|
with pytest.raises(ValueError, match="Frozen"):
|
|
cli.main(["--prepare", "--output", str(ROOT / "releases/new")])
|
|
cfg = copy.deepcopy(protocol.PILOT)
|
|
cfg["seeds"] = [1]
|
|
path = write(tmp_path / "bad.json", cfg)
|
|
with pytest.raises(ValueError):
|
|
cli.main(["--train", "--output", str(tmp_path / "no-output"), "--config", str(path)])
|
|
assert not (tmp_path / "no-output").exists()
|
|
|
|
|
|
def test_cli_preparation_never_imports_kit(cli, tmp_path, monkeypatch):
|
|
calls = []
|
|
monkeypatch.setattr(protocol, "prepare", lambda *args: calls.append(args))
|
|
cli.main(["--prepare", "--output", str(tmp_path / "synthetic")])
|
|
assert len(calls) == 1
|
|
assert "isaaclab.app" not in sys.modules
|
|
|
|
|
|
@pytest.mark.parametrize("reset_fails", [False, True])
|
|
def test_validation_rng_restores_all_streams_even_on_reset_error(reset_fails):
|
|
import random
|
|
|
|
import numpy as np
|
|
|
|
calls = []
|
|
class Env:
|
|
def seed(self, seed):
|
|
calls.append(("seed", seed))
|
|
random.seed(seed)
|
|
np.random.seed(seed)
|
|
torch.manual_seed(seed)
|
|
def reset(self):
|
|
calls.append(("reset",))
|
|
if reset_fails:
|
|
raise RuntimeError("synthetic reset failure")
|
|
torch_before, python_before, numpy_before = torch.get_rng_state(), random.getstate(), np.random.get_state()
|
|
try:
|
|
with protocol.isolated_validation_rng(Env(), 93501, 91501):
|
|
torch.rand(5)
|
|
np.random.random(5)
|
|
random.random()
|
|
except RuntimeError:
|
|
assert reset_fails
|
|
assert calls == [("seed", 93501), ("seed", 91501), ("reset",)]
|
|
assert torch.equal(torch_before, torch.get_rng_state()) and python_before == random.getstate()
|
|
after = np.random.get_state()
|
|
assert numpy_before[0] == after[0] and np.array_equal(numpy_before[1], after[1])
|
|
assert numpy_before[2:] == after[2:]
|
|
|
|
|
|
def test_snapshot_is_checked_during_execution_not_only_preparation(prepared):
|
|
_, out, manifest = prepared
|
|
protocol.verify_snapshot(out, manifest)
|
|
(out / "source_snapshot/input.txt").write_text("changed after startup")
|
|
with pytest.raises(ValueError, match="snapshot"):
|
|
protocol.verify_snapshot(out, manifest)
|
|
|
|
|
|
def test_u0_failure_aborts_before_any_reservation_or_optimizer(cli, tmp_path, monkeypatch):
|
|
# Synthetic recorder/environment exercise runner ordering, never import the simulator.
|
|
from types import SimpleNamespace
|
|
|
|
state = SimpleNamespace(robot=SimpleNamespace(is_fixed_base=False), scenarios=torch.zeros(128, dtype=torch.long),
|
|
stand_only=torch.zeros(128, dtype=torch.bool), nominal=torch.zeros(3))
|
|
class Env:
|
|
cfg = SimpleNamespace(sim=SimpleNamespace(physics=SimpleNamespace(
|
|
solver_type=0, enable_external_forces_every_iteration=False)),
|
|
terminations=SimpleNamespace(failed=object()), episode_length_s=9)
|
|
num_envs, device, physics_dt, step_dt = 128, "cpu", .002, .01
|
|
def seed(self, seed):
|
|
torch.manual_seed(seed)
|
|
def reset(self):
|
|
self.steps = 0
|
|
return {"policy": torch.zeros(128, 46)}, {}
|
|
def step(self, _):
|
|
self.steps += 1
|
|
return ({"policy": torch.zeros(128, 46)}, torch.zeros(128), torch.zeros(128, dtype=torch.bool),
|
|
torch.full((128,), self.steps == 900, dtype=torch.bool), {})
|
|
class Recorder:
|
|
def __init__(self, _):
|
|
self.frames, self.stops, self.early = [], [None] * 128, [False] * 128
|
|
self.initial_state = {"actor_input": torch.zeros(128, 46)}
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, *_):
|
|
pass
|
|
def before_control(self, x, *_):
|
|
self.frames.append({"actor_input": x.clone()})
|
|
def end_control(self, _, truncated, final):
|
|
if final:
|
|
assert truncated.all()
|
|
self.stops = [len(self.frames)] * 128
|
|
monkeypatch.setitem(sys.modules, "SkillFactory.tasks.manager_based.contact_cycle_sim.mdp",
|
|
SimpleNamespace(state=lambda _: state))
|
|
monkeypatch.setitem(sys.modules, "SkillFactory.tasks.manager_based.contact_cycle_sim.recording",
|
|
SimpleNamespace(SimCycleRecorder=Recorder))
|
|
monkeypatch.setattr(protocol, "source_identity", lambda *_: {})
|
|
monkeypatch.setattr(protocol, "validation_rows", lambda *_: [{"env": e, "success": False} for e in range(128)])
|
|
def forbidden(*_):
|
|
pytest.fail("u0 failure must abort before first rollout/optimizer")
|
|
monkeypatch.setattr(protocol, "optimize_reserved_rollout", forbidden)
|
|
(tmp_path / "console.log").touch()
|
|
args = SimpleNamespace(output=tmp_path, config=CFG, console_log=tmp_path / "console.log")
|
|
manifest = {**small_manifest(), "sources": {}}
|
|
with protocol.FreshBudgetLedger(tmp_path, manifest, now=lambda: 100., boot_now=lambda: 10.,
|
|
boot_id="synthetic") as ledger:
|
|
with pytest.raises(RuntimeError, match="u0 did not pass"):
|
|
cli.execute(Env(), protocol.PILOT, args, manifest, ledger, ledger.guard)
|
|
assert ledger.data["reserved_transitions"] == 0 and ledger.data["updates"]["91501"] == 0
|
|
baseline = json.loads((tmp_path / "validation_u0000.json").read_text())
|
|
assert baseline["kind"] == "untrained_zero_residual_network" and not baseline["trained"]
|
|
assert len(baseline["rows"]) == 128
|
|
assert (tmp_path / "untrained_u0000.pt").exists() and not (tmp_path / "latest.pt").exists()
|
|
assert not (tmp_path / "locked_candidate.json").exists()
|