171 lines
5.1 KiB
Python
171 lines
5.1 KiB
Python
"""Pin silent contract failures without requiring a simulator or a GPU."""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from universal_humanoid.artifacts import canonical_hash, digest, tree_hash, write_json
|
|
from universal_humanoid.cli import parser, run
|
|
from universal_humanoid.config import Experiment
|
|
from universal_humanoid.deployment import verify_bundle
|
|
from universal_humanoid.registry import TASKS
|
|
|
|
|
|
def test_registry_is_lightweight():
|
|
command = (
|
|
"import sys; from universal_humanoid.cli import main; main(['list']); "
|
|
"assert 'torch' not in sys.modules; assert 'mjlab' not in sys.modules"
|
|
)
|
|
env = {**os.environ, "PYTHONPATH": str(Path(__file__).parents[1] / "src")}
|
|
subprocess.run([sys.executable, "-c", command], env=env, check=True, capture_output=True)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"patch",
|
|
[
|
|
{"iterations": 0},
|
|
{"num_envs": True},
|
|
{"steps_per_env": -1},
|
|
{"device": "cpu"},
|
|
{"device": "cuda:bad"},
|
|
{"task": "imaginary.flat"},
|
|
{"episode_seconds": float("nan")},
|
|
{"reward_weights": {"a": float("inf")}},
|
|
{"reward_params": {"a": [1]}},
|
|
{"extra_rewards": {"a": {"function": "f", "weight": True}}},
|
|
{"reward_params": {"a": {"std": float("nan")}}},
|
|
{"seed": -1},
|
|
],
|
|
)
|
|
def test_invalid_experiments_rejected(patch):
|
|
with pytest.raises(ValueError):
|
|
Experiment(**patch).validate()
|
|
|
|
|
|
def test_unknown_json_keys_rejected(tmp_path):
|
|
path = tmp_path / "config.json"
|
|
path.write_text('{"num_env": 32}')
|
|
with pytest.raises(ValueError, match="Unknown experiment keys"):
|
|
Experiment.load(path)
|
|
|
|
|
|
def test_json_roundtrip_and_defaults(tmp_path):
|
|
exp = Experiment().validate()
|
|
path = tmp_path / "config.json"
|
|
write_json(path, exp.to_dict())
|
|
assert Experiment.load(path) == exp
|
|
assert len(TASKS) >= 2
|
|
|
|
|
|
def test_fingerprint_detects_untracked_source(tmp_path):
|
|
source = tmp_path / "reward.py"
|
|
source.write_text("weight = 1\n")
|
|
original = tree_hash(tmp_path)
|
|
source.write_text("weight = 2\n")
|
|
assert tree_hash(tmp_path) != original
|
|
assert canonical_hash({"a": 1, "b": 2}) == canonical_hash({"b": 2, "a": 1})
|
|
|
|
|
|
def minimal_bundle(path):
|
|
path.mkdir()
|
|
for name in (
|
|
"policy.onnx",
|
|
"evaluation.json",
|
|
"reference.json",
|
|
"experiment.json",
|
|
"provenance.json",
|
|
"agent.json",
|
|
):
|
|
(path / name).write_text("{}")
|
|
write_json(
|
|
path / "contract.json",
|
|
{
|
|
"schema_version": 1,
|
|
"hardware_approved": False,
|
|
"action": {
|
|
"size": 2,
|
|
"joint_names": ["left", "right"],
|
|
"scale": [0.2, 0.2],
|
|
"offset": [0, 0],
|
|
},
|
|
},
|
|
)
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"hardware_approved": False,
|
|
"files": {p.name: digest(p) for p in path.iterdir()},
|
|
}
|
|
write_json(path / "manifest.json", manifest)
|
|
return manifest
|
|
|
|
|
|
def test_bundle_tamper_rejected(tmp_path):
|
|
bundle = tmp_path / "bundle"
|
|
minimal_bundle(bundle)
|
|
verify_bundle(bundle) # Positive control: an intact bundle passes.
|
|
(bundle / "policy.onnx").write_text("changed")
|
|
with pytest.raises(ValueError, match="integrity"):
|
|
verify_bundle(bundle)
|
|
|
|
|
|
def test_bundle_traversal_rejected(tmp_path):
|
|
bundle = tmp_path / "bundle"
|
|
manifest = minimal_bundle(bundle)
|
|
manifest["files"]["../outside"] = "hash"
|
|
write_json(bundle / "manifest.json", manifest)
|
|
with pytest.raises(ValueError, match="non-local"):
|
|
verify_bundle(bundle)
|
|
|
|
|
|
def test_bundle_cannot_claim_hardware_approval(tmp_path):
|
|
bundle = tmp_path / "bundle"
|
|
manifest = minimal_bundle(bundle)
|
|
manifest["hardware_approved"] = True
|
|
write_json(bundle / "manifest.json", manifest)
|
|
with pytest.raises(ValueError, match="unsafe"):
|
|
verify_bundle(bundle)
|
|
|
|
|
|
def test_comparison_requires_identical_protocol(tmp_path):
|
|
report = {
|
|
"task": "g1.flat",
|
|
"status": "completed",
|
|
"protocol": {"steps": 500},
|
|
"metrics": {
|
|
"tracking_xy_rmse_m_s": 0.1,
|
|
"tracking_yaw_rmse_rad_s": 0.2,
|
|
"failure_events": 0,
|
|
},
|
|
}
|
|
a, b = tmp_path / "a.json", tmp_path / "b.json"
|
|
write_json(a, report)
|
|
write_json(b, report)
|
|
args = parser().parse_args(["compare", str(a), str(b)])
|
|
assert run(args)["candidate_minus_baseline"]["failure_events"] == 0
|
|
report["protocol"]["steps"] = 50
|
|
write_json(b, report)
|
|
with pytest.raises(ValueError, match="identical"):
|
|
run(args)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"command",
|
|
[
|
|
["smoke", "--config", "x", "--steps", "0"],
|
|
["evaluate", "--checkpoint", "x", "--output", "y", "--command", "nan", "0", "0"],
|
|
],
|
|
)
|
|
def test_invalid_cli_bounds_rejected(command):
|
|
with pytest.raises(SystemExit):
|
|
parser().parse_args(command)
|
|
|
|
|
|
def test_all_experiment_examples_parse():
|
|
for path in (Path(__file__).parents[1] / "experiments").glob("*.json"):
|
|
data = json.loads(path.read_text())
|
|
assert Experiment.load(path).task == data["task"]
|