203 lines
6.9 KiB
Python
203 lines
6.9 KiB
Python
"""Statistics, CLI and process lifecycle checks without a GPU."""
|
|
|
|
import copy
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from universal_humanoid import artifacts, jobs
|
|
from universal_humanoid.benchmark import (
|
|
COMMANDS,
|
|
METRICS,
|
|
aggregate,
|
|
compare_benchmarks,
|
|
moments,
|
|
select_checkpoint,
|
|
)
|
|
from universal_humanoid.cli import parser, run
|
|
from universal_humanoid.curves import series_summary, write_svg
|
|
from universal_humanoid.evaluation import profile_spec
|
|
|
|
|
|
def record(seed=0, value=1, command="forward", profile="randomized", checkpoint="c000"):
|
|
metrics = dict.fromkeys(METRICS, value)
|
|
metrics.update(completed_episodes=4, failure_events=2, first_episode_censored_count=1)
|
|
return {
|
|
"checkpoint_id": checkpoint,
|
|
"command_name": command,
|
|
"profile": profile,
|
|
"report": {
|
|
"status": "completed",
|
|
"task": "g1.flat",
|
|
"metrics": metrics,
|
|
"protocol": {
|
|
"seed": seed,
|
|
"command": list(COMMANDS[command]),
|
|
"randomization": {"profile": profile},
|
|
"steps": 500,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def test_aggregate_is_across_seeds_not_worlds():
|
|
rows = aggregate([record(0, 1), record(1, 3)])
|
|
stats = rows[0]["metrics"][METRICS[0]]
|
|
assert stats == {"n": 2, "mean": 2, "variance": 2}
|
|
assert rows[0]["first_episode_censored_count"] == 2
|
|
assert moments([None, None]) == {"n": 0, "mean": None, "variance": None}
|
|
assert moments([2])["variance"] is None
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["duplicate", "protocol", "status"])
|
|
def test_reject_invalid_aggregation(mutation):
|
|
a, b = record(0), record(1)
|
|
if mutation == "duplicate":
|
|
b["report"]["protocol"]["seed"] = 0
|
|
elif mutation == "protocol":
|
|
b["report"]["protocol"]["steps"] = 3
|
|
else:
|
|
b["report"]["status"] = "failed"
|
|
with pytest.raises(ValueError):
|
|
aggregate([a, b])
|
|
|
|
|
|
def matrix(tmp_path, name, value):
|
|
directory = tmp_path / name
|
|
directory.mkdir()
|
|
records = [record(seed, value, command) for command in COMMANDS for seed in (0, 1)]
|
|
filenames = []
|
|
for i, item in enumerate(records):
|
|
filename = f"report-{i}.json"
|
|
artifacts.write_json(directory / filename, item["report"])
|
|
filenames.append(filename)
|
|
summary = {
|
|
"status": "completed",
|
|
"task": "g1.flat",
|
|
"checkpoints": [{"id": "c000"}],
|
|
"rows": aggregate(records),
|
|
"reports": filenames,
|
|
"protocol": {"profiles": ["randomized"], "seeds": [0, 1]},
|
|
}
|
|
path = directory / "summary.json"
|
|
artifacts.write_json(path, summary)
|
|
return path, summary
|
|
|
|
|
|
def test_paired_comparison_and_selection(tmp_path):
|
|
a, _ = matrix(tmp_path, "a", 3)
|
|
b, _ = matrix(tmp_path, "b", 1)
|
|
compared = compare_benchmarks(a, b)
|
|
assert len(compared["paired_candidate_minus_baseline"]) == 4
|
|
stats = compared["paired_candidate_minus_baseline"][0]["metrics"][METRICS[0]]
|
|
assert stats == {"n": 2, "mean": -2, "variance": 0}
|
|
selected = select_checkpoint(b, tmp_path / "selected.json")
|
|
assert selected["selected"]["score"] == 1
|
|
assert not selected["hardware_approved"]
|
|
|
|
|
|
def test_selection_rejects_missing_seed_or_cell(tmp_path):
|
|
path, summary = matrix(tmp_path, "a", 1)
|
|
summary["rows"][0]["seeds"] = [0]
|
|
artifacts.write_json(path, summary)
|
|
with pytest.raises(ValueError, match="seed coverage"):
|
|
select_checkpoint(path, tmp_path / "selection.json")
|
|
summary["rows"].pop()
|
|
artifacts.write_json(path, summary)
|
|
with pytest.raises(ValueError, match="Incomplete"):
|
|
select_checkpoint(path, tmp_path / "selection.json")
|
|
|
|
|
|
def test_comparison_rejects_protocol_mismatch(tmp_path):
|
|
a, _ = matrix(tmp_path, "a", 1)
|
|
b, summary = matrix(tmp_path, "b", 1)
|
|
summary["protocol"]["seeds"] = [10, 11]
|
|
artifacts.write_json(b, summary)
|
|
with pytest.raises(ValueError, match="identical protocol"):
|
|
compare_benchmarks(a, b)
|
|
|
|
|
|
def test_curve_summary_and_escaped_svg(tmp_path):
|
|
points = [{"step": i, "value": 2 * i} for i in range(4)]
|
|
summary = series_summary(points, 3)
|
|
assert summary["tail_slope_per_iteration"] == 2
|
|
assert summary["tail_points"] == 3
|
|
assert series_summary(points[:1], 3)["tail_slope_per_iteration"] is None
|
|
path = tmp_path / "curves.svg"
|
|
write_svg(path, {"<bad&tag>": {"points": points}})
|
|
assert "<bad&tag>" in path.read_text()
|
|
|
|
|
|
def test_profile_specs_are_independent():
|
|
a = profile_spec("stress")
|
|
a["reset_pose_range"]["yaw"] = (10, 20)
|
|
assert profile_spec("stress")["reset_pose_range"]["yaw"] == (-0.2, 0.2)
|
|
assert not profile_spec("nominal")["actor_observation_noise"]
|
|
with pytest.raises(ValueError):
|
|
profile_spec("typo")
|
|
|
|
|
|
def test_resume_requires_explicit_budget():
|
|
args = parser().parse_args(["train", "--resume", "missing.pt", "--output", "unused"])
|
|
with pytest.raises(ValueError, match="explicit --iterations"):
|
|
run(args)
|
|
|
|
|
|
def test_new_cli_arguments():
|
|
args = parser().parse_args(
|
|
[
|
|
"benchmark",
|
|
"--checkpoints",
|
|
"a.pt",
|
|
"b.pt",
|
|
"--output",
|
|
"result",
|
|
"--seeds",
|
|
"1",
|
|
"2",
|
|
"--profiles",
|
|
"randomized",
|
|
"stress",
|
|
]
|
|
)
|
|
assert args.seeds == [1, 2] and len(args.checkpoints) == 2
|
|
args = parser().parse_args(["play", "--checkpoint", "a.pt", "--measure", "joint.csv"])
|
|
assert args.measure == Path("joint.csv")
|
|
|
|
|
|
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux /proc lifecycle")
|
|
def test_pid_identity_stop_and_lost_job(tmp_path):
|
|
output = tmp_path / "run"
|
|
job = jobs.job_directory(output)
|
|
job.mkdir()
|
|
child = subprocess.Popen(
|
|
[sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True
|
|
)
|
|
try:
|
|
identity = jobs.process_identity(child.pid)
|
|
artifacts.write_json(job / "launch.json", {"identity": identity})
|
|
artifacts.write_json(job / "state.json", {"status": "running"})
|
|
assert jobs.status(output)["process_alive"]
|
|
mismatched = copy.deepcopy(identity)
|
|
mismatched["start_ticks"] = "wrong"
|
|
artifacts.write_json(job / "launch.json", {"identity": mismatched})
|
|
with pytest.raises(ValueError, match="PID identity"):
|
|
jobs.stop(output)
|
|
assert child.poll() is None
|
|
artifacts.write_json(job / "launch.json", {"identity": identity})
|
|
if hasattr(signal, "pidfd_send_signal"):
|
|
assert jobs.stop(output)["status"] == "stop-requested"
|
|
else:
|
|
child.terminate()
|
|
child.wait(timeout=5)
|
|
assert jobs.status(output)["effective_status"] == "lost"
|
|
finally:
|
|
if child.poll() is None:
|
|
child.kill()
|
|
child.wait()
|
|
assert jobs.process_identity(os.getpid()) is not None
|