Files
2026-11-03 19:46:08 +08:00

203 lines
6.9 KiB
Python

"""Statistics, CLI and process lifecycle checks without a GPU."""
import copy
import os
import signal
import subprocess
import sys
from pathlib import Path
import pytest
from universal_humanoid import artifacts, jobs
from universal_humanoid.benchmark import (
COMMANDS,
METRICS,
aggregate,
compare_benchmarks,
moments,
select_checkpoint,
)
from universal_humanoid.cli import parser, run
from universal_humanoid.curves import series_summary, write_svg
from universal_humanoid.evaluation import profile_spec
def record(seed=0, value=1, command="forward", profile="randomized", checkpoint="c000"):
metrics = dict.fromkeys(METRICS, value)
metrics.update(completed_episodes=4, failure_events=2, first_episode_censored_count=1)
return {
"checkpoint_id": checkpoint,
"command_name": command,
"profile": profile,
"report": {
"status": "completed",
"task": "g1.flat",
"metrics": metrics,
"protocol": {
"seed": seed,
"command": list(COMMANDS[command]),
"randomization": {"profile": profile},
"steps": 500,
},
},
}
def test_aggregate_is_across_seeds_not_worlds():
rows = aggregate([record(0, 1), record(1, 3)])
stats = rows[0]["metrics"][METRICS[0]]
assert stats == {"n": 2, "mean": 2, "variance": 2}
assert rows[0]["first_episode_censored_count"] == 2
assert moments([None, None]) == {"n": 0, "mean": None, "variance": None}
assert moments([2])["variance"] is None
@pytest.mark.parametrize("mutation", ["duplicate", "protocol", "status"])
def test_reject_invalid_aggregation(mutation):
a, b = record(0), record(1)
if mutation == "duplicate":
b["report"]["protocol"]["seed"] = 0
elif mutation == "protocol":
b["report"]["protocol"]["steps"] = 3
else:
b["report"]["status"] = "failed"
with pytest.raises(ValueError):
aggregate([a, b])
def matrix(tmp_path, name, value):
directory = tmp_path / name
directory.mkdir()
records = [record(seed, value, command) for command in COMMANDS for seed in (0, 1)]
filenames = []
for i, item in enumerate(records):
filename = f"report-{i}.json"
artifacts.write_json(directory / filename, item["report"])
filenames.append(filename)
summary = {
"status": "completed",
"task": "g1.flat",
"checkpoints": [{"id": "c000"}],
"rows": aggregate(records),
"reports": filenames,
"protocol": {"profiles": ["randomized"], "seeds": [0, 1]},
}
path = directory / "summary.json"
artifacts.write_json(path, summary)
return path, summary
def test_paired_comparison_and_selection(tmp_path):
a, _ = matrix(tmp_path, "a", 3)
b, _ = matrix(tmp_path, "b", 1)
compared = compare_benchmarks(a, b)
assert len(compared["paired_candidate_minus_baseline"]) == 4
stats = compared["paired_candidate_minus_baseline"][0]["metrics"][METRICS[0]]
assert stats == {"n": 2, "mean": -2, "variance": 0}
selected = select_checkpoint(b, tmp_path / "selected.json")
assert selected["selected"]["score"] == 1
assert not selected["hardware_approved"]
def test_selection_rejects_missing_seed_or_cell(tmp_path):
path, summary = matrix(tmp_path, "a", 1)
summary["rows"][0]["seeds"] = [0]
artifacts.write_json(path, summary)
with pytest.raises(ValueError, match="seed coverage"):
select_checkpoint(path, tmp_path / "selection.json")
summary["rows"].pop()
artifacts.write_json(path, summary)
with pytest.raises(ValueError, match="Incomplete"):
select_checkpoint(path, tmp_path / "selection.json")
def test_comparison_rejects_protocol_mismatch(tmp_path):
a, _ = matrix(tmp_path, "a", 1)
b, summary = matrix(tmp_path, "b", 1)
summary["protocol"]["seeds"] = [10, 11]
artifacts.write_json(b, summary)
with pytest.raises(ValueError, match="identical protocol"):
compare_benchmarks(a, b)
def test_curve_summary_and_escaped_svg(tmp_path):
points = [{"step": i, "value": 2 * i} for i in range(4)]
summary = series_summary(points, 3)
assert summary["tail_slope_per_iteration"] == 2
assert summary["tail_points"] == 3
assert series_summary(points[:1], 3)["tail_slope_per_iteration"] is None
path = tmp_path / "curves.svg"
write_svg(path, {"<bad&tag>": {"points": points}})
assert "&lt;bad&amp;tag&gt;" in path.read_text()
def test_profile_specs_are_independent():
a = profile_spec("stress")
a["reset_pose_range"]["yaw"] = (10, 20)
assert profile_spec("stress")["reset_pose_range"]["yaw"] == (-0.2, 0.2)
assert not profile_spec("nominal")["actor_observation_noise"]
with pytest.raises(ValueError):
profile_spec("typo")
def test_resume_requires_explicit_budget():
args = parser().parse_args(["train", "--resume", "missing.pt", "--output", "unused"])
with pytest.raises(ValueError, match="explicit --iterations"):
run(args)
def test_new_cli_arguments():
args = parser().parse_args(
[
"benchmark",
"--checkpoints",
"a.pt",
"b.pt",
"--output",
"result",
"--seeds",
"1",
"2",
"--profiles",
"randomized",
"stress",
]
)
assert args.seeds == [1, 2] and len(args.checkpoints) == 2
args = parser().parse_args(["play", "--checkpoint", "a.pt", "--measure", "joint.csv"])
assert args.measure == Path("joint.csv")
@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux /proc lifecycle")
def test_pid_identity_stop_and_lost_job(tmp_path):
output = tmp_path / "run"
job = jobs.job_directory(output)
job.mkdir()
child = subprocess.Popen(
[sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True
)
try:
identity = jobs.process_identity(child.pid)
artifacts.write_json(job / "launch.json", {"identity": identity})
artifacts.write_json(job / "state.json", {"status": "running"})
assert jobs.status(output)["process_alive"]
mismatched = copy.deepcopy(identity)
mismatched["start_ticks"] = "wrong"
artifacts.write_json(job / "launch.json", {"identity": mismatched})
with pytest.raises(ValueError, match="PID identity"):
jobs.stop(output)
assert child.poll() is None
artifacts.write_json(job / "launch.json", {"identity": identity})
if hasattr(signal, "pidfd_send_signal"):
assert jobs.stop(output)["status"] == "stop-requested"
else:
child.terminate()
child.wait(timeout=5)
assert jobs.status(output)["effective_status"] == "lost"
finally:
if child.poll() is None:
child.kill()
child.wait()
assert jobs.process_identity(os.getpid()) is not None