"""Statistics, CLI and process lifecycle checks without a GPU.""" import copy import os import signal import subprocess import sys from pathlib import Path import pytest from universal_humanoid import artifacts, jobs from universal_humanoid.benchmark import ( COMMANDS, METRICS, aggregate, compare_benchmarks, moments, select_checkpoint, ) from universal_humanoid.cli import parser, run from universal_humanoid.curves import series_summary, write_svg from universal_humanoid.evaluation import profile_spec def record(seed=0, value=1, command="forward", profile="randomized", checkpoint="c000"): metrics = dict.fromkeys(METRICS, value) metrics.update(completed_episodes=4, failure_events=2, first_episode_censored_count=1) return { "checkpoint_id": checkpoint, "command_name": command, "profile": profile, "report": { "status": "completed", "task": "g1.flat", "metrics": metrics, "protocol": { "seed": seed, "command": list(COMMANDS[command]), "randomization": {"profile": profile}, "steps": 500, }, }, } def test_aggregate_is_across_seeds_not_worlds(): rows = aggregate([record(0, 1), record(1, 3)]) stats = rows[0]["metrics"][METRICS[0]] assert stats == {"n": 2, "mean": 2, "variance": 2} assert rows[0]["first_episode_censored_count"] == 2 assert moments([None, None]) == {"n": 0, "mean": None, "variance": None} assert moments([2])["variance"] is None @pytest.mark.parametrize("mutation", ["duplicate", "protocol", "status"]) def test_reject_invalid_aggregation(mutation): a, b = record(0), record(1) if mutation == "duplicate": b["report"]["protocol"]["seed"] = 0 elif mutation == "protocol": b["report"]["protocol"]["steps"] = 3 else: b["report"]["status"] = "failed" with pytest.raises(ValueError): aggregate([a, b]) def matrix(tmp_path, name, value): directory = tmp_path / name directory.mkdir() records = [record(seed, value, command) for command in COMMANDS for seed in (0, 1)] filenames = [] for i, item in enumerate(records): filename = f"report-{i}.json" artifacts.write_json(directory / filename, item["report"]) filenames.append(filename) summary = { "status": "completed", "task": "g1.flat", "checkpoints": [{"id": "c000"}], "rows": aggregate(records), "reports": filenames, "protocol": {"profiles": ["randomized"], "seeds": [0, 1]}, } path = directory / "summary.json" artifacts.write_json(path, summary) return path, summary def test_paired_comparison_and_selection(tmp_path): a, _ = matrix(tmp_path, "a", 3) b, _ = matrix(tmp_path, "b", 1) compared = compare_benchmarks(a, b) assert len(compared["paired_candidate_minus_baseline"]) == 4 stats = compared["paired_candidate_minus_baseline"][0]["metrics"][METRICS[0]] assert stats == {"n": 2, "mean": -2, "variance": 0} selected = select_checkpoint(b, tmp_path / "selected.json") assert selected["selected"]["score"] == 1 assert not selected["hardware_approved"] def test_selection_rejects_missing_seed_or_cell(tmp_path): path, summary = matrix(tmp_path, "a", 1) summary["rows"][0]["seeds"] = [0] artifacts.write_json(path, summary) with pytest.raises(ValueError, match="seed coverage"): select_checkpoint(path, tmp_path / "selection.json") summary["rows"].pop() artifacts.write_json(path, summary) with pytest.raises(ValueError, match="Incomplete"): select_checkpoint(path, tmp_path / "selection.json") def test_comparison_rejects_protocol_mismatch(tmp_path): a, _ = matrix(tmp_path, "a", 1) b, summary = matrix(tmp_path, "b", 1) summary["protocol"]["seeds"] = [10, 11] artifacts.write_json(b, summary) with pytest.raises(ValueError, match="identical protocol"): compare_benchmarks(a, b) def test_curve_summary_and_escaped_svg(tmp_path): points = [{"step": i, "value": 2 * i} for i in range(4)] summary = series_summary(points, 3) assert summary["tail_slope_per_iteration"] == 2 assert summary["tail_points"] == 3 assert series_summary(points[:1], 3)["tail_slope_per_iteration"] is None path = tmp_path / "curves.svg" write_svg(path, {"": {"points": points}}) assert "<bad&tag>" in path.read_text() def test_profile_specs_are_independent(): a = profile_spec("stress") a["reset_pose_range"]["yaw"] = (10, 20) assert profile_spec("stress")["reset_pose_range"]["yaw"] == (-0.2, 0.2) assert not profile_spec("nominal")["actor_observation_noise"] with pytest.raises(ValueError): profile_spec("typo") def test_resume_requires_explicit_budget(): args = parser().parse_args(["train", "--resume", "missing.pt", "--output", "unused"]) with pytest.raises(ValueError, match="explicit --iterations"): run(args) def test_new_cli_arguments(): args = parser().parse_args( [ "benchmark", "--checkpoints", "a.pt", "b.pt", "--output", "result", "--seeds", "1", "2", "--profiles", "randomized", "stress", ] ) assert args.seeds == [1, 2] and len(args.checkpoints) == 2 args = parser().parse_args(["play", "--checkpoint", "a.pt", "--measure", "joint.csv"]) assert args.measure == Path("joint.csv") @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux /proc lifecycle") def test_pid_identity_stop_and_lost_job(tmp_path): output = tmp_path / "run" job = jobs.job_directory(output) job.mkdir() child = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True ) try: identity = jobs.process_identity(child.pid) artifacts.write_json(job / "launch.json", {"identity": identity}) artifacts.write_json(job / "state.json", {"status": "running"}) assert jobs.status(output)["process_alive"] mismatched = copy.deepcopy(identity) mismatched["start_ticks"] = "wrong" artifacts.write_json(job / "launch.json", {"identity": mismatched}) with pytest.raises(ValueError, match="PID identity"): jobs.stop(output) assert child.poll() is None artifacts.write_json(job / "launch.json", {"identity": identity}) if hasattr(signal, "pidfd_send_signal"): assert jobs.stop(output)["status"] == "stop-requested" else: child.terminate() child.wait(timeout=5) assert jobs.status(output)["effective_status"] == "lost" finally: if child.poll() is None: child.kill() child.wait() assert jobs.process_identity(os.getpid()) is not None