Mujoco_WASM/training_server/tests/test_server.py

import subprocess
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from server import (  # noqa: E402
    MAX_JOBS,
    ApiError,
    TrainingJob,
    TrainingManager,
    TrainingRequestHandler,
    termination_signal_handler,
)


class TrainingManagerTest(unittest.TestCase):
    def setUp(self):
        self.temporary = tempfile.TemporaryDirectory()
        self.root = Path(self.temporary.name)
        (self.root / "scripts").mkdir()
        (self.root / "scripts" / "train.py").write_text(
            """import os, pathlib, time
print('WANDB_MODE=' + os.environ.get('WANDB_MODE', ''), flush=True)
print('Learning iteration 1 / 2', flush=True)
time.sleep(0.02)
print('Learning iteration 2 / 2', flush=True)
out=pathlib.Path('logs/rsl_rl/test/run/policy.onnx')
out.parent.mkdir(parents=True, exist_ok=True)
out.write_bytes(b'onnx')
""",
            encoding="utf-8",
        )
        self.manager = TrainingManager(
            self.root, sys.executable, ("Unitree-Go2-Flat",), check_environment=False
        )

    def tearDown(self):
        self.temporary.cleanup()

    @staticmethod
    def payload(**patch):
        value = {
            "taskId": "Unitree-Go2-Flat",
            "numEnvs": 16,
            "maxIterations": 2,
            "seed": 42,
            "runName": "browser-test",
            "device": "cpu",
            "gpuIds": [],
            "wandbMode": "offline",
        }
        value.update(patch)
        return value

    def test_validates_allowlist_and_limits(self):
        with self.assertRaises(ApiError):
            self.manager.parse_config(self.payload(taskId="shell injection"))
        with self.assertRaises(ApiError):
            self.manager.parse_config(self.payload(numEnvs=0))
        with self.assertRaises(ApiError):
            self.manager.parse_config(self.payload(runName="bad name"))
        with self.assertRaises(ApiError):
            self.manager.parse_config(self.payload(wandbMode="login"))

    def test_builds_argument_array_without_shell(self):
        config = self.manager.parse_config(self.payload(device="gpu", gpuIds=[0, 2]))
        command = self.manager.command_for(config)
        self.assertEqual(
            command[:4], [sys.executable, "-u", "scripts/train.py", "Unitree-Go2-Flat"]
        )
        self.assertEqual(command[-2:], ["--gpu-ids", "[0,2]"])

    def test_requires_local_host_origin_and_bearer_token(self):
        handler = object.__new__(TrainingRequestHandler)
        handler.access_token = "secret-token-1234"
        handler.allowed_origins = ()
        handler.headers = {
            "Host": "127.0.0.1:8765",
            "Origin": "http://localhost:5173",
            "Authorization": "Bearer secret-token-1234",
        }
        handler._ensure_request()
        handler.headers["Authorization"] = "Bearer wrong-token"
        with self.assertRaises(ApiError) as error:
            handler._ensure_request()
        self.assertEqual(error.exception.status, 401)
        handler.headers["Authorization"] = "Bearer secret-token-1234"
        handler.headers["Host"] = "attacker.example"
        with self.assertRaises(ApiError) as error:
            handler._ensure_request()
        self.assertEqual(error.exception.status, 403)

    def test_caps_completed_job_history(self):
        config = self.manager.parse_config(self.payload())
        for index in range(MAX_JOBS):
            job_id = f"{index:032x}"
            self.manager.jobs[job_id] = TrainingJob(
                id=job_id,
                config=config,
                state="succeeded",
            )
        with patch("server.threading.Thread") as thread:
            created = self.manager.start(self.payload())
        self.assertEqual(len(self.manager.jobs), MAX_JOBS)
        self.assertNotIn(f"{0:032x}", self.manager.jobs)
        self.assertIn(created["id"], self.manager.jobs)
        thread.return_value.start.assert_called_once()

    def test_cancel_waits_until_starting_process_is_registered(self):
        entered_popen = threading.Event()
        release_popen = threading.Event()
        terminated = threading.Event()

        class FakeStdout:
            def __iter__(self):
                terminated.wait(2)
                return iter(())

            def close(self):
                pass

        class FakeProcess:
            pid = 1234
            stdout = FakeStdout()

            @staticmethod
            def poll():
                return -15 if terminated.is_set() else None

            @staticmethod
            def wait(timeout=None):
                if not terminated.wait(timeout):
                    raise subprocess.TimeoutExpired("fake-training", timeout)
                return -15

        def create_process(*_args, **_kwargs):
            entered_popen.set()
            self.assertTrue(release_popen.wait(2))
            return FakeProcess()

        config = self.manager.parse_config(self.payload())
        job = TrainingJob(id="a" * 32, config=config)
        self.manager.jobs[job.id] = job
        runner = threading.Thread(target=self.manager._run, args=(job,))
        cancel_done = threading.Event()

        def cancel():
            self.manager.cancel(job.id)
            cancel_done.set()

        with (
            patch("server.subprocess.Popen", side_effect=create_process),
            patch("server.os.killpg", side_effect=lambda *_args: terminated.set()) as killpg,
        ):
            runner.start()
            self.assertTrue(entered_popen.wait(2))
            canceller = threading.Thread(target=cancel)
            canceller.start()
            self.assertFalse(cancel_done.wait(0.05))
            release_popen.set()
            canceller.join(2)
            runner.join(2)

        self.assertFalse(runner.is_alive())
        self.assertFalse(canceller.is_alive())
        killpg.assert_called_once_with(FakeProcess.pid, 15)
        self.assertEqual(self.manager.get(job.id)["state"], "cancelled")

    def test_sigterm_enters_controlled_shutdown(self):
        with self.assertRaises(KeyboardInterrupt):
            termination_signal_handler(15, None)

    def test_runs_job_and_exposes_new_onnx_artifact(self):
        job = self.manager.start(self.payload())
        deadline = time.monotonic() + 5
        while time.monotonic() < deadline:
            job = self.manager.get(job["id"])
            if job["state"] not in ("queued", "running"):
                break
            time.sleep(0.02)
        self.assertEqual(job["state"], "succeeded")
        self.assertEqual(job["iteration"], 2)
        self.assertIn("WANDB_MODE=offline", job["logs"])
        self.assertTrue(job["artifactReady"])
        self.assertEqual(self.manager.artifact(job["id"]).read_bytes(), b"onnx")


if __name__ == "__main__":
    unittest.main()