Files
chenlin c32df70326
Checks / Layout, translation and Colab checks (push) Has been cancelled
Checks / Ruff (push) Has been cancelled
Checks / Framework tests (native backend, CPU) (push) Has been cancelled
Checks / Portable Rust controller (push) Has been cancelled
release: v0.1 generic robot training and application platform
2026-11-05 11:05:01 +08:00

755 lines
35 KiB
Python

"""Colab must preserve the selected run, not silently choose one that happens to exist.
These tests use valid ZIP-shaped fake checkpoints and real short-lived subprocesses.
They need no torch, MuJoCo, CUDA or Colab kernel; GPU acceptance is a separate gate.
"""
from __future__ import annotations
import ast
import importlib.util
import json
import signal
import stat
import subprocess
import sys
import textwrap
import threading
import time
import zipfile
from collections import deque
from pathlib import Path
from types import SimpleNamespace
import pytest
REPO = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("colab_helpers", REPO / "tools/colab.py")
assert SPEC is not None and SPEC.loader is not None
colab = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(colab)
def config() -> dict:
return {
"task": "robot.walk", "model": "robot", "num_envs": 256,
"backend": "warp", "device": "cuda:0", "revision": "a" * 40,
"source_sha256": "b" * 64, "runtime_versions": {"torch": "test"},
"mjrl_environment": {},
}
def checkpoint(path: Path, value: str = "weights") -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(path, "w") as archive:
archive.writestr("model/data.pkl", value)
def saved_run(path: Path) -> Path:
model = path / "model_10.pt"
checkpoint(model)
colab.atomic_json(path / colab.MANIFEST, {
"schema": 1, "config": config(), "status": "finished",
"checkpoints": {model.name: colab.sha256(model)},
})
return model
def test_resume_checks_every_compatibility_field_and_the_selected_file(tmp_path):
"""A valid unrelated latest run is a control: it must never make a mismatch pass."""
chosen = saved_run(tmp_path / "chosen")
unrelated = saved_run(tmp_path / "unrelated")
checkpoint(unrelated.parent / "model_999.pt")
colab.validate_resume(chosen, config())
for key in colab.COMPATIBILITY:
changed = config()
changed[key] = "different"
with pytest.raises(ValueError, match=key):
colab.validate_resume(chosen, changed)
checkpoint(chosen, "replacement")
with pytest.raises(ValueError, match="does not match"):
colab.validate_resume(chosen, config())
with pytest.raises(ValueError, match="explicit"):
colab.validate_resume(chosen.parent, config())
def test_latest_checkpoint_is_numeric_and_confined_to_one_run(tmp_path):
checkpoint(tmp_path / "chosen/model_9.pt")
checkpoint(tmp_path / "chosen/model_100.pt")
checkpoint(tmp_path / "unrelated/model_9999.pt")
assert colab.latest_checkpoint(tmp_path / "chosen").name == "model_100.pt"
(tmp_path / "chosen/model_101.pt").write_bytes(b"partially written")
assert colab.latest_checkpoint(tmp_path / "chosen").name == "model_100.pt"
def test_persistence_publishes_checkpoint_and_manifest_together(tmp_path, monkeypatch):
model = saved_run(tmp_path / "run")
manifest = json.loads((model.parent / colab.MANIFEST).read_text())
destination = tmp_path / "drive"
published = colab.persist_checkpoint(model, manifest, destination)
assert (published / model.name).read_bytes() == model.read_bytes()
colab.validate_resume(published / model.name, config())
def interrupted_copy(source, target):
Path(target).write_bytes(b"partial")
raise OSError("drive disconnected")
monkeypatch.setattr(colab.shutil, "copy2", interrupted_copy)
with pytest.raises(OSError, match="disconnected"):
colab.persist_checkpoint(model, manifest, destination)
assert list(destination.iterdir()) == [published]
def test_backup_round_trip_preserves_checkpoint_export_and_existing_destination(tmp_path):
model = saved_run(tmp_path / "run")
export = tmp_path / "export"
export.mkdir()
(export / "actor.onnx").write_bytes(b"onnx")
video = tmp_path / "policy.mp4"
video.write_bytes(b"mp4")
archive = colab.create_backup(model.parent, tmp_path / "backup.zip",
export=export, video=video)
restored = colab.restore_backup(archive, tmp_path / "restored")
assert restored.read_bytes() == model.read_bytes()
assert (tmp_path / "restored/export/actor.onnx").read_bytes() == b"onnx"
assert (tmp_path / "restored/video/policy.mp4").read_bytes() == b"mp4"
colab.validate_resume(restored, config())
with pytest.raises(FileExistsError):
colab.restore_backup(archive, tmp_path / "restored")
before = archive.read_bytes()
with pytest.raises(FileExistsError):
colab.create_backup(model.parent, archive)
assert archive.read_bytes() == before
def test_interrupted_backup_skips_partial_unrecorded_and_replaced_highest_saves(tmp_path):
"""An interrupted last save must not make the preceding complete save undownloadable."""
model = saved_run(tmp_path / "run")
partial = model.parent / "model_11.pt"
partial.write_bytes(b"interrupted torch.save")
checkpoint(model.parent / "model_12.pt", "complete but never recorded")
replaced = model.parent / "model_13.pt"
checkpoint(replaced, "original")
manifest = json.loads((model.parent / colab.MANIFEST).read_text())
manifest["checkpoints"][replaced.name] = colab.sha256(replaced)
checkpoint(replaced, "replaced")
colab.atomic_json(model.parent / colab.MANIFEST, manifest)
assert colab.latest_checkpoint(model.parent) == model
archive = colab.create_backup(model.parent, tmp_path / "interrupted.zip")
with zipfile.ZipFile(archive) as content:
weights = [name for name in content.namelist() if name.endswith(".pt")]
assert weights == ["run/model_10.pt"]
restored = colab.restore_backup(archive, tmp_path / "restored-interrupted")
assert restored.name == "model_10.pt"
colab.validate_resume(restored, config())
@pytest.mark.parametrize("drive_failure", [False, True])
def test_interruption_records_complete_save_not_yet_polled_and_preserves_error(
tmp_path, monkeypatch, drive_failure
):
"""Interrupt immediately after a save, before the next watcher poll can see it."""
script = tmp_path / "fake_train.py"
script.write_text(textwrap.dedent('''
import pathlib, time, zipfile
run = pathlib.Path("logs/robot/robot.walk/interrupted-run")
run.mkdir(parents=True)
print("[mjrl] log directory", run, flush=True)
with zipfile.ZipFile(run / "model_1.pt", "w") as archive:
archive.writestr("model/data.pkl", "complete")
(run / "model_2.pt").write_bytes(b"partial")
pathlib.Path("saved").touch()
while True:
time.sleep(0.1)
'''))
real_queue = colab.queue.Queue
class InterruptAfterAnnouncement(real_queue):
reads = 0
def get(self, *args, **kwargs):
if self.reads:
deadline = time.monotonic() + 5
while not (tmp_path / "saved").exists():
if time.monotonic() > deadline:
raise RuntimeError("fake subprocess did not finish saving")
time.sleep(0.01)
raise KeyboardInterrupt("intentional notebook interruption")
line = super().get(*args, **kwargs)
self.reads += 1
return line
monkeypatch.setattr(colab.queue, "Queue", InterruptAfterAnnouncement)
if drive_failure:
def disconnected(*args, **kwargs):
raise OSError("Drive disconnected during cleanup")
monkeypatch.setattr(colab, "persist_checkpoint", disconnected)
with pytest.raises(KeyboardInterrupt, match="intentional"):
colab.run_training([sys.executable, str(script)], repo=tmp_path,
manifest_path=tmp_path / "operation.json", config=config(),
drive_root=tmp_path / "drive", poll_interval=60)
manifest = json.loads((tmp_path / "operation.json").read_text())
assert manifest["status"] == "interrupted"
assert manifest["error"].startswith("KeyboardInterrupt:")
assert set(manifest["checkpoints"]) == {"model_1.pt"}
run = Path(manifest["run_directory"])
colab.validate_resume(run / "model_1.pt", config())
archive = colab.create_backup(run, tmp_path / "after-interruption.zip")
assert colab.restore_backup(archive, tmp_path / "restored").name == "model_1.pt"
if drive_failure:
assert any("Drive disconnected" in error for error in manifest["cleanup_errors"])
else:
snapshots = list((tmp_path / "drive").glob("*/model_1-*/model_1.pt"))
assert len(snapshots) == 1
colab.validate_resume(snapshots[0], config())
@pytest.mark.parametrize("name", [
"../outside", "/absolute", "C:/Windows/file", "run\\..\\outside",
"run/../outside", "run/./model.pt", "run//model.pt",
])
def test_restore_refuses_traversal_before_extracting_any_file(tmp_path, name):
archive = tmp_path / "bad.zip"
with zipfile.ZipFile(archive, "w") as stream:
stream.writestr("an-otherwise-safe-file", "safe")
stream.writestr(name, "escape")
with pytest.raises(ValueError, match="unsafe"):
colab.restore_backup(archive, tmp_path / "destination")
assert not (tmp_path / "destination").exists()
assert not list(tmp_path.glob(".restore-*"))
assert not (tmp_path / "outside").exists()
def test_restore_refuses_duplicate_symlink_and_oversized_entries(tmp_path):
for kind in ("duplicate", "symlink", "oversized"):
archive = tmp_path / f"{kind}.zip"
with zipfile.ZipFile(archive, "w") as stream:
if kind == "duplicate":
stream.writestr("same", "first")
with pytest.warns(UserWarning, match="Duplicate"):
stream.writestr("same", "second")
elif kind == "symlink":
entry = zipfile.ZipInfo("link")
entry.create_system = 3
entry.external_attr = (stat.S_IFLNK | 0o777) << 16
stream.writestr(entry, "../../outside")
else:
stream.writestr("large", "x" * 100)
with pytest.raises(ValueError):
colab.restore_backup(archive, tmp_path / kind, max_bytes=20)
assert not (tmp_path / kind).exists()
def test_source_identity_tracks_runtime_edits_but_not_a_later_document_edit(tmp_path):
"""Documentation edits must not invalidate the same runtime's saved checkpoint."""
subprocess.run(["git", "init", "-q", tmp_path], check=True)
(tmp_path / "rl").mkdir()
(tmp_path / "rl/runtime.py").write_text("original\n")
(tmp_path / "README.md").write_text("original\n")
subprocess.run(["git", "add", "."], cwd=tmp_path, check=True)
subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.test",
"commit", "-qm", "initial"], cwd=tmp_path, check=True)
original = colab.source_identity(tmp_path)
(tmp_path / "README.md").write_text("documentation edit\n")
assert colab.source_identity(tmp_path) == original
(tmp_path / "rl/runtime.py").write_text("runtime edit\n")
assert colab.source_identity(tmp_path)["source_sha256"] != original["source_sha256"]
(tmp_path / "rl/runtime.py").write_text("original\n")
(tmp_path / ".env.local").write_text("MJRL_COMMAND_LEVEL=1\n")
assert colab.source_identity(tmp_path)["source_sha256"] != original["source_sha256"]
def test_checkpoint_is_persisted_before_training_subprocess_finishes(tmp_path):
"""A final-only backup would pass a simple round trip, but fails this timing check."""
repo = tmp_path / "repo"
repo.mkdir()
script = repo / "fake_train.py"
release = repo / "release"
script.write_text(textwrap.dedent('''
import pathlib, time, zipfile
run = pathlib.Path("logs/robot/robot.walk/new-run")
run.mkdir(parents=True)
print("[mjrl] log directory", run, flush=True)
with zipfile.ZipFile(run / "model_0.pt", "w") as archive:
archive.writestr("model/data.pkl", "checkpoint")
deadline = time.monotonic() + 10
while not pathlib.Path("release").exists():
if time.monotonic() > deadline:
raise RuntimeError("checkpoint was not persisted during training")
time.sleep(0.02)
print("training complete", flush=True)
'''))
observed = []
def acknowledge_snapshot():
deadline = time.monotonic() + 8
while time.monotonic() < deadline:
snapshots = list((tmp_path / "drive").glob("*/model_0-*/model_0.pt"))
if snapshots:
observed.append(snapshots[0])
release.touch()
return
time.sleep(0.02)
observer = threading.Thread(target=acknowledge_snapshot, daemon=True)
observer.start()
result = colab.run_training([sys.executable, str(script)], repo=repo,
manifest_path=tmp_path / "operation.json", config=config(),
drive_root=tmp_path / "drive", poll_interval=0.03)
observer.join(timeout=1)
assert observed and result["status"] == "finished"
colab.validate_resume(observed[0], config())
assert json.loads((tmp_path / "operation.json").read_text())["status"] == "finished"
def test_failed_subprocess_keeps_a_loud_failure_manifest(tmp_path):
with pytest.raises(subprocess.CalledProcessError) as failure:
colab.run_training([sys.executable, "-c", "raise SystemExit(7)"], repo=tmp_path,
manifest_path=tmp_path / "operation.json", config=config())
assert failure.value.returncode == 7
manifest = json.loads((tmp_path / "operation.json").read_text())
assert manifest["status"] == "failed"
assert "CalledProcessError" in manifest["error"]
def test_notebook_has_plain_python_cells_and_runs_repo_scripts_in_isolated_processes():
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
code_cells = ["".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code"]
for index, source in enumerate(code_cells):
ast.parse(source, filename=f"cell_{index}")
assert not any(line.startswith(("!pip", "%pip", "%run"))
for line in source.splitlines())
source = "\n".join(code_cells)
assert 'system_site_packages=False' in source
assert 'PYTHON, "scripts/play.py"' in source
assert 'PYTHON, "scripts/export.py"' in source
assert 'PYTHON, "tools/colab.py", "train"' in source
assert '"--iterations", "5"' in source and 'str(min(NUM_ENVS, 256))' in source
assert '"--backend", "warp", "--device", "cuda:0"' in source
assert '"--physics-hz", str(rates["physics_hz"])' in source
assert '"--steps", "500"' in source
assert 'git", "reset' not in source
assert "import torch" not in source
assert "files.download(str(BACKUP_ZIP))" in source
backup_cell = next(cell for cell in code_cells if "files.download(str(BACKUP_ZIP))" in cell)
assert "from google.colab import files" in backup_cell
def notebook_cell(fragment):
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
return next("".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code" and fragment in "".join(cell["source"]))
@pytest.mark.parametrize("overrides", [
{"CHECKOUT_FOLDER": "../another"},
{"CHECKOUT_FOLDER": "a/b"},
{"SIM_SECONDS": 0},
{"SIM_SECONDS": float("inf")},
{"SIM_FPS": -1},
{"SIM_WIDTH": 641},
{"SIM_HEIGHT": 0},
{"SIM_CAMERA_DISTANCE": -1},
{"SIM_SCENE": "invented"},
{"CORE_VERSION_OVERRIDES": '{"untrusted-package":"1.0"}'},
{"CORE_VERSION_OVERRIDES": '{"numpy":"2.5.2 --extra-index-url attacker"}'},
{"CORE_VERSION_OVERRIDES": '{"torch":"2.8.0"}'},
])
def test_notebook_form_refuses_unsafe_or_invalid_choices_before_setup(overrides):
"""A valid form is a control; reject edited parameters before any GPU subprocess."""
# Forms are separate; invalid values still have to stop before setup or replay.
sources = [notebook_cell("WORKFLOW_MODE ="),
notebook_cell("CORE_VERSION_OVERRIDES ="), notebook_cell("SIM_SECONDS =")]
source = "\n".join(sources)
namespace = {}
exec(compile(source, "valid_colab_form", "exec"), namespace) # noqa: S102
assert namespace["NUM_ENVS"] == 256 and namespace["SIM_SCENE"] == "task default"
tree = ast.parse(source)
for node in tree.body:
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
if isinstance(target, ast.Name) and target.id in overrides:
node.value = ast.Constant(overrides[target.id])
with pytest.raises(ValueError):
exec(compile(ast.fix_missing_locations(tree), "invalid_colab_form", "exec"), {}) # noqa: S102
@pytest.mark.parametrize("scene,distance,measure", [
("task default", 0.0, False), ("ice", 1.4, True),
])
def test_notebook_simulation_uses_control_time_and_records_the_selected_checkpoint(
tmp_path, monkeypatch, scene, distance, measure
):
"""Changing recording FPS must not change control steps or the policy's physics rate."""
session = tmp_path / "session"
session.mkdir()
run_dir = tmp_path / "selected-run"
run_dir.mkdir()
model = run_dir / "model_20.pt"
model.touch()
artifacts = session / "artifacts"
artifacts.mkdir()
commands = []
def run(command, **kwargs):
commands.append(command)
if "rates" in command:
Path(command[command.index("--out") + 1]).write_text(
json.dumps({"physics_hz": 200.0, "control_hz": 50.0,
"supports_velocity_command": True})
)
else:
Path(command[command.index("--video") + 1]).write_bytes(b"new MP4")
Path(command[command.index("--replay-info-out") + 1]).write_text(
json.dumps({"command_ranges": {"status": "matched", "curriculum_level": 0}})
)
if "--measure" in command:
directory = model.parent / "measure"
directory.mkdir()
(directory / "measure.csv").write_text("t,pos/joint\n0.02,0.1\n")
(directory / "measure.png").write_bytes(b"plot")
namespace = {
"SESSION": session, "ARTIFACTS": artifacts, "REPO": tmp_path,
"PYTHON": tmp_path / ".venv/python", "TASK": "robot.walk", "MODEL": "robot",
"CHECKPOINT_PATH": model, "RUN_DIR": run_dir,
"training": {"operation": "selected", "checkpoint": str(model),
"config": {"task": "robot.walk", "model": "robot"}},
"Path": Path,
"SIM_SECONDS": 1.01, "SIM_FPS": 12.0, "SIM_WIDTH": 800, "SIM_HEIGHT": 600,
"SIM_SCENE": scene, "SIM_CAMERA_DISTANCE": distance, "RECORD_JOINTS": measure,
"SIM_COMMAND": "Sampled commands",
"run": run, "json": json, "math": __import__("math"), "shutil": __import__("shutil"),
"uuid": __import__("uuid"),
"display": lambda value: None, "Video": lambda *args, **kwargs: None,
"Image": lambda **kwargs: None,
}
monkeypatch.setitem(sys.modules, "IPython", SimpleNamespace())
monkeypatch.setitem(sys.modules, "IPython.display", SimpleNamespace(
display=namespace["display"], Video=namespace["Video"], Image=namespace["Image"],
))
exec(compile(notebook_cell("SIM_STEPS ="), "record_simulation", "exec"), namespace) # noqa: S102
command = commands[-1]
assert command[command.index("--steps") + 1] == "51"
assert command[command.index("--physics-hz") + 1] == "200.0"
assert command[command.index("--video-fps") + 1] == "12.0"
assert command[command.index("--video-width") + 1] == "800"
assert "--checkpoint-command-ranges" in command
assert "--measure" in command if measure else "--measure" not in command
assert "--scene" not in command if scene == "task default" else command[
command.index("--scene") + 1
] == scene
assert "--video-distance" not in command if distance == 0 else command[
command.index("--video-distance") + 1
] == str(distance)
saved = json.loads((artifacts / "simulation-settings.json").read_text())
assert saved["checkpoint"] == str(model) and saved["control_steps"] == 51
assert saved["simulated_seconds"] == 1.02 and saved["observation_noise"] is True
assert namespace["SIMULATION_CHECKPOINT"] == str(model)
if measure:
assert namespace["MEASUREMENT_CHECKPOINT"] == str(model)
assert (artifacts / "measure.csv").is_file()
first_video = namespace["TRAIN_VIDEO"]
namespace.update(RECORD_JOINTS=False, SIM_SCENE="rubber")
exec(compile(notebook_cell("SIM_STEPS ="), "repeat_simulation", "exec"), namespace) # noqa: S102
# A new unmeasured scene must not inherit the earlier ice scene's measurements.
assert namespace["TRAIN_VIDEO"] != first_video and first_video.is_file()
assert not (artifacts / "measure.csv").exists()
assert not (artifacts / "measure.png").exists()
assert not (model.parent / "measure").exists()
assert namespace["MEASUREMENT_CHECKPOINT"] == ""
previous = namespace["REPLAY_HISTORY"]
assert (previous / "checkpoint-measure/measure.csv").is_file()
assert (previous / "measure.csv").is_file()
assert json.loads((previous / "simulation-settings.json").read_text())["scene"] == "ice"
current = json.loads((artifacts / "simulation-settings.json").read_text())
assert current["scene"] == "rubber" and current["record_joints"] is False
@pytest.mark.parametrize("changed", [{"TASK": "robot.other-gait"}, {"MODEL": "other-robot"}])
def test_notebook_replay_rejects_task_or_model_changes_before_archiving_or_gpu(changed):
"""Matching tensor dimensions cannot establish a checkpoint's gait or robot semantics."""
namespace = {"TASK": "robot.walk", "MODEL": "robot", "training": {
"config": {"task": "robot.walk", "model": "robot"},
}, **changed}
with pytest.raises(ValueError, match="Change only SIM_"):
# No Path, shutil or GPU runner is supplied: the task/model gate must come first.
exec(compile(notebook_cell("SIM_STEPS ="), "refuse_wrong_task", "exec"), namespace) # noqa: S102
@pytest.mark.parametrize("changed", [{}, {"TASK": "robot.other-gait"}, {"MODEL": "other-robot"}])
def test_notebook_export_keeps_the_recorded_task_and_model(tmp_path, changed):
"""Equal tensor dimensions must not let a checkpoint acquire another task's contract."""
session = tmp_path / "session"
session.mkdir()
artifacts = session / "artifacts"
artifacts.mkdir()
run_dir = tmp_path / "selected-run"
run_dir.mkdir()
model = run_dir / "model_20.pt"
model.touch()
commands = []
def run(command, **kwargs):
commands.append(command)
output = Path(command[command.index("--out") + 1])
output.mkdir()
(output / "actor.onnx").write_bytes(b"exported actor")
(output / "layout.json").write_text("{}")
namespace = {
"SESSION": session, "ARTIFACTS": artifacts, "REPO": tmp_path,
"PYTHON": tmp_path / ".venv/python", "TASK": "robot.walk", "MODEL": "robot",
"CHECKPOINT_PATH": model, "RUN_DIR": run_dir,
"training": {"config": {"task": "robot.walk", "model": "robot"}},
"run": run, "json": json, "uuid": __import__("uuid"),
"EXPORT_SUCCESS": True, "EXPORTED_CHECKPOINT": "earlier-checkpoint",
**changed,
}
source = compile(notebook_cell("EXPORT_DIR = SESSION"), "export_selected_policy", "exec")
if changed:
with pytest.raises(ValueError, match="different task/model"):
exec(source, namespace) # noqa: S102
assert commands == [] and "EXPORT_DIR" not in namespace
assert not namespace["EXPORT_SUCCESS"] and namespace["EXPORTED_CHECKPOINT"] == ""
assert list(artifacts.iterdir()) == []
else:
# The matching control must reach export and associate the contract with this checkpoint.
exec(source, namespace) # noqa: S102
assert len(commands) == 1
command = commands[0]
assert command[command.index("--task") + 1] == "robot.walk"
assert command[command.index("--model") + 1] == "robot"
assert command[command.index("--checkpoint") + 1] == model
assert namespace["EXPORT_SUCCESS"] and namespace["EXPORTED_CHECKPOINT"] == str(model)
saved = json.loads((artifacts / "export-settings.json").read_text())
assert saved["checkpoint"] == str(model) and saved["run"] == str(run_dir)
@pytest.mark.parametrize("matching", [False, True])
def test_notebook_backup_refuses_stale_video_and_export_without_blocking_checkpoints(
tmp_path, monkeypatch, matching
):
"""Old preview/export globals must not contaminate a newer training run's portable ZIP."""
session = tmp_path / "session"
session.mkdir()
for name in ("runtime.json", "graphics.json"):
(session / name).write_text("{}")
run_dir = tmp_path / "new-run"
run_dir.mkdir()
model = run_dir / "model_20.pt"
model.touch()
manifest = session / "training.json"
manifest.write_text(json.dumps({
"run_directory": str(run_dir), "operation": "new", "checkpoint": str(model),
"checkpoints": {model.name: "recorded"},
}))
video = session / "old-video.mp4"
video.write_bytes(b"MP4")
commands, downloads = [], []
files = SimpleNamespace(download=lambda name: downloads.append(name))
monkeypatch.setitem(sys.modules, "google", SimpleNamespace())
monkeypatch.setitem(sys.modules, "google.colab", SimpleNamespace(files=files))
def run(command, **kwargs):
commands.append(command)
Path(command[command.index("--out") + 1]).write_bytes(b"ZIP")
chosen = str(model) if matching else str(tmp_path / "old-run/model_20.pt")
namespace = {
"SESSION": session, "REPO": tmp_path, "PYTHON": tmp_path / ".venv/python",
"TRAIN_MANIFEST": manifest, "TASK": "robot.walk", "DOWNLOAD_FILES": False,
"EXPORT_SUCCESS": True, "EXPORTED_CHECKPOINT": chosen,
"SIMULATION_CHECKPOINT": chosen, "EXPORT_DIR": session / "old-export",
"TRAIN_VIDEO": video, "json": json, "Path": Path, "uuid": __import__("uuid"),
"run": run,
}
exec(compile(notebook_cell("BACKUP_CHECKPOINT ="), "package_selected_run", "exec"), namespace) # noqa: S102
command = commands[0]
assert "--export" in command if matching else "--export" not in command
assert "--video" in command if matching else "--video" not in command
assert command[command.index("--run") + 1] == run_dir
assert command[command.index("--extras") + 1] == session / "artifacts/new"
assert downloads == []
@pytest.mark.parametrize("existing_interpreter", [False, True])
def test_notebook_bootstraps_only_target_venv_when_ensurepip_is_unavailable(
tmp_path, existing_interpreter
):
"""Model Colab's missing ensurepip and an earlier half-created pip-less environment."""
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
source = next("".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code" and "PYTHON = REPO" in "".join(cell["source"]))
bootstrap = source[source.index("PYTHON = REPO"):source.index("SESSION =")]
python = tmp_path / ".venv/bin/python"
if existing_interpreter:
python.parent.mkdir(parents=True)
python.touch()
builds, commands = [], []
class PiplessEnvBuilder:
def __init__(self, *, with_pip, system_site_packages):
# This is the failure the previous with_pip=True notebook encountered.
if with_pip:
raise RuntimeError("ensurepip is unavailable in this Colab runtime")
if system_site_packages:
raise RuntimeError("kernel site packages must remain isolated")
def create(self, directory):
builds.append(directory)
python.parent.mkdir(parents=True)
python.touch()
namespace = {
"REPO": tmp_path, "sys": sys,
"venv": SimpleNamespace(EnvBuilder=PiplessEnvBuilder),
"subprocess": SimpleNamespace(check_output=lambda *a, **k: "pip 25.2 from kernel"),
"run": lambda command: commands.append(command),
}
exec(compile(bootstrap, "notebook_venv_bootstrap", "exec"), namespace) # noqa: S102
assert bool(builds) is not existing_interpreter
assert commands[-1] == [
sys.executable, "-m", "pip", "--python", python, "install", "--upgrade", "pip",
]
assert all("--python" in command for command in commands if "install" in command)
def test_notebook_reemits_subprocess_logs_and_keeps_a_bounded_failure_tail(capsys):
"""Colab's RPC captures print outputs but misses inherited child file descriptors."""
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
source = next("".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code" and "def run(command" in "".join(cell["source"]))
function = source[source.index("def run(command"):source.index("# Fail before downloads")]
namespace = {"subprocess": subprocess, "deque": deque, "signal": signal}
exec(compile(function, "notebook_process_run", "exec"), namespace) # noqa: S102
result = namespace["run"]([sys.executable, "-c", "print('visible child output')"])
assert isinstance(result, subprocess.CompletedProcess) and result.returncode == 0
assert "visible child output" in capsys.readouterr().out
with pytest.raises(subprocess.CalledProcessError) as failure:
namespace["run"]([
sys.executable, "-c",
("import sys; [print('line', i) for i in range(70)]; "
"print('visible stderr', file=sys.stderr); raise SystemExit(7)"),
])
output = capsys.readouterr().out
assert "visible stderr" in output and "line 0" in output
assert failure.value.returncode == 7
assert "line 69" in failure.value.output
assert "line 0\n" not in failure.value.output
@pytest.mark.parametrize("nvidia_available", [False, True])
def test_notebook_egl_override_is_session_local_and_never_selects_mesa(
tmp_path, nvidia_available
):
"""A GPU driver can exist while GLVND's only registered vendor remains Mesa."""
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
source = next("".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code"
and "def configure_nvidia_egl(" in "".join(cell["source"]))
function = source[source.index("def configure_nvidia_egl("):source.index("NVIDIA_EGL =")]
session = tmp_path / "session"
session.mkdir()
vendor_directory = tmp_path / "system/egl_vendor.d"
vendor_directory.mkdir(parents=True)
mesa_config = vendor_directory / "50_mesa.json"
mesa_config.write_text('{"ICD":{"library_path":"libEGL_mesa.so.0"}}')
mesa_library = tmp_path / "libEGL_mesa.so.0"
mesa_library.write_text("existing Mesa library")
nvidia_library = tmp_path / "driver/libEGL_nvidia.so.0"
if nvidia_available:
nvidia_library.parent.mkdir()
nvidia_library.write_text("existing NVIDIA library")
candidates = [tmp_path / "missing/libEGL_nvidia.so.0", nvidia_library, mesa_library]
environment = {"LD_LIBRARY_PATH": "/existing/path",
"__EGL_VENDOR_LIBRARY_FILENAMES": str(mesa_config)}
original_environment = dict(environment)
original_mesa = mesa_config.read_bytes()
namespace = {"Path": Path, "json": json, "os": SimpleNamespace(environ=environment)}
exec(compile(function, "notebook_egl_configuration", "exec"), namespace) # noqa: S102
if not nvidia_available:
with pytest.raises(RuntimeError, match="No preinstalled NVIDIA EGL"):
namespace["configure_nvidia_egl"](session, candidates)
assert environment == original_environment
assert not list(session.iterdir())
else:
selected = namespace["configure_nvidia_egl"](session, candidates)
assert selected == nvidia_library.resolve()
override = session / "nvidia-egl.json"
assert json.loads(override.read_text()) == {
"file_format_version": "1.0.0",
"ICD": {"library_path": str(nvidia_library.resolve())},
}
assert environment["__EGL_VENDOR_LIBRARY_FILENAMES"] == str(override.resolve())
assert environment["LD_LIBRARY_PATH"] == f"{selected.parent}:/existing/path"
assert environment["MUJOCO_GL"] == environment["PYOPENGL_PLATFORM"] == "egl"
assert list(session.iterdir()) == [override]
assert mesa_config.read_bytes() == original_mesa
assert list(vendor_directory.iterdir()) == [mesa_config]
@pytest.mark.parametrize("actual_vendor", ["NVIDIA Corporation", "Mesa/X.org"])
def test_notebook_egl_probe_checks_actual_vendor_before_accepting_pixels(
tmp_path, monkeypatch, actual_vendor
):
"""Nonempty pixels alone also pass with llvmpipe; the actual vendor is the gate."""
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
source = next("".join(cell["source"]) for cell in notebook["cells"]
if cell["cell_type"] == "code"
and "graphics_probe = " in "".join(cell["source"]))
tree = ast.parse(source)
probe = next(ast.literal_eval(node.value) for node in tree.body
if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == "graphics_probe"
for target in node.targets))
result = tmp_path / "graphics.json"
class Pixels:
shape = (64, 64, 3)
def __ne__(self, other):
return self
class Renderer:
closed = False
def update_scene(self, data):
pass
def render(self):
return Pixels()
def close(self):
self.closed = True
renderer = Renderer()
fake_mujoco = SimpleNamespace(
MjModel=SimpleNamespace(from_xml_string=lambda xml: object()),
MjData=lambda model: object(), mj_forward=lambda model, data: None,
Renderer=lambda model, **kwargs: renderer,
)
fake_gl = SimpleNamespace(
GL_VENDOR=1, GL_RENDERER=2,
glGetString=lambda kind: (actual_vendor if kind == 1 else "example renderer").encode(),
)
monkeypatch.setitem(sys.modules, "mujoco", fake_mujoco)
monkeypatch.setitem(sys.modules, "numpy", SimpleNamespace(
any=lambda pixels, **kwargs: pixels, count_nonzero=lambda pixels: 4096,
))
monkeypatch.setitem(sys.modules, "OpenGL", SimpleNamespace(GL=fake_gl))
monkeypatch.setattr(sys, "argv", ["isolated_egl_probe", str(result)])
monkeypatch.setenv("__EGL_VENDOR_LIBRARY_FILENAMES", str(tmp_path / "nvidia-egl.json"))
if actual_vendor.startswith("NVIDIA"):
exec(compile(probe, "isolated_egl_probe", "exec"), {}) # noqa: S102
graphics = json.loads(result.read_text())
assert graphics["gl_vendor"] == actual_vendor
assert graphics["nonzero_pixels"] == 4096 and graphics["shape"] == [64, 64, 3]
else:
with pytest.raises(RuntimeError, match="actual GL vendor=.*Mesa"):
exec(compile(probe, "isolated_egl_probe", "exec"), {}) # noqa: S102
assert not result.exists()
assert renderer.closed