784 lines
38 KiB
Python
784 lines
38 KiB
Python
"""Notebook reruns must preserve an explicit recovery selection without GPU execution.
|
|
|
|
These checks execute the published cells, rather than copied workflow logic. CPU-only
|
|
ZIP-shaped checkpoints exercise the real portable backup/restore implementation;
|
|
Colab UI and GPU subprocesses are replaced at their external boundaries.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import stat
|
|
import sys
|
|
import uuid
|
|
import zipfile
|
|
from pathlib import Path
|
|
from types import ModuleType, SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
SPEC = importlib.util.spec_from_file_location("notebook_workflow_colab", REPO / "tools/colab.py")
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
colab = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(colab)
|
|
|
|
|
|
def cell(fragment):
|
|
notebook = json.loads((REPO / "notebooks/robot_colab.ipynb").read_text(encoding="utf-8"))
|
|
return next("".join(item["source"]) for item in notebook["cells"]
|
|
if item["cell_type"] == "code" and fragment in "".join(item["source"]))
|
|
|
|
|
|
def execute(fragment, namespace, **parameters):
|
|
tree = ast.parse(cell(fragment))
|
|
for node in tree.body:
|
|
if isinstance(node, ast.Assign) and len(node.targets) == 1:
|
|
target = node.targets[0]
|
|
if isinstance(target, ast.Name) and target.id in parameters:
|
|
node.value = ast.Constant(parameters[target.id])
|
|
code = compile(ast.fix_missing_locations(tree), "published_colab_cell", "exec")
|
|
exec(code, namespace) # noqa: S102 -- execute the actual notebook under test
|
|
|
|
|
|
def settings():
|
|
namespace = {}
|
|
execute("# @title Run settings", namespace)
|
|
execute("# @title Advanced source", namespace)
|
|
execute("# @title Continue training: backup selection", namespace)
|
|
return namespace
|
|
|
|
|
|
def recorded_config():
|
|
return {
|
|
"task": "sample_robot", "model": "sample_robot", "num_envs": 64,
|
|
"revision": "a" * 40, "source_sha256": "b" * 64,
|
|
"backend": "warp", "device": "cuda:0", "mjrl_environment": {},
|
|
"repository_url": "https://github.com/example/robot-workbench.git",
|
|
"runtime_versions": {
|
|
"torch": "2.9.1+cu126", "mujoco": "3.11.0", "mujoco-warp": "3.11.0",
|
|
"warp-lang": "1.18.0", "numpy": "2.5.2", "tensordict": "0.14.2",
|
|
},
|
|
}
|
|
|
|
|
|
def saved_run(directory):
|
|
directory.mkdir(parents=True)
|
|
checkpoint = directory / "model_499.pt"
|
|
with zipfile.ZipFile(checkpoint, "w") as archive:
|
|
archive.writestr("model/data.pkl", "saved optimizer and weights fixture")
|
|
manifest = {
|
|
"schema": 1, "config": recorded_config(), "operation": "recorded-operation",
|
|
"status": "interrupted", "run_directory": str(directory),
|
|
"checkpoint": str(checkpoint), "checkpoints": {checkpoint.name: colab.sha256(checkpoint)},
|
|
}
|
|
colab.atomic_json(directory / colab.MANIFEST, manifest)
|
|
return checkpoint, manifest
|
|
|
|
|
|
@pytest.fixture
|
|
def external_ui(monkeypatch):
|
|
downloads, displays = [], []
|
|
google = ModuleType("google")
|
|
google_colab = ModuleType("google.colab")
|
|
google_colab.files = SimpleNamespace(
|
|
download=downloads.append,
|
|
upload=lambda: pytest.fail("A cached selection must not request another upload"),
|
|
)
|
|
google.colab = google_colab
|
|
monkeypatch.setitem(sys.modules, "google", google)
|
|
monkeypatch.setitem(sys.modules, "google.colab", google_colab)
|
|
ipython = ModuleType("IPython")
|
|
display_module = ModuleType("IPython.display")
|
|
display_module.Image = lambda *args, **kwargs: ("image", args, kwargs)
|
|
display_module.Video = lambda *args, **kwargs: ("video", args, kwargs)
|
|
display_module.display = displays.append
|
|
ipython.display = display_module
|
|
monkeypatch.setitem(sys.modules, "IPython", ipython)
|
|
monkeypatch.setitem(sys.modules, "IPython.display", display_module)
|
|
return SimpleNamespace(downloads=downloads, displays=displays)
|
|
|
|
|
|
def test_basic_and_advanced_reruns_preserve_restored_checkpoint_and_configuration(tmp_path):
|
|
"""The old settings assignment silently erased resume and started training from scratch."""
|
|
namespace = settings()
|
|
checkpoint, manifest = saved_run(tmp_path / "selected")
|
|
state = namespace["RUN_STATE"]
|
|
state.update(restored_checkpoint=str(checkpoint), backup_config=manifest["config"])
|
|
execute("# @title Run settings", namespace)
|
|
execute("# @title Advanced source", namespace)
|
|
assert namespace["RUN_STATE"] is state
|
|
assert state["restored_checkpoint"] == str(checkpoint)
|
|
assert state["backup_config"] == manifest["config"]
|
|
execute("# @title Run settings", namespace, WORKFLOW_MODE="Continue training", ITERATIONS=12)
|
|
assert namespace["TASK"] == "sample_robot" and namespace["NUM_ENVS"] == 64
|
|
assert namespace["ITERATIONS"] == 12 and state["restored_checkpoint"] == str(checkpoint)
|
|
|
|
|
|
def test_new_mode_with_a_recovered_run_fails_before_any_training_subprocess(tmp_path):
|
|
namespace = settings()
|
|
checkpoint, manifest = saved_run(tmp_path / "selected")
|
|
namespace["RUN_STATE"].update(backup_config=manifest["config"],
|
|
restored_checkpoint=str(checkpoint))
|
|
namespace.update(SESSION=tmp_path, PYTHON=tmp_path / "python", REPO=tmp_path,
|
|
PERSIST_ROOT=None, run=lambda *args, **kwargs: pytest.fail("Training started"))
|
|
with pytest.raises(RuntimeError, match="Recovered run is preserved"):
|
|
execute('TRAIN_MANIFEST = SESSION / "training.json"', namespace)
|
|
assert checkpoint.is_file()
|
|
|
|
|
|
def test_legitimate_new_training_selects_its_new_run_and_retains_gate_reports(tmp_path):
|
|
"""Control for the recovered-run gate: it must not prohibit a legitimate fresh run."""
|
|
namespace = settings()
|
|
session = tmp_path / "session"
|
|
session.mkdir()
|
|
for name in ("runtime.json", "graphics.json"):
|
|
(session / name).write_text(json.dumps({"report": name}))
|
|
checkpoint, manifest = saved_run(tmp_path / "newly-trained-run")
|
|
manifest["status"] = "finished"
|
|
commands = []
|
|
|
|
def run(command, **kwargs):
|
|
commands.append(command)
|
|
colab.atomic_json(Path(command[command.index("--manifest") + 1]), manifest)
|
|
|
|
namespace.update(SESSION=session, PYTHON=tmp_path / "python", REPO=tmp_path,
|
|
PERSIST_ROOT=None, TASK="sample_robot", NUM_ENVS=64, ITERATIONS=5, run=run)
|
|
execute('TRAIN_MANIFEST = SESSION / "training.json"', namespace)
|
|
command = commands[0]
|
|
assert command[command.index("--task") + 1] == "sample_robot"
|
|
assert command[command.index("--num-envs") + 1] == "64" and "--resume" not in command
|
|
assert namespace["CHECKPOINT_PATH"] == checkpoint
|
|
assert namespace["RUN_STATE"]["training_manifest"] == str(session / "training.json")
|
|
for name in ("runtime.json", "graphics.json"):
|
|
assert (namespace["ARTIFACTS"] / name).read_bytes() == (session / name).read_bytes()
|
|
|
|
|
|
def test_explicit_new_run_clears_selection_but_preserves_existing_files(tmp_path):
|
|
namespace = settings()
|
|
checkpoint, manifest = saved_run(tmp_path / "old-run")
|
|
namespace["RUN_STATE"].update(backup_config=manifest["config"],
|
|
restored_checkpoint=str(checkpoint), backup_archive="old.zip",
|
|
previous_backup_selection={"backup_archive": "earlier.zip"})
|
|
execute("# @title Advanced source", namespace, START_NEW_RUN=True)
|
|
assert "backup_config" not in namespace["RUN_STATE"]
|
|
assert "restored_checkpoint" not in namespace["RUN_STATE"]
|
|
assert "previous_backup_selection" not in namespace["RUN_STATE"]
|
|
assert checkpoint.is_file() and (checkpoint.parent / colab.MANIFEST).is_file()
|
|
|
|
|
|
def test_setup_reruns_reuse_the_saved_session_directory(tmp_path):
|
|
namespace = settings()
|
|
namespace["RUN_STATE"]["session"] = str(tmp_path / "selected-session")
|
|
namespace["uuid"] = uuid
|
|
tree = ast.parse(cell("SESSION = Path(RUN_STATE.setdefault"))
|
|
assignment = next(node for node in tree.body if isinstance(node, ast.Assign)
|
|
and any(isinstance(target, ast.Name) and target.id == "SESSION"
|
|
for target in node.targets))
|
|
# Execute the published state assignment without the preceding GPU/bootstrap commands.
|
|
program = compile(ast.Module(body=[assignment], type_ignores=[]), "session_state", "exec")
|
|
exec(program, namespace) # noqa: S102
|
|
first = namespace["SESSION"]
|
|
exec(program, namespace) # noqa: S102
|
|
assert namespace["SESSION"] == first == tmp_path / "selected-session"
|
|
|
|
|
|
@pytest.mark.parametrize("environments,expected", [(64, 64), (1024, 256)])
|
|
def test_smoke_checks_the_selected_task_and_bounded_environment_count(
|
|
tmp_path, environments, expected
|
|
):
|
|
namespace = settings()
|
|
commands = []
|
|
|
|
def run(command, **kwargs):
|
|
commands.append(command)
|
|
Path(command[command.index("--manifest") + 1]).write_text(json.dumps({
|
|
"status": "finished", "run_directory": str(tmp_path / "smoke-run"),
|
|
}))
|
|
|
|
namespace.update(SESSION=tmp_path, REPO=tmp_path, PYTHON=tmp_path / "python",
|
|
TASK="sample_robot", NUM_ENVS=environments, run=run)
|
|
execute('SMOKE_MANIFEST = SESSION / "smoke.json"', namespace)
|
|
command = commands[0]
|
|
assert command[command.index("--task") + 1] == "sample_robot"
|
|
assert command[command.index("--num-envs") + 1] == str(expected)
|
|
assert command[command.index("--iterations") + 1] == "5"
|
|
assert "--resume" not in command
|
|
|
|
|
|
def test_resume_selection_uses_all_recorded_versions_and_exact_source_before_install(capsys):
|
|
namespace = settings()
|
|
namespace["RUN_STATE"]["backup_config"] = recorded_config()
|
|
namespace["WORKFLOW_MODE"] = "Continue training"
|
|
execute("# @title Continue training: backup selection", namespace)
|
|
assert namespace["SOURCE_REVISION"] == "a" * 40
|
|
assert namespace["CORE_VERSIONS"] == recorded_config()["runtime_versions"]
|
|
assert namespace["REPOSITORY_URL"] == recorded_config()["repository_url"]
|
|
assert namespace["NUM_ENVS"] == 64
|
|
assert "Continue:" in capsys.readouterr().out
|
|
|
|
|
|
def test_older_backup_without_repository_keeps_operator_selection_and_explains_it(capsys):
|
|
namespace = settings()
|
|
cfg = recorded_config()
|
|
del cfg["repository_url"]
|
|
namespace["RUN_STATE"]["backup_config"] = cfg
|
|
namespace.update(WORKFLOW_MODE="Continue training",
|
|
REPOSITORY_URL="https://github.com/operator/chosen-fork.git")
|
|
execute("# @title Continue training: backup selection", namespace)
|
|
assert namespace["REPOSITORY_URL"] == "https://github.com/operator/chosen-fork.git"
|
|
assert namespace["SOURCE_REVISION"] == cfg["revision"]
|
|
assert "Older backup: use Advanced source" in capsys.readouterr().out
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [
|
|
{"revision": "main"}, {"task": "../tasks"}, {"num_envs": True},
|
|
{"backend": "native"}, {"device": "cpu"},
|
|
{"repository_url": "https://github.com/user/repo.git;echo unwanted"},
|
|
{"runtime_versions": {"torch": "2.9.1+cu126"}},
|
|
{"runtime_versions": {**recorded_config()["runtime_versions"], "numpy": "https://bad/wheel"}},
|
|
])
|
|
def test_backup_metadata_refuses_incomplete_or_executable_setup_values(bad):
|
|
namespace = settings()
|
|
valid = {"schema": 1, "config": recorded_config(), "checkpoints": {"model_499.pt": "hash"}}
|
|
assert namespace["validate_backup_metadata"](valid) == valid["config"]
|
|
invalid = {**valid, "config": {**valid["config"], **bad}}
|
|
with pytest.raises(ValueError):
|
|
namespace["validate_backup_metadata"](invalid)
|
|
|
|
|
|
@pytest.mark.parametrize("unsafe", ["../outside", "/outside", "a\\b", "a/../b", "./a", "C:/a"])
|
|
def test_early_backup_reader_rejects_unsafe_paths_without_extracting(tmp_path, unsafe, monkeypatch):
|
|
namespace = settings()
|
|
if "\\" in unsafe:
|
|
class WireNames(zipfile.ZipInfo):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
# Windows ZipInfo normalizes backslashes; Colab reads the raw Linux wire name.
|
|
self.filename = self.orig_filename
|
|
|
|
monkeypatch.setattr(zipfile, "ZipInfo", WireNames)
|
|
archive = tmp_path / "unsafe.zip"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
output.writestr("run/colab-run.json", json.dumps({"schema": 1}))
|
|
output.writestr(unsafe, "must never be extracted")
|
|
with pytest.raises(ValueError, match="Unsafe"):
|
|
namespace["read_backup_metadata"](archive)
|
|
assert sorted(path.name for path in tmp_path.iterdir()) == ["unsafe.zip"]
|
|
|
|
|
|
def test_early_backup_reader_refuses_symlink_and_oversized_manifest(tmp_path):
|
|
namespace = settings()
|
|
for name, link in (("symlink", True), ("large", False)):
|
|
archive = tmp_path / f"{name}.zip"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
if link:
|
|
member = zipfile.ZipInfo("run/link")
|
|
member.external_attr = (stat.S_IFLNK | 0o777) << 16
|
|
output.writestr(member, "outside")
|
|
output.writestr("run/colab-run.json", "{}")
|
|
else:
|
|
output.writestr("run/colab-run.json", " " * (1024**2 + 1))
|
|
with pytest.raises(ValueError):
|
|
namespace["read_backup_metadata"](archive)
|
|
|
|
|
|
@pytest.mark.parametrize("schema", [True, False, 1.0, "1", None])
|
|
def test_early_manifest_requires_integer_schema_without_boolean_coercion(schema):
|
|
namespace = settings()
|
|
valid = {"schema": 1, "config": recorded_config(), "checkpoints": {"model_499.pt": "hash"}}
|
|
assert namespace["validate_backup_metadata"](valid) == valid["config"]
|
|
with pytest.raises(ValueError, match="Unsupported backup manifest"):
|
|
namespace["validate_backup_metadata"]({**valid, "schema": schema})
|
|
|
|
|
|
def test_early_reader_rejects_a_nul_truncated_zip_name(tmp_path):
|
|
namespace = settings()
|
|
archive = tmp_path / "nul-name.zip"
|
|
original = b"run/dataXsuffix"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
output.writestr("run/colab-run.json", json.dumps({"schema": 1}))
|
|
output.writestr(original.decode(), "payload")
|
|
raw = archive.read_bytes()
|
|
assert raw.count(original) == 2 # Local header and central directory are both altered.
|
|
archive.write_bytes(raw.replace(original, b"run/data\0suffix"))
|
|
with pytest.raises(ValueError, match="Truncated backup member name"):
|
|
namespace["read_backup_metadata"](archive)
|
|
assert sorted(item.name for item in tmp_path.iterdir()) == ["nul-name.zip"]
|
|
|
|
|
|
@pytest.mark.parametrize("environment", [
|
|
{"HOME": "/not-allowed"}, {"MJRL_lowercase": "1"}, {"MJRL_": "1"},
|
|
{"MJRL_TASK": 1}, {"MJRL_TASK": "x" * 4097},
|
|
{"MJRL_TASK": "invalid\0environment"},
|
|
{f"MJRL_KEY_{index}": "1" for index in range(101)}, [],
|
|
])
|
|
def test_manifest_environment_accepts_only_bounded_mjrl_strings(environment):
|
|
namespace = settings()
|
|
valid = {"schema": 1, "config": recorded_config(), "checkpoints": {"model_499.pt": "hash"}}
|
|
valid["config"]["mjrl_environment"] = {"MJRL_COMMAND_LEVEL": "2", "MJRL_TASK": "sample_robot"}
|
|
assert namespace["validate_backup_metadata"](valid) == valid["config"]
|
|
invalid = {**valid, "config": {**valid["config"], "mjrl_environment": environment}}
|
|
with pytest.raises(ValueError, match="Invalid recorded MJRL environment"):
|
|
namespace["validate_backup_metadata"](invalid)
|
|
|
|
|
|
def test_continuation_bootstrap_restores_only_mjrl_environment(monkeypatch):
|
|
namespace = settings()
|
|
cfg = recorded_config()
|
|
cfg["mjrl_environment"] = {"MJRL_COMMAND_LEVEL": "2", "MJRL_TASK": "sample_robot"}
|
|
namespace["RUN_STATE"]["backup_config"] = cfg
|
|
namespace["WORKFLOW_MODE"] = "Continue training"
|
|
environment = {
|
|
"MJRL_COMMAND_LEVEL": "9", "MJRL_OLD_OVERRIDE": "old", "HOME": "preserved-home",
|
|
"CUDA_VISIBLE_DEVICES": "0", "__EGL_VENDOR_LIBRARY_FILENAMES": "preserved-egl",
|
|
}
|
|
monkeypatch.setattr(os, "environ", environment)
|
|
first_statement = ast.parse(cell("SESSION = Path(RUN_STATE.setdefault")).body[0]
|
|
# Execute the real continuation branch before its GPU and clone subprocesses.
|
|
program = compile(ast.Module(body=[first_statement], type_ignores=[]), "restore_environment", "exec")
|
|
exec(program, namespace) # noqa: S102
|
|
assert {key: value for key, value in environment.items() if key.startswith("MJRL_")} == cfg[
|
|
"mjrl_environment"
|
|
]
|
|
assert environment["HOME"] == "preserved-home"
|
|
assert environment["CUDA_VISIBLE_DEVICES"] == "0"
|
|
assert environment["__EGL_VENDOR_LIBRARY_FILENAMES"] == "preserved-egl"
|
|
|
|
|
|
@pytest.mark.parametrize("bad_selection", [
|
|
"boolean-schema", "foreign-environment", "nul-environment", "truncated-name",
|
|
])
|
|
def test_invalid_replacement_upload_keeps_the_existing_verified_selection(
|
|
tmp_path, external_ui, monkeypatch, bad_selection
|
|
):
|
|
checkpoint, manifest = saved_run(tmp_path / "previously-selected")
|
|
namespace = settings()
|
|
state = namespace["RUN_STATE"]
|
|
state.update(backup_config=manifest["config"], backup_manifest=manifest,
|
|
backup_archive="previous.zip", restored_checkpoint=str(checkpoint))
|
|
preserved = dict(state)
|
|
invalid = {**manifest, "config": dict(manifest["config"])}
|
|
if bad_selection == "boolean-schema":
|
|
invalid["schema"] = True
|
|
elif bad_selection == "foreign-environment":
|
|
invalid["config"]["mjrl_environment"] = {"HOME": "not-a-training-setting"}
|
|
elif bad_selection == "nul-environment":
|
|
invalid["config"]["mjrl_environment"] = {"MJRL_TASK": "invalid\0environment"}
|
|
archive = tmp_path / "replacement.zip"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
output.writestr("run/colab-run.json", json.dumps(invalid))
|
|
if bad_selection == "truncated-name":
|
|
output.writestr("run/dataXsuffix", "payload")
|
|
payload = archive.read_bytes()
|
|
if bad_selection == "truncated-name":
|
|
payload = payload.replace(b"run/dataXsuffix", b"run/data\0suffix")
|
|
monkeypatch.setattr(sys.modules["google.colab"].files, "upload", lambda: {"replacement.zip": payload})
|
|
namespace["WORKFLOW_MODE"] = "Continue training"
|
|
namespace["Path"] = lambda value: tmp_path / "upload-inbox" if str(value) == (
|
|
"/content/mjrl-backups"
|
|
) else Path(value)
|
|
with pytest.raises(ValueError):
|
|
execute("# @title Continue training: backup selection", namespace,
|
|
REPLACE_BACKUP_SELECTION=True)
|
|
assert state == preserved and checkpoint.is_file()
|
|
|
|
|
|
@pytest.mark.parametrize("pending", [False, True])
|
|
def test_drive_selection_requires_a_completed_snapshot_and_preserves_failed_replacement(
|
|
tmp_path, external_ui, monkeypatch, pending
|
|
):
|
|
checkpoint, manifest = saved_run(tmp_path / "previous")
|
|
drive_root = tmp_path / "drive/MyDrive"
|
|
folder = ".pending-upload" if pending else "model_499-published"
|
|
candidate, _ = saved_run(drive_root / "operation" / folder)
|
|
namespace = settings()
|
|
state = namespace["RUN_STATE"]
|
|
state.update(backup_config=manifest["config"], backup_manifest=manifest,
|
|
backup_archive="previous.zip", restored_checkpoint=str(checkpoint))
|
|
preserved = dict(state)
|
|
namespace["WORKFLOW_MODE"] = "Continue training"
|
|
namespace["Path"] = lambda value: drive_root if str(value) == "/content/drive/MyDrive" else Path(value)
|
|
mounted = []
|
|
monkeypatch.setattr(sys.modules["google.colab"], "drive",
|
|
SimpleNamespace(mount=mounted.append), raising=False)
|
|
parameters = {"BACKUP_SOURCE": "Drive checkpoint", "DRIVE_CHECKPOINT": str(candidate),
|
|
"REPLACE_BACKUP_SELECTION": True}
|
|
if pending:
|
|
with pytest.raises(ValueError, match="completed Drive snapshot"):
|
|
execute("# @title Continue training: backup selection", namespace, **parameters)
|
|
assert state == preserved
|
|
else:
|
|
execute("# @title Continue training: backup selection", namespace, **parameters)
|
|
assert state["restored_checkpoint"] == str(candidate.resolve())
|
|
assert state["backup_archive"] is None
|
|
assert mounted == ["/content/drive"] and checkpoint.is_file()
|
|
|
|
|
|
def test_restore_cell_reuses_the_verified_selection_without_upload_or_reextract(
|
|
tmp_path, external_ui
|
|
):
|
|
checkpoint, manifest = saved_run(tmp_path / "original")
|
|
archive = colab.create_backup(checkpoint.parent, tmp_path / "saved.zip")
|
|
namespace = settings()
|
|
namespace["RUN_STATE"].update(backup_config=manifest["config"], backup_archive=str(archive))
|
|
namespace.update(WORKFLOW_MODE="Continue training", SESSION=tmp_path / "session",
|
|
PYTHON=tmp_path / "python", REPO=tmp_path)
|
|
namespace["SESSION"].mkdir()
|
|
commands = []
|
|
|
|
def run(command, **kwargs):
|
|
commands.append(command)
|
|
if "restore" in command:
|
|
restored = colab.restore_backup(
|
|
Path(command[command.index("--archive") + 1]),
|
|
Path(command[command.index("--destination") + 1]),
|
|
)
|
|
colab.atomic_json(Path(command[command.index("--out") + 1]),
|
|
{"checkpoint": str(restored)})
|
|
|
|
namespace["run"] = run
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
first = namespace["RESTORED_CHECKPOINT"]
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
assert sum("restore" in command for command in commands) == 1
|
|
assert namespace["RESTORED_CHECKPOINT"] == first
|
|
assert Path(first).read_bytes() == checkpoint.read_bytes()
|
|
assert namespace["RUN_STATE"]["restored_checkpoint"] == first
|
|
|
|
|
|
def replacement_selection(tmp_path, monkeypatch, corruption=None):
|
|
"""Select a real uploaded candidate while retaining a usable earlier backup."""
|
|
old_checkpoint, old_manifest = saved_run(tmp_path / "old-run")
|
|
old_archive = colab.create_backup(old_checkpoint.parent, tmp_path / "old.zip")
|
|
candidate_checkpoint, candidate_manifest = saved_run(tmp_path / "candidate-run")
|
|
candidate_manifest["config"]["revision"] = "c" * 40
|
|
candidate_manifest["config"]["task"] = "another_robot"
|
|
colab.atomic_json(candidate_checkpoint.parent / colab.MANIFEST, candidate_manifest)
|
|
candidate_archive = tmp_path / "candidate.zip"
|
|
if corruption:
|
|
checkpoint_bytes = candidate_checkpoint.read_bytes()
|
|
if corruption == "checkpoint":
|
|
# Outer ZIP CRC is valid; the recorded checkpoint's contents are invalid.
|
|
checkpoint_bytes = b"incomplete checkpoint, with a valid outer member CRC"
|
|
with zipfile.ZipFile(candidate_archive, "w", compression=zipfile.ZIP_STORED) as archive:
|
|
archive.writestr("backup.json", json.dumps({"schema": 1, "run": "run"}))
|
|
archive.writestr("run/" + colab.MANIFEST, json.dumps(candidate_manifest))
|
|
archive.writestr("run/" + candidate_checkpoint.name, checkpoint_bytes)
|
|
if corruption == "crc":
|
|
# Corrupt only the stored policy bytes; manifest reading still succeeds.
|
|
payload = candidate_archive.read_bytes()
|
|
marker = b"saved optimizer and weights fixture"
|
|
assert payload.count(marker) == 1
|
|
candidate_archive.write_bytes(payload.replace(marker, b"X" + marker[1:]))
|
|
else:
|
|
colab.create_backup(candidate_checkpoint.parent, candidate_archive)
|
|
namespace = settings()
|
|
previous = {
|
|
"backup_config": old_manifest["config"], "backup_manifest": old_manifest,
|
|
"backup_archive": str(old_archive), "restored_checkpoint": str(old_checkpoint),
|
|
}
|
|
namespace["RUN_STATE"].update(previous)
|
|
namespace.update(WORKFLOW_MODE="Continue training", SESSION=tmp_path / "session",
|
|
PYTHON=tmp_path / "python", REPO=tmp_path)
|
|
namespace["SESSION"].mkdir()
|
|
namespace["Path"] = lambda value: (
|
|
tmp_path / "upload-inbox" if str(value) == "/content/mjrl-backups" else Path(value)
|
|
)
|
|
monkeypatch.setattr(sys.modules["google.colab"].files, "upload",
|
|
lambda: {"candidate.zip": candidate_archive.read_bytes()})
|
|
execute("# @title Continue training: backup selection", namespace,
|
|
REPLACE_BACKUP_SELECTION=True)
|
|
assert namespace["RUN_STATE"]["previous_backup_selection"] == previous
|
|
assert namespace["RUN_STATE"]["backup_config"] == candidate_manifest["config"]
|
|
assert namespace["RUN_STATE"]["restored_checkpoint"] is None
|
|
return namespace, previous, candidate_checkpoint, candidate_manifest
|
|
|
|
|
|
def restore_runner(commands, configuration):
|
|
"""Run the real CPU restore/validator at the notebook's subprocess boundary."""
|
|
def run(command, **kwargs):
|
|
commands.append(command)
|
|
if "restore" in command:
|
|
checkpoint = colab.restore_backup(
|
|
Path(command[command.index("--archive") + 1]),
|
|
Path(command[command.index("--destination") + 1]),
|
|
)
|
|
colab.atomic_json(Path(command[command.index("--out") + 1]),
|
|
{"checkpoint": str(checkpoint)})
|
|
else:
|
|
assert command[1] == "-c"
|
|
# Nested rollback try/except must not indent the child Python program.
|
|
ast.parse(command[2])
|
|
colab.validate_resume(Path(command[-1]), configuration)
|
|
return run
|
|
|
|
|
|
@pytest.mark.parametrize("corruption,error,match", [
|
|
("crc", zipfile.BadZipFile, "Bad CRC"),
|
|
("checkpoint", ValueError, "no complete matching checkpoint"),
|
|
])
|
|
def test_metadata_readable_replacement_rolls_back_when_real_restore_fails(
|
|
tmp_path, monkeypatch, external_ui, corruption, error, match
|
|
):
|
|
namespace, previous, _, candidate_manifest = replacement_selection(
|
|
tmp_path, monkeypatch, corruption
|
|
)
|
|
commands = []
|
|
namespace["run"] = restore_runner(commands, candidate_manifest["config"])
|
|
with pytest.raises(error, match=match):
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
assert len(commands) == 1 and "restore" in commands[0]
|
|
assert all(namespace["RUN_STATE"][key] == value for key, value in previous.items())
|
|
assert "previous_backup_selection" not in namespace["RUN_STATE"]
|
|
assert namespace["RESTORED_CHECKPOINT"] == previous["restored_checkpoint"]
|
|
assert Path(previous["restored_checkpoint"]).is_file()
|
|
colab.validate_resume(Path(previous["restored_checkpoint"]), previous["backup_config"])
|
|
|
|
|
|
def test_replacement_rolls_back_when_real_resume_validation_fails(
|
|
tmp_path, monkeypatch, external_ui
|
|
):
|
|
namespace, previous, _, manifest = replacement_selection(tmp_path, monkeypatch)
|
|
commands = []
|
|
wrong_environment = {**manifest["config"], "num_envs": manifest["config"]["num_envs"] + 1}
|
|
namespace["run"] = restore_runner(commands, wrong_environment)
|
|
with pytest.raises(ValueError, match="resume configuration differs: num_envs"):
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
assert len(commands) == 2 and commands[1][1] == "-c"
|
|
assert Path(commands[1][-1]).is_file() # The candidate was fully restored first.
|
|
assert all(namespace["RUN_STATE"][key] == value for key, value in previous.items())
|
|
assert "previous_backup_selection" not in namespace["RUN_STATE"]
|
|
assert namespace["RESTORED_CHECKPOINT"] == previous["restored_checkpoint"]
|
|
colab.validate_resume(Path(previous["restored_checkpoint"]), previous["backup_config"])
|
|
|
|
|
|
def test_valid_replacement_commits_selection_and_repeat_does_not_reextract(
|
|
tmp_path, monkeypatch, external_ui
|
|
):
|
|
namespace, previous, checkpoint, manifest = replacement_selection(tmp_path, monkeypatch)
|
|
commands = []
|
|
namespace["run"] = restore_runner(commands, manifest["config"])
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
selected = namespace["RESTORED_CHECKPOINT"]
|
|
assert selected != previous["restored_checkpoint"]
|
|
assert Path(selected).read_bytes() == checkpoint.read_bytes()
|
|
assert namespace["RUN_STATE"]["restored_checkpoint"] == selected
|
|
assert namespace["RUN_STATE"]["backup_config"] == manifest["config"]
|
|
assert "previous_backup_selection" not in namespace["RUN_STATE"]
|
|
monkeypatch.setattr(sys.modules["google.colab"].files, "upload",
|
|
lambda: pytest.fail("Verified replacement requested another upload"))
|
|
execute("# @title Continue training: backup selection", namespace)
|
|
execute("# @title Verify the selected continuation checkpoint", namespace)
|
|
assert sum("restore" in command for command in commands) == 1
|
|
assert sum(command[1] == "-c" for command in commands) == 2
|
|
assert namespace["RESTORED_CHECKPOINT"] == selected
|
|
assert Path(previous["restored_checkpoint"]).is_file()
|
|
|
|
|
|
def test_demo_disabled_does_not_block_downloads_of_current_policy(tmp_path, external_ui):
|
|
namespace = settings()
|
|
execute("if RUN_DEMO:", namespace)
|
|
assert "DEMO_VIDEO" not in namespace
|
|
video = tmp_path / "trained.mp4"
|
|
video.write_bytes(b"current recording")
|
|
namespace.update(TRAIN_VIDEO=video, CHECKPOINT_PATH=tmp_path / "model_499.pt",
|
|
MEASUREMENT_CHECKPOINT="", DOWNLOAD_FILES=True)
|
|
execute("files.download(str(TRAIN_VIDEO))", namespace)
|
|
assert external_ui.downloads == [str(video)]
|
|
|
|
|
|
def test_curve_display_imports_are_independent_of_the_skipped_demo(tmp_path, external_ui):
|
|
namespace = settings()
|
|
execute("if RUN_DEMO:", namespace)
|
|
assert "Image" not in namespace and "display" not in namespace
|
|
artifacts, run_directory = tmp_path / "artifacts", tmp_path / "run"
|
|
artifacts.mkdir()
|
|
run_directory.mkdir()
|
|
|
|
def run(command, **kwargs):
|
|
output = Path(command[command.index("--out") + 1])
|
|
output.write_text(json.dumps({"run": str(run_directory), "progress": {}}))
|
|
Path(command[command.index("--plot") + 1]).write_bytes(b"curve fixture")
|
|
|
|
namespace.update(ARTIFACTS=artifacts, RUN_DIR=run_directory, REPO=tmp_path,
|
|
PYTHON=tmp_path / "python", run=run)
|
|
execute("METRICS_JSON =", namespace)
|
|
assert external_ui.displays[0][0] == "image"
|
|
replay = ast.parse(cell("SIM_STEPS ="))
|
|
imports = [node for node in replay.body if isinstance(node, ast.ImportFrom)
|
|
and node.module == "IPython.display"]
|
|
assert {alias.name for node in imports for alias in node.names} >= {"Image", "Video", "display"}
|
|
|
|
|
|
def test_failed_run_backup_works_without_fresh_runtime_or_egl_metadata(tmp_path, external_ui):
|
|
checkpoint, manifest = saved_run(tmp_path / "interrupted-run")
|
|
namespace = settings()
|
|
session = tmp_path / "fresh-session"
|
|
session.mkdir()
|
|
operation_manifest = tmp_path / "last-training.json"
|
|
colab.atomic_json(operation_manifest, manifest)
|
|
namespace["RUN_STATE"]["training_manifest"] = str(operation_manifest)
|
|
namespace.update(SESSION=session, REPO=tmp_path, PYTHON=tmp_path / "python",
|
|
DOWNLOAD_FILES=False, uuid=uuid)
|
|
assert not (session / "runtime.json").exists() and not (session / "graphics.json").exists()
|
|
|
|
def run(command, **kwargs):
|
|
colab.create_backup(
|
|
Path(command[command.index("--run") + 1]),
|
|
Path(command[command.index("--out") + 1]),
|
|
extras=Path(command[command.index("--extras") + 1]),
|
|
)
|
|
|
|
namespace["run"] = run
|
|
execute("BACKUP_CHECKPOINT =", namespace)
|
|
restored = colab.restore_backup(namespace["BACKUP_ZIP"], tmp_path / "verified-restore")
|
|
assert restored.read_bytes() == checkpoint.read_bytes()
|
|
assert external_ui.downloads == []
|
|
|
|
|
|
def test_backup_after_a_second_run_fails_selects_that_runs_saved_checkpoint(
|
|
tmp_path, external_ui
|
|
):
|
|
"""A failed second run leaves old Python globals; its manifest must select the new save."""
|
|
first_checkpoint, first_manifest = saved_run(tmp_path / "successful-A")
|
|
second_checkpoint, second_manifest = saved_run(tmp_path / "interrupted-B")
|
|
first_manifest.update(status="finished", operation="run-A")
|
|
second_manifest.update(status="failed", operation="run-B")
|
|
session = tmp_path / "session"
|
|
session.mkdir()
|
|
for name in ("runtime.json", "graphics.json"):
|
|
(session / name).write_text("{}")
|
|
namespace = settings()
|
|
namespace.update(SESSION=session, REPO=tmp_path, PYTHON=tmp_path / "python",
|
|
TASK="sample_robot", NUM_ENVS=64, PERSIST_ROOT=None, DOWNLOAD_FILES=False)
|
|
|
|
def first_run(command, **kwargs):
|
|
colab.atomic_json(Path(command[command.index("--manifest") + 1]), first_manifest)
|
|
|
|
namespace["run"] = first_run
|
|
execute('TRAIN_MANIFEST = SESSION / "training.json"', namespace)
|
|
assert namespace["CHECKPOINT_PATH"] == first_checkpoint
|
|
|
|
def interrupted_run(command, **kwargs):
|
|
colab.atomic_json(Path(command[command.index("--manifest") + 1]), second_manifest)
|
|
raise RuntimeError("second run interrupted after saving a complete checkpoint")
|
|
|
|
namespace["run"] = interrupted_run
|
|
with pytest.raises(RuntimeError, match="second run interrupted"):
|
|
execute('TRAIN_MANIFEST = SESSION / "training.json"', namespace)
|
|
assert namespace["CHECKPOINT_PATH"] == first_checkpoint # Prove the stale-global trigger.
|
|
|
|
def backup(command, **kwargs):
|
|
colab.create_backup(Path(command[command.index("--run") + 1]),
|
|
Path(command[command.index("--out") + 1]),
|
|
extras=Path(command[command.index("--extras") + 1]))
|
|
|
|
namespace["run"] = backup
|
|
execute("BACKUP_CHECKPOINT =", namespace)
|
|
assert namespace["CHECKPOINT_PATH"] == second_checkpoint
|
|
assert namespace["RUN_DIR"] == second_checkpoint.parent
|
|
assert namespace["ARTIFACTS"] == session / "artifacts/run-B"
|
|
restored = colab.restore_backup(namespace["BACKUP_ZIP"], tmp_path / "restore-B")
|
|
recovered = json.loads((restored.parent / colab.MANIFEST).read_text())
|
|
assert recovered["run_directory"] == str(second_checkpoint.parent)
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["replay", "evaluation"])
|
|
def test_stale_checkpoint_replay_and_evaluation_fail_before_subprocess(
|
|
tmp_path, external_ui, operation
|
|
):
|
|
old_checkpoint, _ = saved_run(tmp_path / "old")
|
|
selected_checkpoint, manifest = saved_run(tmp_path / "selected")
|
|
selected_manifest = tmp_path / "training.json"
|
|
colab.atomic_json(selected_manifest, manifest)
|
|
namespace = settings()
|
|
namespace.update(TASK=manifest["config"]["task"], MODEL=manifest["config"]["model"],
|
|
CHECKPOINT_PATH=old_checkpoint, training=manifest, TRAIN_MANIFEST=selected_manifest,
|
|
RUN_EVALUATION=True, rates={"supports_velocity_command": True},
|
|
run=lambda *args, **kwargs: pytest.fail("Stale checkpoint reached subprocess"))
|
|
with pytest.raises(ValueError, match="checkpoint must match the selected training manifest"):
|
|
execute("SIM_STEPS =" if operation == "replay" else "# @title Fixed-command evaluation", namespace)
|
|
assert selected_checkpoint.is_file() and old_checkpoint.is_file()
|
|
|
|
|
|
@pytest.mark.parametrize("changed", [{"TASK": "other_robot"}, {"MODEL": "different-robot"}])
|
|
def test_evaluation_rejects_other_task_or_model_before_subprocess(tmp_path, external_ui, changed):
|
|
checkpoint, manifest = saved_run(tmp_path / "selected")
|
|
selected_manifest = tmp_path / "training.json"
|
|
colab.atomic_json(selected_manifest, manifest)
|
|
namespace = settings()
|
|
namespace.update(TASK=manifest["config"]["task"], MODEL=manifest["config"]["model"],
|
|
CHECKPOINT_PATH=checkpoint, TRAIN_MANIFEST=selected_manifest,
|
|
RUN_EVALUATION=True, rates={"supports_velocity_command": True},
|
|
run=lambda *args, **kwargs: pytest.fail("Wrong task/model reached subprocess"))
|
|
namespace.update(changed)
|
|
with pytest.raises(ValueError, match="Evaluation task/model must match"):
|
|
execute("# @title Fixed-command evaluation", namespace)
|
|
|
|
|
|
@pytest.mark.parametrize("inactive", [(), ("lin_vel_y",)])
|
|
def test_matching_evaluation_runs_supported_cases_with_selected_checkpoint(
|
|
tmp_path, external_ui, inactive,
|
|
):
|
|
"""The evaluation gates must reject mismatches without blocking legitimate evaluation."""
|
|
checkpoint, manifest = saved_run(tmp_path / "selected")
|
|
selected_manifest = tmp_path / "training.json"
|
|
colab.atomic_json(selected_manifest, manifest)
|
|
artifacts = tmp_path / "artifacts"
|
|
artifacts.mkdir()
|
|
namespace = settings()
|
|
execute("# @title Simulation and evaluation settings", namespace)
|
|
commands = []
|
|
|
|
def run(command, **kwargs):
|
|
commands.append(command)
|
|
Path(command[command.index("--evaluation-out") + 1]).write_text(json.dumps({
|
|
"tracking_error": 0.02, "checkpoint": str(checkpoint),
|
|
}))
|
|
|
|
namespace.update(TASK=manifest["config"]["task"], MODEL=manifest["config"]["model"],
|
|
CHECKPOINT_PATH=checkpoint, TRAIN_MANIFEST=selected_manifest,
|
|
rates={"supports_velocity_command": True, "control_hz": 50.0,
|
|
"physics_hz": 200.0, "inactive_velocity_axes": inactive},
|
|
SIM_SCENE="studio", EVAL_SECONDS=1.01, ARTIFACTS=artifacts,
|
|
REPO=tmp_path, PYTHON=tmp_path / "python", run=run)
|
|
execute("# @title Fixed-command evaluation", namespace)
|
|
expected_cases = ["forward", "sideways", "turn", "stand"]
|
|
expected_vectors = [[0.1, 0.0, 0.0], [0.0, 0.1, 0.0], [0.0, 0.0, 0.2], [0.0, 0.0, 0.0]]
|
|
if inactive:
|
|
expected_cases.pop(1)
|
|
expected_vectors.pop(1)
|
|
assert len(commands) == len(expected_cases)
|
|
vectors = [[float(value) for value in command[command.index("--command") + 1:
|
|
command.index("--command") + 4]]
|
|
for command in commands]
|
|
assert vectors == expected_vectors
|
|
for command in commands:
|
|
assert command[command.index("--checkpoint") + 1] == checkpoint
|
|
assert command[command.index("--task") + 1] == "sample_robot"
|
|
assert command[command.index("--steps") + 1] == "51"
|
|
assert command[command.index("--scene") + 1] == "studio"
|
|
summary = json.loads((namespace["EVALUATION_DIR"] / "summary.json").read_text())
|
|
assert [item["case"] for item in summary] == expected_cases
|
|
assert all(item["scene"] == "studio" for item in summary)
|
|
|
|
|
|
def test_new_training_requires_explicit_repository_robot_and_model():
|
|
namespace = settings()
|
|
validate = namespace["validate_source_inputs"]
|
|
assert namespace["TASK"] == namespace["MODEL"] == namespace["REPOSITORY_URL"] == ""
|
|
with pytest.raises(ValueError, match="REPOSITORY_URL"):
|
|
validate()
|
|
namespace["REPOSITORY_URL"] = "https://github.com/example/robot-workbench.git"
|
|
with pytest.raises(ValueError, match="TASK"):
|
|
validate()
|
|
namespace["TASK"] = "sample_robot"
|
|
with pytest.raises(ValueError, match="MODEL"):
|
|
validate()
|
|
namespace["MODEL"] = "sample_robot"
|
|
validate()
|