#!/usr/bin/env python3 """Unified train entry point. This script **imports no task's config classes**. It knows only the registry at the top of tasks/, and configs are loaded by `tasks.load_env_cfg` / `load_agent_cfg` following the conventions of a task directory. Adding a robot or a task requires no change under scripts/. Pick the task with `--task` and the model asset with `--model`; `--list` shows the choices for both. The backend comes from `--backend` / `--device`, or is auto-detected (see mjrl.backend.resolve). The live viewer is **on by default**, works on both backends, and draws **at most 128 environments** (all of them below 128; `--viewer-env-num` changes it). The two reasons for the cap are in `_resolve_draw_limit` in mjrl/viewer/live.py. Turn it off with `--headless`; with no usable display it goes headless automatically. **TensorBoard is started automatically** on this run's log directory (the whole task by default, so the previous run is there to compare against) and the URL is printed. Turn it off with `--no-tensorboard`. **Training resumes with `--resume`**, from the newest checkpoint under `logs///`, or from the one `--checkpoint` names. What is restored is the whole training state -- policy, value function, optimiser (so the adaptive learning-rate schedule keeps its place), the iteration counter and the environment's step counter, which is what curricula count in -- rather than the weights alone, so a resumed run continues rather than restarts from a warm policy. """ from __future__ import annotations from pathlib import Path from _cli import ( add_scene_args, add_tensorboard_args, apply_scene, add_viewer_args, build_parser, parse_with_task_args, maybe_tensorboard, maybe_viewer, print_task_table, resolve_all, resolve_checkpoint, ) def main() -> None: parser = build_parser("train.py", __doc__ or "") add_viewer_args(parser) add_scene_args(parser) add_tensorboard_args(parser) r = parser.add_argument_group("training") r.add_argument( "--logger", choices=["tensorboard", "wandb"], default="tensorboard", help="logging backend. **tensorboard rather than mjlab's wandb**: the " "start_method='thread' mjlab passes to wandb is rejected by wandb 0.29.0's " "config validation and training crashes right after the model is built " "(see section 6 of docs/AGENT_SETUP.md)", ) r.add_argument( "--max-iterations", type=int, default=None, help="override the iteration count in the task's rl_cfg.py. **How many " "iterations this run performs**, so on a resumed run it is how many more " "to do, on top of the ones the checkpoint already has", ) r.add_argument( "--resume", action="store_true", help="continue training from the newest checkpoint under " "logs/// -- normally the run that just finished. The whole " "training state is restored, not the weights alone. There is deliberately " "no .env key for this: a sticky default would silently continue an old " "run on a launch meant to start a new one", ) r.add_argument( "--checkpoint", default=None, metavar="PATH", help="which checkpoint to resume from: a model_*.pt, or a directory whose " "newest one is taken (a run directory, say). Implies --resume", ) # The selected task may take arguments of its own; see # `tasks.load_cli_args`. Nothing here knows which, or how many. args = parse_with_task_args(parser) if args.list: print_task_table() return spec, res, asset = resolve_all(args, for_training=True) if args.dry_run: print(f"[mjrl] resolved: task={spec.id} -- --dry-run, stopping here") return _run(spec, res, asset, args) def _run(spec, res, asset: Path | None, args) -> None: """Build the environment and train. Every heavy import happens here.""" import tasks from mjrl.backend.select import use_backend from mjrl.backend.warp_allocation import apply_warp_allocation, write_warp_allocation # ── Order matters: the backend must be registered before the env is built ── # `ManagerBasedRlEnv.__init__` consults the registry once, so changing it # after the env exists has no effect. use_backend(res) from mjlab.envs import ManagerBasedRlEnv from mjlab.rl import RslRlVecEnvWrapper from mjlab.rl.runner import MjlabOnPolicyRunner env_cfg = tasks.load_env_cfg(spec.id, asset=asset, task_args=args.task_args) apply_scene(env_cfg, args, for_training=True) env_cfg.scene.num_envs = res.num_envs agent_cfg = tasks.load_agent_cfg(spec.id) runner_cls = tasks.load_runner_cls(spec.id) or MjlabOnPolicyRunner from tasks.paths import log_root_for # Resolved before the directory is created, and long before the scene is # built: a typo in --checkpoint should cost a second rather than the minute # it takes to compile the models, and it should not leave an empty run # directory behind in logs/. ckpt = ( resolve_checkpoint(args.checkpoint, spec.id, asset) if args.resume or args.checkpoint else None ) if ckpt is not None: print(f"[mjrl] resuming from {ckpt}") apply_warp_allocation(env_cfg.sim, args, res.backend, ckpt) log_dir = Path(log_root_for(asset)) / spec.id / _timestamp() log_dir.mkdir(parents=True, exist_ok=True) print(f"[mjrl] log directory {log_dir}") # The runner takes a **dict**, not a dataclass (`if key in train_cfg` in # mjlab/rl/runner.py requires something iterable), and it **mutates** that # config in place -- so build a fresh one every time. Passing the dataclass # directly gives # `TypeError: argument of type 'RslRlOnPolicyRunnerCfg' is not iterable`. from dataclasses import asdict # mjlab defaults the logger to wandb, and that path is broken on wandb 0.29.0. # Override it explicitly rather than letting people hit it and go read docs. agent_cfg.logger = args.logger if args.max_iterations is not None: agent_cfg.max_iterations = args.max_iterations max_iterations = agent_cfg.max_iterations runner_kwargs = asdict(agent_cfg) # TensorBoard starts here rather than after the environment is built: the # event files do not exist yet, but building the scene and compiling the # models takes tens of seconds, and that is exactly the window in which # someone wants the page already open and waiting. with maybe_tensorboard(log_dir, args): env = ManagerBasedRlEnv(cfg=env_cfg, device=res.device) try: if res.backend == "warp": write_warp_allocation(log_dir, env.sim) wrapped = RslRlVecEnvWrapper(env) runner = runner_cls(wrapped, runner_kwargs, str(log_dir), res.device) runner.add_git_repo_to_log(__file__) if ckpt is not None: _resume(runner, ckpt, res.device, max_iterations) with maybe_viewer(env, args): runner.learn( num_learning_iterations=max_iterations, init_at_random_ep_len=True, ) finally: env.close() def _resume(runner, ckpt: Path, device: str, more_iterations: int) -> None: """Load `ckpt` into `runner` and report where training is picking up. A **new** timestamped run directory is used rather than the checkpoint's own, so a resumed run never overwrites the checkpoints it started from. The curves still join up: the iteration counter continues from the checkpoint, so with the default `--tb-scope task` TensorBoard draws the new run as a series that begins where the old one ended. The failure this catches is the one people actually hit: resuming after the observation terms or the network shape changed. `load_state_dict` then raises a wall of missing and mismatched keys, which reads as a bug in the framework rather than as "this checkpoint belongs to a different configuration". """ try: runner.load(str(ckpt), map_location=device) except (RuntimeError, KeyError) as e: raise SystemExit( f"[mjrl] cannot resume from {ckpt}:\n" f" {type(e).__name__}: {e}\n" f"The usual cause is a checkpoint from a different configuration: it " f"has to come from the same task, model and network as this run, " f"observation and action dimensions included. Train from scratch (drop " f"--resume), or point --checkpoint at a matching run." ) from None start = runner.current_learning_iteration print( f"[mjrl] resumed at iteration {start}; running {more_iterations} more " f"(through {start + more_iterations - 1})" ) def _timestamp() -> str: from datetime import datetime return datetime.now().strftime("%Y-%m-%d_%H-%M-%S") if __name__ == "__main__": main()