Files

223 lines
7.2 KiB
Python

"""Script to train RL agent with RSL-RL."""
import logging
import os
import sys
from dataclasses import asdict, dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Literal, cast
import tyro
from mjlab.envs import ManagerBasedRlEnv, ManagerBasedRlEnvCfg
from mjlab.rl import MjlabOnPolicyRunner, RslRlBaseRunnerCfg, RslRlVecEnvWrapper
from mjlab.tasks.registry import list_tasks, load_env_cfg, load_rl_cfg, load_runner_cls
from mjlab.tasks.tracking.mdp import MotionCommandCfg
from mjlab.utils.gpu import select_gpus
from mjlab.utils.os import dump_yaml, get_checkpoint_path
from mjlab.utils.torch import configure_torch_backends
from mjlab.utils.wrappers import VideoRecorder
@dataclass(frozen=True)
class TrainConfig:
env: ManagerBasedRlEnvCfg
agent: RslRlBaseRunnerCfg
motion_file: str | None = None
video: bool = False
video_length: int = 200
video_interval: int = 2000
enable_nan_guard: bool = False
torchrunx_log_dir: str | None = None
gpu_ids: list[int] | Literal["all"] | None = field(default_factory=lambda: [0])
@staticmethod
def from_task(task_id: str) -> "TrainConfig":
env_cfg = load_env_cfg(task_id)
agent_cfg = load_rl_cfg(task_id)
return TrainConfig(env=env_cfg, agent=agent_cfg)
def run_train(task_id: str, cfg: TrainConfig, log_dir: Path) -> None:
cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
if cuda_visible == "":
device = "cpu"
seed = cfg.agent.seed
rank = 0
else:
local_rank = int(os.environ.get("LOCAL_RANK", "0"))
rank = int(os.environ.get("RANK", "0"))
# Set EGL device to match the CUDA device.
os.environ["MUJOCO_EGL_DEVICE_ID"] = str(local_rank)
device = f"cuda:{local_rank}"
# Set seed to have diversity in different processes.
seed = cfg.agent.seed + local_rank
configure_torch_backends()
cfg.agent.seed = seed
cfg.env.seed = seed
print(f"[INFO] Training with: device={device}, seed={seed}, rank={rank}")
# Check if this is a tracking task by checking for motion command.
is_tracking_task = "motion" in cfg.env.commands and isinstance(
cfg.env.commands["motion"], MotionCommandCfg
)
if is_tracking_task:
if not cfg.motion_file:
raise ValueError("For tracking tasks, --motion-file must be set ...")
motion_path = Path(cfg.motion_file).expanduser().resolve()
if not motion_path.exists():
raise FileNotFoundError(f"Motion file not found: {motion_path}")
motion_cmd = cfg.env.commands["motion"]
assert isinstance(motion_cmd, MotionCommandCfg)
motion_cmd.motion_file = str(motion_path)
print(f"[INFO] Using motion file: {motion_cmd.motion_file}")
# Check if motion_file is already set (e.g., via CLI --env.commands.motion.motion-file).
if motion_cmd.motion_file and Path(motion_cmd.motion_file).exists():
print(f"[INFO] Using local motion file: {motion_cmd.motion_file}")
# Enable NaN guard if requested.
if cfg.enable_nan_guard:
cfg.env.sim.nan_guard.enabled = True
print(f"[INFO] NaN guard enabled, output dir: {cfg.env.sim.nan_guard.output_dir}")
if rank == 0:
print(f"[INFO] Logging experiment in directory: {log_dir}")
env = ManagerBasedRlEnv(
cfg=cfg.env, device=device, render_mode="rgb_array" if cfg.video else None
)
log_root_path = log_dir.parent # Go up from specific run dir to experiment dir.
resume_path: Path | None = None
if cfg.agent.resume:
# Load checkpoint from local filesystem.
resume_path = get_checkpoint_path(
log_root_path, cfg.agent.load_run, cfg.agent.load_checkpoint
)
# Only record videos on rank 0 to avoid multiple workers writing to the same files.
if cfg.video and rank == 0:
env = VideoRecorder(
env,
video_folder=Path(log_dir) / "videos" / "train",
step_trigger=lambda step: step % cfg.video_interval == 0,
video_length=cfg.video_length,
disable_logger=True,
)
print("[INFO] Recording videos during training.")
env = RslRlVecEnvWrapper(env, clip_actions=cfg.agent.clip_actions)
agent_cfg = asdict(cfg.agent)
env_cfg = asdict(cfg.env)
runner_cls = load_runner_cls(task_id)
if runner_cls is None:
runner_cls = MjlabOnPolicyRunner
runner_kwargs = {}
runner = runner_cls(env, agent_cfg, str(log_dir), device, **runner_kwargs)
runner.add_git_repo_to_log(__file__)
if resume_path is not None:
print(f"[INFO]: Loading model checkpoint from: {resume_path}")
runner.load(str(resume_path))
# Only write config files from rank 0 to avoid race conditions.
if rank == 0:
dump_yaml(log_dir / "params" / "env.yaml", env_cfg)
dump_yaml(log_dir / "params" / "agent.yaml", agent_cfg)
runner.learn(
num_learning_iterations=cfg.agent.max_iterations, init_at_random_ep_len=True
)
env.close()
def launch_training(task_id: str, args: TrainConfig | None = None):
args = args or TrainConfig.from_task(task_id)
# Create log directory once before launching workers.
log_root_path = Path("logs") / "rsl_rl" / args.agent.experiment_name
log_root_path.resolve()
log_dir_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
if args.agent.run_name:
log_dir_name += f"_{args.agent.run_name}"
log_dir = log_root_path / log_dir_name
# Select GPUs based on CUDA_VISIBLE_DEVICES and user specification.
selected_gpus, num_gpus = select_gpus(args.gpu_ids)
# Set environment variables for all modes.
if selected_gpus is None:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
else:
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, selected_gpus))
os.environ["MUJOCO_GL"] = "egl"
if num_gpus <= 1:
# CPU or single GPU: run directly without torchrunx.
run_train(task_id, args, log_dir)
else:
# Multi-GPU: use torchrunx.
import torchrunx
# torchrunx redirects stdout to logging.
logging.basicConfig(level=logging.INFO)
# Configure torchrunx logging directory.
# Priority: 1) existing env var, 2) user flag, 3) default to {log_dir}/torchrunx.
if "TORCHRUNX_LOG_DIR" not in os.environ:
if args.torchrunx_log_dir is not None:
# User specified a value via flag (could be "" to disable).
os.environ["TORCHRUNX_LOG_DIR"] = args.torchrunx_log_dir
else:
# Default: put logs in training directory.
os.environ["TORCHRUNX_LOG_DIR"] = str(log_dir / "torchrunx")
print(f"[INFO] Launching training with {num_gpus} GPUs", flush=True)
torchrunx.Launcher(
hostnames=["localhost"],
workers_per_host=num_gpus,
backend=None, # Let rsl_rl handle process group initialization.
copy_env_vars=torchrunx.DEFAULT_ENV_VARS_FOR_COPY + ("MUJOCO*",),
).run(run_train, task_id, args, log_dir)
def main():
# Parse first argument to choose the task.
# Import tasks to populate the registry.
import mjlab.tasks # noqa: F401
import src.tasks
all_tasks = list_tasks()
chosen_task, remaining_args = tyro.cli(
tyro.extras.literal_type_from_choices(all_tasks),
add_help=False,
return_unknown_args=True,
config=mjlab.TYRO_FLAGS,
)
args = tyro.cli(
TrainConfig,
args=remaining_args,
default=TrainConfig.from_task(chosen_task),
prog=sys.argv[0] + f" {chosen_task}",
config=mjlab.TYRO_FLAGS,
)
del remaining_args
launch_training(task_id=chosen_task, args=args)
if __name__ == "__main__":
main()