317 lines
12 KiB
Python
317 lines
12 KiB
Python
# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
|
|
# All rights reserved.
|
|
#
|
|
# SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
import warnings
|
|
from tensordict import TensorDict
|
|
|
|
from rsl_rl.env import VecEnv
|
|
from rsl_rl.models import MLPModel
|
|
from rsl_rl.storage import RolloutStorage
|
|
from rsl_rl.utils import (
|
|
compile_model,
|
|
reduce_gradients_in_buckets,
|
|
resolve_class,
|
|
resolve_obs_groups,
|
|
resolve_optimizer,
|
|
)
|
|
|
|
|
|
class Distillation:
|
|
"""Distillation algorithm for training a student model to mimic a teacher model."""
|
|
|
|
student: MLPModel
|
|
"""The student model."""
|
|
|
|
teacher: MLPModel
|
|
"""The teacher model."""
|
|
|
|
teacher_loaded: bool = False
|
|
"""Indicates whether the teacher model parameters have been loaded."""
|
|
|
|
def __init__(
|
|
self,
|
|
student: MLPModel,
|
|
teacher: MLPModel,
|
|
storage: RolloutStorage,
|
|
num_learning_epochs: int = 1,
|
|
gradient_length: int = 15,
|
|
learning_rate: float = 1e-3,
|
|
max_grad_norm: float | None = None,
|
|
loss_type: str = "mse",
|
|
optimizer: str = "adam",
|
|
use_mixed_precision: bool = False,
|
|
device: str = "cpu",
|
|
# Distributed training parameters
|
|
multi_gpu_cfg: dict | None = None,
|
|
grad_reduce_bucket_mb: float = 25,
|
|
**kwargs: dict, # handle unused config parameters
|
|
) -> None:
|
|
"""Initialize the algorithm with models, storage, and optimization settings."""
|
|
# Device-related parameters
|
|
self.device = device
|
|
self.is_multi_gpu = multi_gpu_cfg is not None
|
|
|
|
# Multi-GPU parameters
|
|
if multi_gpu_cfg is not None:
|
|
self.gpu_global_rank = multi_gpu_cfg["global_rank"]
|
|
self.gpu_world_size = multi_gpu_cfg["world_size"]
|
|
else:
|
|
self.gpu_global_rank = 0
|
|
self.gpu_world_size = 1
|
|
self.grad_reduce_bucket_mb = grad_reduce_bucket_mb
|
|
|
|
# Distillation components
|
|
self.student = student.to(self.device)
|
|
self.teacher = teacher.to(self.device)
|
|
|
|
# Handles to the uncompiled modules for state_dict operations and export
|
|
self._raw_student = self.student
|
|
self._raw_teacher = self.teacher
|
|
|
|
# Create the optimizer
|
|
self.optimizer = resolve_optimizer(optimizer)(self.student.parameters(), lr=learning_rate) # type: ignore
|
|
|
|
# Add storage
|
|
self.storage = storage
|
|
self.transition = RolloutStorage.Transition()
|
|
self.last_hidden_states = (None, None)
|
|
|
|
# Distillation parameters
|
|
self.num_learning_epochs = num_learning_epochs
|
|
self.gradient_length = gradient_length
|
|
self.learning_rate = learning_rate
|
|
self.max_grad_norm = max_grad_norm
|
|
self.use_mixed_precision = use_mixed_precision
|
|
|
|
# Warn about rollout steps not used for optimization
|
|
total_steps = num_learning_epochs * storage.num_transitions_per_env
|
|
if total_steps % gradient_length != 0:
|
|
warnings.warn(
|
|
f"The product of 'num_learning_epochs' ({num_learning_epochs}) and 'num_steps_per_env'"
|
|
f" ({storage.num_transitions_per_env}) is not divisible by 'gradient_length' ({gradient_length})."
|
|
f" The last {total_steps % gradient_length} of {total_steps} steps per update are accumulated but"
|
|
f" never backpropagated. Consider setting 'gradient_length' to a divisor of {total_steps}.",
|
|
stacklevel=2,
|
|
)
|
|
|
|
# Initialize the loss function
|
|
loss_fn_dict = {
|
|
"mse": nn.functional.mse_loss,
|
|
"huber": nn.functional.huber_loss,
|
|
}
|
|
if loss_type in loss_fn_dict:
|
|
self.loss_fn = loss_fn_dict[loss_type]
|
|
else:
|
|
raise ValueError(f"Unknown loss type: {loss_type}. Supported types are: {list(loss_fn_dict.keys())}")
|
|
|
|
self.num_updates = 0
|
|
|
|
def act(self, obs: TensorDict) -> torch.Tensor:
|
|
"""Sample actions and store transition data."""
|
|
# Compute the actions
|
|
self.transition.actions = self.student(obs, stochastic_output=True).detach()
|
|
self.transition.privileged_actions = self.teacher(obs).detach()
|
|
# Record the observations
|
|
self.transition.observations = obs
|
|
return self.transition.actions # type: ignore
|
|
|
|
def process_env_step(
|
|
self, obs: TensorDict, rewards: torch.Tensor, dones: torch.Tensor, extras: dict[str, torch.Tensor]
|
|
) -> None:
|
|
"""Record one environment step."""
|
|
# Record the rewards and dones
|
|
self.transition.rewards = rewards
|
|
self.transition.dones = dones
|
|
# Record the transition
|
|
self.storage.add_transition(self.transition)
|
|
self.transition.clear()
|
|
self.student.reset(dones)
|
|
self.teacher.reset(dones)
|
|
|
|
def compute_returns(self, obs: TensorDict) -> None:
|
|
"""No-op since distillation does not use return targets."""
|
|
# Not needed for distillation
|
|
pass
|
|
|
|
def update(self) -> dict[str, float]:
|
|
"""Run optimization epochs over stored batches and return mean losses."""
|
|
self.num_updates += 1
|
|
mean_behavior_loss = 0
|
|
loss = 0
|
|
cnt = 0
|
|
|
|
for epoch in range(self.num_learning_epochs):
|
|
self.student.reset(hidden_state=self.last_hidden_states[0])
|
|
self.teacher.reset(hidden_state=self.last_hidden_states[1])
|
|
self.student.detach_hidden_state()
|
|
for batch in self.storage.generator():
|
|
# Optionally use mixed precision for the forward pass and loss computation
|
|
with torch.amp.autocast( # type: ignore
|
|
device_type=torch.device(self.device).type, enabled=self.use_mixed_precision, dtype=torch.bfloat16
|
|
):
|
|
# Inference of the student for gradient computation
|
|
actions = self.student(batch.observations)
|
|
|
|
# Behavior cloning loss
|
|
behavior_loss = self.loss_fn(actions, batch.privileged_actions)
|
|
|
|
# Total loss
|
|
loss = loss + behavior_loss
|
|
mean_behavior_loss += behavior_loss.item()
|
|
cnt += 1
|
|
|
|
# Gradient step
|
|
if cnt % self.gradient_length == 0:
|
|
self.optimizer.zero_grad()
|
|
loss.backward()
|
|
if self.is_multi_gpu:
|
|
self.reduce_parameters()
|
|
if self.max_grad_norm:
|
|
nn.utils.clip_grad_norm_(self.student.parameters(), self.max_grad_norm)
|
|
self.optimizer.step()
|
|
self.student.detach_hidden_state()
|
|
loss = 0
|
|
|
|
# Reset dones
|
|
self.student.reset(batch.dones.view(-1))
|
|
self.teacher.reset(batch.dones.view(-1))
|
|
self.student.detach_hidden_state(batch.dones.view(-1))
|
|
|
|
# Store the last hidden states for the next update
|
|
self.last_hidden_states = (self.student.get_hidden_state(), self.teacher.get_hidden_state())
|
|
self.student.detach_hidden_state()
|
|
|
|
# Update the normalizer
|
|
self.student.update_normalization(self.storage.observations.flatten(0, 1)) # type: ignore
|
|
|
|
# Divide the loss by the number of updates
|
|
mean_behavior_loss /= cnt
|
|
|
|
# Construct the loss dictionary
|
|
loss_dict = {"behavior": mean_behavior_loss}
|
|
|
|
# Clear the storage
|
|
self.storage.clear()
|
|
|
|
return loss_dict
|
|
|
|
def train_mode(self) -> None:
|
|
"""Set train mode for the student and keep the teacher in eval mode."""
|
|
self.student.train()
|
|
# Teacher is always in eval mode
|
|
self.teacher.eval()
|
|
|
|
def eval_mode(self) -> None:
|
|
"""Set evaluation mode for student and teacher models."""
|
|
self.student.eval()
|
|
self.teacher.eval()
|
|
|
|
def save(self) -> dict:
|
|
"""Return a dict of all models for saving."""
|
|
saved_dict = {
|
|
"student_state_dict": self._raw_student.state_dict(),
|
|
"teacher_state_dict": self._raw_teacher.state_dict(),
|
|
"optimizer_state_dict": self.optimizer.state_dict(),
|
|
}
|
|
return saved_dict
|
|
|
|
def load(self, loaded_dict: dict, load_cfg: dict | None, strict: bool) -> bool:
|
|
"""Load specified models from a saved dict."""
|
|
# If no load_cfg is provided, determine what to load automatically
|
|
if load_cfg is None and any("actor_state_dict" in key for key in loaded_dict): # Load from RL training
|
|
load_cfg = {"teacher": True, "iteration": False} # Only load teacher by default
|
|
elif load_cfg is None: # Load from distillation training
|
|
load_cfg = {
|
|
"student": True,
|
|
"teacher": True,
|
|
"optimizer": True,
|
|
"iteration": True,
|
|
}
|
|
|
|
# Load the specified models
|
|
if load_cfg.get("student"):
|
|
self._raw_student.load_state_dict(loaded_dict["student_state_dict"], strict=strict)
|
|
if load_cfg.get("teacher"):
|
|
self._raw_teacher.load_state_dict(
|
|
loaded_dict.get("teacher_state_dict") or loaded_dict["actor_state_dict"], strict=strict
|
|
)
|
|
self.teacher_loaded = True
|
|
if load_cfg.get("optimizer"):
|
|
self.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"])
|
|
return load_cfg.get("iteration", False)
|
|
|
|
def get_policy(self) -> MLPModel:
|
|
"""Get the policy model."""
|
|
return self._raw_student
|
|
|
|
def compile(self, mode: str | None = None) -> None:
|
|
"""Compile student and teacher with ``torch.compile``.
|
|
|
|
See :func:`~rsl_rl.utils.compile_model` for the set of accepted modes.
|
|
|
|
Args:
|
|
mode: ``torch.compile`` mode. Defaults to ``None``, in which case compilation is disabled.
|
|
"""
|
|
self.student = compile_model(self._raw_student, mode) # type: ignore
|
|
self.teacher = compile_model(self._raw_teacher, mode) # type: ignore
|
|
|
|
@staticmethod
|
|
def construct_algorithm(obs: TensorDict, env: VecEnv, cfg: dict, device: str) -> Distillation:
|
|
"""Construct the distillation algorithm."""
|
|
# Resolve class callables and configs
|
|
alg_class, alg_cfg = resolve_class(cfg["algorithm"])
|
|
student_class, student_cfg = resolve_class(cfg["student"])
|
|
teacher_class, teacher_cfg = resolve_class(cfg["teacher"])
|
|
|
|
# Resolve observation groups
|
|
default_sets = ["student", "teacher"]
|
|
cfg["obs_groups"] = resolve_obs_groups(obs, cfg["obs_groups"], default_sets)
|
|
|
|
# Distillation is not compatible with RND and symmetry extensions
|
|
if alg_cfg.get("rnd_cfg") is not None:
|
|
raise ValueError("The RND extension is not compatible with Distillation.")
|
|
alg_cfg["rnd_cfg"] = None
|
|
if alg_cfg.get("symmetry_cfg") is not None:
|
|
raise ValueError("The symmetry extension is not compatible with Distillation.")
|
|
alg_cfg["symmetry_cfg"] = None
|
|
|
|
# Initialize the policy
|
|
student: MLPModel = student_class(obs, cfg["obs_groups"], "student", env.num_actions, **student_cfg).to(device)
|
|
print(f"Student Model: {student}")
|
|
teacher: MLPModel = teacher_class(obs, cfg["obs_groups"], "teacher", env.num_actions, **teacher_cfg).to(device)
|
|
print(f"Teacher Model: {teacher}")
|
|
|
|
# Initialize the storage
|
|
storage = RolloutStorage("distillation", env.num_envs, cfg["num_steps_per_env"], obs, [env.num_actions], device)
|
|
|
|
# Initialize the algorithm
|
|
alg: Distillation = alg_class(
|
|
student, teacher, storage, device=device, **alg_cfg, multi_gpu_cfg=cfg["multi_gpu"]
|
|
)
|
|
|
|
# Compile the algorithm's models if requested
|
|
alg.compile(cfg.get("torch_compile_mode"))
|
|
|
|
return alg
|
|
|
|
def broadcast_parameters(self) -> None:
|
|
"""Broadcast model parameters to all GPUs."""
|
|
for model in (self._raw_student, self._raw_teacher):
|
|
for tensor in model.state_dict().values():
|
|
torch.distributed.broadcast(tensor, src=0)
|
|
|
|
def reduce_parameters(self) -> None:
|
|
"""Collect gradients from all GPUs and average them.
|
|
|
|
This function is called after the backward pass to synchronize the gradients across all GPUs.
|
|
"""
|
|
# Average the gradients across all GPUs in bounded buckets
|
|
reduce_gradients_in_buckets(self.student.parameters(), self.gpu_world_size, self.grad_reduce_bucket_mb)
|