Files
2026-11-03 19:46:08 +08:00

317 lines
12 KiB
Python

# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
from __future__ import annotations
import torch
import torch.nn as nn
import warnings
from tensordict import TensorDict
from rsl_rl.env import VecEnv
from rsl_rl.models import MLPModel
from rsl_rl.storage import RolloutStorage
from rsl_rl.utils import (
compile_model,
reduce_gradients_in_buckets,
resolve_class,
resolve_obs_groups,
resolve_optimizer,
)
class Distillation:
"""Distillation algorithm for training a student model to mimic a teacher model."""
student: MLPModel
"""The student model."""
teacher: MLPModel
"""The teacher model."""
teacher_loaded: bool = False
"""Indicates whether the teacher model parameters have been loaded."""
def __init__(
self,
student: MLPModel,
teacher: MLPModel,
storage: RolloutStorage,
num_learning_epochs: int = 1,
gradient_length: int = 15,
learning_rate: float = 1e-3,
max_grad_norm: float | None = None,
loss_type: str = "mse",
optimizer: str = "adam",
use_mixed_precision: bool = False,
device: str = "cpu",
# Distributed training parameters
multi_gpu_cfg: dict | None = None,
grad_reduce_bucket_mb: float = 25,
**kwargs: dict, # handle unused config parameters
) -> None:
"""Initialize the algorithm with models, storage, and optimization settings."""
# Device-related parameters
self.device = device
self.is_multi_gpu = multi_gpu_cfg is not None
# Multi-GPU parameters
if multi_gpu_cfg is not None:
self.gpu_global_rank = multi_gpu_cfg["global_rank"]
self.gpu_world_size = multi_gpu_cfg["world_size"]
else:
self.gpu_global_rank = 0
self.gpu_world_size = 1
self.grad_reduce_bucket_mb = grad_reduce_bucket_mb
# Distillation components
self.student = student.to(self.device)
self.teacher = teacher.to(self.device)
# Handles to the uncompiled modules for state_dict operations and export
self._raw_student = self.student
self._raw_teacher = self.teacher
# Create the optimizer
self.optimizer = resolve_optimizer(optimizer)(self.student.parameters(), lr=learning_rate) # type: ignore
# Add storage
self.storage = storage
self.transition = RolloutStorage.Transition()
self.last_hidden_states = (None, None)
# Distillation parameters
self.num_learning_epochs = num_learning_epochs
self.gradient_length = gradient_length
self.learning_rate = learning_rate
self.max_grad_norm = max_grad_norm
self.use_mixed_precision = use_mixed_precision
# Warn about rollout steps not used for optimization
total_steps = num_learning_epochs * storage.num_transitions_per_env
if total_steps % gradient_length != 0:
warnings.warn(
f"The product of 'num_learning_epochs' ({num_learning_epochs}) and 'num_steps_per_env'"
f" ({storage.num_transitions_per_env}) is not divisible by 'gradient_length' ({gradient_length})."
f" The last {total_steps % gradient_length} of {total_steps} steps per update are accumulated but"
f" never backpropagated. Consider setting 'gradient_length' to a divisor of {total_steps}.",
stacklevel=2,
)
# Initialize the loss function
loss_fn_dict = {
"mse": nn.functional.mse_loss,
"huber": nn.functional.huber_loss,
}
if loss_type in loss_fn_dict:
self.loss_fn = loss_fn_dict[loss_type]
else:
raise ValueError(f"Unknown loss type: {loss_type}. Supported types are: {list(loss_fn_dict.keys())}")
self.num_updates = 0
def act(self, obs: TensorDict) -> torch.Tensor:
"""Sample actions and store transition data."""
# Compute the actions
self.transition.actions = self.student(obs, stochastic_output=True).detach()
self.transition.privileged_actions = self.teacher(obs).detach()
# Record the observations
self.transition.observations = obs
return self.transition.actions # type: ignore
def process_env_step(
self, obs: TensorDict, rewards: torch.Tensor, dones: torch.Tensor, extras: dict[str, torch.Tensor]
) -> None:
"""Record one environment step."""
# Record the rewards and dones
self.transition.rewards = rewards
self.transition.dones = dones
# Record the transition
self.storage.add_transition(self.transition)
self.transition.clear()
self.student.reset(dones)
self.teacher.reset(dones)
def compute_returns(self, obs: TensorDict) -> None:
"""No-op since distillation does not use return targets."""
# Not needed for distillation
pass
def update(self) -> dict[str, float]:
"""Run optimization epochs over stored batches and return mean losses."""
self.num_updates += 1
mean_behavior_loss = 0
loss = 0
cnt = 0
for epoch in range(self.num_learning_epochs):
self.student.reset(hidden_state=self.last_hidden_states[0])
self.teacher.reset(hidden_state=self.last_hidden_states[1])
self.student.detach_hidden_state()
for batch in self.storage.generator():
# Optionally use mixed precision for the forward pass and loss computation
with torch.amp.autocast( # type: ignore
device_type=torch.device(self.device).type, enabled=self.use_mixed_precision, dtype=torch.bfloat16
):
# Inference of the student for gradient computation
actions = self.student(batch.observations)
# Behavior cloning loss
behavior_loss = self.loss_fn(actions, batch.privileged_actions)
# Total loss
loss = loss + behavior_loss
mean_behavior_loss += behavior_loss.item()
cnt += 1
# Gradient step
if cnt % self.gradient_length == 0:
self.optimizer.zero_grad()
loss.backward()
if self.is_multi_gpu:
self.reduce_parameters()
if self.max_grad_norm:
nn.utils.clip_grad_norm_(self.student.parameters(), self.max_grad_norm)
self.optimizer.step()
self.student.detach_hidden_state()
loss = 0
# Reset dones
self.student.reset(batch.dones.view(-1))
self.teacher.reset(batch.dones.view(-1))
self.student.detach_hidden_state(batch.dones.view(-1))
# Store the last hidden states for the next update
self.last_hidden_states = (self.student.get_hidden_state(), self.teacher.get_hidden_state())
self.student.detach_hidden_state()
# Update the normalizer
self.student.update_normalization(self.storage.observations.flatten(0, 1)) # type: ignore
# Divide the loss by the number of updates
mean_behavior_loss /= cnt
# Construct the loss dictionary
loss_dict = {"behavior": mean_behavior_loss}
# Clear the storage
self.storage.clear()
return loss_dict
def train_mode(self) -> None:
"""Set train mode for the student and keep the teacher in eval mode."""
self.student.train()
# Teacher is always in eval mode
self.teacher.eval()
def eval_mode(self) -> None:
"""Set evaluation mode for student and teacher models."""
self.student.eval()
self.teacher.eval()
def save(self) -> dict:
"""Return a dict of all models for saving."""
saved_dict = {
"student_state_dict": self._raw_student.state_dict(),
"teacher_state_dict": self._raw_teacher.state_dict(),
"optimizer_state_dict": self.optimizer.state_dict(),
}
return saved_dict
def load(self, loaded_dict: dict, load_cfg: dict | None, strict: bool) -> bool:
"""Load specified models from a saved dict."""
# If no load_cfg is provided, determine what to load automatically
if load_cfg is None and any("actor_state_dict" in key for key in loaded_dict): # Load from RL training
load_cfg = {"teacher": True, "iteration": False} # Only load teacher by default
elif load_cfg is None: # Load from distillation training
load_cfg = {
"student": True,
"teacher": True,
"optimizer": True,
"iteration": True,
}
# Load the specified models
if load_cfg.get("student"):
self._raw_student.load_state_dict(loaded_dict["student_state_dict"], strict=strict)
if load_cfg.get("teacher"):
self._raw_teacher.load_state_dict(
loaded_dict.get("teacher_state_dict") or loaded_dict["actor_state_dict"], strict=strict
)
self.teacher_loaded = True
if load_cfg.get("optimizer"):
self.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"])
return load_cfg.get("iteration", False)
def get_policy(self) -> MLPModel:
"""Get the policy model."""
return self._raw_student
def compile(self, mode: str | None = None) -> None:
"""Compile student and teacher with ``torch.compile``.
See :func:`~rsl_rl.utils.compile_model` for the set of accepted modes.
Args:
mode: ``torch.compile`` mode. Defaults to ``None``, in which case compilation is disabled.
"""
self.student = compile_model(self._raw_student, mode) # type: ignore
self.teacher = compile_model(self._raw_teacher, mode) # type: ignore
@staticmethod
def construct_algorithm(obs: TensorDict, env: VecEnv, cfg: dict, device: str) -> Distillation:
"""Construct the distillation algorithm."""
# Resolve class callables and configs
alg_class, alg_cfg = resolve_class(cfg["algorithm"])
student_class, student_cfg = resolve_class(cfg["student"])
teacher_class, teacher_cfg = resolve_class(cfg["teacher"])
# Resolve observation groups
default_sets = ["student", "teacher"]
cfg["obs_groups"] = resolve_obs_groups(obs, cfg["obs_groups"], default_sets)
# Distillation is not compatible with RND and symmetry extensions
if alg_cfg.get("rnd_cfg") is not None:
raise ValueError("The RND extension is not compatible with Distillation.")
alg_cfg["rnd_cfg"] = None
if alg_cfg.get("symmetry_cfg") is not None:
raise ValueError("The symmetry extension is not compatible with Distillation.")
alg_cfg["symmetry_cfg"] = None
# Initialize the policy
student: MLPModel = student_class(obs, cfg["obs_groups"], "student", env.num_actions, **student_cfg).to(device)
print(f"Student Model: {student}")
teacher: MLPModel = teacher_class(obs, cfg["obs_groups"], "teacher", env.num_actions, **teacher_cfg).to(device)
print(f"Teacher Model: {teacher}")
# Initialize the storage
storage = RolloutStorage("distillation", env.num_envs, cfg["num_steps_per_env"], obs, [env.num_actions], device)
# Initialize the algorithm
alg: Distillation = alg_class(
student, teacher, storage, device=device, **alg_cfg, multi_gpu_cfg=cfg["multi_gpu"]
)
# Compile the algorithm's models if requested
alg.compile(cfg.get("torch_compile_mode"))
return alg
def broadcast_parameters(self) -> None:
"""Broadcast model parameters to all GPUs."""
for model in (self._raw_student, self._raw_teacher):
for tensor in model.state_dict().values():
torch.distributed.broadcast(tensor, src=0)
def reduce_parameters(self) -> None:
"""Collect gradients from all GPUs and average them.
This function is called after the backward pass to synchronize the gradients across all GPUs.
"""
# Average the gradients across all GPUs in bounded buckets
reduce_gradients_in_buckets(self.student.parameters(), self.gpu_world_size, self.grad_reduce_bucket_mb)