485 lines
21 KiB
Python
485 lines
21 KiB
Python
# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
|
|
# All rights reserved.
|
|
#
|
|
# SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from itertools import chain
|
|
from tensordict import TensorDict
|
|
|
|
from rsl_rl.env import VecEnv
|
|
from rsl_rl.extensions import RandomNetworkDistillation, Symmetry, resolve_rnd_config, resolve_symmetry_config
|
|
from rsl_rl.models import MLPModel
|
|
from rsl_rl.storage import RolloutStorage
|
|
from rsl_rl.utils import (
|
|
compile_model,
|
|
reduce_gradients_in_buckets,
|
|
resolve_class,
|
|
resolve_obs_groups,
|
|
resolve_optimizer,
|
|
)
|
|
|
|
|
|
class PPO:
|
|
"""Proximal Policy Optimization algorithm.
|
|
|
|
Reference:
|
|
- Schulman et al. "Proximal policy optimization algorithms." arXiv preprint arXiv:1707.06347 (2017).
|
|
"""
|
|
|
|
actor: MLPModel
|
|
"""The actor model."""
|
|
|
|
critic: MLPModel
|
|
"""The critic model."""
|
|
|
|
def __init__(
|
|
self,
|
|
actor: MLPModel,
|
|
critic: MLPModel,
|
|
storage: RolloutStorage,
|
|
num_learning_epochs: int = 5,
|
|
num_mini_batches: int = 4,
|
|
clip_param: float = 0.2,
|
|
gamma: float = 0.99,
|
|
lam: float = 0.95,
|
|
value_loss_coef: float = 1.0,
|
|
entropy_coef: float = 0.01,
|
|
learning_rate: float = 0.001,
|
|
max_grad_norm: float = 1.0,
|
|
optimizer: str = "adam",
|
|
use_clipped_value_loss: bool = True,
|
|
schedule: str = "adaptive",
|
|
desired_kl: float = 0.01,
|
|
normalize_advantage_per_mini_batch: bool = False,
|
|
use_mixed_precision: bool = False,
|
|
device: str = "cpu",
|
|
# RND parameters
|
|
rnd_cfg: dict | None = None,
|
|
# Symmetry parameters
|
|
symmetry_cfg: dict | None = None,
|
|
# Distributed training parameters
|
|
multi_gpu_cfg: dict | None = None,
|
|
grad_reduce_bucket_mb: float = 25,
|
|
) -> None:
|
|
"""Initialize the algorithm with models, storage, and optimization settings."""
|
|
# Device-related parameters
|
|
self.device = device
|
|
self.is_multi_gpu = multi_gpu_cfg is not None
|
|
|
|
# Multi-GPU parameters
|
|
if multi_gpu_cfg is not None:
|
|
self.gpu_global_rank = multi_gpu_cfg["global_rank"]
|
|
self.gpu_world_size = multi_gpu_cfg["world_size"]
|
|
else:
|
|
self.gpu_global_rank = 0
|
|
self.gpu_world_size = 1
|
|
self.grad_reduce_bucket_mb = grad_reduce_bucket_mb
|
|
|
|
# RND extension
|
|
self.rnd = RandomNetworkDistillation(device=self.device, **rnd_cfg) if rnd_cfg else None
|
|
|
|
# Symmetry extension
|
|
if symmetry_cfg is not None and (actor.is_recurrent or critic.is_recurrent):
|
|
raise ValueError("Symmetry augmentation is not supported for recurrent policies.")
|
|
self.symmetry = Symmetry(**symmetry_cfg) if symmetry_cfg else None
|
|
|
|
# PPO components
|
|
self.actor = actor.to(self.device)
|
|
self.critic = critic.to(self.device)
|
|
|
|
# Handles to the uncompiled modules for state_dict operations and export
|
|
self._raw_actor = self.actor
|
|
self._raw_critic = self.critic
|
|
|
|
# Create the optimizer
|
|
self.optimizer = resolve_optimizer(optimizer)(
|
|
chain(self.actor.parameters(), self.critic.parameters()), lr=learning_rate
|
|
) # type: ignore
|
|
|
|
# Add storage
|
|
self.storage = storage
|
|
self.transition = RolloutStorage.Transition()
|
|
|
|
# PPO parameters
|
|
self.clip_param = clip_param
|
|
self.num_learning_epochs = num_learning_epochs
|
|
self.num_mini_batches = num_mini_batches
|
|
self.value_loss_coef = value_loss_coef
|
|
self.entropy_coef = entropy_coef
|
|
self.gamma = gamma
|
|
self.lam = lam
|
|
self.max_grad_norm = max_grad_norm
|
|
self.use_clipped_value_loss = use_clipped_value_loss
|
|
self.desired_kl = desired_kl
|
|
self.schedule = schedule
|
|
self.learning_rate = learning_rate
|
|
self.normalize_advantage_per_mini_batch = normalize_advantage_per_mini_batch
|
|
self.use_mixed_precision = use_mixed_precision
|
|
|
|
def act(self, obs: TensorDict) -> torch.Tensor:
|
|
"""Sample actions and store transition data."""
|
|
# Record the hidden states for recurrent policies
|
|
self.transition.hidden_states = (self.actor.get_hidden_state(), self.critic.get_hidden_state())
|
|
# Compute the actions and values
|
|
self.transition.actions = self.actor(obs, stochastic_output=True).detach()
|
|
self.transition.values = self.critic(obs).detach()
|
|
self.transition.actions_log_prob = self.actor.get_output_log_prob(self.transition.actions).detach() # type: ignore
|
|
self.transition.distribution_params = tuple(p.detach() for p in self.actor.output_distribution_params)
|
|
# Record observations before env.step()
|
|
self.transition.observations = obs
|
|
return self.transition.actions # type: ignore
|
|
|
|
def process_env_step(
|
|
self, obs: TensorDict, rewards: torch.Tensor, dones: torch.Tensor, extras: dict[str, torch.Tensor]
|
|
) -> None:
|
|
"""Record one environment step."""
|
|
# Record the rewards and dones
|
|
# Note: We clone here because later on we bootstrap the rewards based on timeouts
|
|
self.transition.rewards = rewards.clone()
|
|
self.transition.dones = dones
|
|
|
|
# Compute the intrinsic rewards and add to extrinsic rewards
|
|
if self.rnd:
|
|
# Compute the intrinsic rewards
|
|
self.intrinsic_rewards = self.rnd.get_intrinsic_reward(obs)
|
|
# Add intrinsic rewards to extrinsic rewards
|
|
self.transition.rewards += self.intrinsic_rewards
|
|
|
|
# Bootstrapping on time outs
|
|
if "time_outs" in extras:
|
|
self.transition.rewards += self.gamma * torch.squeeze(
|
|
self.transition.values * extras["time_outs"].unsqueeze(1).to(self.device), # type: ignore
|
|
1,
|
|
)
|
|
|
|
# Record the transition
|
|
self.storage.add_transition(self.transition)
|
|
self.transition.clear()
|
|
self.actor.reset(dones)
|
|
self.critic.reset(dones)
|
|
|
|
def compute_returns(self, obs: TensorDict) -> None:
|
|
"""Compute return and advantage targets from stored transitions."""
|
|
st = self.storage
|
|
# Compute values for the last step
|
|
critic_hidden_state = self.critic.get_hidden_state()
|
|
last_values = self.critic(obs).detach()
|
|
# Restore the critic's hidden state so the next rollout is not affected by the forward pass
|
|
self.critic.reset(hidden_state=critic_hidden_state)
|
|
# Compute returns and advantages
|
|
advantage = 0
|
|
for step in reversed(range(st.num_transitions_per_env)):
|
|
# If we are at the last step, bootstrap the return value
|
|
next_values = last_values if step == st.num_transitions_per_env - 1 else st.values[step + 1]
|
|
# 1 if we are not in a terminal state, 0 otherwise
|
|
next_is_not_terminal = 1.0 - st.dones[step].float()
|
|
# TD error: r_t + gamma * V(s_{t+1}) - V(s_t)
|
|
delta = st.rewards[step] + next_is_not_terminal * self.gamma * next_values - st.values[step]
|
|
# Advantage: A(s_t, a_t) = delta_t + gamma * lambda * A(s_{t+1}, a_{t+1})
|
|
advantage = delta + next_is_not_terminal * self.gamma * self.lam * advantage
|
|
# Return: R_t = A(s_t, a_t) + V(s_t)
|
|
st.returns[step] = advantage + st.values[step]
|
|
# Compute the advantages
|
|
st.advantages = st.returns - st.values
|
|
# Normalize the advantages if per minibatch normalization is not used
|
|
if not self.normalize_advantage_per_mini_batch:
|
|
st.advantages = (st.advantages - st.advantages.mean()) / (st.advantages.std() + 1e-8)
|
|
|
|
def update(self) -> dict[str, float]:
|
|
"""Run optimization epochs over stored batches and return mean losses."""
|
|
mean_value_loss = 0
|
|
mean_surrogate_loss = 0
|
|
mean_entropy = 0
|
|
# RND loss
|
|
mean_rnd_loss = 0 if self.rnd else None
|
|
# Symmetry loss
|
|
mean_symmetry_loss = 0 if self.symmetry else None
|
|
|
|
# Get mini-batch generator
|
|
if self.actor.is_recurrent or self.critic.is_recurrent:
|
|
generator = self.storage.recurrent_mini_batch_generator(self.num_mini_batches, self.num_learning_epochs)
|
|
else:
|
|
generator = self.storage.mini_batch_generator(self.num_mini_batches, self.num_learning_epochs)
|
|
|
|
# Iterate over mini-batches
|
|
for batch in generator:
|
|
original_batch_size = batch.observations.batch_size[0]
|
|
|
|
# Check if we should normalize advantages per mini-batch
|
|
if self.normalize_advantage_per_mini_batch:
|
|
with torch.no_grad():
|
|
batch.advantages = (batch.advantages - batch.advantages.mean()) / (batch.advantages.std() + 1e-8) # type: ignore
|
|
|
|
# Perform symmetric augmentation if enabled
|
|
if self.symmetry:
|
|
self.symmetry.augment_batch(batch, original_batch_size)
|
|
|
|
# Optionally use mixed precision for the forward pass and loss computation
|
|
with torch.amp.autocast( # type: ignore
|
|
device_type=torch.device(self.device).type, enabled=self.use_mixed_precision, dtype=torch.bfloat16
|
|
):
|
|
# Recompute actions log prob and entropy for current batch of transitions
|
|
# Note: We need to do this because we updated the policy with new parameters
|
|
self.actor(
|
|
batch.observations,
|
|
masks=batch.masks,
|
|
hidden_state=batch.hidden_states[0],
|
|
stochastic_output=True,
|
|
)
|
|
actions_log_prob = self.actor.get_output_log_prob(batch.actions) # type: ignore
|
|
values = self.critic(batch.observations, masks=batch.masks, hidden_state=batch.hidden_states[1])
|
|
# Note: We only keep the following tensors for the original samples in case of symmetry augmentation
|
|
distribution_params = tuple(p[:original_batch_size] for p in self.actor.output_distribution_params)
|
|
entropy = self.actor.output_entropy[:original_batch_size]
|
|
|
|
# Compute KL divergence and adapt the learning rate
|
|
if self.desired_kl is not None and self.schedule == "adaptive":
|
|
with torch.inference_mode():
|
|
kl = self.actor.get_kl_divergence(batch.old_distribution_params, distribution_params) # type: ignore
|
|
kl_mean = torch.mean(kl)
|
|
|
|
# Reduce the KL divergence across all GPUs
|
|
if self.is_multi_gpu:
|
|
torch.distributed.all_reduce(kl_mean, op=torch.distributed.ReduceOp.SUM)
|
|
kl_mean /= self.gpu_world_size
|
|
|
|
# Update the learning rate only on the main process
|
|
if self.gpu_global_rank == 0:
|
|
if kl_mean > self.desired_kl * 2.0:
|
|
self.learning_rate = max(1e-5, self.learning_rate / 1.5)
|
|
elif kl_mean < self.desired_kl / 2.0 and kl_mean > 0.0:
|
|
self.learning_rate = min(1e-2, self.learning_rate * 1.5)
|
|
|
|
# Update the learning rate for all GPUs
|
|
if self.is_multi_gpu:
|
|
lr_tensor = torch.tensor(self.learning_rate, device=self.device)
|
|
torch.distributed.broadcast(lr_tensor, src=0)
|
|
self.learning_rate = lr_tensor.item()
|
|
|
|
# Update the learning rate for all parameter groups
|
|
for param_group in self.optimizer.param_groups:
|
|
param_group["lr"] = self.learning_rate
|
|
|
|
# Surrogate loss
|
|
ratio = torch.exp(actions_log_prob - torch.squeeze(batch.old_actions_log_prob)) # type: ignore
|
|
surrogate = -torch.squeeze(batch.advantages) * ratio # type: ignore
|
|
surrogate_clipped = -torch.squeeze(batch.advantages) * torch.clamp( # type: ignore
|
|
ratio, 1.0 - self.clip_param, 1.0 + self.clip_param
|
|
)
|
|
surrogate_loss = torch.max(surrogate, surrogate_clipped).mean()
|
|
|
|
# Value function loss
|
|
if self.use_clipped_value_loss:
|
|
value_clipped = batch.values + (values - batch.values).clamp(-self.clip_param, self.clip_param)
|
|
value_losses = (values - batch.returns).pow(2)
|
|
value_losses_clipped = (value_clipped - batch.returns).pow(2)
|
|
value_loss = torch.max(value_losses, value_losses_clipped).mean()
|
|
else:
|
|
value_loss = (batch.returns - values).pow(2).mean()
|
|
|
|
loss = surrogate_loss + self.value_loss_coef * value_loss - self.entropy_coef * entropy.mean()
|
|
|
|
# RND loss
|
|
rnd_loss = self.rnd.compute_loss(batch.observations[:original_batch_size]) if self.rnd else None # type: ignore
|
|
|
|
# Symmetry loss
|
|
if self.symmetry:
|
|
symmetry_loss = self.symmetry.compute_loss(self.actor, batch, original_batch_size)
|
|
if self.symmetry.use_mirror_loss:
|
|
loss = loss + self.symmetry.mirror_loss_coeff * symmetry_loss
|
|
|
|
# Compute the gradients for PPO
|
|
self.optimizer.zero_grad()
|
|
loss.backward()
|
|
# Compute the gradients for RND
|
|
if self.rnd:
|
|
self.rnd.optimizer.zero_grad()
|
|
rnd_loss.backward()
|
|
|
|
# Collect gradients from all GPUs
|
|
if self.is_multi_gpu:
|
|
self.reduce_parameters()
|
|
|
|
# Apply the gradients for PPO
|
|
nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
|
|
nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
|
|
self.optimizer.step()
|
|
# Apply the gradients for RND
|
|
if self.rnd:
|
|
self.rnd.optimizer.step()
|
|
|
|
# Store the losses
|
|
mean_value_loss += value_loss.item()
|
|
mean_surrogate_loss += surrogate_loss.item()
|
|
mean_entropy += entropy.mean().item()
|
|
# RND loss
|
|
if mean_rnd_loss is not None:
|
|
mean_rnd_loss += rnd_loss.item()
|
|
# Symmetry loss
|
|
if mean_symmetry_loss is not None:
|
|
mean_symmetry_loss += symmetry_loss.item()
|
|
|
|
# Update the normalizers
|
|
obs = self.storage.observations.flatten(0, 1)
|
|
self.actor.update_normalization(obs) # type: ignore
|
|
self.critic.update_normalization(obs) # type: ignore
|
|
if self.rnd:
|
|
self.rnd.update_normalization(obs) # type: ignore
|
|
|
|
# Divide the losses by the number of updates
|
|
num_updates = self.num_learning_epochs * self.num_mini_batches
|
|
mean_value_loss /= num_updates
|
|
mean_surrogate_loss /= num_updates
|
|
mean_entropy /= num_updates
|
|
if mean_rnd_loss is not None:
|
|
mean_rnd_loss /= num_updates
|
|
if mean_symmetry_loss is not None:
|
|
mean_symmetry_loss /= num_updates
|
|
|
|
# Construct the loss dictionary
|
|
loss_dict = {
|
|
"value": mean_value_loss,
|
|
"surrogate": mean_surrogate_loss,
|
|
"entropy": mean_entropy,
|
|
}
|
|
if self.rnd:
|
|
loss_dict["rnd"] = mean_rnd_loss
|
|
if self.symmetry:
|
|
loss_dict["symmetry"] = mean_symmetry_loss
|
|
|
|
# Clear the storage
|
|
self.storage.clear()
|
|
|
|
return loss_dict
|
|
|
|
def train_mode(self) -> None:
|
|
"""Set train mode for learnable models."""
|
|
self.actor.train()
|
|
self.critic.train()
|
|
if self.rnd:
|
|
self.rnd.train()
|
|
|
|
def eval_mode(self) -> None:
|
|
"""Set evaluation mode for learnable models."""
|
|
self.actor.eval()
|
|
self.critic.eval()
|
|
if self.rnd:
|
|
self.rnd.eval()
|
|
|
|
def save(self) -> dict:
|
|
"""Return a dict of all models for saving."""
|
|
saved_dict = {
|
|
"actor_state_dict": self._raw_actor.state_dict(),
|
|
"critic_state_dict": self._raw_critic.state_dict(),
|
|
"optimizer_state_dict": self.optimizer.state_dict(),
|
|
}
|
|
if self.rnd:
|
|
saved_dict["rnd_state_dict"] = self.rnd.state_dict()
|
|
saved_dict["rnd_optimizer_state_dict"] = self.rnd.optimizer.state_dict()
|
|
return saved_dict
|
|
|
|
def load(self, loaded_dict: dict, load_cfg: dict | None, strict: bool) -> bool:
|
|
"""Load specified models from a saved dict."""
|
|
# If no load_cfg is provided, load all models and states
|
|
if load_cfg is None:
|
|
load_cfg = {
|
|
"actor": True,
|
|
"critic": True,
|
|
"optimizer": True,
|
|
"iteration": True,
|
|
"rnd": True,
|
|
}
|
|
|
|
# Load the specified models
|
|
if load_cfg.get("actor"):
|
|
self._raw_actor.load_state_dict(loaded_dict["actor_state_dict"], strict=strict)
|
|
if load_cfg.get("critic"):
|
|
self._raw_critic.load_state_dict(loaded_dict["critic_state_dict"], strict=strict)
|
|
if load_cfg.get("optimizer"):
|
|
self.optimizer.load_state_dict(loaded_dict["optimizer_state_dict"])
|
|
self.learning_rate = self.optimizer.param_groups[0]["lr"]
|
|
if load_cfg.get("rnd") and self.rnd:
|
|
self.rnd.load_state_dict(loaded_dict["rnd_state_dict"], strict=strict)
|
|
self.rnd.optimizer.load_state_dict(loaded_dict["rnd_optimizer_state_dict"])
|
|
return load_cfg.get("iteration", False)
|
|
|
|
def get_policy(self) -> MLPModel:
|
|
"""Get the policy model."""
|
|
return self._raw_actor
|
|
|
|
def compile(self, mode: str | None = None) -> None:
|
|
"""Compile actor and critic with ``torch.compile``.
|
|
|
|
See :func:`~rsl_rl.utils.compile_model` for the set of accepted modes.
|
|
|
|
Args:
|
|
mode: ``torch.compile`` mode. Defaults to ``None``, in which case compilation is disabled.
|
|
"""
|
|
self.actor = compile_model(self._raw_actor, mode) # type: ignore
|
|
self.critic = compile_model(self._raw_critic, mode) # type: ignore
|
|
|
|
@staticmethod
|
|
def construct_algorithm(obs: TensorDict, env: VecEnv, cfg: dict, device: str) -> PPO:
|
|
"""Construct the PPO algorithm."""
|
|
# Resolve class callables and configs
|
|
alg_class, alg_cfg = resolve_class(cfg["algorithm"])
|
|
actor_class, actor_cfg = resolve_class(cfg["actor"])
|
|
critic_class, critic_cfg = resolve_class(cfg["critic"])
|
|
|
|
# Resolve observation groups
|
|
default_sets = ["actor", "critic"]
|
|
if "rnd_cfg" in alg_cfg and alg_cfg["rnd_cfg"] is not None:
|
|
default_sets.append("rnd_state")
|
|
cfg["obs_groups"] = resolve_obs_groups(obs, cfg["obs_groups"], default_sets)
|
|
|
|
# Resolve RND config if used
|
|
alg_cfg = resolve_rnd_config(alg_cfg, obs, cfg["obs_groups"], env)
|
|
|
|
# Resolve symmetry config if used
|
|
alg_cfg = resolve_symmetry_config(alg_cfg, env)
|
|
|
|
# Initialize the policy
|
|
actor: MLPModel = actor_class(obs, cfg["obs_groups"], "actor", env.num_actions, **actor_cfg).to(device)
|
|
print(f"Actor Model: {actor}")
|
|
if alg_cfg.pop("share_cnn_encoders", None): # Share CNN encoders between actor and critic
|
|
critic_cfg["cnns"] = actor.cnns
|
|
critic: MLPModel = critic_class(obs, cfg["obs_groups"], "critic", 1, **critic_cfg).to(device)
|
|
print(f"Critic Model: {critic}")
|
|
|
|
# Initialize the storage
|
|
storage = RolloutStorage("rl", env.num_envs, cfg["num_steps_per_env"], obs, [env.num_actions], device)
|
|
|
|
# Initialize the algorithm
|
|
alg: PPO = alg_class(actor, critic, storage, device=device, **alg_cfg, multi_gpu_cfg=cfg["multi_gpu"])
|
|
|
|
# Compile the algorithm's models if requested
|
|
alg.compile(cfg.get("torch_compile_mode"))
|
|
|
|
return alg
|
|
|
|
def broadcast_parameters(self) -> None:
|
|
"""Broadcast model parameters to all GPUs."""
|
|
models = [self._raw_actor, self._raw_critic]
|
|
if self.rnd:
|
|
models += [self.rnd.predictor, self.rnd.target]
|
|
for model in models:
|
|
for tensor in model.state_dict().values():
|
|
torch.distributed.broadcast(tensor, src=0)
|
|
|
|
def reduce_parameters(self) -> None:
|
|
"""Collect gradients from all GPUs and average them.
|
|
|
|
This function is called after the backward pass to synchronize the gradients across all GPUs.
|
|
"""
|
|
# Create a tensor to store the gradients
|
|
params = chain(self.actor.parameters(), self.critic.parameters())
|
|
if self.rnd:
|
|
params = chain(params, self.rnd.parameters())
|
|
# Average the gradients across all GPUs in bounded buckets
|
|
reduce_gradients_in_buckets(params, self.gpu_world_size, self.grad_reduce_bucket_mb)
|