Files
2026-11-03 19:46:08 +08:00

254 lines
10 KiB
Python

# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
from __future__ import annotations
import copy
import torch
import torch.nn as nn
from tensordict import TensorDict
from typing import Any
from rsl_rl.models.mlp_model import MLPModel
from rsl_rl.modules import CNN, HiddenState
class CNNModel(MLPModel):
"""CNN-based neural model.
This model uses one or more convolutional neural network (CNN) encoders to process one or more 2D observation groups
before passing the resulting latent to an MLP. Any 1D observation groups are directly concatenated with the CNN
latent and passed to the MLP. 1D observations can be normalized before being passed to the MLP. The output of the
model can be either deterministic or stochastic, in which case a distribution module is used to sample the outputs.
"""
def __init__(
self,
obs: TensorDict,
obs_groups: dict[str, list[str]],
obs_set: str,
output_dim: int,
hidden_dims: tuple[int, ...] | list[int] = (256, 256, 256),
activation: str = "elu",
obs_normalization: bool = False,
distribution_cfg: dict | None = None,
cnn_cfg: dict[str, dict] | dict[str, Any] | None = None,
cnns: nn.ModuleDict | dict[str, nn.Module] | None = None,
) -> None:
"""Initialize the CNN-based model.
Args:
obs: Observation Dictionary.
obs_groups: Dictionary mapping observation sets to lists of observation groups.
obs_set: Observation set to use for this model (e.g., "actor" or "critic").
output_dim: Dimension of the output.
hidden_dims: Hidden dimensions of the MLP.
activation: Activation function of the CNN and MLP.
obs_normalization: Whether to normalize the observations before feeding them to the MLP.
distribution_cfg: Configuration dictionary for the output distribution.
cnn_cfg: Configuration of the CNN encoder(s).
cnns: CNN modules to use, e.g., for sharing CNNs between actor and critic. If None, new CNNs are created.
"""
# Resolve observation groups and dimensions
self._get_obs_dim(obs, obs_groups, obs_set)
# Create or validate CNN encoders
if cnns is not None:
# Check compatibility if CNNs are provided
if set(cnns.keys()) != set(self.obs_groups_2d):
raise ValueError("The 2D observations must be identical for all models sharing CNN encoders.")
print("Sharing CNN encoders between models, the CNN configurations of the receiving model are ignored.")
else:
if cnn_cfg is None:
raise ValueError("CNN configurations must be provided if CNNs are not shared.")
# Create a cnn config for each 2D observation group in case only one is provided
if not all(isinstance(v, dict) for v in cnn_cfg.values()):
cnn_cfg = {group: cnn_cfg for group in self.obs_groups_2d}
# Check that the number of configs matches the number of observation groups
if len(cnn_cfg) != len(self.obs_groups_2d):
raise ValueError("The number of CNN configurations must match the number of 2D observation groups.")
# Create CNNs for each 2D observation
cnns = {}
for idx, obs_group in enumerate(self.obs_groups_2d):
cnns[obs_group] = CNN(
input_dim=self.obs_dims_2d[idx],
input_channels=self.obs_channels_2d[idx],
**cnn_cfg[obs_group],
)
# Compute latent dimension of the CNNs
self.cnn_latent_dim = 0
for cnn in cnns.values():
if cnn.output_channels is not None:
raise ValueError("The output of the CNN must be flattened before passing it to the MLP.")
self.cnn_latent_dim += int(cnn.output_dim) # type: ignore
# Initialize the parent MLP model
super().__init__(
obs,
obs_groups,
obs_set,
output_dim,
hidden_dims,
activation,
obs_normalization,
distribution_cfg,
)
# Register CNN encoders
if isinstance(cnns, nn.ModuleDict):
self.cnns = cnns
else:
self.cnns = nn.ModuleDict(cnns)
def get_latent(
self, obs: TensorDict, masks: torch.Tensor | None = None, hidden_state: HiddenState = None
) -> torch.Tensor:
"""Build the model latent by combining (optional) normalized 1D and CNN-encoded 2D observation groups."""
# Process 2D observation groups with CNNs
latent_cnn = torch.cat([self.cnns[obs_group](obs[obs_group]) for obs_group in self.obs_groups_2d], dim=-1)
# If there are no 1D observations, return the CNN latent directly
if not self.obs_groups:
return latent_cnn
# Select 1D observation groups and normalize
latent_1d = super().get_latent(obs)
# Concatenate 1D and CNN latents
return torch.cat([latent_1d, latent_cnn], dim=-1)
def as_jit(self) -> nn.Module:
"""Return a version of the model compatible with Torch JIT export."""
return _TorchCNNModel(self)
def as_onnx(self, verbose: bool = False) -> nn.Module:
"""Return a version of the model compatible with ONNX export."""
return _OnnxCNNModel(self, verbose)
def _get_obs_dim(self, obs: TensorDict, obs_groups: dict[str, list[str]], obs_set: str) -> tuple[list[str], int]:
"""Select active observation groups and compute observation dimension."""
active_obs_groups = obs_groups[obs_set]
obs_dim_1d = 0
obs_groups_1d = []
obs_dims_2d = []
obs_channels_2d = []
obs_groups_2d = []
# Iterate through active observation groups and separate 1D and 2D observations
for obs_group in active_obs_groups:
if len(obs[obs_group].shape) == 4: # B, C, H, W
obs_groups_2d.append(obs_group)
obs_dims_2d.append(obs[obs_group].shape[2:4])
obs_channels_2d.append(obs[obs_group].shape[1])
elif len(obs[obs_group].shape) == 2: # B, C
obs_groups_1d.append(obs_group)
obs_dim_1d += obs[obs_group].shape[-1]
else:
raise ValueError(f"Invalid observation shape for {obs_group}: {obs[obs_group].shape}")
if not obs_groups_2d:
raise ValueError("No 2D observations are provided. If this is intentional, use the MLP model instead.")
# Store active 2D observation groups and dimensions directly as attributes
self.obs_dims_2d = obs_dims_2d
self.obs_channels_2d = obs_channels_2d
self.obs_groups_2d = obs_groups_2d
# Return active 1D observation groups and dimension for parent class
return obs_groups_1d, obs_dim_1d
def _get_latent_dim(self) -> int:
"""Return the latent dimensionality consumed by the MLP head."""
return self.obs_dim + self.cnn_latent_dim
class _TorchCNNModel(nn.Module):
"""Exportable CNN model for JIT."""
def __init__(self, model: CNNModel) -> None:
"""Create a TorchScript-friendly copy of a CNNModel."""
super().__init__()
self.obs_normalizer = copy.deepcopy(model.obs_normalizer)
# Convert ModuleDict to ModuleList for ordered iteration
self.cnns = nn.ModuleList([copy.deepcopy(model.cnns[g]) for g in model.obs_groups_2d])
self.mlp = copy.deepcopy(model.mlp)
if model.distribution is not None:
self.deterministic_output = model.distribution.as_deterministic_output_module()
else:
self.deterministic_output = nn.Identity()
def forward(self, obs_1d: torch.Tensor, obs_2d: list[torch.Tensor]) -> torch.Tensor:
"""Run deterministic inference from separated 1D and 2D inputs."""
latent_1d = self.obs_normalizer(obs_1d)
latent_cnn_list = []
for i, cnn in enumerate(self.cnns): # We assume obs_2d list matches the order of obs_groups_2d
latent_cnn_list.append(cnn(obs_2d[i]))
latent_cnn = torch.cat(latent_cnn_list, dim=-1)
latent = torch.cat([latent_1d, latent_cnn], dim=-1)
out = self.mlp(latent)
return self.deterministic_output(out)
@torch.jit.export
def reset(self) -> None:
"""Reset recurrent export state (no-op for CNN exports)."""
pass
class _OnnxCNNModel(nn.Module):
"""Exportable CNN model for ONNX."""
def __init__(self, model: CNNModel, verbose: bool) -> None:
"""Create an ONNX-export wrapper around a CNNModel."""
super().__init__()
self.verbose = verbose
self.obs_normalizer = copy.deepcopy(model.obs_normalizer)
# Convert ModuleDict to ModuleList for ordered iteration
self.cnns = nn.ModuleList([copy.deepcopy(model.cnns[g]) for g in model.obs_groups_2d])
self.mlp = copy.deepcopy(model.mlp)
if model.distribution is not None:
self.deterministic_output = model.distribution.as_deterministic_output_module()
else:
self.deterministic_output = nn.Identity()
self.obs_groups_2d = model.obs_groups_2d
self.obs_dims_2d = model.obs_dims_2d
self.obs_channels_2d = model.obs_channels_2d
self.obs_dim_1d = model.obs_dim
def forward(self, obs_1d: torch.Tensor, *obs_2d: torch.Tensor) -> torch.Tensor:
"""Run deterministic inference for ONNX export."""
latent_1d = self.obs_normalizer(obs_1d)
latent_cnn_list = []
for i, cnn in enumerate(self.cnns):
latent_cnn_list.append(cnn(obs_2d[i]))
latent_cnn = torch.cat(latent_cnn_list, dim=-1)
latent = torch.cat([latent_1d, latent_cnn], dim=-1)
out = self.mlp(latent)
return self.deterministic_output(out)
def get_dummy_inputs(self) -> tuple[torch.Tensor, ...]:
"""Return representative dummy inputs for ONNX tracing."""
dummy_1d = torch.zeros(1, self.obs_dim_1d)
dummy_2d = []
for i in range(len(self.obs_groups_2d)):
h, w = self.obs_dims_2d[i]
c = self.obs_channels_2d[i]
dummy_2d.append(torch.zeros(1, c, h, w))
return (dummy_1d, *dummy_2d)
@property
def input_names(self) -> list[str]:
"""Return ONNX input tensor names."""
return ["obs", *self.obs_groups_2d]
@property
def output_names(self) -> list[str]:
"""Return ONNX output tensor names."""
return ["actions"]