87 lines
3.8 KiB
Python
87 lines
3.8 KiB
Python
# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
|
|
# All rights reserved.
|
|
#
|
|
# SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from typing import Union
|
|
|
|
from rsl_rl.utils import unpad_trajectories
|
|
|
|
HiddenState = Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor], None] # Using Union due to Python <3.10
|
|
"""Type alias for the hidden state of RNNs (GRU/LSTM).
|
|
|
|
For GRUs, this is a single tensor while for LSTMs, this is a tuple of two tensors (hidden state and cell state).
|
|
"""
|
|
|
|
|
|
class RNN(nn.Module):
|
|
"""Recurrent Neural Network.
|
|
|
|
This network is used to store the hidden state of the policy. It currently supports GRU and LSTM.
|
|
"""
|
|
|
|
def __init__(self, input_size: int, hidden_dim: int = 256, num_layers: int = 1, type: str = "lstm") -> None:
|
|
"""Initialize a GRU or LSTM module with internal hidden-state storage."""
|
|
super().__init__()
|
|
rnn_cls = nn.GRU if type.lower() == "gru" else nn.LSTM
|
|
self.rnn = rnn_cls(input_size=input_size, hidden_size=hidden_dim, num_layers=num_layers)
|
|
self.hidden_state = None
|
|
|
|
def forward(
|
|
self,
|
|
input: torch.Tensor,
|
|
masks: torch.Tensor | None = None,
|
|
hidden_state: HiddenState = None,
|
|
) -> torch.Tensor:
|
|
"""Run recurrent inference in rollout mode or batched update mode."""
|
|
batch_mode = masks is not None
|
|
if batch_mode:
|
|
# Batch mode needs saved hidden states
|
|
if hidden_state is None:
|
|
raise ValueError("Hidden states not passed to RNN module during policy update")
|
|
out, _ = self.rnn(input, hidden_state)
|
|
out = unpad_trajectories(out, masks)
|
|
else:
|
|
# Inference/distillation mode uses hidden state of last step
|
|
out, self.hidden_state = self.rnn(input.unsqueeze(0), self.hidden_state)
|
|
return out
|
|
|
|
def reset(self, dones: torch.Tensor | None = None, hidden_state: HiddenState = None) -> None:
|
|
"""Reset hidden states for all or done environments."""
|
|
if dones is None: # Reset hidden state
|
|
if hidden_state is None:
|
|
self.hidden_state = None
|
|
else:
|
|
self.hidden_state = hidden_state
|
|
elif self.hidden_state is not None: # Reset hidden state of done environments
|
|
if hidden_state is None:
|
|
if isinstance(self.hidden_state, tuple): # Tuple in case of LSTM
|
|
for hidden_state in self.hidden_state:
|
|
hidden_state[..., dones == 1, :] = 0.0 # type: ignore
|
|
else:
|
|
self.hidden_state[..., dones == 1, :] = 0.0
|
|
else:
|
|
raise NotImplementedError(
|
|
"Resetting the hidden state of done environments with a custom hidden state is not implemented"
|
|
)
|
|
|
|
def detach_hidden_state(self, dones: torch.Tensor | None = None) -> None:
|
|
"""Detach hidden states for all or done environments from the computation graph."""
|
|
if self.hidden_state is not None:
|
|
if dones is None: # Detach hidden state
|
|
if isinstance(self.hidden_state, tuple): # Tuple in case of LSTM
|
|
self.hidden_state = tuple(hidden_state.detach() for hidden_state in self.hidden_state)
|
|
else:
|
|
self.hidden_state = self.hidden_state.detach()
|
|
else: # Detach hidden state of done environments
|
|
if isinstance(self.hidden_state, tuple): # Tuple in case of LSTM
|
|
for hidden_state in self.hidden_state:
|
|
hidden_state[..., dones == 1, :] = hidden_state[..., dones == 1, :].detach()
|
|
else:
|
|
self.hidden_state[..., dones == 1, :] = self.hidden_state[..., dones == 1, :].detach()
|