Files
2026-11-03 19:46:08 +08:00

465 lines
20 KiB
Python

# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
from __future__ import annotations
import numpy as np
import torch
import torch.nn as nn
from torch.distributions import Beta, Normal
class Distribution(nn.Module):
"""Base class for distribution modules.
Distribution modules encapsulate the stochastic output of a neural model. They define the output structure expected
from the MLP, manage learnable distribution parameters, and provide methods for sampling, log probability
computation, and entropy calculation.
Subclasses must implement all abstract methods and properties to define a specific distribution type.
"""
def __init__(self, output_dim: int) -> None:
"""Initialize the distribution module.
Args:
output_dim: Dimension of the action/output space.
"""
super().__init__()
self.output_dim = output_dim
def update(self, mlp_output: torch.Tensor) -> None:
"""Update the distribution parameters given the MLP output.
Args:
mlp_output: Raw output from the MLP.
"""
raise NotImplementedError
def sample(self) -> torch.Tensor:
"""Sample from the distribution.
Returns:
Sampled values.
"""
raise NotImplementedError
def deterministic_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
"""Extract the deterministic (mean) output from the raw MLP output.
Args:
mlp_output: Raw output from the MLP.
Returns:
The deterministic output (typically the distribution mean).
"""
raise NotImplementedError
def as_deterministic_output_module(self) -> nn.Module:
"""Return an export-friendly module that extracts the deterministic output from the MLP output."""
raise NotImplementedError
@property
def input_dim(self) -> int | list[int]:
"""Return the input dimension required by the distribution."""
raise NotImplementedError
@property
def mean(self) -> torch.Tensor:
"""Return the mean of the distribution."""
raise NotImplementedError
@property
def std(self) -> torch.Tensor:
"""Return the standard deviation (or spread measure) of the distribution."""
raise NotImplementedError
@property
def entropy(self) -> torch.Tensor:
"""Return the entropy of the distribution, summed over the last dimension."""
raise NotImplementedError
@property
def params(self) -> tuple[torch.Tensor, ...]:
"""Return the distribution parameters as a tuple of tensors.
These are the distribution-specific parameters needed to reconstruct the distribution (e.g., mean and std for
Gaussian, alpha and beta for Beta). They are stored during rollouts and used for KL divergence computation.
"""
raise NotImplementedError
def log_prob(self, outputs: torch.Tensor) -> torch.Tensor:
"""Compute the log probability of the given outputs, summed over the last dimension.
Args:
outputs: Values to compute the log probability for.
Returns:
Log probability summed over the last dimension.
"""
raise NotImplementedError
def kl_divergence(self, old_params: tuple[torch.Tensor, ...], new_params: tuple[torch.Tensor, ...]) -> torch.Tensor:
"""Compute the KL divergence KL(old || new) between two distributions of this type.
The KL divergence measures how the old distribution diverges from the new distribution.
This is used for adaptive learning rate scheduling in policy optimization.
Args:
old_params: Parameters of the old distribution (as returned by :attr:`params`).
new_params: Parameters of the new distribution (as returned by :attr:`params`).
Returns:
KL divergence summed over the last dimension.
"""
raise NotImplementedError
def init_mlp_weights(self, mlp: nn.Module) -> None:
"""Initialize distribution-specific weights in the MLP.
This is called after MLP creation to set up any special weight initialization
required by the distribution (e.g., initializing std head weights).
Args:
mlp: The MLP module whose weights may need initialization.
"""
pass
class GaussianDistribution(Distribution):
"""Gaussian distribution module with state-independent standard deviation.
This distribution parameterizes stochastic outputs using a multivariate Gaussian with diagonal covariance. The
standard deviation can be a learnable parameter or a constant. It can be parameterized in either "scalar" space or
"log" space and is clamped to a specified range.
.. note::
If the standard deviation type is set to "log", the provided arguments are still interpreted in scalar space,
and converted to log space internally.
"""
def __init__(
self,
output_dim: int,
init_std: float = 1.0,
std_range: tuple[float, float] = (1e-6, 1e6),
std_type: str = "scalar",
learn_std: bool = True,
) -> None:
"""Initialize the Gaussian distribution module.
Args:
output_dim: Dimension of the action/output space.
init_std: Initial standard deviation.
std_range: Range for the standard deviation. Should be a tuple of (min, max) values for clamping.
std_type: Parameterization of the standard deviation: "scalar" or "log".
learn_std: Whether the standard deviation should be learnable. If False, it will be fixed to `init_std`.
"""
super().__init__(output_dim)
self.std_type = std_type
# Learnable std parameters
if std_type == "scalar":
self.std_param = nn.Parameter(init_std * torch.ones(output_dim), requires_grad=learn_std)
elif std_type == "log":
self.log_std_param = nn.Parameter(torch.log(init_std * torch.ones(output_dim)), requires_grad=learn_std)
else:
raise ValueError(f"Unknown standard deviation type: {std_type}. Should be 'scalar' or 'log'.")
# Clamp the std range to ensure numerical stability and store log space range if needed
self.std_range = list(std_range)
self.std_range[0] = max(self.std_range[0], 1e-6) # Avoid zero std for numerical stability
self.log_std_range = [float(np.log(self.std_range[0])), float(np.log(self.std_range[1]))]
# Internal torch distribution (populated by update())
self._distribution: Normal | None = None
# Disable args validation for speedup
Normal.set_default_validate_args(False)
def update(self, mlp_output: torch.Tensor) -> None:
"""Update the Gaussian distribution from MLP output."""
mean = mlp_output
if self.std_type == "scalar":
std = self.std_param.clamp(self.std_range[0], self.std_range[1])
elif self.std_type == "log":
log_std = self.log_std_param.clamp(self.log_std_range[0], self.log_std_range[1])
std = torch.exp(log_std)
self._distribution = Normal(mean, std)
def sample(self) -> torch.Tensor:
"""Sample from the Gaussian distribution."""
return self._distribution.sample() # type: ignore
def deterministic_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
"""Extract the mean from the MLP output."""
return mlp_output
def as_deterministic_output_module(self) -> nn.Module:
"""Return an export-friendly module that extracts the mean from the MLP output."""
return _IdentityDeterministicOutput()
@property
def input_dim(self) -> int:
"""Return the input dimension required by the distribution."""
return self.output_dim
@property
def mean(self) -> torch.Tensor:
"""Return the mean of the Gaussian distribution."""
return self._distribution.mean # type: ignore
@property
def std(self) -> torch.Tensor:
"""Return the standard deviation of the Gaussian distribution."""
return self._distribution.stddev # type: ignore
@property
def entropy(self) -> torch.Tensor:
"""Return the entropy of the Gaussian distribution, summed over the last dimension."""
return self._distribution.entropy().sum(dim=-1) # type: ignore
@property
def params(self) -> tuple[torch.Tensor, ...]:
"""Return (mean, std) of the current Gaussian distribution."""
return (self.mean, self.std)
def log_prob(self, outputs: torch.Tensor) -> torch.Tensor:
"""Compute the log probability under the Gaussian, summed over the last dimension."""
return self._distribution.log_prob(outputs).sum(dim=-1) # type: ignore
def kl_divergence(self, old_params: tuple[torch.Tensor, ...], new_params: tuple[torch.Tensor, ...]) -> torch.Tensor:
"""Compute KL(old || new) between two Gaussian distributions."""
old_mean, old_std = old_params
new_mean, new_std = new_params
return torch.distributions.kl_divergence(Normal(old_mean, old_std), Normal(new_mean, new_std)).sum(dim=-1)
class HeteroscedasticGaussianDistribution(GaussianDistribution):
"""Gaussian distribution module with state-dependent standard deviation.
This distribution parameterizes stochastic outputs using a multivariate Gaussian with diagonal covariance. The
standard deviation is output by the MLP alongside the mean, making it state-dependent. It can be parameterized in
either "scalar" space or "log" space, and is clamped to a specified range.
.. note::
If the standard deviation type is set to "log", the provided arguments are still interpreted in scalar space,
and converted to log space internally.
"""
def __init__(
self,
output_dim: int,
init_std: float = 1.0,
std_range: tuple[float, float] = (1e-6, 1e6),
std_type: str = "scalar",
) -> None:
"""Initialize the heteroscedastic Gaussian distribution module.
Args:
output_dim: Dimension of the action/output space.
init_std: Initial standard deviation (used to initialize the MLP's std head bias).
std_range: Range for the standard deviation. Should be a tuple of (min, max) values for clamping.
std_type: Parameterization of the standard deviation: "scalar" or "log".
"""
# Skip GaussianDistribution.__init__ to avoid creating unnecessary learnable std parameters.
Distribution.__init__(self, output_dim)
self.std_type = std_type
self.init_std = init_std
if std_type not in ("scalar", "log"):
raise ValueError(f"Unknown standard deviation type: {std_type}. Should be 'scalar' or 'log'.")
# Clamp the std range to ensure numerical stability and store log space range if needed
self.std_range = list(std_range)
self.std_range[0] = max(self.std_range[0], 1e-6) # Avoid zero std for numerical stability
self.log_std_range = [float(np.log(self.std_range[0])), float(np.log(self.std_range[1]))]
# Internal torch distribution (populated by update())
self._distribution: Normal | None = None
# Disable args validation for speedup
Normal.set_default_validate_args(False)
def update(self, mlp_output: torch.Tensor) -> None:
"""Update the Gaussian distribution from MLP output."""
if self.std_type == "scalar":
mean, std = torch.unbind(mlp_output, dim=-2)
std = torch.clamp(std, self.std_range[0], self.std_range[1])
elif self.std_type == "log":
mean, log_std = torch.unbind(mlp_output, dim=-2)
log_std = torch.clamp(log_std, self.log_std_range[0], self.log_std_range[1])
std = torch.exp(log_std)
self._distribution = Normal(mean, std)
def deterministic_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
"""Extract the mean from the MLP output (first slice of the second-to-last dimension)."""
return mlp_output[..., 0, :]
def as_deterministic_output_module(self) -> nn.Module:
"""Return export-friendly module that extracts the mean from the MLP output."""
return _FirstSliceDeterministicOutput()
@property
def input_dim(self) -> list[int]:
"""Return the input dimension required by the distribution.
The MLP must output a tensor of shape ``[..., 2, output_dim]`` where the first slice along the second-to-last
dimension is the mean and the second is the standard deviation (or log standard deviation).
"""
return [2, self.output_dim]
def init_mlp_weights(self, mlp: nn.Module) -> None:
"""Initialize the std head weights in the MLP."""
# Initialize weights and biases for the std portion of the last layer
torch.nn.init.zeros_(mlp[-2].weight[self.output_dim :]) # type: ignore
if self.std_type == "scalar":
torch.nn.init.constant_(mlp[-2].bias[self.output_dim :], self.init_std) # type: ignore
elif self.std_type == "log":
init_std_log = torch.log(torch.tensor(self.init_std + 1e-7))
torch.nn.init.constant_(mlp[-2].bias[self.output_dim :], init_std_log) # type: ignore
class BetaDistribution(Distribution):
"""Beta distribution module for bounded action spaces.
This distribution parameterizes stochastic outputs using a Beta distribution, which naturally constrains samples
to [0, 1]. Samples are linearly rescaled to ``action_range``, which defaults to ``(-1.0, 1.0)``.
The MLP must output a tensor of shape ``[..., 2, output_dim]``, where the first slice along the second-to-last
dimension contains the raw alpha parameters and the second contains the raw beta parameters. Both are passed
through ``Softplus + 1`` to ensure they are strictly greater than 1, which guarantees a unimodal distribution.
"""
def __init__(
self,
output_dim: int,
action_range: tuple[float, float] = (-1.0, 1.0),
) -> None:
"""Initialize the Beta distribution module.
Args:
output_dim: Dimension of the action/output space.
action_range: Interval ``(min, max)`` to which Beta samples in ``[0, 1]`` are linearly rescaled.
Defaults to ``(-1.0, 1.0)``.
"""
super().__init__(output_dim)
# Compute scaling and offset for rescaling samples
self.action_range = action_range
self._range_scale = action_range[1] - action_range[0]
self._range_offset = action_range[0]
self._log_range_scale = np.log(self._range_scale)
self._distribution: Beta | None = None
self._alpha: torch.Tensor | None = None
self._beta: torch.Tensor | None = None
# Disable args validation for speedup
Beta.set_default_validate_args(False)
def update(self, mlp_output: torch.Tensor) -> None:
"""Update the Beta distribution from MLP output."""
alpha_raw, beta_raw = torch.unbind(mlp_output, dim=-2)
self._alpha = torch.nn.functional.softplus(alpha_raw) + 1.0
self._beta = torch.nn.functional.softplus(beta_raw) + 1.0
self._distribution = Beta(self._alpha, self._beta)
def sample(self) -> torch.Tensor:
"""Sample from the Beta distribution and rescale to ``action_range``."""
return self._distribution.sample() * self._range_scale + self._range_offset # type: ignore
def deterministic_output(self, mlp_output: torch.Tensor) -> torch.Tensor:
"""Extract the mean from the MLP output and rescale to ``action_range``."""
alpha_raw, beta_raw = torch.unbind(mlp_output, dim=-2)
alpha = torch.nn.functional.softplus(alpha_raw) + 1.0
beta = torch.nn.functional.softplus(beta_raw) + 1.0
return (alpha / (alpha + beta)) * self._range_scale + self._range_offset
def as_deterministic_output_module(self) -> nn.Module:
"""Return export-friendly module that computes the mean from the MLP output."""
return _BetaDeterministicOutput(self._range_scale, self._range_offset)
@property
def input_dim(self) -> list[int]:
"""Return the input dimension required by the distribution.
The MLP must output a tensor of shape ``[..., 2, output_dim]`` where the first slice along the second-to-last
dimension is the raw alpha parameter and the second is the raw beta parameter.
"""
return [2, self.output_dim]
@property
def mean(self) -> torch.Tensor:
"""Return the mean of the Beta distribution, rescaled to ``action_range``."""
return (self._alpha / (self._alpha + self._beta)) * self._range_scale + self._range_offset # type: ignore
@property
def std(self) -> torch.Tensor:
"""Return the standard deviation of the Beta distribution, rescaled to ``action_range``."""
return self._distribution.stddev * self._range_scale # type: ignore
@property
def entropy(self) -> torch.Tensor:
"""Return the entropy of the Beta distribution, summed over the last dimension."""
return self._distribution.entropy().sum(dim=-1) # type: ignore
@property
def params(self) -> tuple[torch.Tensor, ...]:
"""Return ``(alpha, beta)`` of the current Beta distribution."""
return (self._alpha, self._beta) # type: ignore
def log_prob(self, outputs: torch.Tensor) -> torch.Tensor:
"""Compute the log probability under the Beta distribution, summed over the last dimension.
Outputs are unscaled from ``action_range`` back to ``[0, 1]`` before computing the log probability.
The Jacobian correction for the linear rescaling is included.
"""
unscaled = (outputs - self._range_offset) / self._range_scale
unscaled = unscaled.clamp(1e-6, 1.0 - 1e-6)
# Jacobian correction: log p(y) = log p(x) - log(scale), where y = x * scale + offset
return (self._distribution.log_prob(unscaled) - self._log_range_scale).sum(dim=-1) # type: ignore
def kl_divergence(self, old_params: tuple[torch.Tensor, ...], new_params: tuple[torch.Tensor, ...]) -> torch.Tensor:
"""Compute KL(old || new) between two Beta distributions."""
old_alpha, old_beta = old_params
new_alpha, new_beta = new_params
return torch.distributions.kl_divergence(Beta(old_alpha, old_beta), Beta(new_alpha, new_beta)).sum(dim=-1)
def init_mlp_weights(self, mlp: nn.Module) -> None:
"""Initialize the beta-parameter head weights to zero for a near-uniform initial distribution."""
torch.nn.init.zeros_(mlp[-2].weight[self.output_dim :]) # type: ignore
torch.nn.init.zeros_(mlp[-2].bias[self.output_dim :]) # type: ignore
class _IdentityDeterministicOutput(nn.Module):
"""Exportable module that returns the MLP output as is."""
def forward(self, mlp_output: torch.Tensor) -> torch.Tensor:
return mlp_output
class _FirstSliceDeterministicOutput(nn.Module):
"""Exportable module that extracts the mean from the MLP output (first slice of the second-to-last dimension)."""
def forward(self, mlp_output: torch.Tensor) -> torch.Tensor:
return mlp_output[..., 0, :]
class _BetaDeterministicOutput(nn.Module):
"""Exportable module that computes the mean of the Beta distribution from the MLP output."""
def __init__(self, range_scale: float, range_offset: float) -> None:
super().__init__()
self.range_scale = range_scale
self.range_offset = range_offset
def forward(self, mlp_output: torch.Tensor) -> torch.Tensor:
alpha_raw, beta_raw = torch.unbind(mlp_output, dim=-2)
alpha = torch.nn.functional.softplus(alpha_raw) + 1.0
beta = torch.nn.functional.softplus(beta_raw) + 1.0
return (alpha / (alpha + beta)) * self.range_scale + self.range_offset