# Copyright (c) 2021-2026, ETH Zurich and NVIDIA CORPORATION # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2020 Preferred Networks, Inc. from __future__ import annotations import torch from torch import nn class EmpiricalNormalization(nn.Module): """Normalize mean and variance of values based on empirical values.""" def __init__(self, shape: int | tuple[int, ...] | list[int], eps: float = 1e-2, until: int | None = None) -> None: """Initialize EmpiricalNormalization module. .. note:: The normalization parameters are computed over the whole batch, not for each environment separately. Args: shape: Shape of input values except batch axis. eps: Small value for stability. until: If this arg is specified, the module learns input values until the sum of batch sizes exceeds it. """ super().__init__() self.eps = eps self.until = until self.register_buffer("_mean", torch.zeros(shape).unsqueeze(0)) self.register_buffer("_var", torch.ones(shape).unsqueeze(0)) self.register_buffer("_std", torch.ones(shape).unsqueeze(0)) self.register_buffer("count", torch.tensor(0, dtype=torch.long)) @property def mean(self) -> torch.Tensor: """Return the current running mean.""" return self._mean.squeeze(0).clone() # type: ignore @property def std(self) -> torch.Tensor: """Return the current running standard deviation.""" return self._std.squeeze(0).clone() def forward(self, x: torch.Tensor) -> torch.Tensor: """Normalize mean and variance of values based on empirical values.""" return (x - self._mean) / (self._std + self.eps) @torch.jit.unused def update(self, x: torch.Tensor) -> None: """Learn input values without computing the output values of them.""" if not self.training: return if self.until is not None and self.count >= self.until: return count_x = torch.tensor(x.shape[0], dtype=self.count.dtype, device=self.count.device) # type: ignore var_x = torch.var(x, dim=0, unbiased=False, keepdim=True) mean_x = torch.mean(x, dim=0, keepdim=True) if torch.distributed.is_initialized(): # Compute the global mean first, then combine the local variances around that mean local_mean_x = mean_x torch.distributed.all_reduce(count_x) mean_sum_x = mean_x * x.shape[0] torch.distributed.all_reduce(mean_sum_x) mean_x = mean_sum_x / count_x var_sum_x = x.shape[0] * (var_x + (local_mean_x - mean_x).square()) torch.distributed.all_reduce(var_sum_x) var_x = var_sum_x / count_x self.count += count_x rate = count_x / self.count delta_mean = mean_x - self._mean self._mean += rate * delta_mean self._var += rate * (var_x - self._var + delta_mean * (mean_x - self._mean)) self._std = torch.sqrt(self._var) @torch.jit.unused def inverse(self, y: torch.Tensor) -> torch.Tensor: """De-normalize values based on empirical values.""" return y * (self._std + self.eps) + self._mean class EmpiricalDiscountedVariationNormalization(nn.Module): """Reward normalization from Pathak's large scale study on PPO. Reward normalization. Since the reward function is non-stationary, it is useful to normalize the scale of the rewards so that the value function can learn quickly. We did this by dividing the rewards by a running estimate of the standard deviation of the sum of discounted rewards. """ def __init__( self, shape: int | tuple[int, ...] | list[int], eps: float = 1e-2, gamma: float = 0.99, until: int | None = None, ) -> None: """Initialize discounted-reward normalization with running moments.""" super().__init__() self.emp_norm = EmpiricalNormalization(shape, eps, until) self.disc_avg = _DiscountedAverage(gamma) def forward(self, rew: torch.Tensor) -> torch.Tensor: """Normalize rewards using the running std of discounted returns.""" if self.training: # Update discounted rewards avg = self.disc_avg.update(rew) # Update moments from discounted rewards self.emp_norm.update(avg) # Normalize rewards with the empirical std if self.emp_norm._std > 0: # type: ignore return rew / self.emp_norm._std # type: ignore else: return rew class _DiscountedAverage: r"""Discounted average of rewards. The discounted average is defined as: .. math:: \bar{R}_t = \gamma \bar{R}_{t-1} + r_t """ def __init__(self, gamma: float) -> None: """Initialize discounted accumulation with a fixed discount factor.""" self.avg = None self.gamma = gamma def update(self, rew: torch.Tensor) -> torch.Tensor: """Update and return the discounted running average.""" if self.avg is None: self.avg = rew else: self.avg = self.avg * self.gamma + rew return self.avg