348 lines
12 KiB
Python
348 lines
12 KiB
Python
"""Delay buffer for stochastically delayed observations."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
|
|
import torch
|
|
|
|
from mjlab.utils.buffers import CircularBuffer
|
|
|
|
|
|
class DelayBuffer:
|
|
"""Serve stochastically delayed observations from a rolling history.
|
|
|
|
Wraps a CircularBuffer to simulate observation delays by returning frames from T-lag
|
|
timesteps ago, where lag is sampled from [min_lag, max_lag].
|
|
|
|
Core Behavior
|
|
=============
|
|
|
|
At each timestep:
|
|
1. Append new observation to history
|
|
2. Sample or hold lag value (0 = no delay, 3 = 3 timesteps old)
|
|
3. Return observation from T-lag
|
|
|
|
Example with lag=2:
|
|
t=0: append obs_0 → return obs_0 (not enough history)
|
|
t=1: append obs_1 → return obs_0 (clamped to available history)
|
|
t=2: append obs_2 → return obs_0 (lag=2, so T-2 = 0)
|
|
t=3: append obs_3 → return obs_1 (lag=2, so T-2 = 1)
|
|
|
|
Lag Update Policy
|
|
=================
|
|
|
|
Lags can be refreshed every step or periodically:
|
|
|
|
**Every-step updates (update_period=0)**
|
|
Each timestep may sample a new lag (subject to hold_prob).
|
|
|
|
**Periodic updates (update_period=N)**
|
|
Lags refresh only every N steps per environment:
|
|
if (step_count + phase_offset) % N == 0:
|
|
sample new lag
|
|
else:
|
|
keep previous lag
|
|
|
|
**Staggered updates (per_env_phase=True)**
|
|
Each environment gets a random phase_offset ∈ [0, N), causing
|
|
lag updates to occur on different timesteps:
|
|
Env 0: updates at t=0, N, 2N, ...
|
|
Env 1: updates at t=3, N+3, 2N+3, ...
|
|
Env 2: updates at t=7, N+7, 2N+7, ...
|
|
|
|
**Hold probability (hold_prob=0.2)**
|
|
Even when an update would occur, keep previous lag with 20% chance.
|
|
Creates temporal correlation in delay patterns.
|
|
|
|
**First step**
|
|
The first compute() after creation or reset always samples a lag, ignoring
|
|
update phase and hold_prob, unless set_lags() was called first.
|
|
|
|
Per-Environment vs Shared Lags
|
|
==============================
|
|
|
|
**per_env=True** (default)
|
|
Each environment has independent lag:
|
|
Batch 0: lag=1 → returns obs from t-1
|
|
Batch 1: lag=3 → returns obs from t-3
|
|
Batch 2: lag=0 → returns current obs
|
|
|
|
**per_env=False**
|
|
All environments share one sampled lag:
|
|
All batches: lag=2 → all return obs from t-2
|
|
The lag and its update schedule belong to the whole batch, so per_env_phase
|
|
is ignored and a partial reset leaves them untouched.
|
|
|
|
Reset Behavior
|
|
==============
|
|
|
|
reset(batch_ids=[1]) clears history for specified environments:
|
|
- Sets lag and step counter to zero (per_env=True only)
|
|
- Clears circular buffer for those rows
|
|
- Next append backfills their history with first new value
|
|
- Until that append, compute() returns zeros for reset rows
|
|
|
|
Args:
|
|
min_lag (int, optional): Minimum lag (inclusive). Must be >= 0.
|
|
max_lag (int, optional): Maximum lag (inclusive). Must be >= `min_lag`.
|
|
batch_size (int, optional): Number of parallel environments (leading
|
|
dimension of inputs).
|
|
device (str, optional): Torch device for storage and RNG.
|
|
per_env (bool, optional): If True, sample a separate lag per environment;
|
|
otherwise sample one lag and share it across environments.
|
|
hold_prob (float, optional): Probability in `[0.0, 1.0]` to keep the previous
|
|
lag when an update would occur. Creates temporal correlation in delays.
|
|
update_period (int, optional): If > 0, refresh lags every N steps per
|
|
environment; if 0, consider updating every step.
|
|
per_env_phase (bool, optional): If True and `update_period > 0`, each
|
|
environment uses a different phase offset in `[0, update_period)`, causing
|
|
staggered refresh steps across the batch.
|
|
generator (torch.Generator | None, optional): Optional RNG for sampling lags.
|
|
|
|
Examples:
|
|
Constant delay (lag = 2):
|
|
>>> buf = DelayBuffer(min_lag=2, max_lag=2, batch_size=4)
|
|
>>> buf.append(obs) # obs.shape == (4, ...)
|
|
>>> delayed = buf.compute() # delayed[t] = obs[t-2]
|
|
|
|
Stochastic delay (uniform 0-3):
|
|
>>> buf = DelayBuffer(min_lag=0, max_lag=3, batch_size=4)
|
|
>>> buf.append(obs)
|
|
>>> delayed = buf.compute() # per-env lag sampled in {0,1,2,3}
|
|
|
|
Periodic updates with staggering:
|
|
>>> buf = DelayBuffer(
|
|
... min_lag=1, max_lag=5, batch_size=8,
|
|
... update_period=10, # refresh every 10 steps
|
|
... per_env_phase=True, # stagger across envs
|
|
... hold_prob=0.2 # 20% chance to hold lag
|
|
... )
|
|
>>> # Env 0 refreshes at t=0,10,20,...
|
|
>>> # Env 1 refreshes at t=3,13,23,... (random offset)
|
|
>>> # But each refresh has 20% chance to keep previous lag
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
min_lag: int = 0,
|
|
max_lag: int = 3,
|
|
batch_size: int = 1,
|
|
device: str = "cpu",
|
|
per_env: bool = True,
|
|
hold_prob: float = 0.0,
|
|
update_period: int = 0,
|
|
per_env_phase: bool = True,
|
|
generator: torch.Generator | None = None,
|
|
) -> None:
|
|
if min_lag < 0:
|
|
raise ValueError(f"min_lag must be >= 0, got {min_lag}")
|
|
if max_lag < min_lag:
|
|
raise ValueError(f"max_lag ({max_lag}) must be >= min_lag ({min_lag})")
|
|
if not 0.0 <= hold_prob <= 1.0:
|
|
raise ValueError(f"hold_prob must be in [0, 1], got {hold_prob}")
|
|
if update_period < 0:
|
|
raise ValueError(f"update_period must be >= 0, got {update_period}")
|
|
|
|
self.min_lag = min_lag
|
|
self.max_lag = max_lag
|
|
self.batch_size = batch_size
|
|
self.device = device
|
|
self.per_env = per_env
|
|
self.hold_prob = hold_prob
|
|
self.update_period = update_period
|
|
self.per_env_phase = per_env_phase
|
|
self.generator = generator
|
|
self._staggered = per_env and per_env_phase and update_period > 0
|
|
|
|
buffer_size = max_lag + 1 if max_lag > 0 else 1
|
|
self._buffer = CircularBuffer(
|
|
max_len=buffer_size, batch_size=batch_size, device=device
|
|
)
|
|
self._current_lags = torch.zeros(batch_size, dtype=torch.long, device=device)
|
|
self._step_count = torch.zeros(batch_size, dtype=torch.long, device=device)
|
|
# Rows whose lag has not been sampled or set since creation or reset.
|
|
self._needs_lag = torch.ones(batch_size, dtype=torch.bool, device=device)
|
|
|
|
if self._staggered:
|
|
self._phase_offsets = torch.randint(
|
|
0,
|
|
update_period,
|
|
(batch_size,),
|
|
dtype=torch.long,
|
|
device=device,
|
|
generator=generator,
|
|
)
|
|
else:
|
|
self._phase_offsets = torch.zeros(batch_size, dtype=torch.long, device=device)
|
|
|
|
@property
|
|
def is_initialized(self) -> bool:
|
|
"""Check if buffer has been initialized with at least one append."""
|
|
return self._buffer.is_initialized
|
|
|
|
@property
|
|
def current_lags(self) -> torch.Tensor:
|
|
"""Current lag per environment. Shape: (batch_size,)."""
|
|
return self._current_lags
|
|
|
|
def set_lags(
|
|
self,
|
|
lags: torch.Tensor,
|
|
batch_ids: Sequence[int] | torch.Tensor | slice | None = None,
|
|
) -> None:
|
|
"""Set lag values for specified environments.
|
|
|
|
Args:
|
|
lags: Lag values to set. Shape: (num_batch_ids,) or scalar.
|
|
batch_ids: Batch indices to set, or None to set all.
|
|
"""
|
|
idx = slice(None) if batch_ids is None else batch_ids
|
|
self._current_lags[idx] = lags.clamp(self.min_lag, self.max_lag)
|
|
self._needs_lag[idx] = False
|
|
|
|
def reset(
|
|
self, batch_ids: Sequence[int] | torch.Tensor | slice | None = None
|
|
) -> None:
|
|
"""Reset specified environments to initial state.
|
|
|
|
Args:
|
|
batch_ids: Batch indices to reset, or None to reset all.
|
|
"""
|
|
if isinstance(batch_ids, slice):
|
|
indices = range(*batch_ids.indices(self.batch_size))
|
|
batch_ids = list(indices)
|
|
|
|
self._buffer.reset(batch_ids=batch_ids)
|
|
# A shared lag belongs to the whole batch, so only a full reset clears it.
|
|
if not self.per_env and batch_ids is not None:
|
|
return
|
|
idx = slice(None) if batch_ids is None else batch_ids
|
|
self._current_lags[idx] = 0
|
|
self._needs_lag[idx] = True
|
|
self._step_count[idx] = 0
|
|
if self._staggered:
|
|
new_phases = torch.randint(
|
|
0,
|
|
self.update_period,
|
|
(self.batch_size,),
|
|
dtype=torch.long,
|
|
device=self.device,
|
|
generator=self.generator,
|
|
)
|
|
self._phase_offsets[idx] = new_phases[idx]
|
|
|
|
def append(self, data: torch.Tensor) -> None:
|
|
"""Append new observation to buffer.
|
|
|
|
Args:
|
|
data: Observation tensor of shape (batch_size, ...).
|
|
"""
|
|
self._buffer.append(data)
|
|
|
|
def backfill(self, data: torch.Tensor, batch_ids: torch.Tensor) -> None:
|
|
"""Backfill the given rows with one frame, without advancing time.
|
|
|
|
Used after a partial reset: the reset rows (whose lags and step counters
|
|
were just zeroed by reset) get their first post-reset frame while the
|
|
remaining rows keep their history, lags, and update schedule intact.
|
|
|
|
Args:
|
|
data: Tensor of shape (batch_size, ...); only rows at batch_ids are read.
|
|
batch_ids: Batch indices to backfill.
|
|
"""
|
|
self._buffer.backfill(data, batch_ids)
|
|
|
|
def compute(self) -> torch.Tensor:
|
|
"""Compute delayed observation for current step.
|
|
|
|
Advances the lag update schedule, then returns the delayed observation.
|
|
|
|
Returns:
|
|
Delayed observation with shape (batch_size, ...).
|
|
"""
|
|
if not self.is_initialized:
|
|
raise RuntimeError("Buffer not initialized. Call append() first.")
|
|
|
|
self._update_lags()
|
|
return self.peek()
|
|
|
|
def peek(self) -> torch.Tensor:
|
|
"""Return the delayed observation using current lags, without advancing.
|
|
|
|
Unlike compute, this neither steps the update schedule nor resamples lags,
|
|
so it is safe to call outside the once-per-step cadence.
|
|
|
|
Returns:
|
|
Delayed observation with shape (batch_size, ...).
|
|
"""
|
|
if not self.is_initialized:
|
|
raise RuntimeError("Buffer not initialized. Call append() first.")
|
|
|
|
# Clamp lags to valid range [0, buffer_length - 1].
|
|
# Buffer may not be full yet (e.g., only 2 frames but sampled lag=3).
|
|
valid_lags = torch.minimum(self._current_lags, self._buffer.current_length - 1)
|
|
valid_lags = valid_lags.clamp_min(0)
|
|
|
|
return self._buffer[valid_lags]
|
|
|
|
def _update_lags(self) -> None:
|
|
"""Update current lags according to configured policy."""
|
|
if self.update_period > 0:
|
|
phase_adjusted_count = (self._step_count + self._phase_offsets) % (
|
|
self.update_period
|
|
)
|
|
should_update = phase_adjusted_count == 0
|
|
else:
|
|
should_update = torch.ones(self.batch_size, dtype=torch.bool, device=self.device)
|
|
self._current_lags = self._sample_lags(should_update)
|
|
self._needs_lag.zero_()
|
|
self._step_count += 1
|
|
|
|
def _sample_lags(self, mask: torch.Tensor) -> torch.Tensor:
|
|
"""Sample new lags for specified environments.
|
|
|
|
Args:
|
|
mask: Boolean mask of shape (batch_size,) indicating which envs to sample.
|
|
|
|
Returns:
|
|
New lags with shape (batch_size,).
|
|
"""
|
|
if self.per_env:
|
|
candidate_lags = torch.randint(
|
|
self.min_lag,
|
|
self.max_lag + 1,
|
|
(self.batch_size,),
|
|
dtype=torch.long,
|
|
device=self.device,
|
|
generator=self.generator,
|
|
)
|
|
else:
|
|
shared_lag = torch.randint(
|
|
self.min_lag,
|
|
self.max_lag + 1,
|
|
(1,),
|
|
dtype=torch.long,
|
|
device=self.device,
|
|
generator=self.generator,
|
|
)
|
|
candidate_lags = shared_lag.expand(self.batch_size)
|
|
|
|
if self.hold_prob > 0.0:
|
|
# In shared mode the hold decision is shared too, otherwise some envs would
|
|
# adopt the new lag while others keep their old one.
|
|
draws = torch.rand(
|
|
self.batch_size if self.per_env else 1,
|
|
dtype=torch.float32,
|
|
device=self.device,
|
|
generator=self.generator,
|
|
)
|
|
should_sample = (draws >= self.hold_prob).expand(self.batch_size)
|
|
update_mask = mask & should_sample
|
|
else:
|
|
update_mask = mask
|
|
|
|
# Rows without a lag have nothing to hold, so they always take the new sample.
|
|
update_mask = update_mask | self._needs_lag
|
|
return torch.where(update_mask, candidate_lags, self._current_lags)
|