"""Delay buffer for stochastically delayed observations.""" from __future__ import annotations from collections.abc import Sequence import torch from mjlab.utils.buffers import CircularBuffer class DelayBuffer: """Serve stochastically delayed observations from a rolling history. Wraps a CircularBuffer to simulate observation delays by returning frames from T-lag timesteps ago, where lag is sampled from [min_lag, max_lag]. Core Behavior ============= At each timestep: 1. Append new observation to history 2. Sample or hold lag value (0 = no delay, 3 = 3 timesteps old) 3. Return observation from T-lag Example with lag=2: t=0: append obs_0 → return obs_0 (not enough history) t=1: append obs_1 → return obs_0 (clamped to available history) t=2: append obs_2 → return obs_0 (lag=2, so T-2 = 0) t=3: append obs_3 → return obs_1 (lag=2, so T-2 = 1) Lag Update Policy ================= Lags can be refreshed every step or periodically: **Every-step updates (update_period=0)** Each timestep may sample a new lag (subject to hold_prob). **Periodic updates (update_period=N)** Lags refresh only every N steps per environment: if (step_count + phase_offset) % N == 0: sample new lag else: keep previous lag **Staggered updates (per_env_phase=True)** Each environment gets a random phase_offset ∈ [0, N), causing lag updates to occur on different timesteps: Env 0: updates at t=0, N, 2N, ... Env 1: updates at t=3, N+3, 2N+3, ... Env 2: updates at t=7, N+7, 2N+7, ... **Hold probability (hold_prob=0.2)** Even when an update would occur, keep previous lag with 20% chance. Creates temporal correlation in delay patterns. **First step** The first compute() after creation or reset always samples a lag, ignoring update phase and hold_prob, unless set_lags() was called first. Per-Environment vs Shared Lags ============================== **per_env=True** (default) Each environment has independent lag: Batch 0: lag=1 → returns obs from t-1 Batch 1: lag=3 → returns obs from t-3 Batch 2: lag=0 → returns current obs **per_env=False** All environments share one sampled lag: All batches: lag=2 → all return obs from t-2 The lag and its update schedule belong to the whole batch, so per_env_phase is ignored and a partial reset leaves them untouched. Reset Behavior ============== reset(batch_ids=[1]) clears history for specified environments: - Sets lag and step counter to zero (per_env=True only) - Clears circular buffer for those rows - Next append backfills their history with first new value - Until that append, compute() returns zeros for reset rows Args: min_lag (int, optional): Minimum lag (inclusive). Must be >= 0. max_lag (int, optional): Maximum lag (inclusive). Must be >= `min_lag`. batch_size (int, optional): Number of parallel environments (leading dimension of inputs). device (str, optional): Torch device for storage and RNG. per_env (bool, optional): If True, sample a separate lag per environment; otherwise sample one lag and share it across environments. hold_prob (float, optional): Probability in `[0.0, 1.0]` to keep the previous lag when an update would occur. Creates temporal correlation in delays. update_period (int, optional): If > 0, refresh lags every N steps per environment; if 0, consider updating every step. per_env_phase (bool, optional): If True and `update_period > 0`, each environment uses a different phase offset in `[0, update_period)`, causing staggered refresh steps across the batch. generator (torch.Generator | None, optional): Optional RNG for sampling lags. Examples: Constant delay (lag = 2): >>> buf = DelayBuffer(min_lag=2, max_lag=2, batch_size=4) >>> buf.append(obs) # obs.shape == (4, ...) >>> delayed = buf.compute() # delayed[t] = obs[t-2] Stochastic delay (uniform 0-3): >>> buf = DelayBuffer(min_lag=0, max_lag=3, batch_size=4) >>> buf.append(obs) >>> delayed = buf.compute() # per-env lag sampled in {0,1,2,3} Periodic updates with staggering: >>> buf = DelayBuffer( ... min_lag=1, max_lag=5, batch_size=8, ... update_period=10, # refresh every 10 steps ... per_env_phase=True, # stagger across envs ... hold_prob=0.2 # 20% chance to hold lag ... ) >>> # Env 0 refreshes at t=0,10,20,... >>> # Env 1 refreshes at t=3,13,23,... (random offset) >>> # But each refresh has 20% chance to keep previous lag """ def __init__( self, min_lag: int = 0, max_lag: int = 3, batch_size: int = 1, device: str = "cpu", per_env: bool = True, hold_prob: float = 0.0, update_period: int = 0, per_env_phase: bool = True, generator: torch.Generator | None = None, ) -> None: if min_lag < 0: raise ValueError(f"min_lag must be >= 0, got {min_lag}") if max_lag < min_lag: raise ValueError(f"max_lag ({max_lag}) must be >= min_lag ({min_lag})") if not 0.0 <= hold_prob <= 1.0: raise ValueError(f"hold_prob must be in [0, 1], got {hold_prob}") if update_period < 0: raise ValueError(f"update_period must be >= 0, got {update_period}") self.min_lag = min_lag self.max_lag = max_lag self.batch_size = batch_size self.device = device self.per_env = per_env self.hold_prob = hold_prob self.update_period = update_period self.per_env_phase = per_env_phase self.generator = generator self._staggered = per_env and per_env_phase and update_period > 0 buffer_size = max_lag + 1 if max_lag > 0 else 1 self._buffer = CircularBuffer( max_len=buffer_size, batch_size=batch_size, device=device ) self._current_lags = torch.zeros(batch_size, dtype=torch.long, device=device) self._step_count = torch.zeros(batch_size, dtype=torch.long, device=device) # Rows whose lag has not been sampled or set since creation or reset. self._needs_lag = torch.ones(batch_size, dtype=torch.bool, device=device) if self._staggered: self._phase_offsets = torch.randint( 0, update_period, (batch_size,), dtype=torch.long, device=device, generator=generator, ) else: self._phase_offsets = torch.zeros(batch_size, dtype=torch.long, device=device) @property def is_initialized(self) -> bool: """Check if buffer has been initialized with at least one append.""" return self._buffer.is_initialized @property def current_lags(self) -> torch.Tensor: """Current lag per environment. Shape: (batch_size,).""" return self._current_lags def set_lags( self, lags: torch.Tensor, batch_ids: Sequence[int] | torch.Tensor | slice | None = None, ) -> None: """Set lag values for specified environments. Args: lags: Lag values to set. Shape: (num_batch_ids,) or scalar. batch_ids: Batch indices to set, or None to set all. """ idx = slice(None) if batch_ids is None else batch_ids self._current_lags[idx] = lags.clamp(self.min_lag, self.max_lag) self._needs_lag[idx] = False def reset( self, batch_ids: Sequence[int] | torch.Tensor | slice | None = None ) -> None: """Reset specified environments to initial state. Args: batch_ids: Batch indices to reset, or None to reset all. """ if isinstance(batch_ids, slice): indices = range(*batch_ids.indices(self.batch_size)) batch_ids = list(indices) self._buffer.reset(batch_ids=batch_ids) # A shared lag belongs to the whole batch, so only a full reset clears it. if not self.per_env and batch_ids is not None: return idx = slice(None) if batch_ids is None else batch_ids self._current_lags[idx] = 0 self._needs_lag[idx] = True self._step_count[idx] = 0 if self._staggered: new_phases = torch.randint( 0, self.update_period, (self.batch_size,), dtype=torch.long, device=self.device, generator=self.generator, ) self._phase_offsets[idx] = new_phases[idx] def append(self, data: torch.Tensor) -> None: """Append new observation to buffer. Args: data: Observation tensor of shape (batch_size, ...). """ self._buffer.append(data) def backfill(self, data: torch.Tensor, batch_ids: torch.Tensor) -> None: """Backfill the given rows with one frame, without advancing time. Used after a partial reset: the reset rows (whose lags and step counters were just zeroed by reset) get their first post-reset frame while the remaining rows keep their history, lags, and update schedule intact. Args: data: Tensor of shape (batch_size, ...); only rows at batch_ids are read. batch_ids: Batch indices to backfill. """ self._buffer.backfill(data, batch_ids) def compute(self) -> torch.Tensor: """Compute delayed observation for current step. Advances the lag update schedule, then returns the delayed observation. Returns: Delayed observation with shape (batch_size, ...). """ if not self.is_initialized: raise RuntimeError("Buffer not initialized. Call append() first.") self._update_lags() return self.peek() def peek(self) -> torch.Tensor: """Return the delayed observation using current lags, without advancing. Unlike compute, this neither steps the update schedule nor resamples lags, so it is safe to call outside the once-per-step cadence. Returns: Delayed observation with shape (batch_size, ...). """ if not self.is_initialized: raise RuntimeError("Buffer not initialized. Call append() first.") # Clamp lags to valid range [0, buffer_length - 1]. # Buffer may not be full yet (e.g., only 2 frames but sampled lag=3). valid_lags = torch.minimum(self._current_lags, self._buffer.current_length - 1) valid_lags = valid_lags.clamp_min(0) return self._buffer[valid_lags] def _update_lags(self) -> None: """Update current lags according to configured policy.""" if self.update_period > 0: phase_adjusted_count = (self._step_count + self._phase_offsets) % ( self.update_period ) should_update = phase_adjusted_count == 0 else: should_update = torch.ones(self.batch_size, dtype=torch.bool, device=self.device) self._current_lags = self._sample_lags(should_update) self._needs_lag.zero_() self._step_count += 1 def _sample_lags(self, mask: torch.Tensor) -> torch.Tensor: """Sample new lags for specified environments. Args: mask: Boolean mask of shape (batch_size,) indicating which envs to sample. Returns: New lags with shape (batch_size,). """ if self.per_env: candidate_lags = torch.randint( self.min_lag, self.max_lag + 1, (self.batch_size,), dtype=torch.long, device=self.device, generator=self.generator, ) else: shared_lag = torch.randint( self.min_lag, self.max_lag + 1, (1,), dtype=torch.long, device=self.device, generator=self.generator, ) candidate_lags = shared_lag.expand(self.batch_size) if self.hold_prob > 0.0: # In shared mode the hold decision is shared too, otherwise some envs would # adopt the new lag while others keep their old one. draws = torch.rand( self.batch_size if self.per_env else 1, dtype=torch.float32, device=self.device, generator=self.generator, ) should_sample = (draws >= self.hold_prob).expand(self.batch_size) update_mask = mask & should_sample else: update_mask = mask # Rows without a lag have nothing to hold, so they always take the new sample. update_mask = update_mask | self._needs_lag return torch.where(update_mask, candidate_lags, self._current_lags)