Files
2026-11-03 19:46:08 +08:00

209 lines
6.6 KiB
Python

"""Action manager for processing actions sent to the environment."""
from __future__ import annotations
import abc
from dataclasses import dataclass
from typing import TYPE_CHECKING, Sequence
import torch
from prettytable import PrettyTable
from mjlab.managers.manager_base import ManagerBase, ManagerTermBase
if TYPE_CHECKING:
from mjlab.envs import ManagerBasedRlEnv
@dataclass(kw_only=True)
class ActionTermCfg(abc.ABC):
"""Configuration for an action term.
Action terms process raw actions from the policy and apply them to entities
in the scene (e.g., setting joint positions, velocities, or efforts).
"""
entity_name: str
"""Name of the entity in the scene that this action term controls."""
clip: dict[str, tuple] | None = None
"""Optional clipping bounds applied to processed actions (after scale
and offset). Dict maps actuator name regex patterns to (min, max)
tuples, resolved the same way as ``scale`` and ``offset``."""
@abc.abstractmethod
def build(self, env: ManagerBasedRlEnv) -> ActionTerm:
"""Build the action term from this config."""
raise NotImplementedError
class ActionTerm(ManagerTermBase):
"""Base class for action terms.
The action term is responsible for processing the raw actions sent to the environment
and applying them to the entity managed by the term.
"""
def __init__(self, cfg: ActionTermCfg, env: ManagerBasedRlEnv):
self.cfg = cfg
super().__init__(env)
self._entity = self._env.scene[self.cfg.entity_name]
@property
@abc.abstractmethod
def action_dim(self) -> int:
raise NotImplementedError
@abc.abstractmethod
def process_actions(self, actions: torch.Tensor) -> None:
raise NotImplementedError
@abc.abstractmethod
def apply_actions(self) -> None:
raise NotImplementedError
@property
@abc.abstractmethod
def raw_action(self) -> torch.Tensor:
raise NotImplementedError
class ActionManager(ManagerBase):
"""Manages action processing for the environment.
The action manager aggregates multiple action terms, each controlling a different
entity or aspect of the simulation. It splits the policy's action tensor and
routes each slice to the appropriate action term.
"""
def __init__(self, cfg: dict[str, ActionTermCfg], env: ManagerBasedRlEnv):
self.cfg = cfg
super().__init__(env=env)
# Create buffers to store actions.
self._action = torch.zeros(
(self.num_envs, self.total_action_dim), device=self.device
)
self._prev_action = torch.zeros_like(self._action)
self._prev_prev_action = torch.zeros_like(self._action)
def __str__(self) -> str:
msg = f"<ActionManager> contains {len(self._term_names)} active terms.\n"
table = PrettyTable()
table.title = f"Active Action Terms (shape: {self.total_action_dim})"
table.field_names = ["Index", "Name", "Dimension"]
table.align["Name"] = "l"
table.align["Dimension"] = "r"
for index, (name, term) in enumerate(self._terms.items()):
table.add_row([index, name, term.action_dim])
msg += table.get_string()
msg += "\n"
return msg
# Properties.
@property
def total_action_dim(self) -> int:
return sum(self.action_term_dim)
@property
def action_term_dim(self) -> list[int]:
return [term.action_dim for term in self._terms.values()]
@property
def action(self) -> torch.Tensor:
"""Raw policy output from the current step, before per-term
scale/offset. Shape: ``(num_envs, total_action_dim)``."""
return self._action
@property
def prev_action(self) -> torch.Tensor:
"""Raw policy output from the previous step, before per-term
scale/offset. Shape: ``(num_envs, total_action_dim)``."""
return self._prev_action
@property
def prev_prev_action(self) -> torch.Tensor:
"""Raw policy output from two steps ago, before per-term
scale/offset. Shape: ``(num_envs, total_action_dim)``."""
return self._prev_prev_action
@property
def active_terms(self) -> list[str]:
return self._term_names
# Methods.
def get_term(self, name: str) -> ActionTerm:
return self._terms[name]
def reset(self, env_ids: torch.Tensor | slice | None = None) -> dict[str, float]:
if env_ids is None:
env_ids = slice(None)
# Reset action history.
self._prev_action[env_ids] = 0.0
self._prev_prev_action[env_ids] = 0.0
self._action[env_ids] = 0.0
# Reset action terms.
for term in self._terms.values():
term.reset(env_ids=env_ids)
return {}
def process_action(self, action: torch.Tensor) -> None:
"""Store the raw policy output and route slices to each action term.
Called once per policy step. The raw action tensor is saved into the
history buffers (``action``, ``prev_action``, ``prev_prev_action``) *before*
any per-term scale/offset is applied. Each term then receives its slice and
independently applies its own affine transformation via
:meth:`ActionTerm.process_actions`.
"""
if self.total_action_dim != action.shape[1]:
raise ValueError(
f"Invalid action shape, expected: {self.total_action_dim},"
f" received: {action.shape[1]}."
)
# Shift history: prev_prev ← prev ← current ← new.
self._prev_prev_action[:] = self._prev_action
self._prev_action[:] = self._action
self._action[:] = action.to(self.device)
# Split the flat action vector and route each slice to its term.
idx = 0
for term in self._terms.values():
term_actions = action[:, idx : idx + term.action_dim]
term.process_actions(term_actions)
idx += term.action_dim
def apply_action(self) -> None:
"""Write processed actions to entity actuator targets.
Called on every decimation substep (physics step), not just once per policy
step. Each term writes its most recently processed targets to the simulation.
"""
for term in self._terms.values():
term.apply_actions()
def get_active_iterable_terms(
self, env_idx: int
) -> Sequence[tuple[str, Sequence[float]]]:
terms = []
idx = 0
for name, term in self._terms.items():
term_actions = self._action[env_idx, idx : idx + term.action_dim].cpu()
terms.append((name, term_actions.tolist()))
idx += term.action_dim
return terms
def _prepare_terms(self):
self._term_names: list[str] = list()
self._terms: dict[str, ActionTerm] = dict()
for term_name, term_cfg in self.cfg.items():
term_cfg: ActionTermCfg | None
if term_cfg is None:
print(f"term: {term_name} set to None, skipping...")
continue
term = term_cfg.build(self._env)
self._term_names.append(term_name)
self._terms[term_name] = term