231 lines
6.7 KiB
Python
231 lines
6.7 KiB
Python
"""Camera sensor for RGB, depth, and segmentation rendering."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Literal
|
|
|
|
import mujoco
|
|
import mujoco_warp as mjwarp
|
|
import torch
|
|
|
|
from mjlab.entity import Entity
|
|
from mjlab.sensor.sensor import Sensor, SensorCfg
|
|
|
|
if TYPE_CHECKING:
|
|
from mjlab.sensor.sensor_context import SensorContext
|
|
|
|
CameraDataType = Literal["rgb", "depth", "segmentation"]
|
|
|
|
# Default MuJoCo fov, in degrees.
|
|
_DEFAULT_CAM_FOVY = 45.0
|
|
|
|
|
|
@dataclass
|
|
class CameraSensorCfg(SensorCfg):
|
|
"""Configuration for a camera sensor.
|
|
|
|
A camera sensor can either wrap an existing MuJoCo camera
|
|
(``camera_name``) or create a new one at the specified
|
|
pos/quat. New cameras are added to the worldbody by default,
|
|
or to a specific body via ``parent_body``.
|
|
|
|
Note:
|
|
All camera sensors in a scene must share identical values for
|
|
use_textures, use_shadows, and enabled_geom_groups. This is a
|
|
constraint of the underlying mujoco_warp rendering system.
|
|
"""
|
|
|
|
camera_name: str | None = None
|
|
"""Name of an existing MuJoCo camera to wrap.
|
|
|
|
If None, a new camera is created using pos/quat/fovy.
|
|
If set, the sensor wraps the named camera instead of creating one.
|
|
"""
|
|
|
|
parent_body: str | None = None
|
|
"""Parent body to attach a new camera to.
|
|
|
|
Only used when ``camera_name`` is None (creating a new camera).
|
|
If None, the camera is added to the worldbody. Use the full
|
|
prefixed name (e.g., "robot/link_6") to attach to an entity's
|
|
body. The pos/quat are then relative to the parent body frame.
|
|
"""
|
|
|
|
pos: tuple[float, float, float] = (0.0, 0.0, 1.0)
|
|
"""Camera position (used when creating a new camera).
|
|
|
|
World-frame if parent_body is None, otherwise relative to the
|
|
parent body frame.
|
|
"""
|
|
|
|
quat: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0)
|
|
"""Camera orientation quaternion (w, x, y, z)."""
|
|
|
|
fovy: float | None = None
|
|
"""Vertical field of view in degrees. None uses MuJoCo default."""
|
|
|
|
width: int = 160
|
|
"""Image width in pixels."""
|
|
|
|
height: int = 120
|
|
"""Image height in pixels."""
|
|
|
|
data_types: tuple[CameraDataType, ...] = ("rgb",)
|
|
"""Data types to capture: any combination of "rgb", "depth",
|
|
and "segmentation"."""
|
|
|
|
use_textures: bool = True
|
|
"""Whether to use textures in rendering."""
|
|
|
|
use_shadows: bool = False
|
|
"""Whether to use shadows in rendering."""
|
|
|
|
enabled_geom_groups: tuple[int, ...] = (0, 1, 2)
|
|
"""Geom groups (0-5) visible to the camera."""
|
|
|
|
orthographic: bool = False
|
|
"""Use orthographic projection instead of perspective."""
|
|
|
|
clone_data: bool = False
|
|
"""If True, clone tensors on each access.
|
|
|
|
Set to True if you modify the returned data in-place.
|
|
"""
|
|
|
|
def __post_init__(self) -> None:
|
|
valid = {"rgb", "depth", "segmentation"}
|
|
invalid = {dt for dt in self.data_types if dt not in valid}
|
|
if invalid:
|
|
raise ValueError(f"Invalid camera data types: {invalid}. Valid types: {valid}")
|
|
if not self.data_types:
|
|
raise ValueError("At least one data type must be specified.")
|
|
|
|
def build(self) -> CameraSensor:
|
|
return CameraSensor(self)
|
|
|
|
|
|
@dataclass
|
|
class CameraSensorData:
|
|
"""Camera sensor output data.
|
|
|
|
Shapes:
|
|
- rgb: [num_envs, height, width, 3] (uint8)
|
|
- depth: [num_envs, height, width, 1] (float32)
|
|
- segmentation: [num_envs, height, width, 2] (int32)
|
|
"""
|
|
|
|
rgb: torch.Tensor | None = None
|
|
"""RGB image [num_envs, height, width, 3] (uint8). None if not
|
|
enabled."""
|
|
|
|
depth: torch.Tensor | None = None
|
|
"""Depth image [num_envs, height, width, 1] (float32). None if not
|
|
enabled."""
|
|
|
|
segmentation: torch.Tensor | None = None
|
|
"""Per-pixel typed segmentation [num_envs, height, width, 2] (int32).
|
|
|
|
Channel 0 stores MuJoCo object IDs. Channel 1 stores the corresponding
|
|
MuJoCo object types. Background pixels are ``(-1, -1)``.
|
|
"""
|
|
|
|
|
|
class CameraSensor(Sensor[CameraSensorData]):
|
|
"""Camera sensor for RGB and depth rendering."""
|
|
|
|
requires_sensor_context = True
|
|
|
|
def __init__(self, cfg: CameraSensorCfg) -> None:
|
|
super().__init__()
|
|
self.cfg = cfg
|
|
self._camera_name = cfg.camera_name if cfg.camera_name is not None else cfg.name
|
|
self._is_wrapping_existing = cfg.camera_name is not None
|
|
self._ctx: SensorContext | None = None
|
|
self._camera_idx: int = -1
|
|
|
|
@property
|
|
def camera_name(self) -> str:
|
|
return self._camera_name
|
|
|
|
@property
|
|
def camera_idx(self) -> int:
|
|
return self._camera_idx
|
|
|
|
def edit_spec(
|
|
self,
|
|
scene_spec: mujoco.MjSpec,
|
|
entities: dict[str, Entity],
|
|
) -> None:
|
|
del entities
|
|
|
|
if self._is_wrapping_existing:
|
|
cam = scene_spec.camera(self._camera_name)
|
|
assert isinstance(cam, mujoco.MjsCamera)
|
|
if self.cfg.fovy is not None:
|
|
cam.fovy = self.cfg.fovy
|
|
if self.cfg.orthographic:
|
|
cam.proj = mujoco.mjtProjection.mjPROJ_ORTHOGRAPHIC
|
|
return
|
|
|
|
if self.cfg.parent_body is not None:
|
|
parent = scene_spec.body(self.cfg.parent_body)
|
|
else:
|
|
parent = scene_spec.worldbody
|
|
|
|
proj = (
|
|
mujoco.mjtProjection.mjPROJ_ORTHOGRAPHIC
|
|
if self.cfg.orthographic
|
|
else mujoco.mjtProjection.mjPROJ_PERSPECTIVE
|
|
)
|
|
parent.add_camera(
|
|
name=self.cfg.name,
|
|
pos=self.cfg.pos,
|
|
quat=self.cfg.quat,
|
|
fovy=self.cfg.fovy if self.cfg.fovy is not None else _DEFAULT_CAM_FOVY,
|
|
resolution=[self.cfg.width, self.cfg.height],
|
|
proj=proj,
|
|
)
|
|
|
|
def initialize(
|
|
self,
|
|
mj_model: mujoco.MjModel,
|
|
model: mjwarp.Model,
|
|
data: mjwarp.Data,
|
|
device: str,
|
|
) -> None:
|
|
del model, data, device
|
|
|
|
try:
|
|
cam = mj_model.camera(self._camera_name)
|
|
self._camera_idx = cam.id
|
|
except KeyError as e:
|
|
available = [mj_model.camera(i).name for i in range(mj_model.ncam)]
|
|
raise ValueError(
|
|
f"Camera '{self._camera_name}' not found in model. Available: {available}"
|
|
) from e
|
|
|
|
def set_context(self, ctx: SensorContext) -> None:
|
|
self._ctx = ctx
|
|
|
|
def _compute_data(self) -> CameraSensorData:
|
|
if self._ctx is None:
|
|
raise RuntimeError(
|
|
"CameraSensor requires a SensorContext. "
|
|
"Ensure the sensor is part of a scene with "
|
|
"sim.sense() calls."
|
|
)
|
|
rgb_data = None
|
|
depth_data = None
|
|
seg_data = None
|
|
if "rgb" in self.cfg.data_types:
|
|
rgb = self._ctx.get_rgb(self._camera_idx)
|
|
rgb_data = rgb.clone() if self.cfg.clone_data else rgb
|
|
if "depth" in self.cfg.data_types:
|
|
depth = self._ctx.get_depth(self._camera_idx)
|
|
depth_data = depth.clone() if self.cfg.clone_data else depth
|
|
if "segmentation" in self.cfg.data_types:
|
|
seg = self._ctx.get_segmentation(self._camera_idx)
|
|
seg_data = seg.clone() if self.cfg.clone_data else seg
|
|
return CameraSensorData(rgb=rgb_data, depth=depth_data, segmentation=seg_data)
|