Files
UniversalHumanoid/rl/mjlab/viewer/offscreen_renderer.py
T
2026-11-03 19:46:08 +08:00

256 lines
8.7 KiB
Python

"""MuJoCo offscreen renderer for headless visualization."""
import copy
from typing import Any, Callable
import mujoco
import numpy as np
from mjlab.entity import Entity
from mjlab.scene import Scene
from mjlab.viewer.model_sync import (
VIEWER_MODEL_FIELDS,
disable_model_sameframe_shortcuts,
sync_model_fields,
)
from mjlab.viewer.native.visualizer import MujocoNativeDebugVisualizer
from mjlab.viewer.viewer_config import ViewerConfig
def _get_camera_body_id(cfg: ViewerConfig, entities: dict[str, Entity]) -> int:
"""Resolve the body id the camera tracks, or -1 for free cameras."""
if cfg.origin_type in (
ViewerConfig.OriginType.AUTO,
ViewerConfig.OriginType.WORLD,
):
return -1
entity_name = cfg.entity_name
if entity_name is None:
if len(entities) != 1:
raise ValueError(
f"Cannot auto-detect entity: scene has {len(entities)} entities "
f"({list(entities.keys())}). Set ViewerConfig.entity_name to choose one."
)
entity_name = next(iter(entities))
elif entity_name not in entities:
raise ValueError(
f"Entity '{entity_name}' not found in scene. "
f"Available entities: {list(entities.keys())}."
)
entity = entities[entity_name]
if cfg.origin_type == ViewerConfig.OriginType.ASSET_ROOT:
return entity.indexing.root_body_id
assert cfg.origin_type == ViewerConfig.OriginType.ASSET_BODY
if not cfg.body_name:
raise ValueError("body_name is required for the ASSET_BODY origin type.")
if cfg.body_name not in entity.body_names:
raise ValueError(f"Body '{cfg.body_name}' not found in entity '{entity_name}'.")
body_ids, _ = entity.find_bodies(cfg.body_name)
return entity.indexing.bodies[body_ids[0]].id
class OffscreenRenderer:
def __init__(
self,
model: mujoco.MjModel,
cfg: ViewerConfig,
scene: Scene,
sim_model: Any | None = None,
expanded_fields: set[str] | None = None,
) -> None:
self._cfg = cfg
self._scene = scene
self._sim_model = sim_model
self._expanded_fields = expanded_fields
# Work on a copy so render-only tweaks (extent, shadows, reflections,
# sameframe shortcuts) never leak into the shared model.
self._model = copy.copy(model)
self._data = mujoco.MjData(self._model)
if self._sim_model is not None:
disable_model_sameframe_shortcuts(self._model)
# Keep extent override local to offscreen rendering so shadow/camera scaling
# is not dominated by the full multi-env world bounds.
self._model.stat.extent = self._compute_render_extent(cfg.distance)
self._model.vis.global_.offheight = cfg.height
self._model.vis.global_.offwidth = cfg.width
if cfg.fovy is not None:
self._model.vis.global_.fovy = cfg.fovy
if not cfg.enable_shadows:
self._model.light_castshadow[:] = False
if not cfg.enable_reflections:
self._model.mat_reflectance[:] = 0.0
self._cam = self._setup_camera(cfg, self._model, scene.entities)
self._env_ids: tuple[int, ...] | None = None
self._renderer: mujoco.Renderer | None = None
self._opt = mujoco.MjvOption()
self._opt.geomgroup[:] = np.array(cfg.geom_group, dtype=np.uint8)
self._opt.sitegroup[:] = np.array(cfg.site_group, dtype=np.uint8)
self._pert = mujoco.MjvPerturb()
self._catmask = mujoco.mjtCatBit.mjCAT_DYNAMIC
@property
def renderer(self) -> mujoco.Renderer:
if self._renderer is None:
raise ValueError("Renderer not initialized. Call 'initialize()' first.")
return self._renderer
def initialize(self) -> None:
if self._renderer is not None:
raise RuntimeError(
"Renderer is already initialized. Call 'close()' first to reinitialize."
)
self._renderer = mujoco.Renderer(
model=self._model, height=self._cfg.height, width=self._cfg.width
)
def update(
self,
data: Any,
debug_vis_callback: Callable[[MujocoNativeDebugVisualizer], None] | None = None,
camera: str | None = None,
) -> None:
"""Update renderer with simulation data."""
if self._renderer is None:
raise ValueError("Renderer not initialized. Call 'initialize()' first.")
if int(data.nworld) <= 0:
return
if self._env_ids is None:
self._env_ids = self._get_env_ids(
self._cfg, self._scene.env_origins.cpu().numpy()
)
# The primary env frames the camera and receives the debug overlays.
primary_env = self._env_ids[0]
self._sync_model_fields(primary_env)
self._sync_data_fields(data, primary_env)
cam = camera if camera is not None else self._cam
self._renderer.update_scene(self._data, camera=cam, scene_option=self._opt)
# Note: update_scene() resets the scene each frame, so no need to manually clear.
if debug_vis_callback is not None:
visualizer = MujocoNativeDebugVisualizer(
self._renderer.scene, self._model, env_idx=primary_env
)
debug_vis_callback(visualizer)
# Add neighboring environments as geoms for context.
for env_id in self._env_ids[1:]:
self._sync_model_fields(env_id)
self._sync_data_fields(data, env_id)
mujoco.mjv_addGeoms(
self._model,
self._data,
self._opt,
self._pert,
self._catmask.value,
self._renderer.scene,
)
def render(self) -> np.ndarray:
if self._renderer is None:
raise ValueError("Renderer not initialized. Call 'initialize()' first.")
return self._renderer.render()
def close(self) -> None:
if self._renderer is not None:
self._renderer.close()
self._renderer = None
def _sync_data_fields(self, data: Any, env_idx: int) -> None:
"""Copy one env's state into the host MjData and refresh kinematics."""
if self._model.nq > 0:
self._data.qpos[:] = data.qpos[env_idx].cpu().numpy()
self._data.qvel[:] = data.qvel[env_idx].cpu().numpy()
if self._model.nmocap > 0:
self._data.mocap_pos[:] = data.mocap_pos[env_idx].cpu().numpy()
self._data.mocap_quat[:] = data.mocap_quat[env_idx].cpu().numpy()
mujoco.mj_forward(self._model, self._data)
def _sync_model_fields(self, env_idx: int) -> None:
"""Sync visually relevant per-world model fields into the host MjModel."""
if self._sim_model is None or self._expanded_fields is None:
return
fields = self._expanded_fields & VIEWER_MODEL_FIELDS
sync_model_fields(self._model, self._sim_model, fields, env_idx)
@staticmethod
def _get_env_ids(cfg: ViewerConfig, env_origins: np.ndarray) -> tuple[int, ...]:
"""Return the ids of the environments to render, primary env first.
The primary env is cfg.env_idx (clamped to the available worlds), followed
by up to max_extra_envs nearest neighbors by env origin, so videos stay
focused on the tracked robot and nearby peers.
The set is resolved once, on first use, and kept stable: env_origins can
mutate during training (e.g. the terrain curriculum reassigns origins on
reset), and recomputing every frame would make the context robots pop in
and out, causing video flicker.
"""
if cfg.max_extra_envs < 0:
raise ValueError(
f"'max_extra_envs' must be non-negative, got {cfg.max_extra_envs}."
)
n_world = env_origins.shape[0]
env_idx = max(0, min(int(cfg.env_idx), n_world - 1))
k = min(cfg.max_extra_envs, n_world - 1)
if k <= 0:
return (env_idx,)
dist2 = np.sum((env_origins - env_origins[env_idx]) ** 2, axis=1)
dist2[env_idx] = np.inf
nearest = np.argpartition(dist2, kth=k - 1)[:k]
nearest = nearest[np.argsort(dist2[nearest])]
return (env_idx, *(int(i) for i in nearest))
@staticmethod
def _setup_camera(
cfg: ViewerConfig,
model: mujoco.MjModel,
entities: dict[str, Entity],
) -> mujoco.MjvCamera:
"""Setup camera based on config's origin_type."""
body_id = _get_camera_body_id(cfg, entities)
camera = mujoco.MjvCamera()
mujoco.mjv_defaultFreeCamera(model, camera)
if body_id == -1:
camera.type = mujoco.mjtCamera.mjCAMERA_FREE.value
else:
camera.type = mujoco.mjtCamera.mjCAMERA_TRACKING.value
camera.trackbodyid = body_id
camera.fixedcamid = -1
camera.lookat[:] = cfg.lookat
camera.elevation = cfg.elevation
camera.azimuth = cfg.azimuth
camera.distance = cfg.distance
return camera
@staticmethod
def _compute_render_extent(distance: float) -> float:
"""Compute a stable extent for offscreen rendering.
MuJoCo scales z-near/z-far and shadow clip with ``model.stat.extent``. In
large scenes this auto extent can become very large, which causes
shadow-map precision artifacts in offscreen video rendering. We therefore
use a local extent tied to the camera distance, keeping enough room for
the tracked subject and camera motion.
"""
return max(4.0, 1.5 * float(distance))