# Copyright 2022 DeepMind Technologies Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Defines a renderer class for the MuJoCo Python native bindings."""
from typing import Optional, Union
from mujoco import _enums
from mujoco import _functions
from mujoco import _render
from mujoco import _structs
from mujoco import gl_context
import numpy as np
class Renderer:
"""Renders MuJoCo scenes."""
def __init__(
self,
model: _structs.MjModel,
height: int = 240,
width: int = 320,
max_geom: int = 10000
) -> None:
"""Initializes a new `Renderer`.
Args:
model: an MjModel instance.
height: image height in pixels.
width: image width in pixels.
max_geom: Optional integer specifying the maximum number of geoms that can
be rendered in the same scene. If None this will be chosen automatically
based on the estimated maximum number of renderable geoms in the model.
Raises:
ValueError: If `camera_id` is outside the valid range, or if `width` or
`height` exceed the dimensions of MuJoCo's offscreen framebuffer.
"""
buffer_width = model.vis.global_.offwidth
buffer_height = model.vis.global_.offheight
if width > buffer_width:
raise ValueError(f"""
Image width {width} > framebuffer width {buffer_width}. Either reduce the image
width or specify a larger offscreen framebuffer in the model XML using the
clause:
""".lstrip())
if height > buffer_height:
raise ValueError(f"""
Image height {height} > framebuffer height {buffer_height}. Either reduce the
image height or specify a larger offscreen framebuffer in the model XML using
the clause:
""".lstrip())
self._width = width
self._height = height
self._model = model
self._scene = _structs.MjvScene(model=model, maxgeom=max_geom)
self._scene_option = _structs.MjvOption()
self._rect = _render.MjrRect(0, 0, self._width, self._height)
# Internal buffers.
self._rgb_buffer = np.empty((self._height, self._width, 3), dtype=np.uint8)
self._depth_buffer = np.empty((self._height, self._width), dtype=np.float32)
# Create render contexts.
self._gl_context = gl_context.GLContext(width, height)
self._gl_context.make_current()
self._mjr_context = _render.MjrContext(
model, _enums.mjtFontScale.mjFONTSCALE_150
)
_render.mjr_setBuffer(
_enums.mjtFramebuffer.mjFB_OFFSCREEN, self._mjr_context
)
# Default render flags.
self._depth_rendering = False
self._segmentation_rendering = False
@property
def model(self):
return self._model
@property
def scene(self) -> _structs.MjvScene:
return self._scene
@property
def height(self):
return self._height
@property
def width(self):
return self._width
def enable_depth_rendering(self):
self._segmentation_rendering = False
self._depth_rendering = True
def disable_depth_rendering(self):
self._depth_rendering = False
def enable_segmentation_rendering(self):
self._segmentation_rendering = True
self._depth_rendering = False
def disable_segmentation_rendering(self):
self._segmentation_rendering = False
def render(self) -> np.ndarray:
"""Renders the scene as a numpy array of pixel values.
Returns:
A numpy array of pixels with dimensions (H, W, 3). The array will be
mutated by future calls to `render`.
"""
original_flags = self._scene.flags.copy()
if self._segmentation_rendering:
self._scene.flags[_enums.mjtRndFlag.mjRND_SEGMENT] = True
self._scene.flags[_enums.mjtRndFlag.mjRND_IDCOLOR] = True
self._gl_context.make_current()
# Render scene and read contents of RGB and depth buffers.
_render.mjr_render(self._rect, self._scene, self._mjr_context)
_render.mjr_readPixels(self._rgb_buffer, self._depth_buffer, self._rect,
self._mjr_context)
if self._depth_rendering:
# Get the distances to the near and far clipping planes.
extent = self._model.stat.extent
near = self._model.vis.map.znear * extent
far = self._model.vis.map.zfar * extent
# Convert from [0 1] to depth in meters, see links below:
# http://stackoverflow.com/a/6657284/1461210
# https://www.khronos.org/opengl/wiki/Depth_Buffer_Precision
pixels = near / (1 - self._depth_buffer * (1 - near / far))
elif self._segmentation_rendering:
# Convert 3-channel uint8 to 1-channel uint32.
image3 = self._rgb_buffer.astype(np.uint32)
segimage = (image3[:, :, 0] +
image3[:, :, 1] * (2**8) +
image3[:, :, 2] * (2**16))
# Remap segid to 2-channel (object ID, object type) pair.
# Seg ID 0 is background -- will be remapped to (-1, -1).
ngeoms = self._scene.ngeom
segid2output = np.full((ngeoms + 1, 2), fill_value=-1,
dtype=np.int32) # Seg id cannot be > ngeom + 1.
visible_geoms = [g for g in self._scene.geoms[:ngeoms] if g.segid != -1]
visible_segids = np.array([g.segid + 1 for g in visible_geoms], np.int32)
visible_objid = np.array([g.objid for g in visible_geoms], np.int32)
visible_objtype = np.array([g.objtype for g in visible_geoms], np.int32)
segid2output[visible_segids, 0] = visible_objid
segid2output[visible_segids, 1] = visible_objtype
pixels = segid2output[segimage]
# Reset scene flags.
np.copyto(self._scene.flags, original_flags)
else:
pixels = self._rgb_buffer
return np.flipud(pixels)
def update_scene(
self,
data: _structs.MjData,
camera: Union[int, str, _structs.MjvCamera] = -1,
scene_option: Optional[_structs.MjvOption] = None
):
"""Updates geometry used for rendering.
Args:
data: An instance of `MjData`.
camera: An instance of `MjvCamera`, a string or an integer
scene_option: A custom `MjvOption` instance to use to render
the scene instead of the default.
"""
if not isinstance(camera, _structs.MjvCamera):
camera_id = camera
if isinstance(camera_id, str):
camera_id = _functions.mj_name2id(self._model,
_enums.mjtObj.mjOBJ_CAMERA, camera_id)
if camera_id < -1:
raise ValueError('camera_id cannot be smaller than -1.')
if camera_id >= self._model.ncam:
raise ValueError(
f'model has {self._model.ncam} fixed cameras. '
f'camera_id={camera_id} is invalid.'
)
# Render camera.
camera = _structs.MjvCamera()
camera.fixedcamid = camera_id
# Defaults to mjCAMERA_FREE, otherwise mjCAMERA_FIXED refers to a
# camera explicitly defined in the model.
if camera_id == -1:
camera.type = _enums.mjtCamera.mjCAMERA_FREE
_functions.mjv_defaultFreeCamera(self._model, camera)
else:
camera.type = _enums.mjtCamera.mjCAMERA_FIXED
scene_option = scene_option or self._scene_option
_functions.mjv_updateScene(
self._model,
data,
scene_option,
None,
camera, _enums.mjtCatBit.mjCAT_ALL,
self._scene,
)