chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge renderer+offscreen mode into unified graphics mode.

Browse Source 
Allow Studio app to switch between graphics modes dynamically.

PiperOrigin-RevId: 878396966
Change-Id: I666fe3d2452de5404b9be2bd1d017eabec789aef
This commit is contained in:
Haroon Qureshi
2026-03-04 03:19:20 -08:00
committed by Copybara-Service
parent 8ab791fdf7
commit 73f73d94f7
11 changed files with 304 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/experimental/platform/CMakeLists.txt
+2 -1
View File
@@ -41,6 +41,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
egl_utils.h
enum_utils.h
file_dialog.h
graphics_mode.cc
graphics_mode.h
gui.cc
gui.h
gui_spec.cc
@@ -59,7 +61,6 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
plugin.h
renderer.cc
renderer.h
renderer_backend.h
sim_history.cc
sim_history.h
sim_profiler.cc
src/experimental/platform/graphics_mode.cc
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2026 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "experimental/platform/graphics_mode.h"
namespace mujoco::platform {
bool IsClassic(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::ClassicOpenGl ||
gfx_mode == GraphicsMode::ClassicOpenGlHeadless;
}
bool IsFilament(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::FilamentOpenGl ||
gfx_mode == GraphicsMode::FilamentVulkan ||
gfx_mode == GraphicsMode::FilamentWebGl ||
gfx_mode == GraphicsMode::FilamentOpenGlHeadless;
}
bool IsOpenGl(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::ClassicOpenGl ||
gfx_mode == GraphicsMode::ClassicOpenGlHeadless ||
gfx_mode == GraphicsMode::FilamentOpenGl ||
gfx_mode == GraphicsMode::FilamentOpenGlHeadless;
}
bool IsVulkan(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::FilamentVulkan;
}
bool IsWebGl(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::FilamentWebGl;
}
bool IsHeadless(GraphicsMode gfx_mode) {
return gfx_mode == GraphicsMode::ClassicOpenGlHeadless ||
gfx_mode == GraphicsMode::FilamentOpenGlHeadless;
}
} // namespace mujoco::platform
src/experimental/platform/renderer_backend.h → src/experimental/platform/graphics_mode.h
+15 -12
View File
@@ -12,23 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_
#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_
#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
namespace mujoco::platform {
// Describes the configuration of the renderer backend.
//
// This is generally a combination of two things. The first describes the
// high-level rendering engine that sits behind the MuJoCo API (e.g.
// mjr_render). The second describes the low-level graphics engine that is
// powering the rendering (e.g. OpenGL, Vulkan, etc.).
//
// More details about the backends can be found in renderer.h.
enum class RendererBackend {
// Describes the configuration of the graphics and rendering systems.
enum class GraphicsMode {
// The classic MuJoCo OpenGL renderer.
ClassicOpenGl,
// The classic MuJoCo OpenGL renderer.
ClassicOpenGlHeadless,
// The Filament-based renderer running on OpenGL.
FilamentOpenGl,
@@ -42,6 +38,13 @@ enum class RendererBackend {
FilamentOpenGlHeadless,
};
bool IsClassic(GraphicsMode gfx_mode);
bool IsFilament(GraphicsMode gfx_mode);
bool IsOpenGl(GraphicsMode gfx_mode);
bool IsVulkan(GraphicsMode gfx_mode);
bool IsWebGl(GraphicsMode gfx_mode);
bool IsHeadless(GraphicsMode gfx_mode);
} // namespace mujoco::platform
#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_
#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
src/experimental/platform/renderer.cc
+93 -91
View File
@@ -14,35 +14,41 @@
#include "experimental/platform/renderer.h"
#include <chrono>
#include <cstddef>
#include <span>
#include <mujoco/mujoco.h>
#include "experimental/platform/renderer_backend.h"
#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL)
#include <imgui.h>
#include <backends/imgui_impl_opengl3.h>
#ifndef __EMSCRIPTEN__
#include "experimental/platform/egl_utils.h"
#else
#endif // __EMSCRIPTEN__
#include "experimental/filament/render_context_filament.h"
#include "experimental/platform/graphics_mode.h"
#include "experimental/platform/plugin.h"
#endif
namespace mujoco::platform {
Renderer::Renderer(void* native_window) : native_window_(native_window) {
#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL
if (native_window == nullptr) {
graphics_api_context_ = CreateEglContext();
Renderer::Renderer(void* native_window, GraphicsMode gfx)
: native_window_(native_window), gfx_(gfx) {
if (IsClassic(gfx_)) {
if (native_window == nullptr) {
// graphics_api_context_ = CreateEglContext();
}
if (ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_Init();
}
}
if (ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_Init();
}
#endif
}
Renderer::~Renderer() {
if (IsClassic(gfx_)) {
if (ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_Shutdown();
}
}
Deinit();
graphics_api_context_.reset();
}
@@ -50,24 +56,39 @@ Renderer::~Renderer() {
void Renderer::Init(const mjModel* model) {
Deinit();
if (model) {
mjr_defaultContext(&render_context_);
#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL)
mjr_makeContext(model, &render_context_, mjFONTSCALE_150);
#else
mjrFilamentConfig render_config;
mjrf_defaultFilamentConfig(&render_config);
render_config.native_window = native_window_;
render_config.enable_gui = true;
#if defined(MUJOCO_RENDERER_FILAMENT_OPENGL)
render_config.graphics_api = mjGFX_OPENGL;
#elif defined(MUJOCO_RENDERER_FILAMENT_OPENGL_HEADLESS)
render_config.graphics_api = mjGFX_OPENGL;
#elif defined(MUJOCO_RENDERER_FILAMENT_VULKAN)
render_config.graphics_api = mjGFX_VULKAN;
#endif
mjrf_makeFilamentContext(model, &render_context_, &render_config);
#endif
if (IsClassic(gfx_)) {
mjr_defaultContext(&render_context_);
mjr_makeContext(model, &render_context_, mjFONTSCALE_150);
render_ = [&](mjrRect rect, mjvScene* scene) {
mjr_render(rect, scene, &render_context_);
};
set_buffer_ = [&](int framebuffer) {
mjr_setBuffer(framebuffer, &render_context_);
};
read_pixels_ = [&](unsigned char* pixels, mjrRect rect) {
mjr_readPixels(pixels, nullptr, rect, &render_context_);
mjr_setBuffer(mjFB_WINDOW, &render_context_);
};
} else {
mjrf_defaultContext(&render_context_);
mjrFilamentConfig render_config;
mjrf_defaultFilamentConfig(&render_config);
render_config.native_window = native_window_;
render_config.enable_gui = true;
render_config.graphics_api =
IsOpenGl(gfx_) || IsWebGl(gfx_) ? mjGFX_OPENGL : mjGFX_VULKAN;
mjrf_makeFilamentContext(model, &render_context_, &render_config);
render_ = [&](mjrRect rect, mjvScene* scene) {
mjrf_render(rect, scene, &render_context_);
};
set_buffer_ = [&](int framebuffer) {
mjrf_setBuffer(framebuffer, &render_context_);
};
read_pixels_ = [&](unsigned char* pixels, mjrRect rect) {
mjrf_readPixels(pixels, nullptr, rect, &render_context_);
mjrf_setBuffer(mjFB_WINDOW, &render_context_);
};
}
mjv_defaultScene(&scene_);
mjv_makeScene(model, &scene_, 2000);
@@ -78,7 +99,11 @@ void Renderer::Init(const mjModel* model) {
void Renderer::Deinit() {
if (initialized_) {
mjv_freeScene(&scene_);
mjr_freeContext(&render_context_);
if (IsClassic(gfx_)) {
mjr_freeContext(&render_context_);
} else {
mjrf_freeContext(&render_context_);
}
initialized_ = false;
}
}
@@ -116,35 +141,30 @@ void Renderer::Render(const mjModel* model, mjData* data,
mju_error("Offscreen mode requires a pixel buffer of size %d.",
width * height * 3);
}
#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL
if (IsClassic(gfx_)) {
mjr_resizeOffscreen(width, height, &render_context_);
mjr_setBuffer(mjFB_OFFSCREEN, &render_context_);
#else
// The filament backend supports two offscreen framebuffers.
// mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the
// ImGui draw data.
mjr_setBuffer(mjFB_OFFSCREEN + 1, &render_context_);
#endif
}
// The filament backend supports two offscreen framebuffers.
// mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the
// ImGui draw data.
set_buffer_(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1);
}
const mjrRect viewport = {0, 0, width, height};
mjr_render(viewport, &scene_, &render_context_);
render_(viewport, &scene_);
// The filament backend knows how to renders the ImGui draw data. For the
// classic backend, we need to render the ImGui draw data ourselves.
#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL
if (ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_NewFrame();
ImGui::Render();
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
}
#endif
if (IsClassic(gfx_) && ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_NewFrame();
ImGui::Render();
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
}
if (render_to_texture) {
unsigned char* ptr = reinterpret_cast<unsigned char*>(pixels.data());
mjr_readPixels(ptr, nullptr, viewport, &render_context_);
mjr_setBuffer(mjFB_WINDOW, &render_context_);
read_pixels_(ptr, viewport);
}
UpdateFps();
@@ -158,61 +178,44 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data,
}
const mjrRect viewport = {0, 0, width, height};
mjr_setBuffer(mjFB_OFFSCREEN, &render_context_);
mjv_updateCamera(model, data, camera, &scene_);
mjr_render(viewport, &scene_, &render_context_);
mjr_readPixels((unsigned char*)output, nullptr, viewport, &render_context_);
mjr_setBuffer(mjFB_WINDOW, &render_context_);
set_buffer_(mjFB_OFFSCREEN);
render_(viewport, &scene_);
read_pixels_((unsigned char*)output, viewport);
}
int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
int height, int bpp) {
#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL)
return 0;
#else
return mjrf_uploadGuiImage(texture_id,
reinterpret_cast<const unsigned char*>(pixels),
width, height, bpp, &render_context_);
#endif
if (IsClassic(gfx_)) {
return 0;
} else {
return mjrf_uploadGuiImage(texture_id,
reinterpret_cast<const unsigned char*>(pixels),
width, height, bpp, &render_context_);
}
}
double Renderer::GetFps() { return fps_; }
void Renderer::UpdateFps() {
#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL
TimePoint now = std::chrono::steady_clock::now();
TimePoint::duration delta_time = now - last_fps_update_;
const double interval = std::chrono::duration<double>(delta_time).count();
++frames_;
if (interval > 0.2) { // only update FPS stat at most 5 times per second
last_fps_update_ = now;
fps_ = frames_ / interval;
frames_ = 0;
if (IsClassic(gfx_)) {
TimePoint now = std::chrono::steady_clock::now();
TimePoint::duration delta_time = now - last_fps_update_;
const double interval = std::chrono::duration<double>(delta_time).count();
++frames_;
if (interval > 0.2) { // only update FPS stat at most 5 times per second
last_fps_update_ = now;
fps_ = frames_ / interval;
frames_ = 0;
}
} else {
fps_ = mjrf_getFrameRate(&render_context_);
}
#else
fps_ = mjrf_getFrameRate(&render_context_);
#endif
}
RendererBackend Renderer::GetBackend() {
#if defined(MUJOCO_RENDERER_FILAMENT_OPENGL_HEADLESS)
return RendererBackend::FilamentOpenGlHeadless;
#elif defined(MUJOCO_RENDERER_FILAMENT_OPENGL)
return RendererBackend::FilamentOpenGl;
#elif defined(MUJOCO_RENDERER_FILAMENT_VULKAN)
return RendererBackend::FilamentVulkan;
#elif defined(MUJOCO_RENDERER_FILAMENT_WEBGL)
return RendererBackend::FilamentWebGl;
#elif defined(MUJOCO_RENDERER_CLASSIC_OPENGL)
return RendererBackend::ClassicOpenGl;
#else
#error "Unsupported renderer backend."
#endif
}
} // namespace mujoco::platform
#if !defined(MUJOCO_RENDERER_CLASSIC_OPENGL)
mjPLUGIN_LIB_INIT {
mujoco::platform::GuiPlugin plugin;
plugin.name = "Filament";
@@ -221,4 +224,3 @@ mjPLUGIN_LIB_INIT {
};
mujoco::platform::RegisterPlugin(plugin);
}
#endif
src/experimental/platform/renderer.h
+8 -7
View File
@@ -17,13 +17,13 @@
#include <chrono>
#include <cstddef>
#include <functional>
#include <memory>
#include <ratio>
#include <span>
#include <mujoco/mujoco.h>
#include "experimental/platform/renderer_backend.h"
#include "experimental/platform/graphics_mode.h"
namespace mujoco::platform {
@@ -57,7 +57,8 @@ class Renderer {
using Seconds = std::chrono::duration<double>;
using Milliseconds = std::chrono::duration<double, std::milli>;
explicit Renderer(void* native_window = nullptr);
explicit Renderer(void* native_window = nullptr,
GraphicsMode gfx_mode = GraphicsMode::FilamentVulkan);
~Renderer();
Renderer(const Renderer&) = delete;
@@ -90,10 +91,6 @@ class Renderer {
// Returns the current frame rate.
double GetFps();
// Returns the statically-defined backend for which this renderer is
// configured.
static RendererBackend GetBackend();
private:
// Resets the renderer; no rendering will occur until Init() is called again.
void Deinit();
@@ -101,7 +98,11 @@ class Renderer {
void UpdateFps();
void* native_window_ = nullptr;
GraphicsMode gfx_ = GraphicsMode::FilamentVulkan;
std::shared_ptr<void> graphics_api_context_ = nullptr;
std::function<void(mjrRect, mjvScene*)> render_;
std::function<void(int)> set_buffer_;
std::function<void(unsigned char*, mjrRect)> read_pixels_;
mjrContext render_context_;
mjvScene scene_;
bool initialized_ = false;
src/experimental/platform/window.cc
+35 -30
View File
@@ -32,9 +32,10 @@
#include <backends/imgui_impl_sdl2.h>
#include <imgui.h>
#include <mujoco/mujoco.h>
#include "experimental/platform/renderer_backend.h"
#include "experimental/platform/graphics_mode.h"
#include "user/user_resource.h"
// Because X11/Xlib.h defines Status.
#ifdef Status
#undef Status
@@ -95,12 +96,7 @@ static void InitImGui(SDL_Window* window, float content_scale, bool load_fonts,
Window::Window(std::string_view title, int width, int height, Config config)
: width_(width), height_(height), config_(config) {
const RendererBackend renderer_backend = config_.renderer_backend;
if (renderer_backend == RendererBackend::FilamentOpenGlHeadless) {
config_.offscreen_mode = true;
}
if (config_.offscreen_mode) {
if (IsHeadless(config_.gfx_mode)) {
SDL_SetHint(SDL_HINT_RENDER_DRIVER, "software");
SDL_SetHint(SDL_HINT_FRAMEBUFFER_ACCELERATION, "0");
} else {
@@ -114,21 +110,19 @@ Window::Window(std::string_view title, int width, int height, Config config)
}
int window_flags = SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI;
if (renderer_backend == RendererBackend::FilamentVulkan) {
if (IsVulkan(config_.gfx_mode)) {
window_flags |= SDL_WINDOW_VULKAN;
} else if (renderer_backend == RendererBackend::FilamentWebGl) {
} else if (IsWebGl(config_.gfx_mode)) {
window_flags |= SDL_WINDOW_OPENGL;
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
} else if (renderer_backend == RendererBackend::ClassicOpenGl ||
renderer_backend == RendererBackend::FilamentOpenGl ||
renderer_backend == RendererBackend::FilamentOpenGlHeadless) {
} else if (IsOpenGl(config_.gfx_mode)) {
window_flags |= SDL_WINDOW_OPENGL;
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
} else {
mju_error("Unsupported window config: %d", renderer_backend);
mju_error("Unsupported window config: %d", config_.gfx_mode);
}
const float content_scale = ImGui_ImplSDL2_GetContentScaleForDisplay(0);
@@ -140,24 +134,29 @@ Window::Window(std::string_view title, int width, int height, Config config)
}
InitImGui(sdl_window_, content_scale, config.load_fonts,
(renderer_backend != RendererBackend::ClassicOpenGl));
(config_.gfx_mode != GraphicsMode::ClassicOpenGl &&
config_.gfx_mode != GraphicsMode::ClassicOpenGlHeadless));
if (renderer_backend == RendererBackend::FilamentWebGl ||
(renderer_backend == RendererBackend::ClassicOpenGl &&
!config.offscreen_mode)) {
// Filament (except WebGL) manages its own swap chain including when to swap.
// In all other cases, we'll use SDL to manage the swap chain.
if (config_.gfx_mode == GraphicsMode::FilamentWebGl ||
config_.gfx_mode == GraphicsMode::ClassicOpenGl ||
config_.gfx_mode == GraphicsMode::ClassicOpenGlHeadless) {
SDL_GLContext gl_context = SDL_GL_CreateContext(sdl_window_);
SDL_GL_MakeCurrent(sdl_window_, gl_context);
}
if (config_.offscreen_mode) {
// In headless mode, we'll render to a texture and then blit the texture onto
// the window surface. In this case, we'll use SDL's software renderer to
// perform the blitting. Since we're in charge of the window, we don't need
// to get the native window handle for the renderer.
if (IsHeadless(config_.gfx_mode)) {
sdl_renderer_ = SDL_CreateRenderer(sdl_window_, -1, SDL_RENDERER_SOFTWARE);
}
} else {
SDL_SysWMinfo wmi;
SDL_VERSION(&wmi.version);
SDL_GetWindowWMInfo(sdl_window_, &wmi);
SDL_SysWMinfo wmi;
SDL_VERSION(&wmi.version);
SDL_GetWindowWMInfo(sdl_window_, &wmi);
if (!config_.offscreen_mode) {
#if defined(__linux__)
native_window_ = reinterpret_cast<void*>(wmi.info.x11.window);
#elif defined(__WIN32__)
@@ -175,6 +174,7 @@ Window::Window(std::string_view title, int width, int height, Config config)
}
Window::~Window() {
ImGui_ImplSDL2_Shutdown();
SDL_DestroyWindow(sdl_window_);
SDL_Quit();
}
@@ -234,7 +234,9 @@ void Window::EndFrame() {
}
void Window::Present(std::span<const std::byte> pixels) {
if (config_.offscreen_mode) {
// In headless mode, we assume the caller has renderered the scene to an RGB
// buffer of size width_ * height_ * 3.
if (IsHeadless(config_.gfx_mode)) {
if (pixels.size() != width_ * height_ * 3) {
mju_error("Offscreen mode expects RGB buffer of size %d",
width_ * height_ * 3);
@@ -258,7 +260,9 @@ void Window::Present(std::span<const std::byte> pixels) {
}
}
if (config_.renderer_backend == RendererBackend::ClassicOpenGl) {
// Flip the image vertically because the classic renderer uses a bottom-left
// coordinate system while SDL assumes a top-left coordinate system.
if (config_.gfx_mode == GraphicsMode::ClassicOpenGlHeadless) {
dst = static_cast<unsigned char*>(surface->pixels);
for (int r = 0; r < height_ / 2; ++r) {
unsigned char* top_row = &dst[4 * width_ * r];
@@ -268,13 +272,14 @@ void Window::Present(std::span<const std::byte> pixels) {
}
SDL_RenderPresent(sdl_renderer_);
} else if (config_.renderer_backend != RendererBackend::FilamentVulkan
&& config_.renderer_backend != RendererBackend::FilamentOpenGl) {
} else if (config_.gfx_mode != GraphicsMode::FilamentVulkan
&& config_.gfx_mode != GraphicsMode::FilamentOpenGl) {
SDL_GL_SwapWindow(sdl_window_);
}
}
bool Window::IsOffscreenMode() const {
return config_.offscreen_mode;
GraphicsMode Window::GetGraphicsMode() const {
return config_.gfx_mode;
}
} // namespace mujoco::platform
src/experimental/platform/window.h
+4 -6
View File
@@ -20,10 +20,9 @@
#include <string>
#include <string_view>
#include "experimental/platform/renderer_backend.h"
#include <SDL_video.h>
#include <SDL_render.h>
#include "experimental/platform/graphics_mode.h"
namespace mujoco::platform {
@@ -34,9 +33,8 @@ namespace mujoco::platform {
class Window {
public:
struct Config {
RendererBackend renderer_backend = RendererBackend::ClassicOpenGl;
GraphicsMode gfx_mode = GraphicsMode::FilamentVulkan;
bool load_fonts = true;
bool offscreen_mode = false;
};
Window(std::string_view title, int width, int height, Config config);
@@ -78,8 +76,8 @@ class Window {
// the value will be cleared until the next time a file is dropped.
std::string GetDropFile();
// Returns true if the window is in offscreen mode.
bool IsOffscreenMode() const;
// Returns the graphics configuration of the window.
GraphicsMode GetGraphicsMode() const;
// Enables window resizing.
void EnableWindowResizing();