From 73f73d94f73a3e9f17ad5872a1fb66107635a0df Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Wed, 4 Mar 2026 03:19:20 -0800 Subject: [PATCH] Merge renderer+offscreen mode into unified graphics mode. Allow Studio app to switch between graphics modes dynamically. PiperOrigin-RevId: 878396966 Change-Id: I666fe3d2452de5404b9be2bd1d017eabec789aef --- src/experimental/platform/CMakeLists.txt | 3 +- src/experimental/platform/graphics_mode.cc | 51 +++++ .../{renderer_backend.h => graphics_mode.h} | 27 +-- src/experimental/platform/renderer.cc | 184 +++++++++--------- src/experimental/platform/renderer.h | 15 +- src/experimental/platform/window.cc | 65 ++++--- src/experimental/platform/window.h | 10 +- src/experimental/studio/app.cc | 79 ++++++-- src/experimental/studio/app.h | 10 +- src/experimental/studio/main.cc | 24 ++- src/experimental/studio/wasm.cc | 2 + 11 files changed, 304 insertions(+), 166 deletions(-) create mode 100644 src/experimental/platform/graphics_mode.cc rename src/experimental/platform/{renderer_backend.h => graphics_mode.h} (62%) diff --git a/src/experimental/platform/CMakeLists.txt b/src/experimental/platform/CMakeLists.txt index d953c929..c3fc3b38 100644 --- a/src/experimental/platform/CMakeLists.txt +++ b/src/experimental/platform/CMakeLists.txt @@ -41,6 +41,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME} egl_utils.h enum_utils.h file_dialog.h + graphics_mode.cc + graphics_mode.h gui.cc gui.h gui_spec.cc @@ -59,7 +61,6 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME} plugin.h renderer.cc renderer.h - renderer_backend.h sim_history.cc sim_history.h sim_profiler.cc diff --git a/src/experimental/platform/graphics_mode.cc b/src/experimental/platform/graphics_mode.cc new file mode 100644 index 00000000..37c2e110 --- /dev/null +++ b/src/experimental/platform/graphics_mode.cc @@ -0,0 +1,51 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/platform/graphics_mode.h" + +namespace mujoco::platform { + +bool IsClassic(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::ClassicOpenGl || + gfx_mode == GraphicsMode::ClassicOpenGlHeadless; +} + +bool IsFilament(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::FilamentOpenGl || + gfx_mode == GraphicsMode::FilamentVulkan || + gfx_mode == GraphicsMode::FilamentWebGl || + gfx_mode == GraphicsMode::FilamentOpenGlHeadless; +} + +bool IsOpenGl(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::ClassicOpenGl || + gfx_mode == GraphicsMode::ClassicOpenGlHeadless || + gfx_mode == GraphicsMode::FilamentOpenGl || + gfx_mode == GraphicsMode::FilamentOpenGlHeadless; +} + +bool IsVulkan(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::FilamentVulkan; +} + +bool IsWebGl(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::FilamentWebGl; +} + +bool IsHeadless(GraphicsMode gfx_mode) { + return gfx_mode == GraphicsMode::ClassicOpenGlHeadless || + gfx_mode == GraphicsMode::FilamentOpenGlHeadless; +} + +} // namespace mujoco::platform diff --git a/src/experimental/platform/renderer_backend.h b/src/experimental/platform/graphics_mode.h similarity index 62% rename from src/experimental/platform/renderer_backend.h rename to src/experimental/platform/graphics_mode.h index 32ddaee0..1e6ba796 100644 --- a/src/experimental/platform/renderer_backend.h +++ b/src/experimental/platform/graphics_mode.h @@ -12,23 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_ -#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_ +#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_ +#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_ namespace mujoco::platform { -// Describes the configuration of the renderer backend. -// -// This is generally a combination of two things. The first describes the -// high-level rendering engine that sits behind the MuJoCo API (e.g. -// mjr_render). The second describes the low-level graphics engine that is -// powering the rendering (e.g. OpenGL, Vulkan, etc.). -// -// More details about the backends can be found in renderer.h. -enum class RendererBackend { +// Describes the configuration of the graphics and rendering systems. +enum class GraphicsMode { // The classic MuJoCo OpenGL renderer. ClassicOpenGl, + // The classic MuJoCo OpenGL renderer. + ClassicOpenGlHeadless, + // The Filament-based renderer running on OpenGL. FilamentOpenGl, @@ -42,6 +38,13 @@ enum class RendererBackend { FilamentOpenGlHeadless, }; +bool IsClassic(GraphicsMode gfx_mode); +bool IsFilament(GraphicsMode gfx_mode); +bool IsOpenGl(GraphicsMode gfx_mode); +bool IsVulkan(GraphicsMode gfx_mode); +bool IsWebGl(GraphicsMode gfx_mode); +bool IsHeadless(GraphicsMode gfx_mode); + } // namespace mujoco::platform -#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_BACKEND_H_ +#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_ diff --git a/src/experimental/platform/renderer.cc b/src/experimental/platform/renderer.cc index 2188d86f..b00637cd 100644 --- a/src/experimental/platform/renderer.cc +++ b/src/experimental/platform/renderer.cc @@ -14,35 +14,41 @@ #include "experimental/platform/renderer.h" +#include #include #include #include -#include "experimental/platform/renderer_backend.h" -#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL) #include #include +#ifndef __EMSCRIPTEN__ #include "experimental/platform/egl_utils.h" -#else +#endif // __EMSCRIPTEN__ #include "experimental/filament/render_context_filament.h" +#include "experimental/platform/graphics_mode.h" #include "experimental/platform/plugin.h" -#endif namespace mujoco::platform { -Renderer::Renderer(void* native_window) : native_window_(native_window) { -#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL - if (native_window == nullptr) { - graphics_api_context_ = CreateEglContext(); +Renderer::Renderer(void* native_window, GraphicsMode gfx) + : native_window_(native_window), gfx_(gfx) { + if (IsClassic(gfx_)) { + if (native_window == nullptr) { + // graphics_api_context_ = CreateEglContext(); + } + if (ImGui::GetCurrentContext()) { + ImGui_ImplOpenGL3_Init(); + } } - if (ImGui::GetCurrentContext()) { - ImGui_ImplOpenGL3_Init(); - } -#endif } Renderer::~Renderer() { + if (IsClassic(gfx_)) { + if (ImGui::GetCurrentContext()) { + ImGui_ImplOpenGL3_Shutdown(); + } + } Deinit(); graphics_api_context_.reset(); } @@ -50,24 +56,39 @@ Renderer::~Renderer() { void Renderer::Init(const mjModel* model) { Deinit(); if (model) { - mjr_defaultContext(&render_context_); - -#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL) - mjr_makeContext(model, &render_context_, mjFONTSCALE_150); -#else - mjrFilamentConfig render_config; - mjrf_defaultFilamentConfig(&render_config); - render_config.native_window = native_window_; - render_config.enable_gui = true; -#if defined(MUJOCO_RENDERER_FILAMENT_OPENGL) - render_config.graphics_api = mjGFX_OPENGL; -#elif defined(MUJOCO_RENDERER_FILAMENT_OPENGL_HEADLESS) - render_config.graphics_api = mjGFX_OPENGL; -#elif defined(MUJOCO_RENDERER_FILAMENT_VULKAN) - render_config.graphics_api = mjGFX_VULKAN; -#endif - mjrf_makeFilamentContext(model, &render_context_, &render_config); -#endif + if (IsClassic(gfx_)) { + mjr_defaultContext(&render_context_); + mjr_makeContext(model, &render_context_, mjFONTSCALE_150); + render_ = [&](mjrRect rect, mjvScene* scene) { + mjr_render(rect, scene, &render_context_); + }; + set_buffer_ = [&](int framebuffer) { + mjr_setBuffer(framebuffer, &render_context_); + }; + read_pixels_ = [&](unsigned char* pixels, mjrRect rect) { + mjr_readPixels(pixels, nullptr, rect, &render_context_); + mjr_setBuffer(mjFB_WINDOW, &render_context_); + }; + } else { + mjrf_defaultContext(&render_context_); + mjrFilamentConfig render_config; + mjrf_defaultFilamentConfig(&render_config); + render_config.native_window = native_window_; + render_config.enable_gui = true; + render_config.graphics_api = + IsOpenGl(gfx_) || IsWebGl(gfx_) ? mjGFX_OPENGL : mjGFX_VULKAN; + mjrf_makeFilamentContext(model, &render_context_, &render_config); + render_ = [&](mjrRect rect, mjvScene* scene) { + mjrf_render(rect, scene, &render_context_); + }; + set_buffer_ = [&](int framebuffer) { + mjrf_setBuffer(framebuffer, &render_context_); + }; + read_pixels_ = [&](unsigned char* pixels, mjrRect rect) { + mjrf_readPixels(pixels, nullptr, rect, &render_context_); + mjrf_setBuffer(mjFB_WINDOW, &render_context_); + }; + } mjv_defaultScene(&scene_); mjv_makeScene(model, &scene_, 2000); @@ -78,7 +99,11 @@ void Renderer::Init(const mjModel* model) { void Renderer::Deinit() { if (initialized_) { mjv_freeScene(&scene_); - mjr_freeContext(&render_context_); + if (IsClassic(gfx_)) { + mjr_freeContext(&render_context_); + } else { + mjrf_freeContext(&render_context_); + } initialized_ = false; } } @@ -116,35 +141,30 @@ void Renderer::Render(const mjModel* model, mjData* data, mju_error("Offscreen mode requires a pixel buffer of size %d.", width * height * 3); } - - #ifdef MUJOCO_RENDERER_CLASSIC_OPENGL + if (IsClassic(gfx_)) { mjr_resizeOffscreen(width, height, &render_context_); - mjr_setBuffer(mjFB_OFFSCREEN, &render_context_); - #else - // The filament backend supports two offscreen framebuffers. - // mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the - // ImGui draw data. - mjr_setBuffer(mjFB_OFFSCREEN + 1, &render_context_); - #endif + } + + // The filament backend supports two offscreen framebuffers. + // mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the + // ImGui draw data. + set_buffer_(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1); } const mjrRect viewport = {0, 0, width, height}; - mjr_render(viewport, &scene_, &render_context_); + render_(viewport, &scene_); // The filament backend knows how to renders the ImGui draw data. For the // classic backend, we need to render the ImGui draw data ourselves. - #ifdef MUJOCO_RENDERER_CLASSIC_OPENGL - if (ImGui::GetCurrentContext()) { - ImGui_ImplOpenGL3_NewFrame(); - ImGui::Render(); - ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); - } - #endif + if (IsClassic(gfx_) && ImGui::GetCurrentContext()) { + ImGui_ImplOpenGL3_NewFrame(); + ImGui::Render(); + ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); + } if (render_to_texture) { unsigned char* ptr = reinterpret_cast(pixels.data()); - mjr_readPixels(ptr, nullptr, viewport, &render_context_); - mjr_setBuffer(mjFB_WINDOW, &render_context_); + read_pixels_(ptr, viewport); } UpdateFps(); @@ -158,61 +178,44 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data, } const mjrRect viewport = {0, 0, width, height}; - - mjr_setBuffer(mjFB_OFFSCREEN, &render_context_); mjv_updateCamera(model, data, camera, &scene_); - mjr_render(viewport, &scene_, &render_context_); - mjr_readPixels((unsigned char*)output, nullptr, viewport, &render_context_); - mjr_setBuffer(mjFB_WINDOW, &render_context_); + + set_buffer_(mjFB_OFFSCREEN); + render_(viewport, &scene_); + read_pixels_((unsigned char*)output, viewport); } int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width, int height, int bpp) { -#if defined(MUJOCO_RENDERER_CLASSIC_OPENGL) - return 0; -#else - return mjrf_uploadGuiImage(texture_id, - reinterpret_cast(pixels), - width, height, bpp, &render_context_); -#endif + if (IsClassic(gfx_)) { + return 0; + } else { + return mjrf_uploadGuiImage(texture_id, + reinterpret_cast(pixels), + width, height, bpp, &render_context_); + } } double Renderer::GetFps() { return fps_; } void Renderer::UpdateFps() { -#ifdef MUJOCO_RENDERER_CLASSIC_OPENGL - TimePoint now = std::chrono::steady_clock::now(); - TimePoint::duration delta_time = now - last_fps_update_; - const double interval = std::chrono::duration(delta_time).count(); - ++frames_; - if (interval > 0.2) { // only update FPS stat at most 5 times per second - last_fps_update_ = now; - fps_ = frames_ / interval; - frames_ = 0; + if (IsClassic(gfx_)) { + TimePoint now = std::chrono::steady_clock::now(); + TimePoint::duration delta_time = now - last_fps_update_; + const double interval = std::chrono::duration(delta_time).count(); + ++frames_; + if (interval > 0.2) { // only update FPS stat at most 5 times per second + last_fps_update_ = now; + fps_ = frames_ / interval; + frames_ = 0; + } + } else { + fps_ = mjrf_getFrameRate(&render_context_); } -#else - fps_ = mjrf_getFrameRate(&render_context_); -#endif } -RendererBackend Renderer::GetBackend() { -#if defined(MUJOCO_RENDERER_FILAMENT_OPENGL_HEADLESS) - return RendererBackend::FilamentOpenGlHeadless; -#elif defined(MUJOCO_RENDERER_FILAMENT_OPENGL) - return RendererBackend::FilamentOpenGl; -#elif defined(MUJOCO_RENDERER_FILAMENT_VULKAN) - return RendererBackend::FilamentVulkan; -#elif defined(MUJOCO_RENDERER_FILAMENT_WEBGL) - return RendererBackend::FilamentWebGl; -#elif defined(MUJOCO_RENDERER_CLASSIC_OPENGL) - return RendererBackend::ClassicOpenGl; -#else - #error "Unsupported renderer backend." -#endif -} } // namespace mujoco::platform -#if !defined(MUJOCO_RENDERER_CLASSIC_OPENGL) mjPLUGIN_LIB_INIT { mujoco::platform::GuiPlugin plugin; plugin.name = "Filament"; @@ -221,4 +224,3 @@ mjPLUGIN_LIB_INIT { }; mujoco::platform::RegisterPlugin(plugin); } -#endif diff --git a/src/experimental/platform/renderer.h b/src/experimental/platform/renderer.h index a43f1d16..739151bd 100644 --- a/src/experimental/platform/renderer.h +++ b/src/experimental/platform/renderer.h @@ -17,13 +17,13 @@ #include #include +#include #include #include #include #include -#include "experimental/platform/renderer_backend.h" - +#include "experimental/platform/graphics_mode.h" namespace mujoco::platform { @@ -57,7 +57,8 @@ class Renderer { using Seconds = std::chrono::duration; using Milliseconds = std::chrono::duration; - explicit Renderer(void* native_window = nullptr); + explicit Renderer(void* native_window = nullptr, + GraphicsMode gfx_mode = GraphicsMode::FilamentVulkan); ~Renderer(); Renderer(const Renderer&) = delete; @@ -90,10 +91,6 @@ class Renderer { // Returns the current frame rate. double GetFps(); - // Returns the statically-defined backend for which this renderer is - // configured. - static RendererBackend GetBackend(); - private: // Resets the renderer; no rendering will occur until Init() is called again. void Deinit(); @@ -101,7 +98,11 @@ class Renderer { void UpdateFps(); void* native_window_ = nullptr; + GraphicsMode gfx_ = GraphicsMode::FilamentVulkan; std::shared_ptr graphics_api_context_ = nullptr; + std::function render_; + std::function set_buffer_; + std::function read_pixels_; mjrContext render_context_; mjvScene scene_; bool initialized_ = false; diff --git a/src/experimental/platform/window.cc b/src/experimental/platform/window.cc index 49a72317..0f849977 100644 --- a/src/experimental/platform/window.cc +++ b/src/experimental/platform/window.cc @@ -32,9 +32,10 @@ #include #include #include -#include "experimental/platform/renderer_backend.h" +#include "experimental/platform/graphics_mode.h" #include "user/user_resource.h" + // Because X11/Xlib.h defines Status. #ifdef Status #undef Status @@ -95,12 +96,7 @@ static void InitImGui(SDL_Window* window, float content_scale, bool load_fonts, Window::Window(std::string_view title, int width, int height, Config config) : width_(width), height_(height), config_(config) { - const RendererBackend renderer_backend = config_.renderer_backend; - if (renderer_backend == RendererBackend::FilamentOpenGlHeadless) { - config_.offscreen_mode = true; - } - - if (config_.offscreen_mode) { + if (IsHeadless(config_.gfx_mode)) { SDL_SetHint(SDL_HINT_RENDER_DRIVER, "software"); SDL_SetHint(SDL_HINT_FRAMEBUFFER_ACCELERATION, "0"); } else { @@ -114,21 +110,19 @@ Window::Window(std::string_view title, int width, int height, Config config) } int window_flags = SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI; - if (renderer_backend == RendererBackend::FilamentVulkan) { + if (IsVulkan(config_.gfx_mode)) { window_flags |= SDL_WINDOW_VULKAN; - } else if (renderer_backend == RendererBackend::FilamentWebGl) { + } else if (IsWebGl(config_.gfx_mode)) { window_flags |= SDL_WINDOW_OPENGL; SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3); SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0); SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES); - } else if (renderer_backend == RendererBackend::ClassicOpenGl || - renderer_backend == RendererBackend::FilamentOpenGl || - renderer_backend == RendererBackend::FilamentOpenGlHeadless) { + } else if (IsOpenGl(config_.gfx_mode)) { window_flags |= SDL_WINDOW_OPENGL; SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3); SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0); } else { - mju_error("Unsupported window config: %d", renderer_backend); + mju_error("Unsupported window config: %d", config_.gfx_mode); } const float content_scale = ImGui_ImplSDL2_GetContentScaleForDisplay(0); @@ -140,24 +134,29 @@ Window::Window(std::string_view title, int width, int height, Config config) } InitImGui(sdl_window_, content_scale, config.load_fonts, - (renderer_backend != RendererBackend::ClassicOpenGl)); + (config_.gfx_mode != GraphicsMode::ClassicOpenGl && + config_.gfx_mode != GraphicsMode::ClassicOpenGlHeadless)); - if (renderer_backend == RendererBackend::FilamentWebGl || - (renderer_backend == RendererBackend::ClassicOpenGl && - !config.offscreen_mode)) { + // Filament (except WebGL) manages its own swap chain including when to swap. + // In all other cases, we'll use SDL to manage the swap chain. + if (config_.gfx_mode == GraphicsMode::FilamentWebGl || + config_.gfx_mode == GraphicsMode::ClassicOpenGl || + config_.gfx_mode == GraphicsMode::ClassicOpenGlHeadless) { SDL_GLContext gl_context = SDL_GL_CreateContext(sdl_window_); SDL_GL_MakeCurrent(sdl_window_, gl_context); } - if (config_.offscreen_mode) { + // In headless mode, we'll render to a texture and then blit the texture onto + // the window surface. In this case, we'll use SDL's software renderer to + // perform the blitting. Since we're in charge of the window, we don't need + // to get the native window handle for the renderer. + if (IsHeadless(config_.gfx_mode)) { sdl_renderer_ = SDL_CreateRenderer(sdl_window_, -1, SDL_RENDERER_SOFTWARE); - } + } else { + SDL_SysWMinfo wmi; + SDL_VERSION(&wmi.version); + SDL_GetWindowWMInfo(sdl_window_, &wmi); - SDL_SysWMinfo wmi; - SDL_VERSION(&wmi.version); - SDL_GetWindowWMInfo(sdl_window_, &wmi); - - if (!config_.offscreen_mode) { #if defined(__linux__) native_window_ = reinterpret_cast(wmi.info.x11.window); #elif defined(__WIN32__) @@ -175,6 +174,7 @@ Window::Window(std::string_view title, int width, int height, Config config) } Window::~Window() { + ImGui_ImplSDL2_Shutdown(); SDL_DestroyWindow(sdl_window_); SDL_Quit(); } @@ -234,7 +234,9 @@ void Window::EndFrame() { } void Window::Present(std::span pixels) { - if (config_.offscreen_mode) { + // In headless mode, we assume the caller has renderered the scene to an RGB + // buffer of size width_ * height_ * 3. + if (IsHeadless(config_.gfx_mode)) { if (pixels.size() != width_ * height_ * 3) { mju_error("Offscreen mode expects RGB buffer of size %d", width_ * height_ * 3); @@ -258,7 +260,9 @@ void Window::Present(std::span pixels) { } } - if (config_.renderer_backend == RendererBackend::ClassicOpenGl) { + // Flip the image vertically because the classic renderer uses a bottom-left + // coordinate system while SDL assumes a top-left coordinate system. + if (config_.gfx_mode == GraphicsMode::ClassicOpenGlHeadless) { dst = static_cast(surface->pixels); for (int r = 0; r < height_ / 2; ++r) { unsigned char* top_row = &dst[4 * width_ * r]; @@ -268,13 +272,14 @@ void Window::Present(std::span pixels) { } SDL_RenderPresent(sdl_renderer_); - } else if (config_.renderer_backend != RendererBackend::FilamentVulkan - && config_.renderer_backend != RendererBackend::FilamentOpenGl) { + } else if (config_.gfx_mode != GraphicsMode::FilamentVulkan + && config_.gfx_mode != GraphicsMode::FilamentOpenGl) { SDL_GL_SwapWindow(sdl_window_); } } -bool Window::IsOffscreenMode() const { - return config_.offscreen_mode; +GraphicsMode Window::GetGraphicsMode() const { + return config_.gfx_mode; } + } // namespace mujoco::platform diff --git a/src/experimental/platform/window.h b/src/experimental/platform/window.h index 544bdd92..63927ccb 100644 --- a/src/experimental/platform/window.h +++ b/src/experimental/platform/window.h @@ -20,10 +20,9 @@ #include #include -#include "experimental/platform/renderer_backend.h" - #include #include +#include "experimental/platform/graphics_mode.h" namespace mujoco::platform { @@ -34,9 +33,8 @@ namespace mujoco::platform { class Window { public: struct Config { - RendererBackend renderer_backend = RendererBackend::ClassicOpenGl; + GraphicsMode gfx_mode = GraphicsMode::FilamentVulkan; bool load_fonts = true; - bool offscreen_mode = false; }; Window(std::string_view title, int width, int height, Config config); @@ -78,8 +76,8 @@ class Window { // the value will be cleared until the next time a file is dropped. std::string GetDropFile(); - // Returns true if the window is in offscreen mode. - bool IsOffscreenMode() const; + // Returns the graphics configuration of the window. + GraphicsMode GetGraphicsMode() const; // Enables window resizing. void EnableWindowResizing(); diff --git a/src/experimental/studio/app.cc b/src/experimental/studio/app.cc index 12bce034..8bfcd879 100644 --- a/src/experimental/studio/app.cc +++ b/src/experimental/studio/app.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include "experimental/platform/file_dialog.h" +#include "experimental/platform/graphics_mode.h" #include "experimental/platform/gui.h" #include "experimental/platform/gui_spec.h" #include "experimental/platform/helpers.h" @@ -111,14 +113,9 @@ static constexpr std::array kPercentRealTime = { }; // clang-format on -App::App(Config config) : ini_path_(std::move(config.ini_path)) { - platform::Window::Config window_config; - window_config.renderer_backend = platform::Renderer::GetBackend(); - window_config.offscreen_mode = config.offscreen_mode; - window_ = std::make_unique("MuJoCo Studio", config.width, - config.height, window_config); - renderer_ = - std::make_unique(window_->GetNativeWindowHandle()); +App::App(Config config) + : ini_path_(std::move(config.ini_path)), gfx_mode_(config.gfx_mode) { + SwitchGraphicsMode(config.width, config.height, config.gfx_mode); ImPlot::CreateContext(); mjv_defaultPerturb(&perturb_); @@ -128,6 +125,20 @@ App::App(Config config) : ini_path_(std::move(config.ini_path)) { profiler_.Clear(); } +void App::SwitchGraphicsMode(int width, int height, + platform::GraphicsMode mode) { + renderer_.reset(); + window_.reset(); + gfx_mode_ = mode; + + platform::Window::Config window_config; + window_config.gfx_mode = gfx_mode_; + window_ = std::make_unique("MuJoCo Studio", width, height, + window_config); + renderer_ = std::make_unique( + window_->GetNativeWindowHandle(), gfx_mode_); +} + void App::ClearModel() { model_holder_.reset(); window_->SetTitle("MuJoCo Studio"); @@ -361,8 +372,7 @@ void App::Render() { const float width = window_->GetWidth(); const float height = window_->GetHeight(); const float scale = window_->GetScale(); - - if (window_->IsOffscreenMode()) { + if (IsHeadless(window_->GetGraphicsMode())) { pixels_.resize(width * height * 3); } else { pixels_.clear(); @@ -615,14 +625,14 @@ void App::HandleKeyboardEvents() { return; } - constexpr auto ImGuiMode_CtrlShift = ImGuiMod_Ctrl | ImGuiMod_Shift; + constexpr auto ImGuiMod_CtrlShift = ImGuiMod_Ctrl | ImGuiMod_Shift; bool is_freecam_wasd = ui_.camera_idx == platform::kFreeCameraIdx; // Menu shortcuts. if (ImGui_IsChordJustPressed(ImGuiKey_O | ImGuiMod_Ctrl)) { tmp_.file_dialog = UiTempState::FileDialog_Load; - } else if (ImGui_IsChordJustPressed(ImGuiKey_S | ImGuiMode_CtrlShift)) { + } else if (ImGui_IsChordJustPressed(ImGuiKey_S | ImGuiMod_CtrlShift)) { tmp_.file_dialog = UiTempState::FileDialog_SaveMjb; } else if (ImGui_IsChordJustPressed(ImGuiKey_S | ImGuiMod_Ctrl)) { tmp_.file_dialog = UiTempState::FileDialog_SaveXml; @@ -1898,6 +1908,51 @@ void App::MainMenuGui() { if (ImGui::MenuItem("Picture-in-Picture")) { tmp_.picture_in_picture = !tmp_.picture_in_picture; } + ImGui::Separator(); + + + if (ImGui::BeginMenu("Graphics Mode (Experimental)")) { + std::optional mode; + if (ImGui::MenuItem( + "Classic OpenGL", nullptr, + gfx_mode_ == platform::GraphicsMode::ClassicOpenGl)) { + mode = platform::GraphicsMode::ClassicOpenGl; + } + if (ImGui::MenuItem( + "Classic OpenGL Headless", nullptr, + gfx_mode_ == platform::GraphicsMode::ClassicOpenGlHeadless)) { + mode = platform::GraphicsMode::ClassicOpenGlHeadless; + } + if (ImGui::MenuItem( + "Filament OpenGL", nullptr, + gfx_mode_ == platform::GraphicsMode::FilamentOpenGl)) { + mode = platform::GraphicsMode::FilamentOpenGl; + } + if (ImGui::MenuItem( + "Filament OpenGL Headless", nullptr, + gfx_mode_ == platform::GraphicsMode::FilamentOpenGlHeadless)) { + mode = platform::GraphicsMode::FilamentOpenGlHeadless; + } + if (ImGui::MenuItem( + "Filament Vulkan", nullptr, + gfx_mode_ == platform::GraphicsMode::FilamentVulkan)) { + mode = platform::GraphicsMode::FilamentVulkan; + } + if (mode.has_value()) { + spec_op_ = [=, this]() { + const int width = window_->GetWidth(); + const int height = window_->GetHeight(); + SwitchGraphicsMode(width, height, *mode); + // TODO: figure out why ImGui doesn't work unless we do this twice. + if (IsClassic(*mode)) { + SwitchGraphicsMode(width, height, *mode); + } + renderer_->Init(model()); + }; + } + ImGui::EndMenu(); + } + ImGui::EndMenu(); } diff --git a/src/experimental/studio/app.h b/src/experimental/studio/app.h index 64e548d0..c11080fb 100644 --- a/src/experimental/studio/app.h +++ b/src/experimental/studio/app.h @@ -26,6 +26,7 @@ #include #include +#include "experimental/platform/graphics_mode.h" #include "experimental/platform/gui.h" #include "experimental/platform/gui_spec.h" #include "experimental/platform/interaction.h" @@ -51,10 +52,8 @@ class App { // The path to the ini file containing the user settings. std::string ini_path; - // By default, we render directly to the window surface. However, in some - // cases, we may want to render to an (offscreen) texture and blit the - // texture to the window surface. - bool offscreen_mode = false; + // The graphics configuration used for initializing the window. + platform::GraphicsMode gfx_mode = platform::GraphicsMode::FilamentVulkan; }; explicit App(Config config); @@ -190,6 +189,8 @@ class App { // then compile the spec to a model. void OnModelLoaded(std::string filename, ModelKind model_kind); + void SwitchGraphicsMode(int width, int height, platform::GraphicsMode mode); + void SetLoadError(std::string error); void UpdateFilePaths(const std::string& resolved_path); @@ -243,6 +244,7 @@ class App { std::optional pending_load_; bool preserve_camera_on_load_ = false; ModelKind model_kind_ = kEmptyModel; + platform::GraphicsMode gfx_mode_ = platform::GraphicsMode::FilamentVulkan; std::unique_ptr window_; std::unique_ptr renderer_; diff --git a/src/experimental/studio/main.cc b/src/experimental/studio/main.cc index 579d400b..2b7bf227 100644 --- a/src/experimental/studio/main.cc +++ b/src/experimental/studio/main.cc @@ -26,12 +26,13 @@ #include #include +#include "experimental/platform/graphics_mode.h" #include "experimental/studio/app.h" ABSL_FLAG(int, window_width, 1400, "Window width"); ABSL_FLAG(int, window_height, 720, "Window height"); ABSL_FLAG(std::string, model_file, "", "MuJoCo model file."); -ABSL_FLAG(bool, offscreen_mode, false, "Offscreen mode"); +ABSL_FLAG(std::string, gfx, "", "Graphics API"); std::string Resolve(std::string_view path) { std::string_view subpath = path.substr(path.find(':') + 1); @@ -99,14 +100,31 @@ int main(int argc, char** argv, char** envp) { resource_provider.prefix = "filament"; mjp_registerResourceProvider(&resource_provider); + std::string gfx = absl::GetFlag(FLAGS_gfx); + + mujoco::platform::GraphicsMode gfx_mode = + mujoco::platform::GraphicsMode::FilamentVulkan; + if (gfx == "classic") { + gfx_mode = mujoco::platform::GraphicsMode::ClassicOpenGl; + } else if (gfx == "classic_headless") { + gfx_mode = mujoco::platform::GraphicsMode::ClassicOpenGlHeadless; + } else if (gfx == "opengl") { + gfx_mode = mujoco::platform::GraphicsMode::FilamentOpenGl; + } else if (gfx == "opengl_headless") { + gfx_mode = mujoco::platform::GraphicsMode::FilamentOpenGlHeadless; + } else if (gfx == "vulkan" || gfx.empty()) { + gfx_mode = mujoco::platform::GraphicsMode::FilamentVulkan; + } else { + mju_error("Unsupported graphics API: %s", gfx.c_str()); + } + const int width = absl::GetFlag(FLAGS_window_width); const int height = absl::GetFlag(FLAGS_window_height); - const bool offscreen_mode = absl::GetFlag(FLAGS_offscreen_mode); mujoco::studio::App app({ .width = width, .height = height, .ini_path = ini_path, - .offscreen_mode = offscreen_mode, + .gfx_mode = gfx_mode, }); // If the model file is not specified, try to load it from the first argument diff --git a/src/experimental/studio/wasm.cc b/src/experimental/studio/wasm.cc index 0769b78d..1a4738a7 100644 --- a/src/experimental/studio/wasm.cc +++ b/src/experimental/studio/wasm.cc @@ -25,6 +25,7 @@ #include #include +#include "experimental/platform/graphics_mode.h" #include "experimental/studio/app.h" // Global app instance. Lifetime is controlled by Init/Deinit calls which are @@ -95,6 +96,7 @@ void Init() { .width = width, .height = height, .ini_path = ini_path, + .gfx_mode = mujoco::platform::GraphicsMode::FilamentWebGl, }); g_app->InitEmptyModel(); }