From c92746641d5ccced93a36d23df1996b673cf37a1 Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Thu, 15 Jan 2026 04:41:34 -0800 Subject: [PATCH] Implement picture-in-picture mode. Adds a new window that allows the user to display one or more renders from the different cameras in the scene. PiperOrigin-RevId: 856607089 Change-Id: I7268ffd3ef62b9b4d62ce8d004fdbbec92b42af4 --- src/experimental/platform/CMakeLists.txt | 2 + src/experimental/platform/imgui_widgets.h | 12 ++- src/experimental/platform/picture_gui.cc | 122 ++++++++++++++++++++++ src/experimental/platform/picture_gui.h | 40 +++++++ src/experimental/platform/renderer.cc | 13 ++- src/experimental/platform/renderer.h | 7 ++ src/experimental/studio/app.cc | 15 ++- src/experimental/studio/app.h | 5 + 8 files changed, 209 insertions(+), 7 deletions(-) create mode 100644 src/experimental/platform/picture_gui.cc create mode 100644 src/experimental/platform/picture_gui.h diff --git a/src/experimental/platform/CMakeLists.txt b/src/experimental/platform/CMakeLists.txt index 72dab8a4..f5eb81cb 100644 --- a/src/experimental/platform/CMakeLists.txt +++ b/src/experimental/platform/CMakeLists.txt @@ -45,6 +45,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME} imgui_widgets.h interaction.cc interaction.h + picture_gui.h + picture_gui.cc renderer.cc renderer.h sim_history.cc diff --git a/src/experimental/platform/imgui_widgets.h b/src/experimental/platform/imgui_widgets.h index 498fe0a6..40b8c47e 100644 --- a/src/experimental/platform/imgui_widgets.h +++ b/src/experimental/platform/imgui_widgets.h @@ -16,10 +16,10 @@ #define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_IMGUI_WIDGETS_H_ #include -#include #include #include #include +#include #include #include @@ -45,6 +45,7 @@ static constexpr const char ICON_FA_REFRESH[] = "\xEF\x80\xA1"; static constexpr const char ICON_FA_SQUARE_O[] = "\xEF\x87\x9B"; static constexpr const char ICON_FA_SUN[] = "\xEF\x86\x85"; static constexpr const char ICON_FA_TACHOMETER[] = "\xEF\x83\xA4"; +static constexpr const char ICON_FA_TRASH_CAN[] = "\xEF\x87\xB8"; static constexpr const char ICON_FA_UNDO[] = "\xEF\x83\xA2"; using KeyValues = std::unordered_map; @@ -86,14 +87,15 @@ T ReadIniValue(const KeyValues& key_values, const std::string& key, T def) { // styles when going out of scope. struct ScopedStyle { ScopedStyle() = default; - ~ScopedStyle() { - Reset(); - } + ~ScopedStyle() { Reset(); } ScopedStyle(const ScopedStyle&) = delete; ScopedStyle& operator=(const ScopedStyle&) = delete; ScopedStyle(ScopedStyle&& other) { Swap(other); } - ScopedStyle& operator=(ScopedStyle&& other) { Swap(other); return *this; } + ScopedStyle& operator=(ScopedStyle&& other) { + Swap(other); + return *this; + } void Swap(ScopedStyle& other) { std::swap(num_colors, other.num_colors); diff --git a/src/experimental/platform/picture_gui.cc b/src/experimental/platform/picture_gui.cc new file mode 100644 index 00000000..33f169af --- /dev/null +++ b/src/experimental/platform/picture_gui.cc @@ -0,0 +1,122 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/platform/picture_gui.h" + +#include +#include + +#include +#include +#include "experimental/platform/imgui_widgets.h" +#include "experimental/platform/renderer.h" +#include "experimental/platform/window.h" + +namespace mujoco::platform { + +// Returns false if the user requests that this picture-in-picture widget be +// removed from the GUI. +static bool PipGuiImpl(const mjModel* model, mjData* data, + platform::Window* window, platform::Renderer* renderer, + PipState* pip) { + bool result = true; + + auto get_camera_name = [model](int i) -> const char* { + if (model->names[model->name_camadr[i]]) { + return model->names + model->name_camadr[i]; + } else { + return "Unnamed"; + } + }; + + const int width = ImGui::GetContentRegionAvail().x; + const int height = width / window->GetAspectRatio(); + std::vector output(width * height * 3); + + const int combo_width = (width-30) / 2; + + ImGui::PushID(pip); + ImGui::SetNextItemWidth(combo_width); + if (ImGui::BeginCombo("##PipCamera", get_camera_name(pip->camera))) { + for (int i = 0; i < model->ncam; i++) { + if (ImGui::Selectable(get_camera_name(i), (pip->camera == i))) { + pip->camera = i; + } + } + ImGui::EndCombo(); + } + + mjvCamera camera; + mjv_defaultCamera(&camera); + camera.type = mjCAMERA_FIXED; + camera.fixedcamid = pip->camera; + + const char* mode_names[] = {"Color", "Depth", "Segmentation"}; + ImGui::SameLine(); + ImGui::SetNextItemWidth(combo_width); + if (ImGui::BeginCombo("##PipMode", mode_names[pip->mode])) { + for (int i = 0; i < 3; i++) { + if (ImGui::Selectable(mode_names[i], (pip->mode == i))) { + pip->mode = static_cast(i); + } + } + ImGui::EndCombo(); + } + + ImGui::SameLine(); + if (ImGui::Button(ICON_FA_TRASH_CAN)) { + result = false; + } + + mjtByte* flags = renderer->GetRenderFlags(); + const int prev_depth = flags[mjRND_DEPTH]; + const int prev_segment = flags[mjRND_SEGMENT]; + flags[mjRND_DEPTH] = (pip->mode == PipState::Depth) ? 1 : 0; + flags[mjRND_SEGMENT] = (pip->mode == PipState::Segmentation) ? 1 : 0; + renderer->RenderToTexture(model, data, &camera, width, height, output.data()); + pip->texture = + renderer->UploadImage(pip->texture, output.data(), width, height, 3); + + // Restore previous render flags. + flags[mjRND_DEPTH] = prev_depth; + flags[mjRND_SEGMENT] = prev_segment; + + ImGui::Image(pip->texture, {(float)width, (float)height}); + ImGui::PopID(); + return result; +} + +void PipGui(const mjModel* model, mjData* data, platform::Window* window, + platform::Renderer* renderer, std::vector* pips) { + if (pips->empty()) { + pips->emplace_back(); + } + + std::vector to_delete; + for (int i = 0; i < pips->size(); ++i) { + PipState& pip = pips->at(i); + if (PipGuiImpl(model, data, window, renderer, &pip) == false) { + to_delete.push_back(i); + }; + ImGui::Separator(); + } + for (int i = to_delete.size() - 1; i >= 0; --i) { + pips->erase(pips->begin() + to_delete[i]); + } + if (ImGui::Button("+")) { + pips->emplace_back(); + }; +} + +} // namespace mujoco::platform diff --git a/src/experimental/platform/picture_gui.h b/src/experimental/platform/picture_gui.h new file mode 100644 index 00000000..73abdb2b --- /dev/null +++ b/src/experimental/platform/picture_gui.h @@ -0,0 +1,40 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_ +#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_ + +#include + +#include +#include "experimental/platform/renderer.h" +#include "experimental/platform/window.h" + +namespace mujoco::platform { + +// Manages the state of a single picture-in-picture widget. +struct PipState { + enum Mode { Color, Depth, Segmentation }; + int camera = 0; + int texture = 0; + Mode mode = Color; +}; + +// Renders the GUI for a set of picture-in-picture widgets. +void PipGui(const mjModel* model, mjData* data, platform::Window* window, + platform::Renderer* renderer, std::vector* pips); + +} // namespace mujoco::platform + +#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_ diff --git a/src/experimental/platform/renderer.cc b/src/experimental/platform/renderer.cc index 0bb1569d..ed5a821b 100644 --- a/src/experimental/platform/renderer.cc +++ b/src/experimental/platform/renderer.cc @@ -109,7 +109,7 @@ void Renderer::Render(const mjModel* model, mjData* data, void Renderer::RenderToTexture(const mjModel* model, mjData* data, mjvCamera* camera, int width, int height, std::byte* output) { - if (!initialized_ || last_update_time_ == -1) { + if (!initialized_) { return; } @@ -122,6 +122,17 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data, mjr_setBuffer(mjFB_WINDOW, &render_context_); } +int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width, + int height, int bpp) { +#if defined(USE_CLASSIC_OPENGL) + return 0; +#else + return mjr_uploadGuiImage(texture_id, + reinterpret_cast(pixels), + width, height, bpp, &render_context_); +#endif +} + double Renderer::GetFps() { return fps_; } } // namespace mujoco::platform diff --git a/src/experimental/platform/renderer.h b/src/experimental/platform/renderer.h index 928cabe7..e688533e 100644 --- a/src/experimental/platform/renderer.h +++ b/src/experimental/platform/renderer.h @@ -52,6 +52,13 @@ class Renderer { void RenderToTexture(const mjModel* model, mjData* data, mjvCamera* camera, int width, int height, std::byte* output); + // Uploads an image to the backend for GUI rendering, returning the texture + // ID for the texture. The ID can be used in subsequent calls to update the + // texture data. A nullptr pixels argument will free the texture if it exists. + // A texture ID of 0 will create a new texture. + int UploadImage(int texture_id, const std::byte* pixels, int width, + int height, int bpp); + // Rendering flags. mjtByte* GetRenderFlags() { return scene_.flags; } diff --git a/src/experimental/studio/app.cc b/src/experimental/studio/app.cc index 6d9f0082..c5aa81b1 100644 --- a/src/experimental/studio/app.cc +++ b/src/experimental/studio/app.cc @@ -36,6 +36,7 @@ #include "experimental/platform/helpers.h" #include "experimental/platform/imgui_widgets.h" #include "experimental/platform/interaction.h" +#include "experimental/platform/picture_gui.h" #include "experimental/platform/renderer.h" #include "experimental/platform/step_control.h" #include "experimental/platform/window.h" @@ -788,6 +789,13 @@ void App::BuildGui() { ImGui::End(); } + if (tmp_.picture_in_picture) { + if (ImGui::Begin("Picture-in-Picture", &tmp_.picture_in_picture)) { + PipGui(model_, data_, window_.get(), renderer_.get(), &tmp_.pips); + } + ImGui::End(); + } + if (tmp_.help) { platform::ScopedStyle style; style.Var(ImGuiStyleVar_Alpha, 0.6f); @@ -1258,7 +1266,7 @@ void App::ToolBarGui() { if (ImGui::BeginCombo("##Camera", cameras[camera_idx], combo_flags)) { for (int n = 0; n < cameras.size(); n++) { if (ImGui::Selectable(cameras[n], (camera_idx == n))) { - ui_.camera_idx = ::mujoco::platform::SetCamera( + ui_.camera_idx = platform::SetCamera( model_, &camera_, camera_idx + platform::kTumbleCameraIdx); } } @@ -1505,6 +1513,11 @@ void App::MainMenuGui() { if (ImGui::MenuItem("Full Screen", "F11")) { tmp_.full_screen = !tmp_.full_screen; } + ImGui::Separator(); + + if (ImGui::MenuItem("Picture-in-Picture")) { + tmp_.picture_in_picture = !tmp_.picture_in_picture; + } ImGui::EndMenu(); } if (ImGui::BeginMenu("Charts")) { diff --git a/src/experimental/studio/app.h b/src/experimental/studio/app.h index 42a7ce0b..aeda3e3f 100644 --- a/src/experimental/studio/app.h +++ b/src/experimental/studio/app.h @@ -25,6 +25,7 @@ #include "experimental/platform/gui.h" #include "experimental/platform/helpers.h" #include "experimental/platform/interaction.h" +#include "experimental/platform/picture_gui.h" #include "experimental/platform/renderer.h" #include "experimental/platform/sim_history.h" #include "experimental/platform/sim_profiler.h" @@ -84,6 +85,7 @@ class App { bool stats = false; bool chart_solver = false; bool chart_performance = false; + bool picture_in_picture = false; bool options_panel = true; bool inspector_panel = true; bool full_screen = false; @@ -111,6 +113,9 @@ class App { int state_sig = 0; std::vector state; + // Picture-in-Picture. + std::vector pips; + // File dialogs. char filename[1000] = ""; std::string last_load_file;