Implement picture-in-picture mode.

Adds a new window that allows the user to display one or more renders from
the different cameras in the scene.

PiperOrigin-RevId: 856607089
Change-Id: I7268ffd3ef62b9b4d62ce8d004fdbbec92b42af4

src/experimental/platform/CMakeLists.txt

@@ -45,6 +45,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
         imgui_widgets.h
         interaction.cc
         interaction.h
+        picture_gui.h
+        picture_gui.cc
         renderer.cc
         renderer.h
         sim_history.cc

src/experimental/platform/imgui_widgets.h

@@ -16,10 +16,10 @@
 #define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_IMGUI_WIDGETS_H_
 
 #include <optional>
-#include <utility>
 #include <string>
 #include <type_traits>
 #include <unordered_map>
+#include <utility>
 
 #include <imgui.h>
 #include <imgui_internal.h>
@@ -45,6 +45,7 @@ static constexpr const char ICON_FA_REFRESH[] = "\xEF\x80\xA1";
 static constexpr const char ICON_FA_SQUARE_O[] = "\xEF\x87\x9B";
 static constexpr const char ICON_FA_SUN[] = "\xEF\x86\x85";
 static constexpr const char ICON_FA_TACHOMETER[] = "\xEF\x83\xA4";
+static constexpr const char ICON_FA_TRASH_CAN[] = "\xEF\x87\xB8";
 static constexpr const char ICON_FA_UNDO[] = "\xEF\x83\xA2";
 
 using KeyValues = std::unordered_map<std::string, std::string>;
@@ -86,14 +87,15 @@ T ReadIniValue(const KeyValues& key_values, const std::string& key, T def) {
 // styles when going out of scope.
 struct ScopedStyle {
   ScopedStyle() = default;
-  ~ScopedStyle() {
-    Reset();
-  }
+  ~ScopedStyle() { Reset(); }
 
   ScopedStyle(const ScopedStyle&) = delete;
   ScopedStyle& operator=(const ScopedStyle&) = delete;
   ScopedStyle(ScopedStyle&& other) { Swap(other); }
-  ScopedStyle& operator=(ScopedStyle&& other) { Swap(other); return *this; }
+  ScopedStyle& operator=(ScopedStyle&& other) {
+    Swap(other);
+    return *this;
+  }
 
   void Swap(ScopedStyle& other) {
     std::swap(num_colors, other.num_colors);

src/experimental/platform/picture_gui.cc

@@ -0,0 +1,122 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/platform/picture_gui.h"
+
+#include <cstddef>
+#include <vector>
+
+#include <imgui.h>
+#include <mujoco/mujoco.h>
+#include "experimental/platform/imgui_widgets.h"
+#include "experimental/platform/renderer.h"
+#include "experimental/platform/window.h"
+
+namespace mujoco::platform {
+
+// Returns false if the user requests that this picture-in-picture widget be
+// removed from the GUI.
+static bool PipGuiImpl(const mjModel* model, mjData* data,
+                       platform::Window* window, platform::Renderer* renderer,
+                       PipState* pip) {
+  bool result = true;
+
+  auto get_camera_name = [model](int i) -> const char* {
+    if (model->names[model->name_camadr[i]]) {
+      return model->names + model->name_camadr[i];
+    } else {
+      return "Unnamed";
+    }
+  };
+
+  const int width = ImGui::GetContentRegionAvail().x;
+  const int height = width / window->GetAspectRatio();
+  std::vector<std::byte> output(width * height * 3);
+
+  const int combo_width = (width-30) / 2;
+
+  ImGui::PushID(pip);
+  ImGui::SetNextItemWidth(combo_width);
+  if (ImGui::BeginCombo("##PipCamera", get_camera_name(pip->camera))) {
+    for (int i = 0; i < model->ncam; i++) {
+      if (ImGui::Selectable(get_camera_name(i), (pip->camera == i))) {
+        pip->camera = i;
+      }
+    }
+    ImGui::EndCombo();
+  }
+
+  mjvCamera camera;
+  mjv_defaultCamera(&camera);
+  camera.type = mjCAMERA_FIXED;
+  camera.fixedcamid = pip->camera;
+
+  const char* mode_names[] = {"Color", "Depth", "Segmentation"};
+  ImGui::SameLine();
+  ImGui::SetNextItemWidth(combo_width);
+  if (ImGui::BeginCombo("##PipMode", mode_names[pip->mode])) {
+    for (int i = 0; i < 3; i++) {
+      if (ImGui::Selectable(mode_names[i], (pip->mode == i))) {
+        pip->mode = static_cast<PipState::Mode>(i);
+      }
+    }
+    ImGui::EndCombo();
+  }
+
+  ImGui::SameLine();
+  if (ImGui::Button(ICON_FA_TRASH_CAN)) {
+    result = false;
+  }
+
+  mjtByte* flags = renderer->GetRenderFlags();
+  const int prev_depth = flags[mjRND_DEPTH];
+  const int prev_segment = flags[mjRND_SEGMENT];
+  flags[mjRND_DEPTH] = (pip->mode == PipState::Depth) ? 1 : 0;
+  flags[mjRND_SEGMENT] = (pip->mode == PipState::Segmentation) ? 1 : 0;
+  renderer->RenderToTexture(model, data, &camera, width, height, output.data());
+  pip->texture =
+      renderer->UploadImage(pip->texture, output.data(), width, height, 3);
+
+  // Restore previous render flags.
+  flags[mjRND_DEPTH] = prev_depth;
+  flags[mjRND_SEGMENT] = prev_segment;
+
+  ImGui::Image(pip->texture, {(float)width, (float)height});
+  ImGui::PopID();
+  return result;
+}
+
+void PipGui(const mjModel* model, mjData* data, platform::Window* window,
+            platform::Renderer* renderer, std::vector<PipState>* pips) {
+  if (pips->empty()) {
+    pips->emplace_back();
+  }
+
+  std::vector<int> to_delete;
+  for (int i = 0; i < pips->size(); ++i) {
+    PipState& pip = pips->at(i);
+    if (PipGuiImpl(model, data, window, renderer, &pip) == false) {
+      to_delete.push_back(i);
+    };
+    ImGui::Separator();
+  }
+  for (int i = to_delete.size() - 1; i >= 0; --i) {
+    pips->erase(pips->begin() + to_delete[i]);
+  }
+  if (ImGui::Button("+")) {
+    pips->emplace_back();
+  };
+}
+
+}  // namespace mujoco::platform

src/experimental/platform/picture_gui.h

@@ -0,0 +1,40 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_
+
+#include <vector>
+
+#include <mujoco/mujoco.h>
+#include "experimental/platform/renderer.h"
+#include "experimental/platform/window.h"
+
+namespace mujoco::platform {
+
+// Manages the state of a single picture-in-picture widget.
+struct PipState {
+  enum Mode { Color, Depth, Segmentation };
+  int camera = 0;
+  int texture = 0;
+  Mode mode = Color;
+};
+
+// Renders the GUI for a set of picture-in-picture widgets.
+void PipGui(const mjModel* model, mjData* data, platform::Window* window,
+            platform::Renderer* renderer, std::vector<PipState>* pips);
+
+}  // namespace mujoco::platform
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_PICTURE_GUI_H_

src/experimental/platform/renderer.cc

@@ -109,7 +109,7 @@ void Renderer::Render(const mjModel* model, mjData* data,
 void Renderer::RenderToTexture(const mjModel* model, mjData* data,
                                mjvCamera* camera, int width, int height,
                                std::byte* output) {
-  if (!initialized_ || last_update_time_ == -1) {
+  if (!initialized_) {
     return;
   }
 
@@ -122,6 +122,17 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data,
   mjr_setBuffer(mjFB_WINDOW, &render_context_);
 }
 
+int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
+                          int height, int bpp) {
+#if defined(USE_CLASSIC_OPENGL)
+  return 0;
+#else
+  return mjr_uploadGuiImage(texture_id,
+                            reinterpret_cast<const unsigned char*>(pixels),
+                            width, height, bpp, &render_context_);
+#endif
+}
+
 double Renderer::GetFps() { return fps_; }
 
 }  // namespace mujoco::platform

src/experimental/platform/renderer.h

@@ -52,6 +52,13 @@ class Renderer {
   void RenderToTexture(const mjModel* model, mjData* data, mjvCamera* camera,
                        int width, int height, std::byte* output);
 
+  // Uploads an image to the backend for GUI rendering, returning the texture
+  // ID for the texture. The ID can be used in subsequent calls to update the
+  // texture data. A nullptr pixels argument will free the texture if it exists.
+  // A texture ID of 0 will create a new texture.
+  int UploadImage(int texture_id, const std::byte* pixels, int width,
+                  int height, int bpp);
+
   // Rendering flags.
   mjtByte* GetRenderFlags() { return scene_.flags; }
 

src/experimental/studio/app.cc

@@ -36,6 +36,7 @@
 #include "experimental/platform/helpers.h"
 #include "experimental/platform/imgui_widgets.h"
 #include "experimental/platform/interaction.h"
+#include "experimental/platform/picture_gui.h"
 #include "experimental/platform/renderer.h"
 #include "experimental/platform/step_control.h"
 #include "experimental/platform/window.h"
@@ -788,6 +789,13 @@ void App::BuildGui() {
     ImGui::End();
   }
 
+  if (tmp_.picture_in_picture) {
+    if (ImGui::Begin("Picture-in-Picture", &tmp_.picture_in_picture)) {
+      PipGui(model_, data_, window_.get(), renderer_.get(), &tmp_.pips);
+    }
+    ImGui::End();
+  }
+
   if (tmp_.help) {
     platform::ScopedStyle style;
     style.Var(ImGuiStyleVar_Alpha, 0.6f);
@@ -1258,7 +1266,7 @@ void App::ToolBarGui() {
     if (ImGui::BeginCombo("##Camera", cameras[camera_idx], combo_flags)) {
       for (int n = 0; n < cameras.size(); n++) {
         if (ImGui::Selectable(cameras[n], (camera_idx == n))) {
-          ui_.camera_idx = ::mujoco::platform::SetCamera(
+          ui_.camera_idx = platform::SetCamera(
               model_, &camera_, camera_idx + platform::kTumbleCameraIdx);
         }
       }
@@ -1505,6 +1513,11 @@ void App::MainMenuGui() {
       if (ImGui::MenuItem("Full Screen", "F11")) {
         tmp_.full_screen = !tmp_.full_screen;
       }
+      ImGui::Separator();
+
+      if (ImGui::MenuItem("Picture-in-Picture")) {
+        tmp_.picture_in_picture = !tmp_.picture_in_picture;
+      }
       ImGui::EndMenu();
     }
     if (ImGui::BeginMenu("Charts")) {

src/experimental/studio/app.h

@@ -25,6 +25,7 @@
 #include "experimental/platform/gui.h"
 #include "experimental/platform/helpers.h"
 #include "experimental/platform/interaction.h"
+#include "experimental/platform/picture_gui.h"
 #include "experimental/platform/renderer.h"
 #include "experimental/platform/sim_history.h"
 #include "experimental/platform/sim_profiler.h"
@@ -84,6 +85,7 @@ class App {
     bool stats = false;
     bool chart_solver = false;
     bool chart_performance = false;
+    bool picture_in_picture = false;
     bool options_panel = true;
     bool inspector_panel = true;
     bool full_screen = false;
@@ -111,6 +113,9 @@ class App {
     int state_sig = 0;
     std::vector<mjtNum> state;
 
+    // Picture-in-Picture.
+    std::vector<platform::PipState> pips;
+
     // File dialogs.
     char filename[1000] = "";
     std::string last_load_file;