From ad482a57bb2241e2872a646cbd872c8ccdf2fd11 Mon Sep 17 00:00:00 2001
From: Haroon Qureshi <haroonq@google.com>
Date: Tue, 27 Jan 2026 04:06:59 -0800
Subject: [PATCH] Add a headless OpenGL renderer target.

PiperOrigin-RevId: 861642051
Change-Id: Ie0b880a2d3d85205dcd41bca5dc270efd35b7767
---
 src/experimental/platform/renderer.cc | 70 +++++++++++++++++++--------
 src/experimental/platform/renderer.h  |  9 ++--
 2 files changed, 57 insertions(+), 22 deletions(-)

diff --git a/src/experimental/platform/renderer.cc b/src/experimental/platform/renderer.cc
index c7ba2e3b..af01667f 100644
--- a/src/experimental/platform/renderer.cc
+++ b/src/experimental/platform/renderer.cc
@@ -15,6 +15,7 @@
 #include "experimental/platform/renderer.h"
 
 #include <cstddef>
+#include <span>
 
 #include <mujoco/mujoco.h>
 
@@ -74,7 +75,8 @@ void Renderer::Deinit() {
 
 void Renderer::Render(const mjModel* model, mjData* data,
                       const mjvPerturb* perturb, mjvCamera* camera,
-                      const mjvOption* vis_option, int width, int height) {
+                      const mjvOption* vis_option, int width, int height,
+                      std::span<std::byte> pixels) {
   if (!initialized_) {
     return;
   }
@@ -82,29 +84,43 @@ void Renderer::Render(const mjModel* model, mjData* data,
   mjv_updateScene(model, data, vis_option, perturb, camera, mjCAT_ALL,
                   &scene_);
 
+  const bool render_to_texture = !pixels.empty();
+  if (render_to_texture) {
+    // mjr_readPixels reads to a RGB buffer (i.e. 3 bytes per pixel).
+    if (pixels.size() != width * height * 3) {
+      mju_error("Offscreen mode requires a pixel buffer of size %d.",
+                width * height * 3);
+    }
+
+    #ifdef USE_CLASSIC_OPENGL
+      mjr_resizeOffscreen(width, height, &render_context_);
+      mjr_setBuffer(mjFB_OFFSCREEN, &render_context_);
+    #else
+      // The filament backend supports two offscreen framebuffers.
+      // mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the
+      // ImGui draw data.
+      mjr_setBuffer(mjFB_OFFSCREEN + 1, &render_context_);
+    #endif
+  }
+
   const mjrRect viewport = {0, 0, width, height};
   mjr_render(viewport, &scene_, &render_context_);
 
-#ifdef USE_CLASSIC_OPENGL
-  ImGui_ImplOpenGL3_NewFrame();
-  ImGui::Render();
-  ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
-#endif
+  // The filament backend knows how to renders the ImGui draw data. For the
+  // classic backend, we need to render the ImGui draw data ourselves.
+  #ifdef USE_CLASSIC_OPENGL
+    ImGui_ImplOpenGL3_NewFrame();
+    ImGui::Render();
+    ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
+  #endif
 
-#ifdef USE_CLASSIC_OPENGL
-  TimePoint now = std::chrono::steady_clock::now();
-  TimePoint::duration delta_time = now - last_fps_update_;
-  const double interval = std::chrono::duration<double>(delta_time).count();
-
-  ++frames_;
-  if (interval > 0.2) {  // only update FPS stat at most 5 times per second
-    last_fps_update_ = now;
-    fps_ = frames_ / interval;
-    frames_ = 0;
+  if (render_to_texture) {
+    unsigned char* ptr = reinterpret_cast<unsigned char*>(pixels.data());
+    mjr_readPixels(ptr, nullptr, viewport, &render_context_);
+    mjr_setBuffer(mjFB_WINDOW, &render_context_);
   }
-#else
-  fps_ = mjr_getFrameRate(&render_context_);
-#endif
+
+  UpdateFps();
 }
 
 void Renderer::RenderToTexture(const mjModel* model, mjData* data,
@@ -136,6 +152,22 @@ int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
 
 double Renderer::GetFps() { return fps_; }
 
+void Renderer::UpdateFps() {
+#ifdef USE_CLASSIC_OPENGL
+  TimePoint now = std::chrono::steady_clock::now();
+  TimePoint::duration delta_time = now - last_fps_update_;
+  const double interval = std::chrono::duration<double>(delta_time).count();
+  ++frames_;
+  if (interval > 0.2) {  // only update FPS stat at most 5 times per second
+    last_fps_update_ = now;
+    fps_ = frames_ / interval;
+    frames_ = 0;
+  }
+#else
+  fps_ = mjr_getFrameRate(&render_context_);
+#endif
+}
+
 }  // namespace mujoco::platform
 
 #if !defined(USE_CLASSIC_OPENGL)
diff --git a/src/experimental/platform/renderer.h b/src/experimental/platform/renderer.h
index e688533e..b3066c8d 100644
--- a/src/experimental/platform/renderer.h
+++ b/src/experimental/platform/renderer.h
@@ -18,6 +18,7 @@
 #include <chrono>
 #include <cstddef>
 #include <ratio>
+#include <span>
 
 #include <mujoco/mujoco.h>
 
@@ -41,11 +42,11 @@ class Renderer {
   // Initializes the renderer with the given mjModel.
   void Init(const mjModel* model);
 
-  // Renders the simulation state into the active window. Also renders the imgui
-  // state, but that is obtained directly from the ImGui library.
+  // Renders the simulation and ux state. Renders into `pixels` if provided,
+  // otherwise renders to the `native_window` provided at construction.
   void Render(const mjModel* model, mjData* data, const mjvPerturb* perturb,
               mjvCamera* camera, const mjvOption* vis_option, int width,
-              int height);
+              int height, std::span<std::byte> pixels = {});
 
   // Populates the given output buffer with RGB888 pixel data. The size of the
   // output buffer must be at least width * height * 3.
@@ -69,6 +70,8 @@ class Renderer {
   // Resets the renderer; no rendering will occur until Init() is called again.
   void Deinit();
 
+  void UpdateFps();
+
   void* native_window_ = nullptr;
   mjrContext render_context_;
   mjvScene scene_;