Merge branch 'main' into newton-schemas

src/experimental/filament/CMakeLists.txt

@@ -23,46 +23,44 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME}
     PUBLIC
         render_context_filament.h
         render_context_filament.cc
-        filament/buffer_util.cc
-        filament/buffer_util.h
+        render_context_filament_cpp.h
+        filament_util.h
+        filament_util.cc
         filament/builtins.cc
         filament/builtins.h
         filament/color_grading_options.cc
         filament/color_grading_options.h
-        filament/drawable.cc
-        filament/drawable.h
         filament/filament_context.cc
         filament/filament_context.h
         filament/filament_platform_factory.cc
         filament/filament_platform_factory.h
-        filament/geom_util.cc
-        filament/geom_util.h
-        filament/gui_view.cc
-        filament/gui_view.h
         filament/imgui_editor.cc
-        filament/imgui_editor.h
         filament/light.cc
         filament/light.h
         filament/material.cc
         filament/material.h
-        filament/math_util.cc
-        filament/math_util.h
-        filament/model_objects.cc
-        filament/model_objects.h
-        filament/model_util.cc
-        filament/model_util.h
+        filament/mesh.cc
+        filament/mesh.h
         filament/object_manager.cc
         filament/object_manager.h
-        filament/render_target_util.cc
-        filament/render_target_util.h
-        filament/renderables.cc
-        filament/renderables.h
+        filament/render_target.cc
+        filament/render_target.h
+        filament/renderable.cc
+        filament/renderable.h
         filament/scene_view.cc
         filament/scene_view.h
-        filament/texture_util.cc
-        filament/texture_util.h
-        filament/vertex_util.cc
-        filament/vertex_util.h
+        filament/texture.cc
+        filament/texture.h
+        compat/imgui_bridge.cc
+        compat/imgui_bridge.h
+        compat/mjr_filament_renderer.cc
+        compat/mjr_filament_renderer.h
+        compat/model_objects.cc
+        compat/model_objects.h
+        compat/scene_bridge.cc
+        compat/scene_bridge.h
+        compat/scene_geom_util.cc
+        compat/scene_geom_util.h
 )
 if(MUJOCO_USE_FILAMENT_MJR_COMPAT)
     target_sources(${MUJOCO_FILAMENT_TARGET_NAME}
@@ -117,8 +115,8 @@ set(MATERIAL_FILES
     phong_cube_fade.mat
     phong_cube.mat
     phong_cube_reflect.mat
+    unlit_decor.mat
     unlit_depth.mat
-    unlit_line.mat
     unlit_segmentation.mat
     unlit_ui.mat
 )
@@ -128,20 +126,32 @@ foreach(MATERIAL_FILE ${MATERIAL_FILES})
     set(INPUT_FILE "${ASSETS_DIR}/${MATERIAL_FILE}")
     set(OUTPUT_FILE "${OUTPUT_ASSETS_DIR}/${MATERIAL_NAME}.filamat")
 
-    add_custom_command(
-        OUTPUT ${OUTPUT_FILE}
-        COMMAND ${MATC_EXECUTABLE}
-            --platform=all
-            --api=vulkan
-            --api=opengl
-            --variant-filter skinning
-            --optimize-size
-            --output ${OUTPUT_FILE}
-            ${INPUT_FILE}
-        DEPENDS ${INPUT_FILE}
-        DEPENDS matc
-        COMMENT "Compiling ${MATERIAL_FILE}"
-    )
+    if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
+        set(PRECOMPILED_FILE "${MUJOCO_NATIVE_BUILD_DIR}/src/experimental/filament/assets/${MATERIAL_NAME}.filamat")
+        add_custom_command(
+            OUTPUT ${OUTPUT_FILE}
+            COMMAND ${CMAKE_COMMAND} -E copy
+                ${PRECOMPILED_FILE}
+                ${OUTPUT_FILE}
+            DEPENDS ${PRECOMPILED_FILE}
+            COMMENT "Copying precompiled material ${MATERIAL_NAME}.filamat"
+        )
+    else()
+        add_custom_command(
+            OUTPUT ${OUTPUT_FILE}
+            COMMAND ${MATC_EXECUTABLE}
+                --platform=all
+                --api=vulkan
+                --api=opengl
+                --variant-filter skinning
+                --optimize-size
+                --output ${OUTPUT_FILE}
+                ${INPUT_FILE}
+            DEPENDS ${INPUT_FILE}
+            DEPENDS matc
+            COMMENT "Compiling ${MATERIAL_FILE}"
+        )
+    endif()
 
     list(APPEND MUJOCO_FILAMENT_ASSET_FILES ${OUTPUT_FILE})
 endforeach()

src/experimental/filament/assets/pbr.mat

@@ -15,7 +15,6 @@
 material {
     name : pbr,
     shadingModel : lit,
-    culling: none,
     flipUV: false,
     parameters : [
         { type : sampler2d, name : BaseColor },

src/experimental/filament/assets/pbr_packed.mat

@@ -15,7 +15,6 @@
 material {
     name : pbr_packed,
     shadingModel : lit,
-    culling: none,
     flipUV: false,
     parameters : [
         { type : sampler2d, name : BaseColor },

src/experimental/filament/assets/phong_2d.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_2d_fade.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d_fade,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     blending: fade,
     parameters : [

src/experimental/filament/assets/phong_2d_reflect.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d_reflect,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_2d_uv.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d_uv,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_2d_uv_fade.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d_uv_fade,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     blending: fade,
     parameters : [

src/experimental/filament/assets/phong_2d_uv_reflect.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_2d_uv_reflect,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_color.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_color,
     shadingModel : specularGlossiness,
-    culling: none,
     parameters : [
         { type : float4, name : BaseColorFactor },
         { type : float, name : SpecularFactor },

src/experimental/filament/assets/phong_color_fade.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_color_fade,
     shadingModel : specularGlossiness,
-    culling: none,
     blending: fade,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_color_reflect.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_color_reflect,
     shadingModel : specularGlossiness,
-    culling: none,
     parameters : [
         { type : float4, name : BaseColorFactor },
         { type : float, name : SpecularFactor },

src/experimental/filament/assets/phong_cube.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_cube,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/phong_cube_fade.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_cube_fade,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     blending: fade,
     parameters : [

src/experimental/filament/assets/phong_cube_reflect.mat

@@ -15,7 +15,6 @@
 material {
     name : phong_cube_reflect,
     shadingModel : specularGlossiness,
-    culling : none,
     flipUV : false,
     parameters : [
         { type : float4, name : BaseColorFactor },

src/experimental/filament/assets/unlit_decor.mat

@@ -13,9 +13,8 @@
 // limitations under the License.
 
 material {
-    name : unlit_segmentation,
+    name : unlit_decor,
     shadingModel : unlit,
-    culling: none,
     parameters : [
         { type : float4, name : BaseColorFactor }
     ]

src/experimental/filament/assets/unlit_depth.mat

@@ -16,7 +16,6 @@ material {
     name : unlit_depth,
     shadingModel : unlit,
     blending : opaque,
-    culling: none,
     depthWrite: true
 }
 

src/experimental/filament/assets/unlit_segmentation.mat

@@ -15,15 +15,14 @@
 material {
     name : unlit_segmentation,
     shadingModel : unlit,
-    culling: none,
     parameters : [
-        { type : float4, name : BaseColorFactor }
+        { type : float4, name : SegmentationColor }
     ]
 }
 
 fragment {
     void material(inout MaterialInputs material) {
       prepareMaterial(material);
-      material.baseColor = materialParams.BaseColorFactor;
+      material.baseColor = materialParams.SegmentationColor;
     }
 }

src/experimental/filament/assets/unlit_ui.mat

@@ -17,7 +17,7 @@ material {
     parameters : [
         {
             type : sampler2d,
-            name : glyph
+            name : BaseColor
         }
     ],
     requires : [
@@ -27,8 +27,7 @@ material {
     shadingModel : unlit,
     culling : none,
     depthCulling: false,
-    blending : transparent,
-    featureLevel : 0
+    blending : transparent
 }
 
 fragment {
@@ -36,7 +35,7 @@ fragment {
         prepareMaterial(material);
         vec2 uv = getUV0();
         uv.y = 1.0 - uv.y;
-        vec4 tex_color = texture2D(materialParams_glyph, uv);
+        vec4 tex_color = texture(materialParams_BaseColor, uv);
         material.baseColor = getColor() * tex_color;
         material.baseColor.rgb *= material.baseColor.a;
     }

src/experimental/filament/compat/imgui_bridge.cc

@@ -0,0 +1,358 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/compat/imgui_bridge.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include <imgui.h>
+#include <math/mat3.h>
+#include <math/vec3.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+
+namespace mujoco {
+
+using filament::math::float3;
+using filament::math::mat3f;
+
+ImguiBridge::ImguiBridge(mjrfContext* ctx) : ctx_(ctx) {
+  mjrSceneParams params;
+  mjr_defaultSceneParams(&params);
+  params.enable_post_processing = false;
+  params.enable_reflections = false;
+  params.enable_shadows = false;
+  scene_ = CreateScene(ctx_, params);
+}
+
+ImguiBridge::~ImguiBridge() {
+  PrepareRenderables(0);
+
+  // Destroy all textures tracked by ImGui.
+  if (ImGui::GetCurrentContext()) {
+    for (ImTextureData* tex : ImGui::GetPlatformIO().Textures) {
+      if (tex->Status != ImTextureStatus_Destroyed) {
+        DestroyTexture(tex);
+      }
+    }
+  }
+}
+
+uintptr_t ImguiBridge::UploadImage(uintptr_t tex_id, const uint8_t* pixels,
+                                   int width, int height, int bpp) {
+  if (bpp != 4 && bpp != 3) {
+    mju_error("Unsupported image bpp. Got %d, wanted 3 or 4", bpp);
+  }
+
+  if (pixels == nullptr) {
+    // If the pixels are nullptr, we destroy the texture.
+    if (tex_id != 0) {
+      textures_.erase(tex_id);
+    }
+    return 0;
+  }
+
+  // Assign a new texture ID.
+  if (tex_id == 0) {
+    tex_id = next_tex_id_++;
+  }
+
+  mjrTexture* texture = GetTexture(tex_id);
+
+  // If the texture does not exist or the dimensions have changed, we create a
+  // new texture.
+  if (texture == nullptr || mjrf_getTextureWidth(texture) != width ||
+      mjrf_getTextureHeight(texture) != height) {
+    mjrTextureConfig config;
+    mjr_defaultTextureConfig(&config);
+    config.width = width;
+    config.height = height;
+    config.sampler_type = mjTEXTURE_2D;
+    config.format = bpp == 4 ? mjPIXEL_FORMAT_RGBA8 : mjPIXEL_FORMAT_RGB8;
+    config.color_space = mjCOLORSPACE_LINEAR;
+    UniquePtr<mjrTexture> new_texture = ::mujoco::CreateTexture(ctx_, config);
+    texture = new_texture.get();
+    textures_.insert_or_assign(tex_id, std::move(new_texture));
+  }
+
+  // Create a copy of the image to pass it to filament as we don't know the
+  // lifetime of the data.
+  const size_t num_bytes = width * height * bpp;
+  std::byte* bytes = new std::byte[num_bytes];
+  const auto callback =
+      +[](void* user) { delete[] reinterpret_cast<std::byte*>(user); };
+
+  mjrTextureData texture_data;
+  mjr_defaultTextureData(&texture_data);
+  texture_data.bytes = bytes;
+  texture_data.nbytes = num_bytes;
+  texture_data.user_data = bytes;
+  texture_data.release_callback = callback;
+
+  std::memcpy(bytes, pixels, num_bytes);
+  mjrf_setTextureData(texture, &texture_data);
+  return tex_id;
+}
+
+void ImguiBridge::CreateTexture(ImTextureData* data) {
+  if (data->Format != ImTextureFormat_RGBA32) {
+    mju_error("Unsupported texture format.");
+  }
+
+  mjrTextureConfig config;
+  mjr_defaultTextureConfig(&config);
+  config.width = data->Width;
+  config.height = data->Height;
+  config.sampler_type = mjTEXTURE_2D;
+  config.format = mjPIXEL_FORMAT_RGBA8;
+  config.color_space = mjCOLORSPACE_LINEAR;
+
+  const uintptr_t tex_id = next_tex_id_++;
+  textures_.insert_or_assign(tex_id, ::mujoco::CreateTexture(ctx_, config));
+  data->SetTexID((ImTextureID)tex_id);
+  UpdateTexture(data);
+}
+
+void ImguiBridge::UpdateTexture(ImTextureData* data) {
+  auto iter = textures_.find(data->TexID);
+  if (iter == textures_.end()) {
+    mju_error("Texture not found: %llu", data->TexID);
+  }
+
+  mjrTextureData texture_data;
+  mjr_defaultTextureData(&texture_data);
+  texture_data.bytes = data->GetPixels();
+  texture_data.nbytes = data->Width * data->Height * 4;
+  texture_data.user_data = nullptr;
+  texture_data.release_callback = nullptr;
+  mjrf_setTextureData(iter->second.get(), &texture_data);
+  data->SetStatus(ImTextureStatus_OK);
+}
+
+void ImguiBridge::DestroyTexture(ImTextureData* data) {
+  auto iter = textures_.find(data->TexID);
+  if (iter != textures_.end()) {
+    textures_.erase(data->TexID);
+    data->SetTexID(ImTextureID_Invalid);
+    data->SetStatus(ImTextureStatus_Destroyed);
+  }
+}
+
+mjrTexture* ImguiBridge::GetTexture(uintptr_t tex_id) const {
+  auto iter = textures_.find(tex_id);
+  if (iter == textures_.end()) {
+    return nullptr;
+  }
+  return iter->second.get();
+}
+
+void ImguiBridge::Update() {
+  if (!ImGui::GetCurrentContext()) {
+    PrepareRenderables(0);
+    return;
+  }
+
+  // Prepare the imgui draw commands. We must call this function even if we do
+  // not plan on rendering anything to ensure imgui state is updated.
+  ImGui::Render();
+
+  ImGuiIO& io = ImGui::GetIO();
+  const ImVec2& size = io.DisplaySize;
+  const ImVec2& scale = io.DisplayFramebufferScale;
+  ImDrawData* commands = ImGui::GetDrawData();
+  if (!commands || size.x == 0 || size.y == 0) {
+    PrepareRenderables(0);
+    return;
+  }
+  commands->ScaleClipRects(scale);
+
+  // 2 floats for position, 2 floats for uv, 4 bytes for color.
+  constexpr size_t kExpectedVertexSize =
+      sizeof(float) * 4 + sizeof(uint8_t) * 4;
+
+  int num_elements = 0;
+  for (int n = 0; n < commands->CmdListsCount; ++n) {
+    const ImDrawList* cmds = commands->CmdLists[n];
+    if (kExpectedVertexSize != sizeof(cmds->VtxBuffer.Data[0])) {
+      mju_error("Invalid vertex buffer size.");
+    }
+    if (sizeof(uint16_t) != sizeof(cmds->IdxBuffer.Data[0])) {
+      mju_error("Invalid index buffer size.");
+    }
+    num_elements += cmds->CmdBuffer.size();
+  }
+
+  if (commands->Textures != nullptr) {
+    for (ImTextureData* tex : *commands->Textures) {
+      if (tex->Status == ImTextureStatus_WantCreate) {
+        CreateTexture(tex);
+      } else if (tex->Status == ImTextureStatus_WantUpdates) {
+        UpdateTexture(tex);
+      } else if (tex->Status == ImTextureStatus_WantDestroy &&
+                 tex->UnusedFrames > 0) {
+        DestroyTexture(tex);
+      }
+    }
+  }
+
+  PrepareRenderables(num_elements);
+  if (num_elements == 0) {
+    return;
+  }
+
+  meshes_.clear();
+  int renderable_index = 0;
+  for (int n = 0; n < commands->CmdListsCount; ++n) {
+    const ImDrawList* cmds = commands->CmdLists[n];
+
+    mjrMeshData data;
+    mjr_defaultMeshData(&data);
+    data.nattributes = 3;
+    data.attributes[0].usage = mjVERTEX_ATTRIBUTE_USAGE_POSITION;
+    data.attributes[0].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT2;
+    data.attributes[0].bytes = cmds->VtxBuffer.Data;
+    data.attributes[1].usage = mjVERTEX_ATTRIBUTE_USAGE_UV;
+    data.attributes[1].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT2;
+    data.attributes[1].bytes = cmds->VtxBuffer.Data + sizeof(float) * 2;
+    data.attributes[2].usage = mjVERTEX_ATTRIBUTE_USAGE_COLOR;
+    data.attributes[2].type = mjVERTEX_ATTRIBUTE_TYPE_UBYTE4;
+    data.attributes[2].bytes = cmds->VtxBuffer.Data + sizeof(float) * 4;
+    data.interleaved = true;
+    data.nvertices = cmds->VtxBuffer.Size;
+    data.nindices = cmds->IdxBuffer.Size;
+    data.indices = cmds->IdxBuffer.Data;
+    data.index_type = mjINDEX_TYPE_U16;
+    data.primitive_type = mjMESH_PRIMITIVE_TYPE_TRIANGLES;
+    meshes_.push_back(CreateMesh(ctx_, data));
+
+    const mjrMesh* mesh = meshes_.back().get();
+
+    int index_offset = 0;
+    for (const ImDrawCmd& command : cmds->CmdBuffer) {
+      const int width = size.x * scale.x;
+      const int height = size.y * scale.y;
+
+      UniquePtr<mjrRenderable>& renderable = renderables_[renderable_index];
+      mjrf_setRenderableMesh(renderable.get(), mesh, index_offset,
+                             command.ElemCount);
+
+      mjrMaterial material;
+      mjr_defaultMaterial(&material);
+      material.color_texture = GetTexture(command.GetTexID());
+
+      material.decor_ux = true;
+      material.scissor[0] = command.ClipRect.x;
+      material.scissor[1] = height - command.ClipRect.w;
+      material.scissor[2] = command.ClipRect.z - command.ClipRect.x;
+      material.scissor[3] = command.ClipRect.w - command.ClipRect.y;
+      // Modal dialogs try to cover the whole window, but also a little outside
+      // of it. This doesn't work well with filament's scissor test, so we clip
+      // them to the window.
+      if (material.scissor[0] < 0 || material.scissor[1] < 0) {
+        material.scissor[0] = 0;
+        material.scissor[1] = 0;
+        material.scissor[2] = width;
+        material.scissor[3] = height;
+      }
+      mjrf_setRenderableMaterial(renderable.get(), &material);
+
+      const float position[] = {0, 0, 0};
+      const float rotation[] = {1, 0, 0, 0, 1, 0, 0, 0, 1};
+      const float size[] = {scale.x, scale.y, 1.0f};
+      mjrf_setRenderableTransform(renderable.get(), position, rotation, size);
+
+      index_offset += command.ElemCount;
+      ++renderable_index;
+    }
+  }
+}
+
+void ImguiBridge::PrepareRenderables(int count) {
+  while (renderables_.size() < count) {
+    mjrRenderableParams params;
+    mjr_defaultRenderableParams(&params);
+    params.cast_shadows = false;
+    params.receive_shadows = false;
+    params.blend_order = static_cast<std::uint16_t>(renderables_.size() + 1);
+    auto& renderable = renderables_.emplace_back(CreateRenderable(ctx_, params));
+    mjrf_addRenderableToScene(scene_.get(), renderable.get());
+  }
+  while (renderables_.size() > count) {
+    mjrf_removeRenderableFromScene(scene_.get(), renderables_.back().get());
+    renderables_.pop_back();
+  }
+}
+
+mjrScene* ImguiBridge::GetScene() const { return scene_.get(); }
+
+mjrCamera ImguiBridge::GetCamera(int width, int height) const {
+  mjrCamera camera;
+  camera.orthographic = true;
+  camera.pos[0] = 0.0f;
+  camera.pos[1] = 0.0f;
+  camera.pos[2] = 1.0f;
+  camera.forward[0] = 0.0f;
+  camera.forward[1] = 0.0f;
+  camera.forward[2] = -1.0f;
+  camera.up[0] = 0.0f;
+  camera.up[1] = 1.0f;
+  camera.up[2] = 0.0f;
+  camera.frustum_top = 0.0f;
+  camera.frustum_near = 0.0f;
+  camera.frustum_far = 1.0f;
+  camera.frustum_center = width / 2.0f;
+  camera.frustum_width = width / 2.0f;
+  camera.frustum_bottom = height;
+  return camera;
+}
+
+static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) {
+  const ImVec2& display_size = ImGui::GetIO().DisplaySize;
+  const float pos_x = display_size.x * ((x + 1) * 0.5f);
+  const float pos_y = display_size.y * (1.0f - ((y + 1) * 0.5f));
+  return ImVec2(pos_x, pos_y);
+}
+
+void DrawTextAt(const char* text, float x, float y, float z) {
+  if (x < -1 || y < -1 || x > 1 || y > 1 || z < -1 || z > 1) {
+    return;
+  }
+
+  const ImVec2 center_pos = ClipSpaceToWindowCoordinates(x, y);
+  const ImVec2 size = ImGui::CalcTextSize(text);
+
+  const ImVec2 pos = ImVec2(center_pos.x - size.x / 2, center_pos.y);
+  const ImVec2 shadow_pos = ImVec2(pos.x + 2, pos.y + 2);
+  const int flags = ImGuiWindowFlags_NoBringToFrontOnFocus |
+                    ImGuiWindowFlags_NoFocusOnAppearing |
+                    ImGuiWindowFlags_NoBackground |
+                    ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoInputs |
+                    ImGuiWindowFlags_NoNav;
+
+  ImGui::Begin("labels", nullptr, flags);
+  ImGui::BeginChild("labels", ImGui::GetIO().DisplaySize, 0, flags);
+  ImDrawList* draw_list = ImGui::GetWindowDrawList();
+  draw_list->AddText(shadow_pos, IM_COL32_BLACK, text);
+  draw_list->AddText(pos, IM_COL32_WHITE, text);
+  ImGui::EndChild();
+  ImGui::End();
+}
+
+}  // namespace mujoco

src/experimental/filament/compat/imgui_bridge.h

@@ -0,0 +1,74 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_IMGUI_BRIDGE_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_IMGUI_BRIDGE_H_
+
+#include <cstdint>
+#include <unordered_map>
+#include <vector>
+
+#include <imgui.h>
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+
+namespace mujoco {
+
+// Creates and manages a SceneView using data read from ImGui.
+class ImguiBridge {
+ public:
+  explicit ImguiBridge(mjrfContext* ctx);
+  ~ImguiBridge();
+
+  // Prepares the Renderables using data from the current ImGui state. This
+  // function must be called once per frame to ensure ImGui state is correctly
+  // synced.
+  void Update();
+
+  // Returns the managed UX scene.
+  mjrScene* GetScene() const;
+  mjrCamera GetCamera(int width, int height) const;
+
+  // Uploads texture to be used with ImGui's Image and ImageButton functions.
+  uintptr_t UploadImage(uintptr_t tex_id, const uint8_t* pixels, int width,
+                        int height, int bpp);
+
+  ImguiBridge(const ImguiBridge&) = delete;
+  ImguiBridge& operator=(const ImguiBridge&) = delete;
+
+ private:
+  // Ensures exactly `count` Renderables exist, creating or destroying them as
+  // needed.
+  void PrepareRenderables(int count);
+
+  void CreateTexture(ImTextureData* data);
+  void UpdateTexture(ImTextureData* data);
+  void DestroyTexture(ImTextureData* data);
+  mjrTexture* GetTexture(uintptr_t tex_id) const;
+
+  mjrfContext* ctx_ = nullptr;
+  UniquePtr<mjrScene> scene_{nullptr, nullptr};
+  std::vector<UniquePtr<mjrRenderable>> renderables_;
+  std::vector<UniquePtr<mjrMesh>> meshes_;
+  std::unordered_map<uintptr_t, UniquePtr<mjrTexture>> textures_;
+  uintptr_t next_tex_id_ = 1;
+};
+
+// Draws text at the given screen coordinates in clip space (i.e. [-1,-1,-1] to
+// [1,1,1]).
+void DrawTextAt(const char* text, float x, float y, float z);
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_IMGUI_BRIDGE_H_

src/experimental/filament/compat/mjr_filament_renderer.cc

@@ -0,0 +1,191 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/compat/mjr_filament_renderer.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+#include <mujoco/mjmodel.h>
+#include <mujoco/mjvisualize.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/compat/imgui_bridge.h"
+#include "experimental/filament/compat/scene_bridge.h"
+#include "experimental/filament/filament/filament_context.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+MjrFilamentRenderer::MjrFilamentRenderer(const mjrFilamentConfig* config) {
+  filament_context_ = std::make_unique<FilamentContext>(config);
+}
+
+void MjrFilamentRenderer::Init(const mjModel* model) {
+  scene_bridge_ = std::make_unique<SceneBridge>(filament_context_.get(), model);
+  imgui_bridge_ = std::make_unique<ImguiBridge>(filament_context_.get());
+  scene_bridge_->SetDrawTextFunction(DrawTextAt);
+}
+
+void MjrFilamentRenderer::Render(const mjrRect& viewport,
+                                 const mjvScene* scene) {
+  scene_bridge_->Update(viewport, scene);
+  // Update the UX renderable entity after processing the scene in case there
+  // are any elements in the scene which generate UX draw calls (e.g. labels).
+  if (mode_ != FrameBufferMode::OffScreen) {
+    imgui_bridge_->Update();
+  }
+
+  if (mode_ == FrameBufferMode::Window) {
+    mjrRenderRequest reqs[2];
+    mjr_defaultRenderRequest(&reqs[0]);
+    reqs[0].scene = scene_bridge_->GetScene();
+    reqs[0].draw_mode = scene_bridge_->GetDrawMode();
+    reqs[0].camera = scene_bridge_->GetCamera();
+    reqs[0].viewport = viewport;
+
+    mjr_defaultRenderRequest(&reqs[1]);
+    reqs[1].scene = imgui_bridge_->GetScene();
+    reqs[1].draw_mode = mjDRAW_MODE_COLOR;
+    reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
+    reqs[1].viewport = viewport;
+    filament_context_->Render(reqs);
+  }
+}
+
+void MjrFilamentRenderer::SetFrameBuffer(int framebuffer) {
+  switch (framebuffer) {
+    case mjFB_WINDOW:
+      mode_ = FrameBufferMode::Window;
+      break;
+    case mjFB_OFFSCREEN:
+      mode_ = FrameBufferMode::OffScreen;
+      break;
+    case 2:  // No official constant fo this.
+      mode_ = FrameBufferMode::OffScreenWithGui;
+      break;
+    default:
+      mju_error("Invalid framebuffer mode: %d", framebuffer);
+  }
+}
+
+void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb,
+                                     float* depth) {
+  if (mode_ == FrameBufferMode::Window) {
+    mju_error("ReadPixels is only supported for offscreen rendering.");
+  }
+
+  mjrRenderRequest reqs[2];
+  mjr_defaultRenderRequest(&reqs[0]);
+
+  reqs[0].scene = scene_bridge_->GetScene();
+  reqs[0].draw_mode = scene_bridge_->GetDrawMode();
+  reqs[0].camera = scene_bridge_->GetCamera();
+  reqs[0].viewport = viewport;
+
+  mjr_defaultRenderRequest(&reqs[1]);
+  reqs[1].scene = imgui_bridge_->GetScene();
+  reqs[1].draw_mode = mjDRAW_MODE_COLOR;
+  reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
+  reqs[1].viewport = viewport;
+
+  if (rgb) {
+    mjrRenderTargetConfig config;
+    mjr_defaultRenderTargetConfig(&config);
+    config.width = viewport.width;
+    config.height = viewport.height;
+    config.color_format = mjPIXEL_FORMAT_RGB8;
+    config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
+    auto target = CreateRenderTarget(filament_context_.get(), config);
+
+    reqs[0].target = target.get();
+    reqs[1].target = target.get();
+
+    mjrReadPixelsRequest read_request;
+    mjr_defaultReadPixelsRequest(&read_request);
+    read_request.target = target.get();
+    read_request.output = rgb;
+    read_request.num_bytes = viewport.width * viewport.height * 3;
+
+    const size_t num_requests =
+        (mode_ == FrameBufferMode::OffScreenWithGui) ? 2 : 1;
+    const mjrFrameHandle frame = filament_context_->Render(
+        {&reqs[0], num_requests}, {&read_request, 1});
+    filament_context_->WaitForFrame(frame);
+  }
+
+  if (depth) {
+    mjrRenderTargetConfig config;
+    mjr_defaultRenderTargetConfig(&config);
+    config.width = viewport.width;
+    config.height = viewport.height;
+    config.color_format = mjPIXEL_FORMAT_R32F;
+    config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
+    auto target = CreateRenderTarget(filament_context_.get(), config);
+
+    reqs[0].draw_mode = mjDRAW_MODE_DEPTH;
+    reqs[0].target = target.get();
+
+    mjrReadPixelsRequest read_request;
+    mjr_defaultReadPixelsRequest(&read_request);
+    read_request.target = target.get();
+    read_request.output = reinterpret_cast<uint8_t*>(depth);
+    read_request.num_bytes = viewport.width * viewport.height * sizeof(float);
+
+    const mjrFrameHandle frame = filament_context_->Render(
+        {&reqs[0], 1}, {&read_request, 1});
+    filament_context_->WaitForFrame(frame);
+  }
+}
+
+void MjrFilamentRenderer::UploadMesh(const mjModel* model, int id) {
+  if (!scene_bridge_) {
+    mju_error("SceneBridge is not initialized.");
+  }
+  scene_bridge_->UploadMesh(model, id);
+}
+
+void MjrFilamentRenderer::UploadTexture(const mjModel* model, int id) {
+  if (!scene_bridge_) {
+    mju_error("SceneBridge is not initialized.");
+  }
+  scene_bridge_->UploadTexture(model, id);
+}
+
+void MjrFilamentRenderer::UploadHeightField(const mjModel* model, int id) {
+  if (!scene_bridge_) {
+    mju_error("SceneBridge is not initialized.");
+  }
+  scene_bridge_->UploadHeightField(model, id);
+}
+
+uintptr_t MjrFilamentRenderer::UploadGuiImage(uintptr_t tex_id,
+                                              const uint8_t* pixels, int width,
+                                              int height, int bpp) {
+  return imgui_bridge_->UploadImage(tex_id, pixels, width, height, bpp);
+}
+
+double MjrFilamentRenderer::GetFrameRate() const {
+  mjrFrameStats stats;
+  mjr_defaultFrameStats(&stats);
+  filament_context_->GetFrameStats(0, &stats);
+  return stats.frame_rate;
+}
+
+void MjrFilamentRenderer::UpdateGui() {
+  mjrf_DEBUG_drawImguiEditor(scene_bridge_->GetScene());
+}
+
+}  // namespace mujoco

src/experimental/filament/compat/mjr_filament_renderer.h

@@ -0,0 +1,88 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_
+
+#include <cstdint>
+#include <memory>
+
+#include <mujoco/mjmodel.h>
+#include <mujoco/mjrender.h>
+#include <mujoco/mjvisualize.h>
+#include "experimental/filament/compat/imgui_bridge.h"
+#include "experimental/filament/compat/scene_bridge.h"
+#include "experimental/filament/filament/filament_context.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+// Subclass of the FilamentContext that implements the legacy mjr API.
+class MjrFilamentRenderer {
+ public:
+  explicit MjrFilamentRenderer(const mjrFilamentConfig* config);
+  ~MjrFilamentRenderer() = default;
+
+  // Initializes the renderer with the given model.
+  void Init(const mjModel* model);
+
+  // Renders the given mjvScene to the viewport.
+  void Render(const mjrRect& viewport, const mjvScene* scene);
+
+  // Configures the renderer to render to the window (0) or an offscreen
+  // texture (1 or 2). Rendering to the window always includes UX data from
+  // ImGui. A value of 1 indicates the UX should not be included in the
+  // offscreen render, whereas 2 indicates that it should.
+  void SetFrameBuffer(int framebuffer);
+
+  // Renders the scene to a texture if the framebuffer is not 0.
+  void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth);
+
+  // Uploads the mesh data from the model to the GPU.
+  void UploadMesh(const mjModel* model, int id);
+
+  // Uploads the texture data from the model to the GPU.
+  void UploadTexture(const mjModel* model, int id);
+
+  // Uploads the height field data from the model to the GPU.
+  void UploadHeightField(const mjModel* model, int id);
+
+  // Uploads a texture that can be used with ImGui to the GPU.
+  uintptr_t UploadGuiImage(uintptr_t tex_id, const uint8_t* pixels, int width,
+                           int height, int bpp);
+
+  // Renders an ImGui window containing Filament-specific editor UI.
+  void UpdateGui();
+
+  double GetFrameRate() const;
+
+  MjrFilamentRenderer(const MjrFilamentRenderer&) = delete;
+  MjrFilamentRenderer& operator=(const MjrFilamentRenderer&) = delete;
+
+ private:
+  enum class FrameBufferMode {
+    Window,
+    OffScreen,
+    OffScreenWithGui,
+  };
+
+  std::unique_ptr<FilamentContext> filament_context_;
+  std::unique_ptr<SceneBridge> scene_bridge_;
+  std::unique_ptr<ImguiBridge> imgui_bridge_;
+  FrameBufferMode mode_ = FrameBufferMode::Window;
+};
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_

src/experimental/filament/compat/model_objects.cc

@@ -0,0 +1,651 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/compat/model_objects.h"
+
+#include <algorithm>
+#include <cfloat>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <span>
+#include <utility>
+#include <vector>
+
+#include <math/TVecHelpers.h>
+#include <math/vec2.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+using filament::math::float2;
+using filament::math::float3;
+using filament::math::float4;
+
+enum class MeshType {
+  kNormal,
+  kConvexHull,
+  kHeightField,
+};
+
+struct MeshBuilder {
+  MeshBuilder(int nvertices) : nvertices(nvertices) {
+    positions.reserve(nvertices);
+    orientations.reserve(nvertices);
+    uvs.reserve(nvertices);
+  }
+
+  void Append(const float3& position, const float4& orientation,
+              const float2& uv) {
+    positions.push_back(position);
+    orientations.push_back(orientation);
+    uvs.push_back(uv);
+    bounds_min = min(bounds_min, position);
+    bounds_max = max(bounds_max, position);
+  }
+
+  int nvertices = 0;
+  float3 bounds_min = {FLT_MAX, FLT_MAX, FLT_MAX};
+  float3 bounds_max = {-FLT_MAX, -FLT_MAX, -FLT_MAX};
+  std::vector<float3> positions;
+  std::vector<float4> orientations;
+  std::vector<float2> uvs;
+};
+
+static bool UseFaceNormal(const float3& face_normal,
+                          const float3& mesh_normal) {
+  // clang-format off
+  return (face_normal[0] * mesh_normal[0] +
+          face_normal[1] * mesh_normal[1] +
+          face_normal[2] * mesh_normal[2]) < 0.8f;
+  // clang-format on
+}
+
+static void FillConvexHullBuffer(MeshBuilder& builder, const mjModel* model,
+                                 int meshid) {
+  const int numvert = model->mesh_graph[model->mesh_graphadr[meshid]];
+  const int numface = model->mesh_graph[model->mesh_graphadr[meshid] + 1];
+  if (builder.nvertices != numface * 3) {
+    mju_error("Invalid vertex count (%d vs %d).", builder.nvertices, numface * 3);
+    return;
+  }
+
+  const int dataadr = model->mesh_graphadr[meshid] + 2;
+  const int vertadr = model->mesh_vertadr[meshid];
+  const float* vertices = model->mesh_vert + (3 * vertadr);
+  const int texcoordadr = model->mesh_texcoordadr[meshid];
+  const float* texcoords = texcoordadr >= 0 ? model->mesh_texcoord + (2 * texcoordadr) : nullptr;
+
+  for (int face = 0; face < numface; ++face) {
+    const int j = dataadr + (3 * numvert) + (3 * numface) + (3 * face);
+    const float3 p1 = ReadFloat3(vertices, model->mesh_graph[j + 0]);
+    const float3 p2 = ReadFloat3(vertices, model->mesh_graph[j + 1]);
+    const float3 p3 = ReadFloat3(vertices, model->mesh_graph[j + 2]);
+    const float4 orientation = CalculateOrientation(p1, p2, p3);
+    const float2 uv1 = texcoords ? ReadFloat2(texcoords, model->mesh_graph[j + 0]) : float2(0, 0);
+    const float2 uv2 = texcoords ? ReadFloat2(texcoords, model->mesh_graph[j + 1]) : float2(0, 0);
+    const float2 uv3 = texcoords ? ReadFloat2(texcoords, model->mesh_graph[j + 2]) : float2(0, 0);
+    builder.Append(p1, orientation, uv1);
+    builder.Append(p2, orientation, uv2);
+    builder.Append(p3, orientation, uv3);
+  }
+}
+
+static void FillMeshBuffer(MeshBuilder& builder, const mjModel* model, int meshid) {
+  const int faceadr = model->mesh_faceadr[meshid];
+  const int facenum = model->mesh_facenum[meshid];
+  if (builder.nvertices != facenum * 3) {
+    mju_error("Invalid vertex count (%d vs %d).", builder.nvertices, facenum * 3);
+    return;
+  }
+
+  const int vertadr = model->mesh_vertadr[meshid];
+  const float* vertices = model->mesh_vert + (3 * vertadr);
+  const int normaladr = model->mesh_normaladr[meshid];
+  const float* normals = model->mesh_normal + 3 * normaladr;
+  const int texcoordadr = model->mesh_texcoordadr[meshid];
+  const float* texcoords = texcoordadr >= 0 ? model->mesh_texcoord + (2 * texcoordadr) : nullptr;
+
+  for (int i = 0; i < facenum; ++i) {
+    const int face = 3 * (faceadr + i);
+
+    const float3 p1 = ReadFloat3(vertices, model->mesh_face[face + 0]);
+    const float3 p2 = ReadFloat3(vertices, model->mesh_face[face + 1]);
+    const float3 p3 = ReadFloat3(vertices, model->mesh_face[face + 2]);
+    const float3 face_normal = CalculateNormal(p1, p2, p3);
+    const float3 n1 = ReadFloat3(normals, model->mesh_facenormal[face + 0]);
+    const float3 n2 = ReadFloat3(normals, model->mesh_facenormal[face + 1]);
+    const float3 n3 = ReadFloat3(normals, model->mesh_facenormal[face + 2]);
+    const float2 uv1 = texcoords ? ReadFloat2(texcoords, model->mesh_facetexcoord[face + 0]) : float2(0, 0);
+    const float2 uv2 = texcoords ? ReadFloat2(texcoords, model->mesh_facetexcoord[face + 1]) : float2(0, 0);
+    const float2 uv3 = texcoords ? ReadFloat2(texcoords, model->mesh_facetexcoord[face + 2]) : float2(0, 0);
+
+    if (UseFaceNormal(face_normal, n1)) {
+      builder.Append(p1, CalculateOrientation(face_normal), uv1);
+    } else {
+      builder.Append(p1, CalculateOrientation(n1), uv1);
+    }
+
+    if (UseFaceNormal(face_normal, n2)) {
+      builder.Append(p2, CalculateOrientation(face_normal), uv2);
+    } else {
+      builder.Append(p2, CalculateOrientation(n2), uv2);
+    }
+
+    if (UseFaceNormal(face_normal, n3)) {
+      builder.Append(p3, CalculateOrientation(face_normal), uv3);
+    } else {
+      builder.Append(p3, CalculateOrientation(n3), uv3);
+    }
+  }
+}
+
+static void FillHeightFieldBuffer(MeshBuilder& builder, const mjModel* model,
+                                  int hfieldid) {
+  auto append_tri = [&](float3 a, float3 b, float3 c) {
+    float4 orientation = CalculateOrientation(a, b, c);
+    builder.Append(a, orientation, float2(0, 0));
+    builder.Append(b, orientation, float2(0, 0));
+    builder.Append(c, orientation, float2(0, 0));
+  };
+  auto append_quad = [&](float3 a, float3 b, float3 c, float3 d) {
+    append_tri(a, b, d);
+    append_tri(d, b, c);
+  };
+
+  const float* data = model->hfield_data + model->hfield_adr[hfieldid];
+  const int nrow = model->hfield_nrow[hfieldid];
+  const int ncol = model->hfield_ncol[hfieldid];
+  const float height = 0.5f * (nrow - 1);
+  const float width = 0.5f * (ncol - 1);
+  float sz[4];
+  for (int i = 0; i < 4; ++i) {
+    sz[i] = static_cast<float>(model->hfield_size[4 * hfieldid + i]);
+  }
+
+  auto get_pos = [=](int r, int c) {
+    const float x = sz[0] * (c / width - 1.0f);
+    const float y = sz[1] * (r / height - 1.0f);
+    const float z = sz[2] * data[(r * ncol) + c];
+    return float3{x, y, z};
+  };
+
+  // For each quad defined by 4 points in the height field, we will create 4
+  // triangles by introducing a vertex in the middle of the quad.
+  //     a---b
+  //     |\ /|
+  //     | m |
+  //     |/ \|
+  //     d---c
+  for (int row = 0; row < nrow - 1; ++row) {
+    for (int col = 0; col < ncol - 1; ++col) {
+      const float3 a = get_pos(row, col);
+      const float3 b = get_pos(row, col + 1);
+      const float3 c = get_pos(row + 1, col + 1);
+      const float3 d = get_pos(row + 1, col);
+
+      const float mid_x = (a.x + b.x) * 0.5f;
+      const float mid_y = (a.y + d.y) * 0.5f;
+
+      // To determine the height of the middle vertex, we look at the heights
+      // of the opposing corners (i.e. {a, c} and {b, d}). Our goal is to avoid
+      // creating any odd bumps or valleys in the height field if possible.
+      //
+      // If one of the two opposing corners are of the same height, then we
+      // set the middle vertex such that we're effectively rendering two
+      // triangles, preventing an odd bump. Otherwise, we use the higher
+      // midpoint between two opposing corners to prevent valleys.
+      //     0---0      0---0      6---4
+      //     |\  |      |  /|      |\ /|
+      //     | 0 |      | 0 |      | 7 |
+      //     |  \|      |/  |      |/ \|
+      //     2---0      0---2      0---8
+      float mid_z = 0;
+      if (a.z == c.z && b.z != d.z) {
+        mid_z = a.z;
+      } else if (a.z != c.z && b.z == d.z) {
+        mid_z = b.z;
+      } else {
+        const float mid_z_ac = (a.z + c.z) * 0.5f;
+        const float mid_z_bd = (b.z + d.z) * 0.5f;
+        mid_z = std::max(mid_z_ac, mid_z_bd);
+      }
+
+      const float3 mid = {mid_x, mid_y, mid_z};
+      append_tri(a, b, mid);
+      append_tri(b, c, mid);
+      append_tri(c, d, mid);
+      append_tri(d, a, mid);
+    }
+  }
+  // Build the left edge.
+  for (int row = 0; row < nrow - 1; ++row) {
+    const float3 a = get_pos(row, 0);
+    const float3 b = get_pos(row + 1, 0);
+    const float3 c = {b.x, b.y, -sz[3]};
+    const float3 d = {a.x, a.y, -sz[3]};
+    append_quad(a, b, c, d);
+  }
+  // Build the right edge.
+  for (int row = 0; row < nrow - 1; ++row) {
+    const float3 a = get_pos(row + 1, ncol - 1);
+    const float3 b = get_pos(row, ncol - 1);
+    const float3 c = {b.x, b.y, -sz[3]};
+    const float3 d = {a.x, a.y, -sz[3]};
+    append_quad(a, b, c, d);
+  }
+  // Build the front edge.
+  for (int col = 0; col < ncol - 1; ++col) {
+    const float3 a = get_pos(0, col);
+    const float3 b = {a.x, a.y, -sz[3]};
+    const float3 d = get_pos(0, col + 1);
+    const float3 c = {d.x, d.y, -sz[3]};
+    append_quad(a, b, c, d);
+  }
+  // Build the back edge.
+  for (int col = 0; col < ncol - 1; ++col) {
+    const float3 a = get_pos(nrow - 1, col + 1);
+    const float3 b = {a.x, a.y, -sz[3]};
+    const float3 d = get_pos(nrow - 1, col);
+    const float3 c = {d.x, d.y, -sz[3]};
+    append_quad(a, b, c, d);
+  }
+  // Build the base. We use the visualization quality as the size rather than
+  // the height field dimensions.
+  const float base_width = (0.5f * model->vis.quality.numquads);
+  const float base_height = (0.5f * model->vis.quality.numquads);
+  for (int row = 0; row < model->vis.quality.numquads; ++row) {
+    for (int col = 0; col < model->vis.quality.numquads; ++col) {
+      const float x0 = sz[0] * ((col + 0) / base_width - 1.0f);
+      const float x1 = sz[0] * ((col + 1) / base_width - 1.0f);
+      const float y0 = sz[1] * ((row + 0) / base_height - 1.0f);
+      const float y1 = sz[1] * ((row + 1) / base_height - 1.0f);
+      append_quad({x0, y0, -sz[3]}, {x0, y1, -sz[3]}, {x1, y1, -sz[3]},
+                  {x1, y0, -sz[3]});
+    }
+  }
+}
+
+static int CalculateHeightFieldVertexCount(const mjModel* model, int hfieldid) {
+  const int nrow = model->hfield_nrow[hfieldid];
+  const int ncol = model->hfield_ncol[hfieldid];
+
+  // For details, see the logic in FillHeightFieldBuffer for how many vertices
+  // we need. But, in general...
+
+  // We use 4 triangles (i.e. 12 vertices) per quad.
+  const int surface_count = 12 * (nrow - 1) * (ncol - 1);
+  // We use 1 quad (i.e. 6 vertices) per edge element. We double this because
+  // we have two edges per dimension (e.g. left/right and front/back).
+  const int edge_count = (12 * (nrow - 1)) + (12 * (ncol - 1));
+  // We use 1 quad (i.e. 6 vertices) per base element. We use the visualization
+  // quality as the size rather than the height field dimensions.
+  const int base_count =
+      6 * model->vis.quality.numquads * model->vis.quality.numquads;
+
+  const int total_count = surface_count + edge_count + base_count;
+  return total_count;
+}
+
+static bool HasUvs(const mjModel* model, int id, MeshType mesh_type) {
+  return mesh_type != MeshType::kHeightField &&
+         model->mesh_texcoordadr[id] >= 0;
+}
+
+static bool IsValidIndex(const mjModel* model, int id, MeshType mesh_type) {
+  switch (mesh_type) {
+    case MeshType::kNormal:
+      return id >= 0 && id < model->nmesh;
+    case MeshType::kConvexHull:
+      return id >= 0 && id < model->nmesh;
+    case MeshType::kHeightField:
+      return id >= 0 && id < model->nhfield;
+  }
+}
+
+static int GetNumVertices(const mjModel* model, int id, MeshType mesh_type) {
+  switch (mesh_type) {
+    case MeshType::kNormal:
+      return 3 * model->mesh_facenum[id];
+    case MeshType::kConvexHull:
+      return 3 * model->mesh_graph[model->mesh_graphadr[id] + 1];
+    case MeshType::kHeightField:
+      return CalculateHeightFieldVertexCount(model, id);
+  }
+}
+
+static std::span<const float> GetPositions(const mjModel* model,
+                                           const mjvScene* scene,
+                                           const mjvGeom& geom) {
+  if (geom.type == mjGEOM_FLEX) {
+    const int num = 9 * scene->flexfaceused[geom.objid];
+    const int addr = scene->flexfaceadr[geom.objid];
+    const float* ptr = scene->flexface + (9 * addr);
+    return {ptr, static_cast<size_t>(num)};
+  } else {
+    const int num = 3 * scene->skinvertnum[geom.objid];
+    const int addr = scene->skinvertadr[geom.objid];
+    const float* ptr = scene->skinvert + (3 * addr);
+    return {ptr, static_cast<size_t>(num)};
+  }
+}
+
+static std::span<const float> GetNormals(const mjModel* model,
+                                         const mjvScene* scene,
+                                         const mjvGeom& geom) {
+  if (geom.type == mjGEOM_FLEX) {
+    const int num = 9 * scene->flexfaceused[geom.objid];
+    const int addr = scene->flexfaceadr[geom.objid];
+    const float* ptr = scene->flexnormal + (9 * addr);
+    return {ptr, static_cast<size_t>(num)};
+  } else {
+    const int num = 3 * scene->skinvertnum[geom.objid];
+    const int addr = scene->skinvertadr[geom.objid];
+    const float* ptr = scene->skinnormal + (3 * addr);
+    return {ptr, static_cast<size_t>(num)};
+  }
+}
+
+static std::span<const float> GetUvs(const mjModel* model,
+                                     const mjvScene* scene,
+                                     const mjvGeom& geom) {
+  if (geom.type == mjGEOM_FLEX) {
+    if (geom.texcoord && geom.matid >= 0) {
+      const int num = 6 * scene->flexfaceused[geom.objid];
+      const int addr = scene->flexfaceadr[geom.objid];
+      const float* ptr = scene->flextexcoord + (6 * addr);
+      return {ptr, static_cast<size_t>(num)};
+    } else {
+      const float* ptr = nullptr;
+      return {ptr, 0};
+    }
+  } else {
+    if (model->skin_texcoordadr[geom.objid] >= 0) {
+      const int num = 3 * scene->skinvertnum[geom.objid];
+      const int addr = model->skin_texcoordadr[geom.objid];
+      const float* ptr = model->skin_texcoord + (2 * addr);
+      return {ptr, static_cast<size_t>(num)};
+    } else {
+      const float* ptr = nullptr;
+      return {ptr, 0};
+    }
+  }
+}
+
+static std::span<const int> GetIndices(const mjModel* model,
+                                       const mjvScene* scene,
+                                       const mjvGeom& geom) {
+  if (geom.type == mjGEOM_FLEX) {
+    const int* ptr = nullptr;
+    return {ptr, 0};
+  } else {
+    const int num = 3 * model->skin_facenum[geom.objid];
+    const int* ptr = model->skin_face + 3 * model->skin_faceadr[geom.objid];
+    return {ptr, static_cast<size_t>(num)};
+  }
+}
+
+static void UpdateMeshData(mjrMeshData* data, const mjModel* model, int id,
+                           MeshType mesh_type) {
+  if (!IsValidIndex(model, id, mesh_type)) {
+    mju_error("Invalid index %d for type %d", id, mesh_type);
+    return;
+  }
+
+  const int num_vertices = GetNumVertices(model, id, mesh_type);
+  const bool has_uvs = HasUvs(model, id, mesh_type);
+
+  MeshBuilder* builder = new MeshBuilder(num_vertices);
+  data->user_data = builder;
+  data->release_callback = [](void* user_data) {
+    delete static_cast<MeshBuilder*>(user_data);
+  };
+
+  switch (mesh_type) {
+    case MeshType::kNormal:
+      FillMeshBuffer(*builder, model, id);
+      break;
+    case MeshType::kConvexHull:
+      FillConvexHullBuffer(*builder, model, id);
+      break;
+    case MeshType::kHeightField:
+      FillHeightFieldBuffer(*builder, model, id);
+      break;
+  }
+
+  data->primitive_type = mjMESH_PRIMITIVE_TYPE_TRIANGLES;
+  data->nvertices = num_vertices;
+  data->nindices = data->nvertices;
+  data->indices = nullptr;
+  data->index_type = data->nvertices >= std::numeric_limits<uint16_t>::max()
+                         ? mjINDEX_TYPE_U32
+                         : mjINDEX_TYPE_U16;
+  data->nattributes = has_uvs ? 3 : 2;
+  data->attributes[0].usage = mjVERTEX_ATTRIBUTE_USAGE_POSITION;
+  data->attributes[0].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT3;
+  data->attributes[0].bytes = builder->positions.data();
+  data->attributes[1].usage = mjVERTEX_ATTRIBUTE_USAGE_TANGENTS;
+  data->attributes[1].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT4;
+  data->attributes[1].bytes = builder->orientations.data();
+  if (has_uvs) {
+    data->attributes[2].usage = mjVERTEX_ATTRIBUTE_USAGE_UV;
+    data->attributes[2].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT2;
+    data->attributes[2].bytes = builder->uvs.data();
+  }
+  data->bounds_min[0] = builder->bounds_min.x;
+  data->bounds_min[1] = builder->bounds_min.y;
+  data->bounds_min[2] = builder->bounds_min.z;
+  data->bounds_max[0] = builder->bounds_max.x;
+  data->bounds_max[1] = builder->bounds_max.y;
+  data->bounds_max[2] = builder->bounds_max.z;
+}
+
+void UpdateSkinFlexMeshData(mjrMeshData* data, const mjModel* model,
+                            const mjvScene* scene, const mjvGeom& geom) {
+  auto positions = GetPositions(model, scene, geom);
+  auto normals = GetNormals(model, scene, geom);
+  auto uvs = GetUvs(model, scene, geom);
+  auto indices = GetIndices(model, scene, geom);
+
+  int num_indices = indices.size();
+  if (num_indices == 0 && geom.type == mjGEOM_FLEX) {
+    num_indices = 3 * scene->flexfaceused[geom.objid];
+  }
+
+  data->nattributes = uvs.data() ? 3 : 2;
+  data->attributes[0].usage = mjVERTEX_ATTRIBUTE_USAGE_POSITION;
+  data->attributes[0].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT3;
+  data->attributes[0].bytes = positions.data();
+  data->attributes[1].usage = mjVERTEX_ATTRIBUTE_USAGE_NORMAL;
+  data->attributes[1].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT3;
+  data->attributes[1].bytes = normals.data();
+  data->attributes[2].usage = mjVERTEX_ATTRIBUTE_USAGE_UV;
+  data->attributes[2].type = mjVERTEX_ATTRIBUTE_TYPE_FLOAT2;
+  data->attributes[2].bytes = uvs.data();
+  data->nvertices = positions.size() / 3;
+  data->nindices = num_indices;
+  data->indices = indices.data();
+  data->index_type = mjINDEX_TYPE_U32;
+  data->primitive_type = mjMESH_PRIMITIVE_TYPE_TRIANGLES;
+  data->compute_bounds = true;
+  data->release_callback = nullptr;
+  data->user_data = nullptr;
+}
+
+ModelObjects::ModelObjects(const mjModel* model, mjrfContext* ctx)
+    : model_(model), ctx_(ctx) {
+
+  for (int i = 0; i < model_->ntex; ++i) {
+    UploadTexture(model_, i);
+  }
+  for (int i = 0; i < model_->nmesh; ++i) {
+    UploadMesh(model_, i);
+  }
+  for (int i = 0; i < model_->nhfield; ++i) {
+    UploadHeightField(model_, i);
+  }
+
+  specular_multiplier_ = ReadElement(
+      model_, "filament.phong.specular_multiplier", specular_multiplier_);
+  shininess_multiplier_ = ReadElement(
+      model_, "filament.phong.shininess_multiplier", shininess_multiplier_);
+  emissive_multiplier_ = ReadElement(
+      model_, "filament.phong.emissive_multiplier", emissive_multiplier_);
+}
+
+void ModelObjects::UploadMesh(const mjModel* model, int id) {
+  if (model != model_) {
+    mju_error("Model mismatch.");
+  }
+  if (id < 0 || id >= model->nmesh) {
+    mju_error("Invalid mesh index %d", id);
+  }
+  meshes_.erase(id);
+  convex_hulls_.erase(id);
+
+  mjrMeshData data;
+  mjr_defaultMeshData(&data);
+  UpdateMeshData(&data, model, id, MeshType::kNormal);
+  meshes_.insert_or_assign(id, CreateMesh(ctx_, data));
+
+  if (model->mesh_graphadr[id] >= 0) {
+    mjrMeshData convex_hull_data;
+    mjr_defaultMeshData(&convex_hull_data);
+    UpdateMeshData(&convex_hull_data, model, id, MeshType::kConvexHull);
+    convex_hulls_.insert_or_assign(id, CreateMesh(ctx_, convex_hull_data));
+  }
+}
+
+void ModelObjects::UploadTexture(const mjModel* model, int id) {
+  if (model != model_) {
+    mju_error("Model mismatch.");
+  }
+  if (id < 0 || id >= model->ntex) {
+    mju_error("Invalid texture index: %d", id);
+  }
+
+  mjrTextureConfig config;
+  mjr_defaultTextureConfig(&config);
+  config.width = model->tex_width[id];
+  config.height = model->tex_height[id];
+  config.sampler_type = (mjtTexture)model->tex_type[id];
+  config.color_space = (mjtColorSpace)model->tex_colorspace[id];
+  switch (model->tex_nchannel[id]) {
+    case 1:
+      config.format = mjPIXEL_FORMAT_R8;
+      break;
+    case 3:
+      config.format = mjPIXEL_FORMAT_RGB8;
+      break;
+    case 4:
+      config.format = mjPIXEL_FORMAT_RGBA8;
+      break;
+    default:
+      mju_error("Unsupported texture format: %d", model->tex_nchannel[id]);
+      break;
+  }
+  if (config.height == 1 && model->tex_nchannel[id] == 1) {
+    config.format = mjPIXEL_FORMAT_KTX;
+  }
+
+  mjrTextureData payload;
+  mjr_defaultTextureData(&payload);
+  payload.bytes = model->tex_data + model->tex_adr[id];
+  payload.nbytes =
+      model->tex_width[id] * model->tex_height[id] * model->tex_nchannel[id];
+  // We assume that the model has the same lifetime as the engine.
+  payload.user_data = nullptr;
+  payload.release_callback = nullptr;
+
+  auto texture = CreateTexture(ctx_, config);
+  mjrf_setTextureData(texture.get(), &payload);
+  textures_.insert_or_assign(id, std::move(texture));
+}
+
+void ModelObjects::UploadHeightField(const mjModel* model, int id) {
+  if (model != model_) {
+    mju_error("Model mismatch.");
+  }
+  if (id < 0 || id >= model->nhfield) {
+    mju_error("Invalid height field index %d", id);
+  }
+
+  height_fields_.erase(id);
+
+  mjrMeshData data;
+  mjr_defaultMeshData(&data);
+  UpdateMeshData(&data, model, id, MeshType::kHeightField);
+  height_fields_.insert_or_assign(id, CreateMesh(ctx_, data));
+}
+
+void ModelObjects::CreateSkinFlexMesh(const mjvScene* scene, const mjvGeom& geom) {
+  mjrMeshData data;
+  mjr_defaultMeshData(&data);
+  UpdateSkinFlexMeshData(&data, model_, scene, geom);
+  dynamic_meshes_.insert_or_assign(geom.objid, CreateMesh(ctx_, data));
+}
+
+const mjrMesh* ModelObjects::GetMeshBuffer(int data_id) const {
+  // As defined by mjv_updateScene:
+  //   original mesh: mesh_id * 2
+  //   convex hull: (mesh_id * 2) + 1
+  const int mesh_id = data_id / 2;
+  if (data_id % 2 == 0) {
+    auto it = meshes_.find(mesh_id);
+    return it != meshes_.end() ? it->second.get() : nullptr;
+  } else {
+    auto it = convex_hulls_.find(mesh_id);
+    return it != convex_hulls_.end() ? it->second.get() : nullptr;
+  }
+}
+
+const mjrMesh* ModelObjects::GetHeightFieldBuffer(int hfield_id) const {
+  auto it = height_fields_.find(hfield_id);
+  return it != height_fields_.end() ? it->second.get() : nullptr;
+}
+
+const mjrMesh* ModelObjects::GetFlexSkinGeomMesh(int geom_id) const {
+  auto it = dynamic_meshes_.find(geom_id);
+  return it != dynamic_meshes_.end() ? it->second.get() : nullptr;
+}
+
+const mjrTexture* ModelObjects::GetTexture(int tex_id) const {
+  auto it = textures_.find(tex_id);
+  return it != textures_.end() ? it->second.get() : nullptr;
+}
+
+const mjrTexture* ModelObjects::GetTexture(int mat_id, int role) const {
+  if (mat_id < 0 || mat_id >= model_->nmat || role < 0 || role >= mjNTEXROLE) {
+    return nullptr;
+  }
+  const int tex_id = model_->mat_texid[mat_id * mjNTEXROLE + role];
+  return GetTexture(tex_id);
+}
+
+const mjrTexture* ModelObjects::GetSkyboxTexture() const {
+  for (auto& iter : textures_) {
+    if (model_->tex_type[iter.first] == mjTEXTURE_SKYBOX) {
+      return iter.second.get();
+    }
+  }
+  return nullptr;
+}
+
+}  // namespace mujoco

src/experimental/filament/compat/model_objects.h

@@ -0,0 +1,72 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MODEL_OBJECTS_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MODEL_OBJECTS_H_
+
+#include <unordered_map>
+
+#include <mujoco/mjmodel.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+
+namespace mujoco {
+
+// Creates and owns various filament objects based on the mjModel.
+class ModelObjects {
+ public:
+  ModelObjects(const mjModel* model, mjrfContext* ctx);
+
+  void UploadMesh(const mjModel* model, int id);
+
+  void UploadTexture(const mjModel* model, int id);
+
+  void UploadHeightField(const mjModel* model, int id);
+
+  void CreateSkinFlexMesh(const mjvScene* scene, const mjvGeom& geom);
+
+  // Returns the cached instance of a filament object created from the mjModel.
+  const mjrMesh* GetMeshBuffer(int data_id) const;
+  const mjrMesh* GetHeightFieldBuffer(int hfield_id) const;
+  const mjrMesh* GetFlexSkinGeomMesh(int geom_id) const;
+  const mjrTexture* GetTexture(int tex_id) const;
+  const mjrTexture* GetTexture(int mat_id, int role) const;
+  const mjrTexture* GetSkyboxTexture() const;
+
+  float GetSpecularMultiplier() const { return specular_multiplier_; }
+  float GetShininessMultiplier() const { return shininess_multiplier_; }
+  float GetEmissiveMultiplier() const { return emissive_multiplier_; }
+
+  const mjModel* GetModel() const { return model_; }
+
+  ModelObjects(const ModelObjects&) = delete;
+  ModelObjects& operator=(const ModelObjects&) = delete;
+
+ private:
+  const mjModel* model_ = nullptr;
+  mjrfContext* ctx_ = nullptr;
+  std::unordered_map<int, UniquePtr<mjrMesh>> meshes_;
+  std::unordered_map<int, UniquePtr<mjrMesh>> convex_hulls_;
+  std::unordered_map<int, UniquePtr<mjrMesh>> height_fields_;
+  std::unordered_map<int, UniquePtr<mjrMesh>> dynamic_meshes_;
+  std::unordered_map<int, UniquePtr<mjrTexture>> textures_;
+  float specular_multiplier_ = 0.2f;
+  float shininess_multiplier_ = 0.1f;
+  float emissive_multiplier_ = 0.3f;
+};
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MODEL_OBJECTS_H_

src/experimental/filament/compat/scene_bridge.cc

@@ -0,0 +1,361 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/compat/scene_bridge.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include <math/TMatHelpers.h>
+#include <math/mat4.h>
+#include <math/mathfwd.h>
+#include <math/TVecHelpers.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/compat/model_objects.h"
+#include "experimental/filament/compat/scene_geom_util.h"
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+using filament::math::float3;
+using filament::math::float4;
+using filament::math::mat3;
+using filament::math::mat4;
+
+static UniquePtr<mjrTexture> CreateFallbackIndirectLightTexture(
+    mjrfContext* ctx) {
+  const std::string filename = ResolveFilamentAssetPath("ibl.ktx");
+  mjResource* resource =
+      mju_openResource("", filename.c_str(), nullptr, nullptr, 0);
+  if (!resource) {
+    mju_error("Failed to open resource: %s", filename.c_str());
+  }
+  const void* bytes = nullptr;
+  const int nbytes = mju_readResource(resource, &bytes);
+  if (bytes == nullptr || nbytes <= 0) {
+    mju_error("Failed to read resource: %s", filename.c_str());
+  }
+
+  mjrTextureConfig config;
+  mjr_defaultTextureConfig(&config);
+  config.width = 1;
+  config.height = 1;
+  config.sampler_type = mjTEXTURE_CUBE;
+  config.format = mjPIXEL_FORMAT_KTX;
+  config.color_space = mjCOLORSPACE_AUTO;
+
+  auto texture = CreateTexture(ctx, config);
+
+  mjrTextureData payload;
+  mjr_defaultTextureData(&payload);
+  payload.bytes = bytes;
+  payload.nbytes = nbytes;
+  payload.release_callback = +[](void* user_data) {
+    mju_closeResource((mjResource*)user_data);
+  };
+  payload.user_data = resource;
+
+  mjrf_setTextureData(texture.get(), &payload);
+  return texture;
+}
+
+SceneBridge::SceneBridge(mjrfContext* ctx, const mjModel* model)
+    : ctx_(ctx) {
+  mjrSceneParams params;
+  mjr_defaultSceneParams(&params);
+  params.layer_mask = mjCAT_ALL;
+  params.reflection_layer_mask = mjCAT_DYNAMIC | mjCAT_STATIC;
+  scene_ = CreateScene(ctx_, params);
+  model_objects_ = std::make_unique<ModelObjects>(model, ctx_);
+
+  mjrf_configureSceneFromModel(scene_.get(), model);
+
+  auto clear_color = ReadElement(model, "filament.clearColor",
+                                 filament::math::float4(0, 0, 0, 1));
+  mjrf_setClearColor(ctx_, &clear_color[0]);
+
+  default_shadow_map_size_ = ReadElement(
+      model, "filament.shadows.map_size", default_shadow_map_size_);
+  default_vsm_blur_width_ = ReadElement(
+      model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_);
+  fallback_head_light_intensity_ =
+      ReadElement(model, "filament.fallback.head_light_intensity",
+                  fallback_head_light_intensity_);
+  fallback_scene_light_intensity_ =
+      ReadElement(model, "filament.fallback.scene_light_intensity",
+                  fallback_scene_light_intensity_);
+  fallback_environment_light_intensity_ =
+      ReadElement(model, "filament.fallback.environment_light_intensity",
+                  fallback_environment_light_intensity_);
+
+  PrepareLights();
+}
+
+SceneBridge::~SceneBridge() {
+  for (auto& iter : lights_) {
+    mjrf_removeLightFromScene(scene_.get(), iter.get());
+  }
+  lights_.clear();
+  if (fallback_ibl_) {
+    mjrf_removeLightFromScene(scene_.get(), fallback_ibl_.get());
+  }
+  fallback_ibl_.reset();
+  for (auto& iter : renderables_) {
+    mjrf_removeRenderableFromScene(scene_.get(), iter.get());
+  }
+  renderables_.clear();
+}
+
+std::optional<float3> SceneBridge::ClipFromWorld(const float3& pos) const{
+  const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f);
+  if (clip_pos.w == 0.0f) {
+    return std::nullopt;
+  }
+  return clip_pos.xyz / clip_pos.w;
+}
+
+void SceneBridge::PrepareLights() {
+  const mjModel* model = model_objects_->GetModel();
+
+  bool has_image_based_light = false;
+  float total_light_intensity = 0.0f;
+  for (int i = 0; i < model->nlight; ++i) {
+    total_light_intensity += model->light_intensity[i];
+
+    if (model->light_type[i] == mjLIGHT_IMAGE) {
+      mjrLightParams params;
+      mjr_defaultLightParams(&params);
+      params.type = mjLIGHT_IMAGE;
+      params.texture = model_objects_->GetTexture(model->light_texid[i]);
+      params.intensity = model->light_intensity[i];
+      auto light_obj = CreateLight(ctx_, params);
+      mjrf_addLightToScene(scene_.get(), light_obj.get());
+      lights_.emplace_back(std::move(light_obj));
+      has_image_based_light = true;
+    } else {
+      mjrLightParams params;
+      mjr_defaultLightParams(&params);
+      params.color[0] = model->light_diffuse[0];
+      params.color[1] = model->light_diffuse[1];
+      params.color[2] = model->light_diffuse[2];
+      params.type = (mjtLightType)model->light_type[i];
+      params.cast_shadows = model->light_castshadow[i];
+      params.bulb_radius = model->light_bulbradius[i];
+      params.range = model->light_range[i];
+      params.intensity = model->light_intensity[i];
+      params.shadow_map_size = default_shadow_map_size_;
+      params.vsm_blur_width = default_vsm_blur_width_;
+      if (params.type == mjLIGHT_SPOT) {
+        params.spot_cone_angle = model->light_cutoff[i];
+      }
+
+      auto light_obj = CreateLight(ctx_, params);
+      mjrf_addLightToScene(scene_.get(), light_obj.get());
+      lights_.emplace_back(std::move(light_obj));
+    }
+  }
+
+  // Add a placeholder (black) headlight as our last light. Going forward, we'll
+  // assume lights_.back() is always the headlight.
+  {
+    mjrLightParams params;
+    mjr_defaultLightParams(&params);
+    // We break with the spec here slightly and use a spot light for the head
+    // light instead of a directional params. This is because filament only
+    // supports a single directional light, and we'd rather allow a scene
+    // light to be that directional params. It's also a bit odd for a
+    // directional light to move with the camera.
+    params.type = mjLIGHT_SPOT;
+    params.cast_shadows = 0;
+    params.intensity = 0.0f;
+    params.spot_cone_angle = 90.0f;
+    auto light_obj = CreateLight(ctx_, params);
+    mjrf_addLightToScene(scene_.get(), light_obj.get());
+    lights_.emplace_back(std::move(light_obj));
+  }
+
+  if (!has_image_based_light && total_light_intensity > 0.0f) {
+    // Create a black indirect light to ensure that the skybox is
+    // oriented to respect mujoco's Z-up convention.
+    mjrLightParams params;
+    mjr_defaultLightParams(&params);
+    params.type = mjLIGHT_IMAGE;
+    params.intensity = 10.0f;
+    fallback_ibl_ = CreateLight(ctx_, params);
+    mjrf_addLightToScene(scene_.get(), fallback_ibl_.get());
+  }
+
+  // There are no "physical" lights in the scene which means we're likely
+  // dealing with a "classic renderer" scene. In this case, let's add a
+  // default environment light and set the light intensity ourselves.
+  if (total_light_intensity == 0.0f) {
+    // Create a fallback environment light.
+    fallback_ibl_texture_ = CreateFallbackIndirectLightTexture(ctx_);
+
+    mjrLightParams params;
+    mjr_defaultLightParams(&params);
+    params.type = mjLIGHT_IMAGE;
+    params.texture = fallback_ibl_texture_.get();
+    params.intensity = fallback_environment_light_intensity_;
+    fallback_ibl_ = CreateLight(ctx_, params);
+    mjrf_addLightToScene(scene_.get(), fallback_ibl_.get());
+
+    // Distribute the fallback scene light intensity among the lights.
+    const float intensity = fallback_scene_light_intensity_ / lights_.size();
+    for (auto& light : lights_) {
+      if (light) {
+        const bool is_headlight = (light == lights_.back());
+        mjrf_setLightIntensity(light.get(),
+                               is_headlight ? fallback_head_light_intensity_
+                                            : intensity);
+      }
+    }
+  }
+
+  mjrf_setSceneSkybox(scene_.get(), model_objects_->GetSkyboxTexture());
+}
+
+mat4 CalculateClipFromWorld(const mjrRect& viewport, const mjrCamera& cam) {
+  const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]);
+  const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]);
+  const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]);
+  const float3 cam_at = cam_pos + cam_fwd;
+  const float aspect_ratio = (float)viewport.width / (float)viewport.height;
+  const float halfwidth =
+      cam.frustum_width
+          ? cam.frustum_width
+          : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom);
+  const float left = cam.frustum_center - halfwidth;
+  const float right = cam.frustum_center + halfwidth;
+
+  mat4 projection;
+  if (cam.orthographic) {
+    projection = mat4::ortho(left, right, cam.frustum_bottom, cam.frustum_top,
+                             cam.frustum_near, cam.frustum_far);
+  } else {
+    projection = mat4::frustum(left, right, cam.frustum_bottom, cam.frustum_top,
+                               cam.frustum_near, cam.frustum_far);
+    projection[2][2] = -1.0f;
+    projection[3][2] = -2.0f * cam.frustum_near;
+  }
+  mat4 look_at = mat4::lookAt(cam_pos, cam_at, cam_up);
+  return projection * inverse(look_at);
+}
+
+void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) {
+  mjrf_setSceneShadowsEnabled(scene_.get(), scene->flags[mjRND_SHADOW]);
+  mjrf_setSceneReflectionsEnabled(scene_.get(), scene->flags[mjRND_REFLECTION]);
+  if (scene->flags[mjRND_SEGMENT]) {
+    draw_mode_ = mjDRAW_MODE_SEGMENTATION;
+  } else if (scene->flags[mjRND_DEPTH]) {
+    draw_mode_ = mjDRAW_MODE_DEPTH;
+  } else {
+    draw_mode_ = mjDRAW_MODE_COLOR;
+  }
+
+  mjtNum hpos[3], hfwd[3];
+  float headpos[3], gazedir[3];
+  mjv_cameraInModel(hpos, hfwd, nullptr, scene);
+  mju_n2f(headpos, hpos, 3);
+  mju_n2f(gazedir, hfwd, 3);
+
+  camera_ = mjv_averageCamera(scene->camera, scene->camera + 1);
+  clip_from_world_ = CalculateClipFromWorld(viewport, camera_);
+
+  // Remove all drawables from previous render and prepare new ones.
+  for (auto& iter : renderables_) {
+    mjrf_removeRenderableFromScene(scene_.get(), iter.get());
+  }
+  renderables_.clear();
+  for (int i = 0; i < scene->ngeom; ++i) {
+    const mjvGeom* geom = scene->geoms + i;
+
+    if (draw_text_callback_ && geom->label[0] != 0) {
+      if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) {
+        draw_text_callback_(geom->label, pos->x, pos->y, pos->z);
+      }
+    }
+
+    if (geom->type == mjGEOM_FLEX || geom->type == mjGEOM_SKIN) {
+      model_objects_->CreateSkinFlexMesh(scene, *geom);
+    }
+
+    UniquePtr<mjrRenderable> renderable = CreateGeomRenderable(
+        *geom, scene, ctx_, model_objects_.get(), headpos);
+
+    mjrf_addRenderableToScene(scene_.get(), renderable.get());
+    renderables_.push_back(std::move(renderable));
+  }
+
+  bool headlight_enabled = false;
+  for (int i = 0; i < scene->nlight; ++i) {
+    const mjvLight& scene_light = scene->lights[i];
+    if (scene_light.id < 0 && scene_light.headlight) {
+      // We position the headlight slightly behind the camera to avoid some
+      // odd clipping issues.
+      headlight_enabled = true;
+      headpos[0] -= gazedir[0] * 0.05f;
+      headpos[1] -= gazedir[1] * 0.05f;
+      headpos[2] -= gazedir[2] * 0.05f;
+
+      // The headlight is always the "back" light.
+      UniquePtr<mjrLight>& light = lights_.back();
+      mjrf_setLightColor(light.get(), scene_light.diffuse);
+      mjrf_setLightTransform(light.get(), headpos, gazedir);
+      continue;
+    } else if (scene_light.id < lights_.size() - 1) {
+      UniquePtr<mjrLight>& light = lights_[scene_light.id];
+      if (light) {
+        mjrf_setLightColor(light.get(), scene_light.diffuse);
+        mjrf_setLightTransform(light.get(), scene_light.pos, scene_light.dir);
+      }
+    } else {
+      mju_error("Unexpected light id: %d", scene_light.id);
+    }
+  }
+
+  // Enable/disable the headlight based on whether or not it's in the scene.
+  mjrf_setLightEnabled(lights_.back().get(), headlight_enabled);
+}
+
+void SceneBridge::UploadMesh(const mjModel* model, int id) {
+  model_objects_->UploadMesh(model, id);
+}
+
+void SceneBridge::UploadTexture(const mjModel* model, int id) {
+  model_objects_->UploadTexture(model, id);
+}
+
+void SceneBridge::UploadHeightField(const mjModel* model, int id) {
+  model_objects_->UploadHeightField(model, id);
+}
+
+void SceneBridge::SetDrawTextFunction(DrawTextAtFn fn) {
+  draw_text_callback_ = std::move(fn);
+}
+
+mjrScene* SceneBridge::GetScene() const { return scene_.get(); }
+
+mjrCamera SceneBridge::GetCamera() const { return camera_; }
+
+mjrDrawMode SceneBridge::GetDrawMode() const { return draw_mode_; }
+
+}  // namespace mujoco

src/experimental/filament/compat/scene_bridge.h

@@ -0,0 +1,88 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_BRIDGE_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_BRIDGE_H_
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string_view>
+#include <vector>
+
+#include <math/mat4.h>
+#include <math/vec3.h>
+#include <mujoco/mjvisualize.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/compat/model_objects.h"
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+
+namespace mujoco {
+
+// Manages all mjModel data and updates a SceneView using an mjvScene.
+class SceneBridge {
+ public:
+  SceneBridge(mjrfContext* ctx, const mjModel* model);
+  ~SceneBridge();
+
+  // Updates the Entities in the filament Scene to match the current mjvScene
+  // state.
+  void Update(const mjrRect& viewport, const mjvScene* scene);
+
+  // Creates the filament objects from the mjModel.
+  void UploadMesh(const mjModel* model, int id);
+  void UploadTexture(const mjModel* model, int id);
+  void UploadHeightField(const mjModel* model, int id);
+
+  using DrawTextAtFn = std::function<void(const char*, float, float, float)>;
+  void SetDrawTextFunction(DrawTextAtFn fn);
+
+  // Returns the managed scene.
+  mjrScene* GetScene() const;
+  mjrCamera GetCamera() const;
+  mjrDrawMode GetDrawMode() const;
+
+  SceneBridge(const SceneBridge&) = delete;
+  SceneBridge& operator=(const SceneBridge&) = delete;
+
+ private:
+  void PrepareLights();
+
+  // Converts a point in world space to clip space, eg. in the range [-1,-1, 0]
+  // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera.
+  std::optional<filament::math::float3> ClipFromWorld(
+      const filament::math::float3& pos) const;
+
+  mjrfContext* ctx_ = nullptr;
+  std::unique_ptr<ModelObjects> model_objects_;
+  mjrCamera camera_;
+  mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR;
+  DrawTextAtFn draw_text_callback_;
+  UniquePtr<mjrScene> scene_{nullptr, nullptr};
+  UniquePtr<mjrLight> fallback_ibl_{nullptr, nullptr};
+  UniquePtr<mjrTexture> fallback_ibl_texture_{nullptr, nullptr};
+  std::vector<UniquePtr<mjrLight>> lights_;
+  std::vector<UniquePtr<mjrRenderable>> renderables_;
+  filament::math::mat4 clip_from_world_;
+  int default_shadow_map_size_ = 2048;
+  float default_vsm_blur_width_ = 0.0f;
+  float fallback_head_light_intensity_ = 0.f;
+  float fallback_scene_light_intensity_ = 80'000.f;
+  float fallback_environment_light_intensity_ = 5'000.f;
+};
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_BRIDGE_H_

src/experimental/filament/compat/scene_geom_util.cc

@@ -0,0 +1,349 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/compat/scene_geom_util.h"
+
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+
+#include <mujoco/mjvisualize.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/compat/model_objects.h"
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
+
+namespace mujoco {
+
+// Returns the tile size for infinite plane texture alignment.
+// This is duplicated from engine_vis_visualize.c (re-center infinite plane)
+// to ensure UV scaling matches the re-centering increments.
+static float GetPlaneTileSize(const mjModel* model, int matid,
+                              float texrepeat) {
+  if (matid >= 0 && texrepeat > 0) {
+    return 2.0f / texrepeat;
+  } else {
+    const float zfar = model->vis.map.zfar * model->stat.extent;
+    return 2.1f * zfar / (mjMAXPLANEGRID - 2);
+  }
+}
+
+static bool IsBehind(const float* headpos, const float* pos, const float* mat) {
+  return ((headpos[0] - pos[0]) * mat[2] + (headpos[1] - pos[1]) * mat[5] +
+              (headpos[2] - pos[2]) * mat[8] <
+          0.0f);
+}
+
+static const mjrMesh* GetMesh(ModelObjects* model_objs, int data_id) {
+  const mjrMesh* mesh = model_objs->GetMeshBuffer(data_id);
+  if (mesh == nullptr) {
+    mju_error("Unknown mesh %d", data_id);
+  }
+  return mesh;
+}
+
+static const mjrMesh* GetSkinFlexMesh(ModelObjects* model_objs, int objid) {
+  return model_objs->GetFlexSkinGeomMesh(objid);
+}
+
+static const mjrMesh* GetHeightField(ModelObjects* model_objs, int hfield_id) {
+  const mjrMesh* mesh = model_objs->GetHeightFieldBuffer(hfield_id);
+  if (mesh == nullptr) {
+    mju_error("Unknown height field %d", hfield_id);
+  }
+  return mesh;
+}
+
+static void PrepareGeomMeshes(mjrRenderable* renderable, const mjvGeom& geom,
+                              const mjvScene* scene,
+                              ModelObjects* model_objects) {
+  const mjModel* model = model_objects->GetModel();
+  const int nstack = model->vis.quality.numstacks;
+  const int nslice = model->vis.quality.numslices;
+  const int nquad = model->vis.quality.numquads;
+
+  float position[3];
+  std::memcpy(position, &geom.pos, 3 * sizeof(float));
+  float rotation[9];
+  std::memcpy(rotation, &geom.mat, 9 * sizeof(float));
+  float size[3];
+  std::memcpy(size, &geom.size, 3 * sizeof(float));
+
+  switch ((mjtGeom)geom.type) {
+    case mjGEOM_MESH:
+    case mjGEOM_SDF:
+      mjrf_setRenderableMesh(renderable, GetMesh(model_objects, geom.dataid), 0, 0);
+      // Ignore size for meshes.
+      size[0] = 1.f;
+      size[1] = 1.f;
+      size[2] = 1.f;
+      break;
+    case mjGEOM_HFIELD:
+      mjrf_setRenderableMesh(renderable, GetHeightField(model_objects, geom.dataid), 0, 0);
+      // Ignore size for meshes.
+      size[0] = 1.f;
+      size[1] = 1.f;
+      size[2] = 1.f;
+      break;
+    case mjGEOM_PLANE: {
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+
+      const bool is_infinite = !(size[0] > 0 && size[1] > 0);
+      if (is_infinite) {
+        // Infinite planes are scaled to match the tile size used by
+        // re-centering in engine_vis_visualize.c.
+        const float plane_scale = static_cast<float>(mjMAXPLANEGRID) / 2.0f;
+        size[0] = plane_scale;
+        size[1] = plane_scale;
+      }
+      // Planes only define an xy size, so set the z-dimension to 1.0f.
+      size[2] = 1.0f;
+      break;
+    }
+    case mjGEOM_SPHERE:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_ELLIPSOID:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_BOX:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_CAPSULE:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_CYLINDER:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_ARROW:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_ARROW1:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_ARROW2:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_LINE:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_LINEBOX:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_TRIANGLE:
+      mjrf_setRenderableGeomMesh(renderable, (mjtGeom)geom.type, nstack, nslice, nquad);
+      break;
+    case mjGEOM_FLEX:
+      mjrf_setRenderableMesh(renderable, GetSkinFlexMesh(model_objects, geom.objid), 0, 0);
+      // Flexes are defined in global space.
+      std::memset(position, 0, sizeof(position));
+      std::memset(rotation, 0, sizeof(rotation));
+      rotation[0] = 1.f;
+      rotation[4] = 1.f;
+      rotation[8] = 1.f;
+      size[0] = 1.f;
+      size[1] = 1.f;
+      size[2] = 1.f;
+      break;
+    case mjGEOM_SKIN:
+      mjrf_setRenderableMesh(renderable, GetSkinFlexMesh(model_objects, geom.objid), 0, 0);
+      // Skins are defined in global space.
+      std::memset(position, 0, sizeof(position));
+      std::memset(rotation, 0, sizeof(rotation));
+      rotation[0] = 1.f;
+      rotation[4] = 1.f;
+      rotation[8] = 1.f;
+      size[0] = 1.f;
+      size[1] = 1.f;
+      size[2] = 1.f;
+      break;
+    case mjGEOM_NONE:
+    case mjGEOM_LABEL:
+      // Do nothing.
+      break;
+    case mjNGEOMTYPES:
+      mju_warning("Unsupported geom type: %d", geom.type);
+      break;
+  }
+
+  mjrf_setRenderableTransform(renderable, position, rotation, size);
+}
+
+static void UpdateGeomMaterial(mjrRenderable* renderable, const mjvGeom& geom,
+                               const mjvScene* scene, ModelObjects* model_objs,
+                               const float headpos[3]) {
+  const mjModel* model = model_objs->GetModel();
+
+  const bool use_segid_color = scene->flags[mjRND_IDCOLOR];
+  mjrMaterial material;
+  mjr_defaultMaterial(&material);
+
+  if (geom.category == mjCAT_DECOR) {
+    material.decor_ux = true;
+  }
+
+  material.color[0] = geom.rgba[0];
+  material.color[1] = geom.rgba[1];
+  material.color[2] = geom.rgba[2];
+  material.color[3] = geom.rgba[3];
+  if (geom.type == mjGEOM_PLANE) {
+    if (IsBehind(headpos, geom.pos, geom.mat)) {
+      material.color[3] *= 0.3;
+      mjrf_setRenderableReceiveShadows(renderable, false);
+      material.reflective = false;
+    } else {
+      mjrf_setRenderableReceiveShadows(renderable, true);
+      material.reflective = geom.reflectance > 0 && material.color[3] == 1.0f;
+    }
+  }
+  mjrf_setRenderableLayerMask(renderable, geom.category);
+  if (geom.category == mjCAT_DECOR) {
+    mjrf_setRenderableCastShadows(renderable, false);
+    mjrf_setRenderableReceiveShadows(renderable, false);
+  } else {
+    mjrf_setRenderableWireframe(renderable, scene->flags[mjRND_WIREFRAME]);
+  }
+
+  if (geom.matid >= 0) {
+    material.color_texture = model_objs->GetTexture(geom.matid, mjTEXROLE_RGB);
+    material.normal_texture =
+        model_objs->GetTexture(geom.matid, mjTEXROLE_NORMAL);
+    material.emissive_texture =
+        model_objs->GetTexture(geom.matid, mjTEXROLE_EMISSIVE);
+    material.orm_texture = model_objs->GetTexture(geom.matid, mjTEXROLE_ORM);
+    material.metallic_texture =
+        model_objs->GetTexture(geom.matid, mjTEXROLE_METALLIC);
+    material.roughness_texture =
+        model_objs->GetTexture(geom.matid, mjTEXROLE_ROUGHNESS);
+    material.occlusion_texture =
+        model_objs->GetTexture(geom.matid, mjTEXROLE_OCCLUSION);
+  }
+
+  material.reflectance = geom.reflectance;
+  material.emissive = geom.emission;
+  material.specular = geom.specular;
+  material.glossiness = geom.shininess;
+  if (geom.matid >= 0) {
+    material.metallic = model->mat_metallic[geom.matid];
+    material.roughness = model->mat_roughness[geom.matid];
+  }
+
+  if (geom.segid >= 0) {
+    uint32_t segmentation_color = geom.segid + 1;
+    if (!use_segid_color) {
+      constexpr double phi1 = 1.61803398874989484820;  // Cached Phi(1).
+      constexpr double coef1 = 1.0 / phi1;
+      const double index = static_cast<double>(geom.segid);
+      const double sample = std::fmod(0.5 + coef1 * index, 1.0);
+      segmentation_color = 0x01000000 * sample;
+    }
+
+    const uint8_t red = (segmentation_color >> 0) & 0xff;
+    const uint8_t green = (segmentation_color >> 8) & 0xff;
+    const uint8_t blue = (segmentation_color >> 16) & 0xff;
+    material.segmentation_color[0] = static_cast<float>(red) / 255.0f;
+    material.segmentation_color[1] = static_cast<float>(green) / 255.0f;
+    material.segmentation_color[2] = static_cast<float>(blue) / 255.0f;
+  }
+
+  // UvScale only applies to objects that don't have explicit UV coordinates
+  // in their vertex buffer. Instead, we set the UV coordinate to be the same
+  // as the vertex position.
+  //
+  // The material's `texuniform` and `texrepeat` parameters allow us to scale
+  // the programmatic UVs.
+
+  if (material.color_texture) {
+    const bool tex_uniform = model->mat_texuniform[geom.matid];
+    if (mjrf_getSamplerType(material.color_texture) == mjTEXTURE_2D) {
+      // For 2D textures, `tex_repeat` specifies how many times the texture
+      // image is repeated. The `tex_uniform` flag determines if the repetition
+      // is applied at in object space (false) or in world space (true).
+      float tex_repeat[2];
+      tex_repeat[0] = model->mat_texrepeat[(geom.matid * 2) + 0];
+      tex_repeat[1] = model->mat_texrepeat[(geom.matid * 2) + 1];
+      material.uv_scale[0] = tex_repeat[0];
+      material.uv_scale[1] = tex_repeat[1];
+
+      if (geom.dataid >= 0 && geom.type != mjGEOM_PLANE) {
+        if (geom.size[0] > mjMINVAL) {
+          material.uv_scale[0] /= geom.size[0];
+        }
+        if (geom.size[1] > mjMINVAL) {
+          material.uv_scale[1] /= geom.size[1];
+        }
+      }
+
+      if (tex_uniform) {
+        if (geom.size[0] > 0) {
+          material.uv_scale[0] *= geom.size[0];
+        }
+        if (geom.size[1] > 0) {
+          material.uv_scale[1] *= geom.size[1];
+        }
+      }
+      const bool is_infinite_plane =
+          geom.type == mjGEOM_PLANE && (geom.size[0] <= 0 || geom.size[1] <= 0);
+      if (is_infinite_plane) {
+        // Infinite planes are scaled to match the tile size used by
+        // re-centering in engine_vis_visualize.c.
+        const float plane_scale = static_cast<float>(mjMAXPLANEGRID) / 2.0f;
+        const float tile_size_x =
+            GetPlaneTileSize(model, geom.matid, tex_repeat[0]);
+        const float tile_size_y =
+            GetPlaneTileSize(model, geom.matid, tex_repeat[1]);
+        material.uv_scale[0] = 2.0f * plane_scale / tile_size_x;
+        material.uv_scale[1] = 2.0f * plane_scale / tile_size_y;
+      }
+
+      // We want to do the equivalent of:
+      //   mjr_setf4(splane, 0.5 * scl.x,  0, 0, -0.5);
+      //   mjr_setf4(tplane, 0, -0.5 * scl.y, 0, -0.5);
+      //   glTexGenfv(GL_S, GL_OBJECT_PLANE, splane);
+      //   glTexGenfv(GL_T, GL_OBJECT_PLANE, tplane);
+      material.uv_scale[0] = 0.5f * material.uv_scale[0];
+      material.uv_scale[1] = -0.5f * material.uv_scale[1];
+      material.uv_offset[0] = -0.5f;
+      material.uv_offset[1] = -0.5f;
+    } else {
+      // For cube maps, if `tex_uniform` is true, then scale the texture so that
+      // it covers a 1x1 area of world space rather than the area of the object.
+      if (tex_uniform) {
+        material.uv_scale[0] = 1.0f / (geom.size[0] ? geom.size[0] : 1.0f);
+        material.uv_scale[1] = 1.0f / (geom.size[1] ? geom.size[1] : 1.0f);
+        material.uv_scale[2] = 1.0f / (geom.size[2] ? geom.size[2] : 1.0f);
+      }
+    }
+  }
+
+  // Apply material multipliers from the model.
+  material.emissive *= model_objs->GetEmissiveMultiplier();
+  material.specular *= model_objs->GetSpecularMultiplier();
+  material.glossiness *= model_objs->GetShininessMultiplier();
+
+  mjrf_setRenderableMaterial(renderable, &material);
+}
+
+UniquePtr<mjrRenderable> CreateGeomRenderable(
+    const mjvGeom& geom, const mjvScene* scene, mjrfContext* ctx,
+    ModelObjects* model_objs, const float headpos[3]) {
+  mjrRenderableParams params;
+  mjr_defaultRenderableParams(&params);
+  auto renderable = CreateRenderable(ctx, params);
+  PrepareGeomMeshes(renderable.get(), geom, scene, model_objs);
+  UpdateGeomMaterial(renderable.get(), geom, scene, model_objs, headpos);
+  return renderable;
+}
+}  // namespace mujoco

src/experimental/filament/compat/scene_geom_util.h

@@ -12,23 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/filament/filament/buffer_util.h"
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_GEOM_UTIL_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_GEOM_UTIL_H_
 
-#include <cstddef>
-
-#include <backend/BufferDescriptor.h>
+#include <mujoco/mjvisualize.h>
+#include "experimental/filament/compat/model_objects.h"
+#include "experimental/filament/render_context_filament.h"
+#include "experimental/filament/render_context_filament_cpp.h"
 
 namespace mujoco {
 
-filament::backend::BufferDescriptor CreateBufferDescriptor(
-    std::size_t num_bytes, const FillBufferFn& fill) {
-  std::byte* bytes = new std::byte[num_bytes];
-  fill(bytes, num_bytes);
-  const auto callback = [](void* buffer, size_t size, void* user) {
-    auto* ptr = reinterpret_cast<std::byte*>(user);
-    delete[] ptr;
-  };
-  return filament::backend::BufferDescriptor(bytes, num_bytes, callback, bytes);
-}
+// Creates a Renderable from the given mjvGeom.
+UniquePtr<mjrRenderable> CreateGeomRenderable(
+    const mjvGeom& geom, const mjvScene* scene, mjrfContext* ctx,
+    ModelObjects* model_objs, const float headpos[3]);
 
 }  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_SCENE_GEOM_UTIL_H_

src/experimental/filament/filament/buffer_util.h

@@ -1,152 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_
-
-#include <cstddef>
-#include <cstdint>
-#include <functional>
-#include <optional>
-#include <type_traits>
-#include <backend/BufferDescriptor.h>
-#include <filament/Box.h>
-#include <filament/Engine.h>
-#include <filament/IndexBuffer.h>
-#include <filament/RenderableManager.h>
-#include <filament/VertexBuffer.h>
-
-// Functions for creating filament vertex and index buffers.
-namespace mujoco {
-
-// Simple tuple-type of a IndexBuffer+VertexBuffer.
-struct FilamentBuffers {
-  filament::IndexBuffer* index_buffer = nullptr;
-  filament::VertexBuffer* vertex_buffer = nullptr;
-  std::optional<filament::Box> bounds = std::nullopt;
-  filament::RenderableManager::PrimitiveType type =
-      filament::RenderableManager::PrimitiveType::TRIANGLES;
-};
-
-// Function that fills in the given buffer with actual data.
-using FillBufferFn = std::function<void(std::byte*, std::size_t)>;
-
-// Creates and populates a BufferDescriptor (for vertex and index buffers).
-filament::backend::BufferDescriptor CreateBufferDescriptor(
-    std::size_t num_bytes, const FillBufferFn& fill);
-
-// Creates a filament::VertexBuffer based on the VertexType. The fill function
-// will be used to populate the buffer.
-template <typename VertexType>
-filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine,
-                                           std::size_t num_vertices,
-                                           const FillBufferFn& fill) {
-  int vertex_size = 0;
-  if constexpr (VertexType::kHasPosition) {
-    vertex_size += sizeof(VertexType::position);
-  }
-  if constexpr (VertexType::kHasPosition2d) {
-    vertex_size += sizeof(VertexType::position);
-  }
-  if constexpr (VertexType::kHasOrientation) {
-    vertex_size += sizeof(VertexType::orientation);
-  }
-  if constexpr (VertexType::kHasUv) {
-    vertex_size += sizeof(VertexType::uv);
-  }
-  if constexpr (VertexType::kHasColor) {
-    vertex_size += sizeof(VertexType::color);
-  }
-
-  auto builder = filament::VertexBuffer::Builder();
-  builder.bufferCount(1);
-  builder.vertexCount(num_vertices);
-
-  int offset = 0;
-  if constexpr (VertexType::kHasPosition) {
-    builder.attribute(filament::VertexAttribute::POSITION, 0,
-                      filament::VertexBuffer::AttributeType::FLOAT3, offset,
-                      vertex_size);
-    offset += sizeof(VertexType::position);
-  }
-  if constexpr (VertexType::kHasPosition2d) {
-    builder.attribute(filament::VertexAttribute::POSITION, 0,
-                      filament::VertexBuffer::AttributeType::FLOAT2, offset,
-                      vertex_size);
-    offset += sizeof(VertexType::position);
-  }
-  if constexpr (VertexType::kHasOrientation) {
-    builder.attribute(filament::VertexAttribute::TANGENTS, 0,
-                      filament::VertexBuffer::AttributeType::FLOAT4, offset,
-                      vertex_size);
-    offset += sizeof(VertexType::orientation);
-  }
-  if constexpr (VertexType::kHasUv) {
-    builder.attribute(filament::VertexAttribute::UV0, 0,
-                      filament::VertexBuffer::AttributeType::FLOAT2, offset,
-                      vertex_size);
-    offset += sizeof(VertexType::uv);
-  }
-  if constexpr (VertexType::kHasColor) {
-    builder.attribute(filament::VertexAttribute::COLOR, 0,
-                      filament::VertexBuffer::AttributeType::UBYTE4, offset,
-                      vertex_size);
-    builder.normalized(filament::VertexAttribute::COLOR);
-    offset += sizeof(VertexType::color);
-  }
-
-  auto vb = builder.build(*engine);
-  const std::size_t buffer_size = num_vertices * vertex_size;
-  vb->setBufferAt(*engine, 0, CreateBufferDescriptor(buffer_size, fill));
-  return vb;
-}
-
-// Creates a filament::IndexBuffer. The IndexType should be either uin16_t or
-// uint32_t. The fill function will be used to populate the buffer.
-template <typename IndexType>
-filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine,
-                                         std::size_t num_indices,
-                                         const FillBufferFn& fill) {
-  static_assert(std::is_same<IndexType, uint16_t>::value ||
-                std::is_same<IndexType, uint32_t>::value);
-
-  constexpr auto type = std::is_same<IndexType, uint16_t>::value
-                            ? filament::IndexBuffer::IndexType::USHORT
-                            : filament::IndexBuffer::IndexType::UINT;
-
-  auto builder = filament::IndexBuffer::Builder();
-  builder.bufferType(type);
-  builder.indexCount(num_indices);
-
-  auto ib = builder.build(*engine);
-
-  const std::size_t buffer_size = num_indices * sizeof(IndexType);
-  ib->setBuffer(*engine, CreateBufferDescriptor(buffer_size, fill));
-  return ib;
-}
-
-// Fills an index buffer with a basic incrementing sequence.
-template <typename T>
-int FillSequence(std::byte* buffer, std::size_t num_bytes) {
-  const T num = num_bytes / sizeof(T);
-  T* ptr = reinterpret_cast<T*>(buffer);
-  for (T i = 0; i < num; ++i) {
-    ptr[i] = i;
-  }
-  return num;
-}
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_

src/experimental/filament/filament/builtins.h

@@ -15,22 +15,42 @@
 #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUILTINS_H_
 #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUILTINS_H_
 
-#include <filament/Engine.h>
-#include "experimental/filament/filament/buffer_util.h"
+#include <memory>
+
+#include <filament/Engine.h>
+#include "experimental/filament/filament/mesh.h"
 
-// Generates buffers for built-in shapes.
 namespace mujoco {
 
-FilamentBuffers CreateLine(filament::Engine* engine);
-FilamentBuffers CreatePlane(filament::Engine* engine, int nquad);
-FilamentBuffers CreateTriangle(filament::Engine* engine);
-FilamentBuffers CreateBox(filament::Engine* engine, int nquad);
-FilamentBuffers CreateLineBox(filament::Engine* engine);
-FilamentBuffers CreateSphere(filament::Engine* engine, int nstack, int nslice);
-FilamentBuffers CreateTube(filament::Engine* engine, int nstack, int nslice);
-FilamentBuffers CreateDisk(filament::Engine* engine, int nslice);
-FilamentBuffers CreateDome(filament::Engine* engine, int nstack, int nslice);
-FilamentBuffers CreateCone(filament::Engine* engine, int nstack, int nslice);
+// A collection of meshes that "built in" to the renderer.
+class Builtins {
+ public:
+  Builtins(filament::Engine* engine, int nstack, int nslice, int nquad);
+
+  // Returns a mesh for the corresponding built-in shape.
+  const Mesh* Line();
+  const Mesh* LineBox();
+  const Mesh* Plane();
+  const Mesh* Triangle();
+  const Mesh* Box();
+  const Mesh* Sphere();
+  const Mesh* Cone();
+  const Mesh* Disk();
+  const Mesh* Dome();
+  const Mesh* Tube();
+
+ private:
+  std::unique_ptr<Mesh> line_;
+  std::unique_ptr<Mesh> line_box_;
+  std::unique_ptr<Mesh> plane_;
+  std::unique_ptr<Mesh> triangle_;
+  std::unique_ptr<Mesh> box_;
+  std::unique_ptr<Mesh> sphere_;
+  std::unique_ptr<Mesh> cone_;
+  std::unique_ptr<Mesh> disk_;
+  std::unique_ptr<Mesh> dome_;
+  std::unique_ptr<Mesh> tube_;
+};
 
 }  // namespace mujoco
 

src/experimental/filament/filament/drawable.cc

@@ -1,557 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/drawable.h"
-
-#include <cmath>
-#include <cstdint>
-#include <numbers>
-#include <utility>
-
-#include <filament/Material.h>
-#include <filament/RenderableManager.h>
-#include <filament/Scene.h>
-#include <filament/Texture.h>
-#include <filament/TransformManager.h>
-#include <math/mat4.h>
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-#include <utils/Entity.h>
-#include <mujoco/mjvisualize.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-#include "experimental/filament/filament/geom_util.h"
-#include "experimental/filament/filament/material.h"
-#include "experimental/filament/filament/math_util.h"
-#include "experimental/filament/filament/model_objects.h"
-#include "experimental/filament/filament/object_manager.h"
-
-namespace mujoco {
-
-using filament::math::float2;
-using filament::math::float3;
-using filament::math::float4;
-using filament::math::mat4;
-
-// An arbitrary scale factor for arrows.
-static constexpr float kArrowScale = 1.f / 6.f;
-static constexpr float kArrowHeadSize = 1.75f;
-
-// Some built-in geometries are actually composed of multiple simple shapes. A
-// capsule, for example, is a open-ended tube with two dome ends. We use these
-// constants to help identify which entity (by index) represents which part of
-// the overall shape.
-static constexpr int kCapsuleTopDome = 1;
-static constexpr int kCapsuleBottomDome = 2;
-static constexpr int kCylinderTopDisk = 1;
-static constexpr int kCylinderBottomDisk = 2;
-static constexpr int kArrow0Cone = 1;
-static constexpr int kArrow0ConeDisk = 2;
-static constexpr int kArrow0BottomDisk = 3;
-static constexpr int kArrow1Cone = 1;
-static constexpr int kArrow1BottomDisk = 2;
-static constexpr int kArrow2TopCone = 1;
-static constexpr int kArrow2BottomCone = 2;
-static constexpr int kArrow2TopConeDisk = 3;
-static constexpr int kArrow2BottomConeDisk = 4;
-
-// Returns the tile size for infinite plane texture alignment.
-// This is duplicated from engine_vis_visualize.c (re-center infinite plane)
-// to ensure UV scaling matches the re-centering increments.
-static float GetPlaneTileSize(const mjModel* model, int matid,
-                              float texrepeat) {
-  if (matid >= 0 && texrepeat > 0) {
-    return 2.0f / texrepeat;
-  } else {
-    const float zfar = model->vis.map.zfar * model->stat.extent;
-    return 2.1f * zfar / (mjMAXPLANEGRID - 2);
-  }
-}
-
-static bool IsBehind(const mjtNum* headpos, const float* pos, const float* mat) {
-  return ((headpos[0] - pos[0]) * mat[2] +
-          (headpos[1] - pos[1]) * mat[5] +
-          (headpos[2] - pos[2]) * mat[8] < 0.0f);
-}
-
-Drawable::Drawable(ObjectManager* object_mgr, ModelObjects* model_objects,
-                   const mjvGeom& geom)
-    : material_(object_mgr),
-      model_objs_(model_objects),
-      renderables_(object_mgr->GetEngine()) {
-  if (geom.category == mjCAT_DECOR) {
-    renderables_.SetCastShadows(false);
-    renderables_.SetReceiveShadows(false);
-  }
-
-  switch ((mjtGeom)geom.type) {
-    case mjGEOM_MESH:
-      AddMesh(geom.dataid);
-      break;
-    case mjGEOM_HFIELD:
-      AddHeightField(geom.dataid);
-      break;
-    case mjGEOM_PLANE:
-      AddShape(ModelObjects::kPlane);
-      break;
-    case mjGEOM_SPHERE:
-      AddShape(ModelObjects::kSphere);
-      break;
-    case mjGEOM_ELLIPSOID:
-      AddShape(ModelObjects::kSphere);
-      break;
-    case mjGEOM_BOX:
-      AddShape(ModelObjects::kBox);
-      break;
-    case mjGEOM_CAPSULE:
-      AddShape(ModelObjects::kTube);
-      AddShape(ModelObjects::kDome);
-      AddShape(ModelObjects::kDome);
-      break;
-    case mjGEOM_CYLINDER:
-      AddShape(ModelObjects::kTube);
-      AddShape(ModelObjects::kDisk);
-      AddShape(ModelObjects::kDisk);
-      break;
-    case mjGEOM_ARROW:
-      AddShape(ModelObjects::kTube);
-      AddShape(ModelObjects::kCone);
-      AddShape(ModelObjects::kDisk);
-      break;
-    case mjGEOM_ARROW1:
-      AddShape(ModelObjects::kTube);
-      AddShape(ModelObjects::kCone);
-      AddShape(ModelObjects::kDisk);
-      AddShape(ModelObjects::kDisk);
-      break;
-    case mjGEOM_ARROW2:
-      AddShape(ModelObjects::kTube);
-      AddShape(ModelObjects::kCone);
-      AddShape(ModelObjects::kCone);
-      AddShape(ModelObjects::kDisk);
-      AddShape(ModelObjects::kDisk);
-      break;
-    case mjGEOM_LINE:
-      AddShape(ModelObjects::kLine);
-      break;
-    case mjGEOM_LINEBOX:
-      AddShape(ModelObjects::kLineBox);
-      break;
-    case mjGEOM_TRIANGLE:
-      AddShape(ModelObjects::kTriangle);
-      break;
-    case mjGEOM_FLEX:
-    case mjGEOM_SKIN:
-      // Flex and skin geometries are dynamically updated every frame.
-      break;
-    case mjGEOM_NONE:
-    case mjGEOM_LABEL:
-      // Do nothing .
-      break;
-    case mjGEOM_SDF:
-    case mjNGEOMTYPES:
-      mju_warning("Unsupported geom type: %d", geom.type);
-      break;
-  }
-}
-
-void Drawable::Update(const mjModel* model, const mjvScene* scene,
-                      const mjvGeom& geom) {
-  if (geom.type == mjGEOM_FLEX || geom.type == mjGEOM_SKIN) {
-    // Flex geometry is updated every frame with new vertex data.
-    filament::Engine* engine = renderables_.GetEngine();
-    FilamentBuffers buffers = CreateGeomBuffers(engine, model, scene, geom);
-
-    if (renderables_.GetNumEntities() == 0) {
-      renderables_.Append(std::move(buffers));
-    } else {
-      renderables_.Update(0, std::move(buffers));
-    }
-  }
-
-  mjtNum head_pos[3];
-  mjv_cameraInModel(head_pos, nullptr, nullptr, scene);
-
-  SetTransform(geom);
-  UpdateMaterial(geom, scene->flags[mjRND_IDCOLOR],
-                 scene->flags[mjRND_REFLECTION], head_pos);
-  renderables_.SetWireframe(scene->flags[mjRND_WIREFRAME]);
-}
-
-void Drawable::AddMesh(int data_id) {
-  const FilamentBuffers* buffers = model_objs_->GetMeshBuffer(data_id);
-  if (buffers == nullptr) {
-    mju_error("Unknown mesh %d", data_id);
-  }
-  renderables_.Append(*buffers);
-}
-
-void Drawable::AddHeightField(int hfield_id) {
-  const FilamentBuffers* buffers = model_objs_->GetHeightFieldBuffer(hfield_id);
-  if (buffers == nullptr) {
-    mju_error("Unknown height field %d", hfield_id);
-  }
-  renderables_.Append(*buffers);
-}
-
-void Drawable::AddShape(ModelObjects::ShapeType shape_type) {
-  const FilamentBuffers* buffers = model_objs_->GetShapeBuffer(shape_type);
-  if (buffers == nullptr) {
-    mju_error("Unknown shape %d", shape_type);
-  }
-  renderables_.Append(*buffers);
-}
-
-void Drawable::AddToScene(filament::Scene* scene) {
-  renderables_.AddToScene(scene);
-}
-
-void Drawable::RemoveFromScene(filament::Scene* scene) {
-  renderables_.RemoveFromScene(scene);
-}
-
-void Drawable::SetDrawMode(Material::DrawMode mode) {
-  renderables_.SetMaterialInstance(material_.GetMaterialInstance(mode));
-}
-
-void Drawable::UpdateReflectionTexture(const filament::Texture* tex) {
-  material_.UpdateReflectionTexture(tex);
-}
-
-void Drawable::SetLayerMask(std::uint8_t mask) {
-  renderables_.SetLayerMask(mask);
-}
-
-void Drawable::SetTransform(const mjvGeom& geom) {
-  // Flex and skin geometries are in global space.
-  if (geom.type == mjGEOM_FLEX || geom.type == mjGEOM_SKIN) {
-    return;
-  }
-
-  transform_ = mat4(ReadMat3(geom.mat), ReadFloat3(geom.pos));
-
-  float3 size = ReadFloat3(geom.size);
-  filament::TransformManager& tm =
-      renderables_.GetEngine()->getTransformManager();
-  for (int j = 0; j < renderables_.GetNumEntities(); ++j) {
-    const utils::Entity& entity = renderables_[j];
-
-    // Update object transform.
-    mat4 entity_transform = transform_;
-
-    // Some built-in drawables are composed of multiple entities. For example,
-    // capsules are a combination of a open tube and two dome end caps.
-
-    if (geom.type == mjGEOM_CYLINDER) {
-      // Cylinders are a tube with two disks at the ends. The "bottom" disk is
-      // rotated so that the normals point outwards.
-      if (j == kCylinderTopDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-      } else if (j == kCylinderBottomDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-      }
-    } else if (geom.type == mjGEOM_CAPSULE) {
-      // Capsules are a tube with two domes at the ends. We apply an inverse
-      // scale to the domes to "counteract" the capsule's overall scale so that
-      // the domes remain spherical in shape.
-      const float xz_size = 0.5f * (size.x + size.y);
-      if (j == kCapsuleTopDome) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-        entity_transform *= mat4::scaling(float3{1, 1, xz_size / size.z});
-      } else if (j == kCapsuleBottomDome) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-        entity_transform *= mat4::scaling(float3{1, 1, xz_size / size.z});
-      }
-    } else if (geom.type == mjGEOM_ARROW) {
-      // An arrow is a tube with a cone at the end and a disk cap at the other
-      // end. Because the cone head's base is larger than the tube, an extra
-      // disk is added to the base of the cone. This disk is rotated such that
-      // its normal points outwards.
-      entity_transform *= mat4::scaling(float3{1, 1, kArrowScale});
-      entity_transform *= mat4::translation(float3{0, 0, size.z});
-      if (j == kArrow0Cone) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      } else if (j == kArrow0ConeDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      } else if (j == kArrow0BottomDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-      }
-    } else if (geom.type == mjGEOM_ARROW1) {
-      // An arrow1 is a tube with a cone at the end and a disk cap at the other
-      // end.
-      entity_transform *= mat4::scaling(float3{1, 1, kArrowScale});
-      entity_transform *= mat4::translation(float3{0, 0, size.z});
-      if (j == kArrow1Cone) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-      } else if (j == kArrow1BottomDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-      }
-    } else if (geom.type == mjGEOM_ARROW2) {
-      // An arrow2 is a tube with a cone at both ends.  Like the standard arrow,
-      // an extra disk is added to the base of each cone.
-      entity_transform *= mat4::scaling(float3{1, 1, kArrowScale});
-      entity_transform *= mat4::translation(float3{0, 0, size.z});
-      if (j == kArrow2TopCone) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      } else if (j == kArrow2BottomCone) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      } else if (j == kArrow2TopConeDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, size.z});
-        entity_transform *= mat4::rotation(std::numbers::pi, float3{1, 0, 0});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      } else if (j == kArrow2BottomConeDisk) {
-        entity_transform *= mat4::translation(float3{0, 0, -size.z});
-        entity_transform *=
-            mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
-      }
-    }
-    if (geom.type == mjGEOM_PLANE) {
-      const bool is_infinite = !(size.x > 0 && size.y > 0);
-      if (is_infinite) {
-        // Infinite planes are scaled to match the tile size used by
-        // re-centering in engine_vis_visualize.c.
-        const float plane_scale = static_cast<float>(mjMAXPLANEGRID) / 2.0f;
-        entity_transform *=
-            mat4::scaling(float3{plane_scale, plane_scale, 1.0f});
-      } else {
-        // Regular planes are scaled by geom.size.
-        entity_transform *= mat4::scaling(float3{size.x, size.y, 1.0f});
-      }
-    } else if (geom.type != mjGEOM_MESH && geom.type != mjGEOM_HFIELD) {
-      entity_transform *= mat4::scaling(size);
-    }
-    tm.setTransform(tm.getInstance(entity), entity_transform);
-  }
-}
-
-void Drawable::UpdateMaterial(const mjvGeom& geom, bool use_segid_color,
-                              bool enable_reflection, const mjtNum* headpos) {
-  const mjModel* model = model_objs_->GetModel();
-
-  float4 color = ReadFloat4(geom.rgba);
-  if (geom.type == mjGEOM_PLANE) {
-    if (IsBehind(headpos, geom.pos, geom.mat)) {
-      color[3] *= 0.3;
-      renderables_.SetReceiveShadows(false);
-      reflective_ = false;
-    } else {
-      renderables_.SetReceiveShadows(true);
-      reflective_ =
-          enable_reflection && geom.reflectance > 0 && color.a == 1.0f;
-    }
-  }
-
-  Material::Textures textures;
-  if (geom.matid >= 0) {
-    textures.color = model_objs_->GetTexture(geom.matid, mjTEXROLE_RGB);
-    textures.normal = model_objs_->GetTexture(geom.matid, mjTEXROLE_NORMAL);
-    textures.emissive = model_objs_->GetTexture(geom.matid, mjTEXROLE_EMISSIVE);
-    textures.orm = model_objs_->GetTexture(geom.matid, mjTEXROLE_ORM);
-    textures.metallic = model_objs_->GetTexture(geom.matid, mjTEXROLE_METALLIC);
-    textures.roughness =
-        model_objs_->GetTexture(geom.matid, mjTEXROLE_ROUGHNESS);
-    textures.occlusion =
-        model_objs_->GetTexture(geom.matid, mjTEXROLE_OCCLUSION);
-    material_.UpdateTextures(textures);
-  }
-
-  if (geom.type == mjGEOM_LINE || geom.type == mjGEOM_LINEBOX) {
-    material_.SetNormalMaterialType(ObjectManager::kUnlitLine);
-  } else {
-    bool material_assigned = false;
-    if (geom.matid >= 0) {
-      material_assigned = true;
-      if (textures.orm) {
-        material_.SetNormalMaterialType(ObjectManager::kPbrPacked);
-      } else if (textures.metallic) {
-        material_.SetNormalMaterialType(ObjectManager::kPbr);
-      } else if (textures.roughness) {
-        material_.SetNormalMaterialType(ObjectManager::kPbr);
-      } else if (model->mat_metallic[geom.matid] >= 0) {
-        material_.SetNormalMaterialType(ObjectManager::kPbr);
-      } else if (model->mat_roughness[geom.matid] >= 0) {
-        material_.SetNormalMaterialType(ObjectManager::kPbr);
-      } else {
-        material_assigned = false;
-      }
-    }
-
-    if (!material_assigned) {
-      // Check to see if we're dealing with a mesh with texture coordinates.
-      // `data_id` is the id of the mesh in model (i.e. the geom has mesh
-      // geometry) and `mesh_texcoordadr` stores the address of the mesh uvs if
-      // it has them.
-      bool has_texcoords = false;
-      if ((geom.type == mjGEOM_MESH || geom.type == mjGEOM_SDF) &&
-          geom.dataid >= 0 && model->mesh_texcoordadr[geom.dataid / 2] >= 0) {
-        has_texcoords = true;
-      }
-
-      if (textures.color == nullptr) {
-        if (color.a < 1.0f) {
-          material_.SetNormalMaterialType(ObjectManager::kPhongColorFade);
-        } else if (reflective_) {
-          material_.SetNormalMaterialType(ObjectManager::kPhongColorReflect);
-        } else {
-          material_.SetNormalMaterialType(ObjectManager::kPhongColor);
-        }
-      } else if (textures.color->getTarget() ==
-                filament::Texture::Sampler::SAMPLER_CUBEMAP) {
-        if (color.a < 1.0f) {
-          material_.SetNormalMaterialType(ObjectManager::kPhongCubeFade);
-        } else if (reflective_) {
-          material_.SetNormalMaterialType(ObjectManager::kPhongCubeReflect);
-        } else {
-          material_.SetNormalMaterialType(ObjectManager::kPhongCube);
-        }
-      } else if (has_texcoords) {
-        if (color.a < 1.0f) {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2dUvFade);
-        } else if (reflective_) {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2dUvReflect);
-        } else {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2dUv);
-        }
-      } else {
-        if (color.a < 1.0f) {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2dFade);
-        } else if (reflective_) {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2dReflect);
-        } else {
-          material_.SetNormalMaterialType(ObjectManager::kPhong2d);
-        }
-      }
-    }
-  }
-
-  Material::Params params;
-  params.color = color;
-  params.reflectance = geom.reflectance;
-  params.emissive = geom.emission;
-  params.specular = geom.specular;
-  params.glossiness = geom.shininess;
-  if (geom.matid >= 0) {
-    params.metallic = model->mat_metallic[geom.matid];
-    params.roughness = model->mat_roughness[geom.matid];
-    params.tex_uniform = model->mat_texuniform[geom.matid];
-    params.tex_repeat = ReadFloat2(model->mat_texrepeat, geom.matid);
-  }
-
-  if (geom.segid >= 0) {
-    uint32_t segmentation_color = geom.segid + 1;
-    if (!use_segid_color) {
-      constexpr double phi1 = 1.61803398874989484820;  // Cached Phi(1).
-      constexpr double coef1 = 1.0 / phi1;
-      const double index = static_cast<double>(geom.segid);
-      const double sample = std::fmod(0.5 + coef1 * index, 1.0);
-      segmentation_color = 0x01000000 * sample;
-    }
-
-    const uint8_t red = (segmentation_color >> 0) & 0xff;
-    const uint8_t green = (segmentation_color >> 8) & 0xff;
-    const uint8_t blue = (segmentation_color >> 16) & 0xff;
-    params.segmentation_color.x = static_cast<float>(red) / 255.0f;
-    params.segmentation_color.y = static_cast<float>(green) / 255.0f;
-    params.segmentation_color.z = static_cast<float>(blue) / 255.0f;
-  }
-
-  // UvScale only applies to objects that don't have explicit UV coordinates
-  // in their vertex buffer. Instead, we set the UV coordinate to be the same
-  // as the vertex position.
-  //
-  // The material's `texuniform` and `texrepeat` parameters allow us to scale
-  // the programmatic UVs.
-
-  if (textures.color) {
-    if (textures.color->getTarget() == filament::Texture::Sampler::SAMPLER_2D) {
-      // For 2D textures, `tex_repeat` specifies how many times the texture
-      // image is repeated. The `tex_uniform` flag determines if the repetition
-      // is applied at in object space (false) or in world space (true).
-      params.uv_scale.x = params.tex_repeat.x;
-      params.uv_scale.y = params.tex_repeat.y;
-
-      if (geom.dataid >= 0 && geom.type != mjGEOM_PLANE) {
-        if (geom.size[0] > mjMINVAL) {
-          params.uv_scale.x /= geom.size[0];
-        }
-        if (geom.size[1] > mjMINVAL) {
-          params.uv_scale.y /= geom.size[1];
-        }
-      }
-      if (params.tex_uniform) {
-        if (geom.size[0] > 0) {
-          params.uv_scale.x *= geom.size[0];
-        }
-        if (geom.size[1] > 0) {
-          params.uv_scale.y *= geom.size[1];
-        }
-      }
-      const bool is_infinite_plane =
-          geom.type == mjGEOM_PLANE && (geom.size[0] <= 0 || geom.size[1] <= 0);
-      if (is_infinite_plane) {
-        // Infinite planes are scaled to match the tile size used by
-        // re-centering in engine_vis_visualize.c.
-        const float plane_scale = static_cast<float>(mjMAXPLANEGRID) / 2.0f;
-        const float tile_size_x =
-            GetPlaneTileSize(model, geom.matid, params.tex_repeat.x);
-        const float tile_size_y =
-            GetPlaneTileSize(model, geom.matid, params.tex_repeat.y);
-        params.uv_scale.x = 2.0f * plane_scale / tile_size_x;
-        params.uv_scale.y = 2.0f * plane_scale / tile_size_y;
-      }
-
-      // We want to do the equivalent of:
-      //   mjr_setf4(splane, 0.5 * scl.x,  0, 0, -0.5);
-      //   mjr_setf4(tplane, 0, -0.5 * scl.y, 0, -0.5);
-      //   glTexGenfv(GL_S, GL_OBJECT_PLANE, splane);
-      //   glTexGenfv(GL_T, GL_OBJECT_PLANE, tplane);
-      params.uv_scale.x = 0.5f * params.uv_scale.x;
-      params.uv_scale.y = -0.5f * params.uv_scale.y;
-      params.uv_offset.x = -0.5f;
-      params.uv_offset.y = -0.5f;
-    } else {
-      // For cube maps, if `tex_uniform` is true, then scale the texture so that
-      // it covers a 1x1 area of world space rather than the area of the object.
-      if (params.tex_uniform) {
-        params.uv_scale.x = 1.0f / (geom.size[0] ? geom.size[0] : 1.0f);
-        params.uv_scale.y = 1.0f / (geom.size[1] ? geom.size[1] : 1.0f);
-        params.uv_scale.z = 1.0f / (geom.size[2] ? geom.size[2] : 1.0f);
-      }
-    }
-  }
-
-  // Apply material multipliers from the model.
-  params.emissive *= model_objs_->GetEmissiveMultiplier();
-  params.specular *= model_objs_->GetSpecularMultiplier();
-  params.glossiness *= model_objs_->GetShininessMultiplier();
-
-  material_.UpdateParams(params);
-}
-}  // namespace mujoco

src/experimental/filament/filament/drawable.h

@@ -1,94 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_
-
-#include <cstdint>
-#include <filament/Engine.h>
-#include <filament/Scene.h>
-#include <math/mat4.h>
-#include <mujoco/mjmodel.h>
-#include <mujoco/mjtnum.h>
-#include <mujoco/mjvisualize.h>
-#include "experimental/filament/filament/material.h"
-#include "experimental/filament/filament/model_objects.h"
-#include "experimental/filament/filament/object_manager.h"
-#include "experimental/filament/filament/renderables.h"
-
-namespace mujoco {
-
-// Manages the filament Entities and MaterialInstances for a single mjvGeom.
-class Drawable {
- public:
-  Drawable(ObjectManager* object_mgr, ModelObjects* model_objects,
-           const mjvGeom& geom);
-  ~Drawable() noexcept = default;
-
-  Drawable(const Drawable&) = delete;
-  Drawable& operator=(const Drawable&) = delete;
-
-  // Adds the Drawable to the given filament Scene. Note that a Drawable can
-  // only be assigned to a single Scene at any given time.
-  void AddToScene(filament::Scene* scene);
-
-  // Removes the Drawable from the given filament Scene.
-  void RemoveFromScene(filament::Scene* scene);
-
-  // Updates the drawable to reflect the current state (e.g. geometry,
-  // transform, material, etc.) of the geom.
-  void Update(const mjModel* model, const mjvScene* scene, const mjvGeom& geom);
-
-  // Returns the transform of the drawable.
-  const filament::math::mat4& GetTransform() const { return transform_; }
-
-  // Swaps the MaterialInstance that will be used to render the Drawable (e.g.
-  // normal, depth, segmentation, etc.). This must be called before the filament
-  // beginFrame/endFrame.
-  void SetDrawMode(Material::DrawMode mode);
-
-  // Sets the layer mask for all managed entities. This can be used to show
-  // or hide the drawable from specific passes. The default layer mask is 0x01.
-  void SetLayerMask(std::uint8_t mask);
-
-  // Returns true if the drawable is reflective.
-  bool IsReflective() const { return reflective_; }
-
-  // Sets the reflection texture for the drawable. We have a separate setter
-  // because we need to render the reflection texture before it can be applied
-  // to the material.
-  void UpdateReflectionTexture(const filament::Texture* tex);
-
- private:
-  void AddMesh(int data_id);
-  void AddHeightField(int hfield_id);
-  void AddShape(ModelObjects::ShapeType shape_type);
-
-  // Updates the transform of the drawable for rendering.
-  void SetTransform(const mjvGeom& geom);
-
-  // Updates the material parameters of the drawable for rendering.
-  void UpdateMaterial(const mjvGeom& geom, bool use_segid_color,
-                      bool enable_reflection, const mjtNum* headpos);
-
-  Material material_;
-  ModelObjects* model_objs_ = nullptr;
-  Renderables renderables_;
-  bool reflective_ = false;
-  filament::math::mat4 transform_;
-};
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_

src/experimental/filament/filament/filament_context.cc

@@ -14,13 +14,11 @@
 
 #include "experimental/filament/filament/filament_context.h"
 
-#include <cstddef>
 #include <cstdint>
-#include <cstdlib>
 #include <memory>
+#include <span>
 #include <utility>
 
-#include <backend/DriverEnums.h>
 #include <filament/Engine.h>
 #include <filament/IndexBuffer.h>
 #include <filament/IndirectLight.h>
@@ -35,15 +33,10 @@
 #include <math/vec4.h>
 #include <utils/FixedCapacityVector.h>
 #include <utils/compiler.h>
-#include <mujoco/mjmodel.h>
-#include <mujoco/mjvisualize.h>
 #include <mujoco/mujoco.h>
 #include "experimental/filament/filament/filament_platform_factory.h"
-#include "experimental/filament/filament/gui_view.h"
-#include "experimental/filament/filament/imgui_editor.h"
-#include "experimental/filament/filament/model_util.h"
 #include "experimental/filament/filament/object_manager.h"
-#include "experimental/filament/filament/render_target_util.h"
+#include "experimental/filament/filament/render_target.h"
 #include "experimental/filament/filament/scene_view.h"
 #include "experimental/filament/render_context_filament.h"
 
@@ -62,24 +55,23 @@ FilamentContext::FilamentContext(const mjrFilamentConfig* config)
   engine_ = engine_builder.build();
 
   renderer_ = engine_->createRenderer();
-  #ifdef __EMSCRIPTEN__
-    window_swap_chain_ = engine_->createSwapChain(nullptr);
-  #else
+#ifdef __EMSCRIPTEN__
+  window_swap_chain_ = engine_->createSwapChain(nullptr);
+#else
   if (config_.native_window) {
     window_swap_chain_ = engine_->createSwapChain(config_.native_window);
   } else {
-    window_swap_chain_ = engine_->createSwapChain(config_.width, config_.height);
+    window_swap_chain_ =
+        engine_->createSwapChain(config_.width, config_.height);
   }
-  #endif
-  offscreen_swap_chain_ = engine_->createSwapChain(config_.width, config_.height);
+#endif
+  offscreen_swap_chain_ =
+      engine_->createSwapChain(config_.width, config_.height);
 
   object_manager_ = std::make_unique<ObjectManager>(engine_);
 }
 
 FilamentContext::~FilamentContext() {
-  DestroyRenderTargets();
-  gui_view_.reset();
-  scene_view_.reset();
   object_manager_.reset();
   engine_->destroy(renderer_);
   engine_->destroy(window_swap_chain_);
@@ -87,231 +79,121 @@ FilamentContext::~FilamentContext() {
   filament::Engine::destroy(engine_);
 }
 
-void FilamentContext::Init(const mjModel* model) {
-  scene_view_ = std::make_unique<SceneView>(object_manager_.get(), model);
-  gui_view_ = std::make_unique<GuiView>(
-      engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi));
+mjrFrameHandle FilamentContext::Render(
+    std::span<const mjrRenderRequest> requests,
+    std::span<const mjrReadPixelsRequest> read_requests) {
+  if (read_requests.size() > 1) {
+    mju_error("Only one read request is supported for now.");
+  }
 
-  // Set clear options.
+  bool render_began = false;
+  mjrRenderTarget* current_target = nullptr;
+  for (const mjrRenderRequest& request : requests) {
+    if (request.target != current_target && render_began) {
+      renderer_->endFrame();
+      render_began = false;
+    }
+    current_target = request.target;
+
+    if (current_target == nullptr) {
+      if (!read_requests.empty()) {
+        mju_error("Cannot read pixels from the window.");
+      }
+
+      if constexpr (UTILS_HAS_THREADING) {
+        // Wait until previous frame is completed before requesting a new frame.
+        engine_->flushAndWait();
+      }
+
+      // If the window size has changed, we need to reacquire the swap chain.
+      if (request.viewport.width != window_width_ ||
+          request.viewport.height != window_height_) {
+        if (window_width_ != 0 && window_height_ != 0) {
+          engine_->destroy(window_swap_chain_);
+          window_swap_chain_ = engine_->createSwapChain(config_.native_window);
+        }
+        window_width_ = request.viewport.width;
+        window_height_ = request.viewport.height;
+      }
+
+      if (!render_began) {
+        render_began = renderer_->beginFrame(window_swap_chain_);
+      }
+      if (!render_began) {
+        break;
+      }
+      if (render_began) {
+        SceneView* scene_view = SceneView::downcast(request.scene);
+        scene_view->Render(renderer_, request);
+      }
+    } else {
+      if (read_requests.empty()) {
+        mju_error(
+            "Rendering to a render target without a read request is pointless.");
+      }
+
+      const mjrReadPixelsRequest& read_request = read_requests[0];
+      if (read_request.num_bytes == 0) {
+        mju_error("Output buffer size is zero.");
+      }
+
+      if (!render_began) {
+        render_began = renderer_->beginFrame(offscreen_swap_chain_);
+      }
+      if (!render_began) {
+        break;
+      }
+      if (render_began) {
+        SceneView* scene_view = SceneView::downcast(request.scene);
+        scene_view->Render(renderer_, request);
+        RenderTarget* render_target = RenderTarget::downcast(request.target);
+        render_target->ReadColorPixels(renderer_, (uint8_t*)read_request.output,
+                                       read_request.num_bytes);
+      }
+    }
+  }
+
+  if (render_began) {
+    renderer_->endFrame();
+  }
+  if constexpr (!UTILS_HAS_THREADING) {
+    engine_->execute();
+  }
+
+  if (!read_requests.empty()) {
+    engine_->flushAndWait();
+    if (read_requests[0].read_completed_callback) {
+      read_requests[0].read_completed_callback(read_requests[0].user_data);
+    }
+  }
+
+  return ++frame_counter_;
+}
+
+void FilamentContext::WaitForFrame(mjrFrameHandle frame_handle) {
+  if (frame_counter_ < frame_handle) {
+    engine_->flushAndWait();
+  }
+}
+
+void FilamentContext::SetClearColor(const filament::math::float4& color) {
   filament::Renderer::ClearOptions opts;
   opts.clear = true;
   opts.discard = true;
-  opts.clearColor = ReadElement(model, "filament.clearColor",
-                                filament::math::float4(0, 0, 0, 1));
+  opts.clearColor = color;
   renderer_->setClearOptions(opts);
 }
 
-void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) {
-  // If we're rendering to the window, and the window size has changed, we need
-  // to reacquire the swap chain.
-  if (scene_swap_chain_target_ == kWindowSwapChain &&
-      (viewport.width != window_width_ || viewport.height != window_height_)) {
-    if (window_width_ != 0 && window_height_ != 0) {
-      if constexpr (UTILS_HAS_THREADING) {
-        engine_->flushAndWait();
-      }
-      engine_->destroy(window_swap_chain_);
-      window_swap_chain_ = engine_->createSwapChain(config_.native_window);
-    }
-    window_width_ = viewport.width;
-    window_height_ = viewport.height;
-  }
-
-  scene_view_->SetViewport(viewport);
-  scene_view_->UpdateScene(scene);
-  // Update the UX renderable entity after processing the scene in case there
-  // are any elements in the scene which generate UX draw calls (e.g. labels).
-  if (gui_view_ && gui_swap_chain_target_ == scene_swap_chain_target_) {
-    // Prepare the filament Renderable that contains the GUI draw commands. We
-    // must call this function even if we do not plan on rendering the GUI to
-    // ensure the ImGui state is updated.
-    gui_view_->UpdateRenderable();
-  }
-
-  last_render_mode_ = SceneView::DrawMode::kNormal;
-  if (scene->flags[mjRND_SEGMENT]) {
-    last_render_mode_ = SceneView::DrawMode::kSegmentation;
-  } else if (scene->flags[mjRND_DEPTH]) {
-    last_render_mode_ = SceneView::DrawMode::kDepth;
-  }
-
-  // Render the frame if we're not rendering to a texture.
-  if (scene_swap_chain_target_ == kWindowSwapChain) {
-    if constexpr (UTILS_HAS_THREADING) {
-      // Wait until previous frame is completed before requesting a new frame.
-      engine_->flushAndWait();
-    }
-
-    if (renderer_->beginFrame(window_swap_chain_)) {
-      scene_view_->Render(renderer_, last_render_mode_);
-
-      if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) {
-        gui_view_->Render(renderer_);
-      }
-
-      renderer_->endFrame();
-    }
-
-    if constexpr (!UTILS_HAS_THREADING) {
-      engine_->execute();
-    }
-  }
-}
-
-void FilamentContext::SetFrameBuffer(int framebuffer) {
-  switch (framebuffer) {
-    case mjFB_WINDOW:
-      scene_swap_chain_target_ = kWindowSwapChain;
-      gui_swap_chain_target_ = kWindowSwapChain;
-      break;
-    case mjFB_OFFSCREEN:
-      scene_swap_chain_target_ = kOffscreenSwapChain;
-      gui_swap_chain_target_ = kWindowSwapChain;
-      break;
-    case 2:  // No official constant fo this.
-      scene_swap_chain_target_ = kOffscreenSwapChain;
-      gui_swap_chain_target_ = kOffscreenSwapChain;
-      break;
-    default:
-      mju_error("Invalid framebuffer mode: %d", framebuffer);
-  }
-
-  if (framebuffer == 0) {
-    DestroyRenderTargets();
-  }
-}
-
-void FilamentContext::PrepareRenderTargets(int width, int height) {
-  color_target_ = std::make_unique<RenderTargetAndTextures>(
-      engine_, kRenderTargetColor, kRenderTargetDepth);
-  color_target_->Prepare(width, height);
-
-  depth_target_ = std::make_unique<RenderTargetAndTextures>(
-      engine_, kRenderTargetDepthColor, kRenderTargetDepth);
-  depth_target_->Prepare(width, height);
-}
-
-void FilamentContext::DestroyRenderTargets() {
-  depth_target_.reset();
-  color_target_.reset();
-}
-
-static void ReadColorPixels(filament::Renderer* renderer,
-                            filament::RenderTarget* target, mjrRect viewport,
-                            unsigned char* buffer, size_t num_bytes) {
-  filament::backend::PixelBufferDescriptor descriptor(
-      buffer, num_bytes, filament::backend::PixelDataFormat::RGB,
-      filament::backend::PixelDataType::UBYTE);
-  renderer->readPixels(target, viewport.left, viewport.bottom, viewport.width,
-                       viewport.height, std::move(descriptor));
-}
-
-static void ReadDepthPixels(filament::Renderer* renderer,
-                            filament::RenderTarget* target, mjrRect viewport,
-                            float* buffer, size_t num_bytes) {
-  filament::backend::PixelBufferDescriptor descriptor(
-      buffer, num_bytes, filament::backend::PixelDataFormat::R,
-      filament::backend::PixelDataType::FLOAT);
-  renderer->readPixels(target, viewport.left, viewport.bottom, viewport.width,
-                       viewport.height, std::move(descriptor));
-}
-
-void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb,
-                                 float* depth) {
-  if (scene_swap_chain_target_ != kOffscreenSwapChain) {
-    mju_error("Cannot read pixels unless framebuffer is set.");
-  }
-  if (color_target_ == nullptr || depth_target_ == nullptr) {
-    if (viewport.left != 0) {
-      mju_error("Reading subpixels not supported.");
-    }
-    if (viewport.bottom != 0) {
-      mju_error("Reading subpixels not supported.");
-    }
-    PrepareRenderTargets(viewport.width, viewport.height);
-  }
-
-  if (rgb) {
-    if (renderer_->beginFrame(offscreen_swap_chain_)) {
-      scene_view_->Render(renderer_, last_render_mode_,
-                          color_target_->GetRenderTarget());
-
-      // Render the GUI to the texture as well if requested.
-      if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) {
-        gui_view_->Render(renderer_, color_target_->GetRenderTarget());
-      }
-
-      const size_t num_bytes = viewport.width * viewport.height * 3;
-      ReadColorPixels(renderer_, color_target_->GetRenderTarget(), viewport,
-                      rgb, num_bytes);
-
-      renderer_->endFrame();
-    }
-  }
-
-  if (depth) {
-    if (renderer_->beginFrame(offscreen_swap_chain_)) {
-      scene_view_->Render(renderer_, SceneView::DrawMode::kDepth,
-                          depth_target_->GetRenderTarget());
-
-      const size_t num_bytes = viewport.width * viewport.height * sizeof(float);
-      ReadDepthPixels(renderer_, depth_target_->GetRenderTarget(), viewport,
-                      depth, num_bytes);
-
-      renderer_->endFrame();
-    }
-  }
-
-  if (rgb || depth) {
-    if constexpr (UTILS_HAS_THREADING) {
-      // Wait for rendering to copy back to buffer to complete.
-      engine_->flushAndWait();
-    }
-  }
-}
-
-void FilamentContext::UploadMesh(const mjModel* model, int id) {
-  if (!scene_view_) {
-    mju_error("SceneView is not initialized.");
-  }
-  scene_view_->UploadMesh(model, id);
-}
-
-void FilamentContext::UploadTexture(const mjModel* model, int id) {
-  if (!scene_view_) {
-    mju_error("SceneView is not initialized.");
-  }
-  scene_view_->UploadTexture(model, id);
-}
-
-void FilamentContext::UploadHeightField(const mjModel* model, int id) {
-  if (!scene_view_) {
-    mju_error("SceneView is not initialized.");
-  }
-  scene_view_->UploadHeightField(model, id);
-}
-
-uintptr_t FilamentContext::UploadGuiImage(uintptr_t tex_id,
-                                          const uint8_t* pixels, int width,
-                                          int height, int bpp) {
-  if (gui_view_) {
-    return gui_view_->UploadImage(tex_id, pixels, width, height, bpp);
-  }
-  return 0;
-}
-
-double FilamentContext::GetFrameRate() const {
+void FilamentContext::GetFrameStats(mjrFrameHandle frame,
+                                    mjrFrameStats* stats_out) const {
   utils::FixedCapacityVector<filament::Renderer::FrameInfo> frame_info =
       renderer_->getFrameInfoHistory(1);
-  if (frame_info.empty()) {
-    return 0;
+  if (!frame_info.empty()) {
+    const int64_t ns = frame_info[0].denoisedGpuFrameDuration;
+    stats_out->frame_rate = 1.0e9 / static_cast<double>(ns);
+  } else {
+    stats_out->frame_rate = 0.0;
   }
-  const int64_t ns = frame_info[0].denoisedGpuFrameDuration;
-  return 1.0e9 / static_cast<double>(ns);
-}
-
-void FilamentContext::UpdateGui() {
-  DrawGui(scene_view_.get());
 }
 
 }  // namespace mujoco

src/experimental/filament/filament/filament_context.h

@@ -16,61 +16,58 @@
 #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_FILAMENT_CONTEXT_H_
 
 #include <cstdint>
+#include <span>
 #include <memory>
 
 #include <backend/Platform.h>
 #include <filament/Engine.h>
 #include <filament/Renderer.h>
-#include <mujoco/mjmodel.h>
-#include <mujoco/mjrender.h>
-#include <mujoco/mjvisualize.h>
-#include "experimental/filament/filament/gui_view.h"
+#include <filament/SwapChain.h>
+#include <math/vec4.h>
 #include "experimental/filament/filament/object_manager.h"
-#include "experimental/filament/filament/render_target_util.h"
-#include "experimental/filament/filament/scene_view.h"
 #include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-// Manages the filament renderer that is exposed via the mjr functions.
-class FilamentContext {
+// Manages the filament::Renderer and provides APIs for rendering scenes.
+class FilamentContext : public mjrfContext {
  public:
   explicit FilamentContext(const mjrFilamentConfig* config);
   ~FilamentContext();
 
-  void Init(const mjModel* model);
-
-  void Render(const mjrRect& viewport, const mjvScene* scene);
-
-  void SetFrameBuffer(int framebuffer);
-
-  void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth);
-
-  void UploadMesh(const mjModel* model, int id);
-
-  void UploadTexture(const mjModel* model, int id);
-
-  void UploadHeightField(const mjModel* model, int id);
-
-  uintptr_t UploadGuiImage(uintptr_t tex_id, const uint8_t* pixels, int width,
-                           int height, int bpp);
-
-  double GetFrameRate() const;
-
-  void UpdateGui();
-
   FilamentContext(const FilamentContext&) = delete;
   FilamentContext& operator=(const FilamentContext&) = delete;
 
+  // Queues the given render requests for rendering. This function copies the
+  // necessary data from the requests into the renderer thread and returns
+  // immediately afterwards. The renderer thread will then perform the actual
+  // rendering on the GPU. Callers can use WaitForFrame to block until the
+  // rendering is complete.
+  mjrFrameHandle Render(
+      std::span<const mjrRenderRequest> render_requests,
+      std::span<const mjrReadPixelsRequest> read_requests = {});
+
+  // Blocks until the given frame has completed rendering.
+  void WaitForFrame(mjrFrameHandle frame_handle);
+
+  // Sets the clear color for the renderer.
+  void SetClearColor(const filament::math::float4& color);
+
+  // Returns information about the frame.
+  void GetFrameStats(mjrFrameHandle frame, mjrFrameStats* stats_out) const;
+
+  filament::Engine* GetEngine() const { return engine_; }
+
+  ObjectManager* GetObjectManager() const { return object_manager_.get(); }
+
+  static FilamentContext* downcast(mjrfContext* context) {
+    return static_cast<FilamentContext*>(context);
+  }
+  static const FilamentContext* downcast(const mjrfContext* context) {
+    return static_cast<const FilamentContext*>(context);
+  }
+
  private:
-  enum SwapChainType {
-    kWindowSwapChain,
-    kOffscreenSwapChain,
-  };
-
-  void PrepareRenderTargets(int width, int height);
-  void DestroyRenderTargets();
-
   mjrFilamentConfig config_;
 
   filament::Engine* engine_ = nullptr;
@@ -78,17 +75,10 @@ class FilamentContext {
   filament::SwapChain* window_swap_chain_ = nullptr;
   filament::SwapChain* offscreen_swap_chain_ = nullptr;
   std::unique_ptr<filament::backend::Platform> platform_;
-
-  SceneView::DrawMode last_render_mode_ = SceneView::DrawMode::kNormal;
-  SwapChainType scene_swap_chain_target_ = kWindowSwapChain;
-  SwapChainType gui_swap_chain_target_ = kWindowSwapChain;
-  std::unique_ptr<RenderTargetAndTextures> color_target_;
-  std::unique_ptr<RenderTargetAndTextures> depth_target_;
   std::unique_ptr<ObjectManager> object_manager_;
-  std::unique_ptr<SceneView> scene_view_;
-  std::unique_ptr<GuiView> gui_view_;
   int window_width_ = 0;
   int window_height_ = 0;
+  std::uint64_t frame_counter_ = 0;
 };
 
 }  // namespace mujoco

src/experimental/filament/filament/filament_platform_factory.cc

@@ -31,13 +31,13 @@ static filament::Engine::Backend ResolveBackend(int graphics_api) {
 #endif
 
   switch (graphics_api) {
-    case mjGFX_DEFAULT:
+    case mjGRAPHICS_API_DEFAULT:
       // Use the default based on the platform above.
       break;
-    case mjGFX_OPENGL:
+    case mjGRAPHICS_API_OPENGL:
       backend = filament::Engine::Backend::OPENGL;
       break;
-    case mjGFX_VULKAN:
+    case mjGRAPHICS_API_VULKAN:
       backend = filament::Engine::Backend::VULKAN;
       break;
     default:

src/experimental/filament/filament/geom_util.cc

@@ -1,188 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/geom_util.h"
-
-#include <cfloat>
-#include <cstddef>
-#include <cstdint>
-#include <cstring>
-#include <span>
-
-#include <filament/Engine.h>
-#include <filament/IndexBuffer.h>
-#include <filament/VertexBuffer.h>
-#include <math/vec3.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-#include "experimental/filament/filament/math_util.h"
-#include "experimental/filament/filament/vertex_util.h"
-
-namespace mujoco {
-
-using filament::math::float3;
-
-static std::span<const float> GetPositions(const mjModel* model,
-                                           const mjvScene* scene,
-                                           const mjvGeom& geom) {
-  if (geom.type == mjGEOM_FLEX) {
-    const int num = 9 * scene->flexfaceused[geom.objid];
-    const int addr = scene->flexfaceadr[geom.objid];
-    const float* ptr = scene->flexface + (9 * addr);
-    return {ptr, static_cast<size_t>(num)};
-  } else {
-    const int num = 3 * scene->skinvertnum[geom.objid];
-    const int addr = scene->skinvertadr[geom.objid];
-    const float* ptr = scene->skinvert + (3 * addr);
-    return {ptr, static_cast<size_t>(num)};
-  }
-}
-
-static std::span<const float> GetNormals(const mjModel* model,
-                                         const mjvScene* scene,
-                                         const mjvGeom& geom) {
-  if (geom.type == mjGEOM_FLEX) {
-    const int num = 9 * scene->flexfaceused[geom.objid];
-    const int addr = scene->flexfaceadr[geom.objid];
-    const float* ptr = scene->flexnormal + (9 * addr);
-    return {ptr, static_cast<size_t>(num)};
-  } else {
-    const int num = 3 * scene->skinvertnum[geom.objid];
-    const int addr = scene->skinvertadr[geom.objid];
-    const float* ptr = scene->skinnormal + (3 * addr);
-    return {ptr, static_cast<size_t>(num)};
-  }
-}
-
-static std::span<const float> GetUvs(const mjModel* model,
-                                     const mjvScene* scene,
-                                     const mjvGeom& geom) {
-  if (geom.type == mjGEOM_FLEX) {
-    if (geom.texcoord && geom.matid >= 0) {
-      const int num = 6 * scene->flexfaceused[geom.objid];
-      const int addr = scene->flexfaceadr[geom.objid];
-      const float* ptr = scene->flextexcoord + (6 * addr);
-      return {ptr, static_cast<size_t>(num)};
-    } else {
-      const float* ptr = nullptr;
-      return {ptr, 0};
-    }
-  } else {
-    if (model->skin_texcoordadr[geom.objid] >= 0) {
-      const int num = 3 * scene->skinvertnum[geom.objid];
-      const int addr = model->skin_texcoordadr[geom.objid];
-      const float* ptr = model->skin_texcoord + (2 * addr);
-      return {ptr, static_cast<size_t>(num)};
-    } else {
-      const float* ptr = nullptr;
-      return {ptr, 0};
-    }
-  }
-}
-
-static std::span<const int> GetIndices(const mjModel* model,
-                                       const mjvScene* scene,
-                                       const mjvGeom& geom) {
-  if (geom.type == mjGEOM_FLEX) {
-    const int* ptr = nullptr;
-    return {ptr, 0};
-  } else {
-    const int num = 3 * model->skin_facenum[geom.objid];
-    const int* ptr = model->skin_face + 3 * model->skin_faceadr[geom.objid];
-    return {ptr, static_cast<size_t>(num)};
-  }
-}
-
-template <typename T>
-static void FillVertices(std::byte* buffer, std::size_t len,
-                         std::span<const float> positions,
-                         std::span<const float> normals,
-                         std::span<const float> uvs,
-                         float3* vmin,
-                         float3* vmax) {
-  const int num_vertices = len / sizeof(T);
-  T* ptr = reinterpret_cast<T*>(buffer);
-  for (int i = 0; i < num_vertices; ++i) {
-    ptr->position = ReadFloat3(positions.data(), i);
-    *vmin = min(*vmin, ptr->position);
-    *vmax = max(*vmax, ptr->position);
-    ptr->orientation = CalculateOrientation(ReadFloat3(normals.data(), i));
-    if constexpr (T::kHasUv) {
-      ptr->uv.x = uvs[i * 2];
-      ptr->uv.y = uvs[i * 2 + 1];
-    }
-    ++ptr;
-  }
-}
-
-static filament::VertexBuffer* BuildVertexBuffer(
-    filament::Engine* engine, std::span<const float> positions,
-    std::span<const float> normals, std::span<const float> uvs, float3* vmin,
-    float3* vmax) {
-  const int num_vertices = positions.size() / 3;
-  if (uvs.data() != nullptr) {
-    using VertexType = VertexWithUv;
-    auto fill = [&](std::byte* buffer, std::size_t len) {
-      FillVertices<VertexType>(buffer, len, positions, normals, uvs, vmin,
-                               vmax);
-    };
-    return CreateVertexBuffer<VertexType>(engine, num_vertices, fill);
-  } else {
-    using VertexType = VertexNoUv;
-    auto fill = [&](std::byte* buffer, std::size_t len) {
-      FillVertices<VertexType>(buffer, len, positions, normals, uvs, vmin,
-                               vmax);
-    };
-    return CreateVertexBuffer<VertexType>(engine, num_vertices, fill);
-  }
-}
-
-static filament::IndexBuffer* BuildIndexBuffer(filament::Engine* engine,
-                                               std::span<const int> indices,
-                                               int num_indices) {
-  if (indices.data() == nullptr) {
-    auto fill_indices = FillSequence<uint32_t>;
-    return CreateIndexBuffer<uint32_t>(engine, num_indices, fill_indices);
-  } else {
-    auto fill_indices = [&](std::byte* buffer, std::size_t len) {
-      std::memcpy(buffer, indices.data(), len);
-    };
-    return CreateIndexBuffer<uint32_t>(engine, indices.size(), fill_indices);
-  }
-}
-
-FilamentBuffers CreateGeomBuffers(filament::Engine* engine,
-                                  const mjModel* model, const mjvScene* scene,
-                                  const mjvGeom& geom) {
-  auto positions = GetPositions(model, scene, geom);
-  auto normals = GetNormals(model, scene, geom);
-  auto uvs = GetUvs(model, scene, geom);
-  auto indices = GetIndices(model, scene, geom);
-
-  int num_indices = indices.size();
-  if (num_indices == 0 && geom.type == mjGEOM_FLEX) {
-    num_indices = 3 * scene->flexfaceused[geom.objid];
-  }
-
-  FilamentBuffers buffers;
-  float3 vmin = {FLT_MAX, FLT_MAX, FLT_MAX};
-  float3 vmax = {-FLT_MAX, -FLT_MAX, -FLT_MAX};
-  buffers.vertex_buffer =
-      BuildVertexBuffer(engine, positions, normals, uvs, &vmin, &vmax);
-  buffers.index_buffer = BuildIndexBuffer(engine, indices, num_indices);
-  buffers.bounds.emplace().set(vmin, vmax);
-  return buffers;
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/geom_util.h

@@ -1,31 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_
-
-#include <filament/Engine.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-
-namespace mujoco {
-
-// Populates the FilamentBuffers for a flex geometry.
-FilamentBuffers CreateGeomBuffers(filament::Engine* engine,
-                                  const mjModel* model, const mjvScene* scene,
-                                  const mjvGeom& geom);
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_

src/experimental/filament/filament/gui_view.cc

@@ -1,413 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/gui_view.h"
-
-#include <cstddef>
-#include <cstdint>
-#include <cstring>
-#include <utility>
-#include <vector>
-
-#include <imgui.h>
-#include <filament/Engine.h>
-#include <filament/RenderableManager.h>
-#include <filament/Renderer.h>
-#include <filament/TextureSampler.h>
-#include <filament/Viewport.h>
-#include <math/vec4.h>
-#include <utils/EntityManager.h>
-#include <mujoco/mjrender.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-#include "experimental/filament/filament/vertex_util.h"
-
-namespace mujoco {
-
-using filament::math::float4;
-
-static constexpr auto kTriangles =
-    filament::RenderableManager::PrimitiveType::TRIANGLES;
-
-GuiView::GuiView(filament::Engine* engine, filament::Material* ui_material)
-    : engine_(engine), material_(ui_material) {
-  auto& em = utils::EntityManager::get();
-  scene_ = engine_->createScene();
-  camera_ = engine_->createCamera(em.create());
-  view_ = engine_->createView();
-  renderable_ = em.create();
-  view_->setScene(scene_);
-  view_->setCamera(camera_);
-  view_->setPostProcessingEnabled(false);
-}
-
-GuiView::~GuiView() {
-  if (num_elements_ > 0) {
-    scene_->remove(renderable_);
-    auto& rm = engine_->getRenderableManager();
-    rm.destroy(renderable_);
-  }
-  auto& em = utils::EntityManager::get();
-  em.destroy(renderable_);
-  for (auto& buffer : buffers_) {
-    engine_->destroy(buffer.vertex_buffer);
-    engine_->destroy(buffer.index_buffer);
-  }
-  for (auto& instance : instances_) {
-    engine_->destroy(instance);
-  }
-  for (auto& texture : textures_) {
-    engine_->destroy(texture.second);
-  }
-  engine_->destroyCameraComponent(camera_->getEntity());
-  engine_->destroy(view_);
-  engine_->destroy(scene_);
-}
-
-void GuiView::ResetRenderable() {
-  auto& em = utils::EntityManager::get();
-  if (!renderable_.isNull()) {
-    scene_->remove(renderable_);
-    auto& rm = engine_->getRenderableManager();
-    rm.destroy(renderable_);
-    em.destroy(renderable_);
-    renderable_ = utils::Entity();
-  }
-
-  for (auto& buffer : buffers_) {
-    engine_->destroy(buffer.vertex_buffer);
-    engine_->destroy(buffer.index_buffer);
-  }
-  buffers_.clear();
-}
-
-uintptr_t GuiView::UploadImage(uintptr_t tex_id, const uint8_t* pixels,
-                               int width, int height, int bpp) {
-  if (bpp != 4 && bpp != 3) {
-    mju_error("Unsupported image bpp. Got %d, wanted 3 or 4", bpp);
-  }
-
-  const auto internal_format =
-      bpp == 4 ? filament::Texture::InternalFormat::RGBA8
-               : filament::Texture::InternalFormat::RGB8;
-  const auto texture_format = bpp == 4 ? filament::Texture::Format::RGBA
-                                       : filament::Texture::Format::RGB;
-
-  filament::Texture* texture = nullptr;
-  if (tex_id == 0) {
-    texture = filament::Texture::Builder()
-                  .width(width)
-                  .height(height)
-                  .levels(1)
-                  .format(internal_format)
-                  .sampler(filament::Texture::Sampler::SAMPLER_2D)
-                  .build(*engine_);
-    tex_id = textures_.size() + 1;
-    textures_[tex_id] = texture;
-  } else {
-    auto iter = textures_.find(tex_id);
-    if (iter == textures_.end()) {
-      mju_error("Texture not found: %lu", tex_id);
-    }
-    texture = iter->second;
-
-    if (pixels == nullptr) {
-      // A nullptr implies that the user wants to destroy the texture.
-      engine_->destroy(texture);
-      textures_.erase(tex_id);
-      return 0;
-    } else if (texture->getWidth() != width || texture->getHeight() != height) {
-      // Recreate the texture if the dimensions have changed.
-      engine_->destroy(texture);
-      texture = filament::Texture::Builder()
-                    .width(width)
-                    .height(height)
-                    .levels(1)
-                    .format(internal_format)
-                    .sampler(filament::Texture::Sampler::SAMPLER_2D)
-                    .build(*engine_);
-      textures_[tex_id] = texture;
-    }
-  }
-
-  // Create a copy of the image to pass it to filament as we don't know the
-  // lifetime of the data.
-  const int num_bytes = width * height * bpp;
-  std::byte* bytes = new std::byte[num_bytes];
-  std::memcpy(bytes, pixels, num_bytes);
-  const auto callback = [](void* buffer, size_t size, void* user) {
-    auto* ptr = reinterpret_cast<std::byte*>(user);
-    delete[] ptr;
-  };
-  filament::Texture::PixelBufferDescriptor pb(bytes, num_bytes, texture_format,
-                                              filament::Texture::Type::UBYTE,
-                                              callback);
-  texture->setImage(*engine_, 0, std::move(pb));
-  return tex_id;
-}
-
-void GuiView::CreateTexture(ImTextureData* data) {
-  if (data->Format != ImTextureFormat_RGBA32) {
-    mju_error("Unsupported texture format.");
-  }
-
-  filament::Texture* texture =
-      filament::Texture::Builder()
-          .width(data->Width)
-          .height(data->Height)
-          .levels(1)
-          .format(filament::Texture::InternalFormat::RGBA8)
-          .sampler(filament::Texture::Sampler::SAMPLER_2D)
-          .build(*engine_);
-
-  const uintptr_t tex_id = textures_.size() + 1;
-  textures_[tex_id] = texture;
-  data->SetTexID((ImTextureID)tex_id);
-  UpdateTexture(data);
-}
-
-void GuiView::UpdateTexture(ImTextureData* data) {
-  const int size = data->Width * data->Height * 4;
-  filament::Texture::PixelBufferDescriptor pb(data->GetPixels(), size,
-                                              filament::Texture::Format::RGBA,
-                                              filament::Texture::Type::UBYTE);
-  auto iter = textures_.find(data->TexID);
-  if (iter == textures_.end()) {
-    mju_error("Texture not found: %llu", data->TexID);
-  }
-
-  filament::Texture* texture = iter->second;
-  texture->setImage(*engine_, 0, std::move(pb));
-  data->SetStatus(ImTextureStatus_OK);
-}
-
-void GuiView::DestroyTexture(ImTextureData* data) {
-  auto iter = textures_.find(data->TexID);
-  if (iter != textures_.end()) {
-    engine_->destroy(iter->second);
-    textures_.erase(data->TexID);
-    data->SetTexID(ImTextureID_Invalid);
-    data->SetStatus(ImTextureStatus_Destroyed);
-  }
-}
-
-void GuiView::UpdateRenderable() {
-  if (!ImGui::GetCurrentContext()) {
-    return;
-  }
-
-  // Prepare the imgui draw commands. We must call this function even if we do
-  // not plan on rendering anything to ensure imgui state is updated.
-  ImGui::Render();
-  auto& rm = engine_->getRenderableManager();
-
-  ImGuiIO& io = ImGui::GetIO();
-  const ImVec2& size = io.DisplaySize;
-  const ImVec2& scale = io.DisplayFramebufferScale;
-  ImDrawData* commands = ImGui::GetDrawData();
-  if (!commands) {
-    return;
-  }
-  commands->ScaleClipRects(scale);
-
-  int num_elements = 0;
-  for (int n = 0; n < commands->CmdListsCount; ++n) {
-    const ImDrawList* cmds = commands->CmdLists[n];
-    if (sizeof(GuiVertex) != sizeof(cmds->VtxBuffer.Data[0])) {
-      mju_error("Invalid vertex buffer size.");
-    }
-    if (sizeof(uint16_t) != sizeof(cmds->IdxBuffer.Data[0])) {
-      mju_error("Invalid index buffer size.");
-    }
-    num_elements += cmds->CmdBuffer.size();
-  }
-
-  if (commands->Textures != nullptr) {
-    for (ImTextureData* tex : *commands->Textures) {
-      if (tex->Status == ImTextureStatus_OK) {
-        // ImGui's lifecycle is independent of the filament context lifecycle.
-        // As such, it is possible to destroy and create a new filament context
-        // while ImGui is still expecting the "OK" textures to work. In this
-        // case, we simply recreate the texture.
-        if (textures_.find(tex->TexID) == textures_.end()) {
-          CreateTexture(tex);
-        }
-      } else if (tex->Status == ImTextureStatus_WantCreate) {
-        CreateTexture(tex);
-      } else if (tex->Status == ImTextureStatus_WantUpdates) {
-        if (textures_.find(tex->TexID) == textures_.end()) {
-          CreateTexture(tex);
-        } else {
-          UpdateTexture(tex);
-        }
-      } else if (tex->Status == ImTextureStatus_WantDestroy &&
-                 tex->UnusedFrames > 0) {
-        DestroyTexture(tex);
-      }
-    }
-  }
-
-  if (size.x == 0 || size.y == 0 || num_elements == 0) {
-    if (num_elements_ > 0) {
-      scene_->remove(renderable_);
-      rm.destroy(renderable_);
-    }
-    num_elements_ = 0;
-    return;
-  }
-
-  view_->setViewport(
-      filament::Viewport(0.f, 0.f, size.x * scale.x, size.y * scale.y));
-  camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0, size.x,
-                         size.y, 0.0, 0.0, 1.0);
-
-  if (num_elements != num_elements_) {
-    if (num_elements_ > 0) {
-      scene_->remove(renderable_);
-      rm.destroy(renderable_);
-    }
-
-    num_elements_ = num_elements;
-
-    filament::RenderableManager::Builder builder(num_elements_);
-    builder.boundingBox({{-100, -100, -100}, {100, 100, 100}});
-    builder.culling(false);
-    builder.build(*engine_, renderable_);
-    scene_->addEntity(renderable_);
-  }
-
-  for (auto& buffer : buffers_) {
-    engine_->destroy(buffer.vertex_buffer);
-    engine_->destroy(buffer.index_buffer);
-  }
-  buffers_.clear();
-
-  auto ri = rm.getInstance(renderable_);
-
-  int drawable_index = 0;
-  for (int n = 0; n < commands->CmdListsCount; ++n) {
-    const ImDrawList* cmds = commands->CmdLists[n];
-    auto vfill = [&](std::byte* dst, std::size_t size) {
-      if (size != cmds->VtxBuffer.size_in_bytes()) {
-        mju_error("Invalid vertex buffer size.");
-      }
-      std::memcpy(dst, cmds->VtxBuffer.Data, size);
-    };
-    auto ifill = [&](std::byte* dst, std::size_t size) {
-      if (size != cmds->IdxBuffer.size_in_bytes()) {
-        mju_error("Invalid index buffer size.");
-      }
-      std::memcpy(dst, cmds->IdxBuffer.Data, size);
-    };
-    buffers_.push_back(
-        {CreateIndexBuffer<uint16_t>(engine_, cmds->IdxBuffer.Size, ifill),
-         CreateVertexBuffer<GuiVertex>(engine_, cmds->VtxBuffer.Size, vfill)});
-    const mujoco::FilamentBuffers& buffer = buffers_.back();
-
-    int index_offset = 0;
-    for (const ImDrawCmd& command : cmds->CmdBuffer) {
-      const int width = size.x * scale.x;
-      const int height = size.y * scale.y;
-
-      int clip_left = command.ClipRect.x;
-      int clip_bottom = height - command.ClipRect.w;
-      int clip_width = command.ClipRect.z - command.ClipRect.x;
-      int clip_height = command.ClipRect.w - command.ClipRect.y;
-      // Modal dialogs try to cover the whole window, but also a little outside
-      // of it. This doesn't work well with filament's scissor test, so we clip
-      // them to the window.
-      if (clip_left < 0 || clip_bottom < 0) {
-        clip_left = 0;
-        clip_bottom = 0;
-        clip_width = width;
-        clip_height = height;
-      }
-
-      mjrRect clip_rect{clip_left, clip_bottom, clip_width, clip_height};
-      rm.setMaterialInstanceAt(
-          ri, drawable_index,
-          GetMaterialInstance(drawable_index, clip_rect, command.GetTexID()));
-      rm.setGeometryAt(ri, drawable_index, kTriangles, buffer.vertex_buffer,
-                       buffer.index_buffer, index_offset, command.ElemCount);
-      rm.setBlendOrderAt(ri, drawable_index, drawable_index);
-
-      index_offset += command.ElemCount;
-      ++drawable_index;
-    }
-  }
-}
-
-filament::MaterialInstance* GuiView::GetMaterialInstance(int index,
-                                                         mjrRect rect,
-                                                         uintptr_t texture_id) {
-  while (index >= instances_.size()) {
-    instances_.push_back(material_->createInstance());
-  }
-
-  auto iter = textures_.find(texture_id);
-  if (iter == textures_.end()) {
-    mju_error("Texture not found: %lu", texture_id);
-  }
-
-  filament::MaterialInstance* instance = instances_[index];
-  instance->setParameter("glyph", iter->second, filament::TextureSampler());
-  instance->setScissor(rect.left, rect.bottom, rect.width, rect.height);
-  return instance;
-}
-
-void GuiView::Render(filament::Renderer* renderer,
-                     filament::RenderTarget* target) {
-  if (num_elements_ == 0) {
-    return;
-  }
-
-  view_->setRenderTarget(target);
-  renderer->render(view_);
-  view_->setRenderTarget(nullptr);
-}
-
-static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) {
-  const ImVec2& display_size = ImGui::GetIO().DisplaySize;
-  const float pos_x = display_size.x * ((x + 1) * 0.5f);
-  const float pos_y = display_size.y * (1.0f - ((y + 1) * 0.5f));
-  return ImVec2(pos_x, pos_y);
-}
-
-void DrawTextAt(const char* text, float x, float y, float z) {
-  if (x < -1 || y < -1 || x > 1 || y > 1 || z < -1 || z > 1) {
-    return;
-  }
-
-  const ImVec2 center_pos = ClipSpaceToWindowCoordinates(x, y);
-  const ImVec2 size = ImGui::CalcTextSize(text);
-
-  const ImVec2 pos = ImVec2(center_pos.x - size.x / 2, center_pos.y);
-  const ImVec2 shadow_pos = ImVec2(pos.x + 2, pos.y + 2);
-  const int flags = ImGuiWindowFlags_NoBringToFrontOnFocus |
-                    ImGuiWindowFlags_NoFocusOnAppearing |
-                    ImGuiWindowFlags_NoBackground |
-                    ImGuiWindowFlags_NoDecoration |
-                    ImGuiWindowFlags_NoInputs |
-                    ImGuiWindowFlags_NoNav;
-
-  ImGui::Begin("labels", nullptr, flags);
-  ImGui::BeginChild("labels", ImGui::GetIO().DisplaySize, 0, flags);
-  ImDrawList* draw_list = ImGui::GetWindowDrawList();
-  draw_list->AddText(shadow_pos, IM_COL32_BLACK, text);
-  draw_list->AddText(pos, IM_COL32_WHITE, text);
-  ImGui::EndChild();
-  ImGui::End();
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/gui_view.h

@@ -1,84 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_
-
-#include <cstdint>
-#include <unordered_map>
-#include <vector>
-
-#include <imgui.h>
-#include <filament/Camera.h>
-#include <filament/Engine.h>
-#include <filament/Material.h>
-#include <filament/MaterialInstance.h>
-#include <filament/Scene.h>
-#include <filament/Texture.h>
-#include <filament/View.h>
-#include <mujoco/mjrender.h>
-#include "experimental/filament/filament/buffer_util.h"
-
-namespace mujoco {
-
-// A filament::View that contains a filament::Scene used for rendering the GUI.
-class GuiView {
- public:
-  GuiView(filament::Engine* engine, filament::Material* ui_material);
-  ~GuiView();
-
-  // Prepares the UX scene renderable using data from the current ImGui state.
-  // This function must be called once per frame to ensure ImGui state is
-  // correctly synced.
-  void UpdateRenderable();
-
-  void Render(filament::Renderer* renderer,
-              filament::RenderTarget* target = nullptr);
-
-  // Uploads texture to be used with ImGui's Image and ImageButton functions.
-  uintptr_t UploadImage(uintptr_t tex_id, const uint8_t* pixels, int width,
-                        int height, int bpp);
-
- private:
-  void CreateTexture(ImTextureData* data);
-  void UpdateTexture(ImTextureData* data);
-  void DestroyTexture(ImTextureData* data);
-
-  // Returns the filament::MaterialInstance configured to draw into the given
-  // scissor rect.
-  filament::MaterialInstance* GetMaterialInstance(int index, mjrRect rect,
-                                                  uintptr_t texture_id);
-
-  // Clears the filament::Scene of the UX renderable and releases all buffers.
-  void ResetRenderable();
-
-  filament::Engine* engine_ = nullptr;
-  filament::Scene* scene_ = nullptr;
-  filament::Camera* camera_ = nullptr;
-  filament::View* view_ = nullptr;
-  filament::Material* material_ = nullptr;
-  utils::Entity renderable_;
-  std::vector<FilamentBuffers> buffers_;
-  std::vector<filament::MaterialInstance*> instances_;
-  std::unordered_map<uintptr_t, filament::Texture*> textures_;
-  int num_elements_ = 0;
-};
-
-// Draws text at the given screen coordinates in clip space (i.e. [-1,-1,-1] to
-// [1,1,1]).
-void DrawTextAt(const char* text, float x, float y, float z);
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_

src/experimental/filament/filament/imgui_editor.cc

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/filament/filament/imgui_editor.h"
 
 #include <algorithm>
 #include <any>
@@ -36,6 +35,7 @@
 #include <math/vec3.h>
 #include "experimental/filament/filament/color_grading_options.h"
 #include "experimental/filament/filament/scene_view.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
@@ -571,12 +571,6 @@ void DrawIndirectLightGui(SceneView* scene_view) {
       ibl->setIntensity(intensity);
     }
   }
-
-  static char filename[256];
-  ImGui::InputText("Filename", filename, sizeof(filename));
-  if (ImGui::Button("Load")) {
-    scene_view->SetEnvironmentLight(filename, intensity);
-  }
 }
 
 void DrawLightGui(filament::LightManager& lm,
@@ -649,74 +643,79 @@ void DrawLightGui(filament::LightManager& lm,
   }
 }
 
-void DrawGui(SceneView* scene_view) {
+}  // namespace mujoco
+
+extern "C" {
+
+void mjrf_DEBUG_drawImguiEditor(mjrScene* scene) {
+  mujoco::SceneView* scene_view = mujoco::SceneView::downcast(scene);
   filament::View* view = scene_view->GetDefaultRenderView();
   filament::Engine* engine = scene_view->GetEngine();
   filament::LightManager& lm = engine->getLightManager();
 
   if (ImGui::TreeNodeEx("Ambient Occlusion")) {
-    DrawAmbientOcclusionGui(scene_view);
+    mujoco::DrawAmbientOcclusionGui(scene_view);
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Screen Space")) {
-    DrawScreenSpaceGui(scene_view);
+    mujoco::DrawScreenSpaceGui(scene_view);
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Shadowing")) {
-    DrawShadowingGui(scene_view);
+    mujoco::DrawShadowingGui(scene_view);
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Post Processing")) {
-    DrawPostProcessingGui(scene_view);
+    mujoco::DrawPostProcessingGui(scene_view);
     if (ImGui::TreeNodeEx("Anti Aliasing (FXAA)")) {
-      DrawFxaaGui(scene_view);
+      mujoco::DrawFxaaGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Anti Aliasing (MSAA)")) {
-      DrawMsaaGui(scene_view);
+      mujoco::DrawMsaaGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Anti Aliasing (Temporal)")) {
-      DrawTaaGui(scene_view);
+      mujoco::DrawTaaGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Bloom")) {
-      DrawBloomGui(scene_view);
+      mujoco::DrawBloomGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Color Grading")) {
-      DrawColorGradingGui(scene_view);
+      mujoco::DrawColorGradingGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Depth of Field")) {
-      DrawDepthOfFieldGui(scene_view);
+      mujoco::DrawDepthOfFieldGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Dithering")) {
-      DrawDitheringGui(scene_view);
+      mujoco::DrawDitheringGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Fog")) {
-      DrawFogGui(scene_view);
+      mujoco::DrawFogGui(scene_view);
       ImGui::TreePop();
     }
     if (ImGui::TreeNodeEx("Vignette")) {
-      DrawVignetteGui(scene_view);
+      mujoco::DrawVignetteGui(scene_view);
       ImGui::TreePop();
     }
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Visibility Layers")) {
-    DrawVisibleLayersGui(scene_view);
+    mujoco::DrawVisibleLayersGui(scene_view);
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Camera")) {
-    DrawCameraGui(scene_view);
+    mujoco::DrawCameraGui(scene_view);
     ImGui::TreePop();
   }
   if (ImGui::TreeNodeEx("Lights")) {
     if (ImGui::TreeNodeEx("Indirect (Image-based) Light")) {
-      DrawIndirectLightGui(scene_view);
+      mujoco::DrawIndirectLightGui(scene_view);
       ImGui::TreePop();
     }
     view->getScene()->forEach([&](utils::Entity entity) {
@@ -730,11 +729,12 @@ void DrawGui(SceneView* scene_view) {
                                                : " (S)";
       const std::string name = "Light " + std::to_string(entity.getId()) + type;
       if (ImGui::TreeNodeEx(name.c_str())) {
-        DrawLightGui(lm, li);
+        mujoco::DrawLightGui(lm, li);
         ImGui::TreePop();
       }
     });
     ImGui::TreePop();
   }
 }
-}  // namespace mujoco
+
+}  // extern "C"

src/experimental/filament/filament/imgui_editor.h

@@ -1,27 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_
-
-#include "experimental/filament/filament/scene_view.h"
-
-namespace mujoco {
-
-// Generates a ImGui Window for the given scene views.
-void DrawGui(SceneView* scene_views);
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_

src/experimental/filament/filament/light.cc

@@ -17,30 +17,53 @@
 #include <numbers>
 
 #include <filament/Engine.h>
+#include <filament/IndirectLight.h>
 #include <filament/LightManager.h>
 #include <filament/Scene.h>
+#include <math/mat3.h>
 #include <math/vec3.h>
 #include <utils/Entity.h>
 #include <utils/EntityManager.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-Light::Light(filament::Engine* engine, const Params& params)
+using filament::math::float3;
+using filament::math::mat3f;
+
+Light::Light(filament::Engine* engine, const mjrLightParams& params)
     : engine_(engine), params_(params) {
+  // Filament treats image-based lights (IBLs) as separate objects (i.e.
+  // filament::IndirectLight) and so we need to handle IBLs specially.
+  if (params.type == mjLIGHT_IMAGE) {
+    filament::IndirectLight::Builder builder;
+    if (params.texture) {
+      // Allow null textures for fallback lights.
+      const Texture* texture = Texture::downcast(params.texture);
+      builder.reflections(texture->GetFilamentTexture());
+      const Texture::SphericalHarmonics* spherical_harmonics =
+          texture->GetSphericalHarmonics();
+      if (spherical_harmonics != nullptr) {
+        builder.irradiance(3, *spherical_harmonics);
+      }
+    }
+    builder.intensity(params.intensity);
+    // Rotate the light to match mujoco's Z-up convention.
+    builder.rotation(mat3f::rotation(std::numbers::pi / 2, float3{1, 0, 0}));
+    ibl_ = builder.build(*engine_);
+    return;
+  }
+
   filament::LightManager::Type type;
   switch (params.type) {
     case mjLIGHT_SPOT:
       type = filament::LightManager::Type::FOCUSED_SPOT;
       break;
     case mjLIGHT_DIRECTIONAL:
-      // We break with the spec here slightly and use a spot light for the head
-      // light instead of a directional params. This is because filament only
-      // supports a single directional light, and we'd rather allow a scene
-      // light to be that directional params. It's also a bit odd for a
-      // directional light to move with the camera.
-      type = params.headlight ? filament::LightManager::Type::FOCUSED_SPOT
-                             : filament::LightManager::Type::DIRECTIONAL;
+      type = filament::LightManager::Type::DIRECTIONAL;
       break;
     case mjLIGHT_POINT:
       type = filament::LightManager::Type::POINT;
@@ -51,16 +74,12 @@ Light::Light(filament::Engine* engine, const Params& params)
   }
 
   filament::LightManager::Builder builder(type);
-  builder.color(params.color);
+  builder.color(ReadFloat3(params.color));
   builder.intensityCandela(params.intensity);
-  builder.castShadows(params.castshadow);
+  builder.castShadows(params.cast_shadows);
   if (type == filament::LightManager::Type::FOCUSED_SPOT) {
-    if (params.headlight) {
-      builder.spotLightCone(0, std::numbers::pi / 2.0f);
-    } else {
-      builder.spotLightCone(0,
-                            params.spot_cone_angle * std::numbers::pi / 180.0f);
-    }
+    builder.spotLightCone(0,
+                          params.spot_cone_angle * std::numbers::pi / 180.0f);
   }
   if (type != filament::LightManager::Type::DIRECTIONAL) {
     builder.falloff(params.range);
@@ -69,7 +88,7 @@ Light::Light(filament::Engine* engine, const Params& params)
   opts.mapSize = 4096;
   opts.shadowCascades =
       type == filament::LightManager::Type::DIRECTIONAL ? 4 : 1;
-  opts.shadowBulbRadius = params.bulbradius;
+  opts.shadowBulbRadius = params.bulb_radius;
   opts.mapSize = params.shadow_map_size;
   if (params.vsm_blur_width > 0.0f) {
     opts.vsm.elvsm = true;
@@ -86,52 +105,89 @@ Light::Light(filament::Engine* engine, const Params& params)
 }
 
 Light::~Light() noexcept {
-  utils::EntityManager& em = utils::EntityManager::get();
-  if (!entity_.isNull()) {
-    engine_->destroy(entity_);
-    em.destroy(entity_);
+  if (ibl_) {
+    engine_->destroy(ibl_);
+  } else {
+    utils::EntityManager& em = utils::EntityManager::get();
+    if (!entity_.isNull()) {
+      engine_->destroy(entity_);
+      em.destroy(entity_);
+    }
   }
 }
 
-void Light::AddToScene(filament::Scene* scene) { scene->addEntity(entity_); }
+void Light::AddToScene(filament::Scene* scene) {
+  if (ibl_) {
+    scene->setIndirectLight(ibl_);
+  } else {
+    scene->addEntity(entity_);
+  }
+}
 
-void Light::RemoveFromScene(filament::Scene* scene) { scene->remove(entity_); }
+void Light::RemoveFromScene(filament::Scene* scene) {
+  if (ibl_) {
+    scene->setIndirectLight(nullptr);
+  } else {
+    scene->remove(entity_);
+  }
+}
 
 void Light::SetTransform(filament::math::float3 position,
                          filament::math::float3 direction) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setPosition(li, position);
-  lm.setDirection(li, direction);
+  if (!ibl_) {
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setPosition(li, position);
+    lm.setDirection(li, direction);
+  }
 }
 
 void Light::SetColor(const filament::math::float3& color) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setColor(li, color);
+  if (!ibl_) {
+    params_.color[0] = color.r;
+    params_.color[1] = color.g;
+    params_.color[2] = color.b;
+
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setColor(li, color);
+  }
 }
 
 void Light::SetIntensity(float intensity) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setIntensityCandela(li, intensity);
+  params_.intensity = intensity;
+  if (ibl_) {
+    ibl_->setIntensity(intensity);
+  } else {
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setIntensityCandela(li, intensity);
+  }
 }
 
 void Light::Enable() {
   if (!enabled_) {
     enabled_ = true;
-    filament::LightManager& lm = engine_->getLightManager();
-    const filament::LightManager::Instance li = lm.getInstance(entity_);
-    lm.setLightChannel(li, 0, enabled_);
+    if (ibl_) {
+      ibl_->setIntensity(params_.intensity);
+    } else {
+      filament::LightManager& lm = engine_->getLightManager();
+      const filament::LightManager::Instance li = lm.getInstance(entity_);
+      lm.setLightChannel(li, 0, enabled_);
+    }
   }
 }
 
 void Light::Disable() {
   if (enabled_) {
     enabled_ = false;
-    filament::LightManager& lm = engine_->getLightManager();
-    const filament::LightManager::Instance li = lm.getInstance(entity_);
-    lm.setLightChannel(li, 0, enabled_);
+    if (ibl_) {
+      ibl_->setIntensity(0.f);
+    } else {
+      filament::LightManager& lm = engine_->getLightManager();
+      const filament::LightManager::Instance li = lm.getInstance(entity_);
+      lm.setLightChannel(li, 0, enabled_);
+    }
   }
 }
 

src/experimental/filament/filament/light.h

@@ -20,70 +20,56 @@
 #include <math/vec3.h>
 #include <utils/Entity.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-// Manages the filament Entities for a single mjvLight.
-class Light {
+// Wrapper around both a "normal" filament Light Entity and a filament
+// IndirectLight.
+class Light : public mjrLight {
  public:
-  // Configuration parameters for a light.
-  struct Params {
-    // The type of light (e.g. spot, point, directional, etc.)
-    mjtLightType type;
-    // The color of the light.
-    filament::math::float3 color = {0, 0, 0};
-    // The intensity of the light, in candela.
-    float intensity = 0.0f;
-    // Whether or not the light casts shadows.
-    bool castshadow = true;
-    // The range/distance in which the light is effective, in meters.
-    float range = 10.0f;
-    // The angle of the spot light cone, in degrees.
-    float spot_cone_angle = 180.f;
-    // The radius of the bulb used for soft shadows.
-    float bulbradius = 0.0f;
-    // The size of the shadow map.
-    int shadow_map_size = 2048;
-    // Blur width for EL VSM.
-    float vsm_blur_width = 0.0f;
-    // Whether or not the light is a headlight.
-    bool headlight = false;
-  };
-
-  Light(filament::Engine* engine, const Params& params);
+  Light(filament::Engine* engine, const mjrLightParams& params);
   ~Light() noexcept;
 
   Light(const Light&) = delete;
   Light& operator=(const Light&) = delete;
 
-  // Adds the filament light Entities to the given filament Scene.
+  // Adds this light to the filament Scene.
   void AddToScene(filament::Scene* scene);
 
-  // Removes the filament light Entities from the given filament Scene.
+  // Removes this light from the filament Scene.
   void RemoveFromScene(filament::Scene* scene);
 
-  // Updates the light's position/rotation.
+  // Updates this light's position and rotation.
   void SetTransform(filament::math::float3 position,
                     filament::math::float3 direction);
 
-  // Sets the color of the light.
+  // Sets the color of this light.
   void SetColor(const filament::math::float3& color);
 
-  // Sets the intensity of the light in candela.
+  // Sets the intensity of this light, in candela.
   void SetIntensity(float intensity);
 
+  // Returns the type of the light.
+  mjtLightType GetType() const { return params_.type; }
+
   // Enables/disables the light in the scene.
   void Enable();
   void Disable();
 
-  // Returns true if the light is a headlight.
-  bool IsHeadlight() const { return params_.headlight; }
+  static Light* downcast(mjrLight* light) {
+    return static_cast<Light*>(light);
+  }
+  static const Light* downcast(const mjrLight* light) {
+    return static_cast<const Light*>(light);
+  }
 
  private:
   filament::Engine* engine_ = nullptr;
+  filament::IndirectLight* ibl_ = nullptr;
   utils::Entity entity_;
+  mjrLightParams params_;
   bool enabled_ = true;
-  Params params_;
 };
 
 }  // namespace mujoco

src/experimental/filament/filament/material.cc

@@ -19,104 +19,56 @@
 #include <filament/MaterialInstance.h>
 #include <filament/RenderableManager.h>
 #include <filament/TextureSampler.h>
-#include <mujoco/mjmodel.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
 #include "experimental/filament/filament/object_manager.h"
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-Material::Material(ObjectManager* object_mgr) : object_mgr_(object_mgr) {
-  instances_[kDepth] =
-      object_mgr_->GetMaterial(ObjectManager::kUnlitDepth)->createInstance();
-  instances_[kSegmentation] =
-      object_mgr_->GetMaterial(ObjectManager::kUnlitSegmentation)
-          ->createInstance();
-}
-
-Material::~Material() noexcept {
-  filament::Engine* engine = object_mgr_->GetEngine();
-  for (int i = 0; i < kNumDrawModes; ++i) {
-    if (instances_[i]) {
-      engine->destroy(instances_[i]);
-    }
-  }
-}
-
-void Material::SetNormalMaterialType(
-    ObjectManager::MaterialType material_type) {
-  filament::Material* material = object_mgr_->GetMaterial(material_type);
-
-  if (instances_[kNormal]) {
-    const filament::Material* current_material =
-        instances_[kNormal]->getMaterial();
-    if (current_material == material) {
-      return;
-    }
-
-    object_mgr_->GetEngine()->destroy(instances_[kNormal]);
-    instances_[kNormal] = nullptr;
-  }
-  if (material) {
-    instances_[kNormal] = material->createInstance();
-    UpdateMaterialInstances();
-  }
-}
-
-void Material::UpdateParams(const Params& params) {
-  params_ = params;
-  UpdateMaterialInstances();
-}
-
-void Material::UpdateTextures(const Textures& textures) {
-  textures_ = textures;
-  UpdateMaterialInstances();
-}
-
-void Material::UpdateReflectionTexture(const filament::Texture* tex) {
-  textures_.reflection = tex;
-  UpdateMaterialInstances();
-}
-
-void Material::UpdateMaterialInstances() {
-  filament::MaterialInstance* instance = instances_[DrawMode::kNormal];
-  if (instance == nullptr) {
-    return;
+void UpdateMaterialInstance(filament::MaterialInstance* instance,
+                            const mjrMaterial& material,
+                            ObjectManager* object_mgr) {
+  if (material.scissor[2] != 0 && material.scissor[3] != 0) {
+    instance->setScissor(material.scissor[0], material.scissor[1],
+                         material.scissor[2], material.scissor[3]);
   }
 
-  const filament::Material* material = instance->getMaterial();
-  if (material->hasParameter("BaseColorFactor")) {
+  const filament::Material* fmaterial = instance->getMaterial();
+  if (fmaterial->hasParameter("BaseColorFactor")) {
     instance->setParameter("BaseColorFactor", filament::RgbaType::sRGB,
-                           params_.color);
+                           ReadFloat4(material.color));
   }
-  if (material->hasParameter("EmissiveFactor")) {
-    instance->setParameter("EmissiveFactor", params_.emissive);
+  if (fmaterial->hasParameter("SegmentationColor")) {
+    instance->setParameter("SegmentationColor", filament::RgbaType::LINEAR,
+                           ReadFloat4(material.segmentation_color));
   }
-  if (material->hasParameter("SpecularFactor")) {
-    instance->setParameter("SpecularFactor", params_.specular);
+  if (fmaterial->hasParameter("EmissiveFactor")) {
+    instance->setParameter("EmissiveFactor", material.emissive);
   }
-  if (material->hasParameter("GlossinessFactor")) {
-    instance->setParameter("GlossinessFactor", params_.glossiness);
+  if (fmaterial->hasParameter("SpecularFactor")) {
+    instance->setParameter("SpecularFactor", material.specular);
   }
-  if (material->hasParameter("MetallicFactor")) {
+  if (fmaterial->hasParameter("GlossinessFactor")) {
+    instance->setParameter("GlossinessFactor", material.glossiness);
+  }
+  if (fmaterial->hasParameter("MetallicFactor")) {
     instance->setParameter("MetallicFactor",
-                           params_.metallic >= 0 ? params_.metallic : 1.0f);
+                           material.metallic >= 0 ? material.metallic : 1.0f);
   }
-  if (material->hasParameter("RoughnessFactor")) {
+  if (fmaterial->hasParameter("RoughnessFactor")) {
     instance->setParameter("RoughnessFactor",
-                           params_.roughness >= 0 ? params_.roughness : 1.0f);
+                           material.roughness >= 0 ? material.roughness : 1.0f);
   }
-  if (material->hasParameter("UvScale")) {
-    instance->setParameter("UvScale", params_.uv_scale);
+  if (fmaterial->hasParameter("UvScale")) {
+    instance->setParameter("UvScale", ReadFloat3(material.uv_scale));
   }
-  if (material->hasParameter("UvOffset")) {
-    instance->setParameter("UvOffset", params_.uv_offset);
+  if (fmaterial->hasParameter("UvOffset")) {
+    instance->setParameter("UvOffset", ReadFloat3(material.uv_offset));
   }
-  if (material->hasParameter("Reflectance")) {
-    instance->setParameter("Reflectance", params_.reflectance);
-  }
-
-  if (instances_[DrawMode::kSegmentation]) {
-    instances_[DrawMode::kSegmentation]->setParameter(
-        "BaseColorFactor", params_.segmentation_color);
+  if (fmaterial->hasParameter("Reflectance")) {
+    instance->setParameter("Reflectance", material.reflectance);
   }
 
   // All textures use the same default sampler.
@@ -128,70 +80,27 @@ void Material::UpdateMaterialInstances() {
   sampler.setMinFilter(
       filament::TextureSampler::MinFilter::LINEAR_MIPMAP_LINEAR);
 
-  if (material->hasParameter("BaseColor")) {
-    if (textures_.color) {
-      instance->setParameter("BaseColor", textures_.color, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_RGB);
-      instance->setParameter("BaseColor", fallback, sampler);
+  auto TrySetTexture = [&](const char* name, const mjrTexture* texture,
+                           mjtTextureRole role) {
+    if (fmaterial->hasParameter(name)) {
+      if (texture != nullptr) {
+        instance->setParameter(
+            name, Texture::downcast(texture)->GetFilamentTexture(), sampler);
+      } else {
+        instance->setParameter(name, object_mgr->GetFallbackTexture(role),
+                               sampler);
+      }
     }
-  }
-  if (material->hasParameter("Normal")) {
-    if (textures_.normal) {
-      instance->setParameter("Normal", textures_.normal, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_NORMAL);
-      instance->setParameter("Normal", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("Metallic")) {
-    if (textures_.metallic) {
-      instance->setParameter("Metallic", textures_.metallic, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_METALLIC);
-      instance->setParameter("Metallic", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("Roughness")) {
-    if (textures_.roughness) {
-      instance->setParameter("Roughness", textures_.roughness, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_ROUGHNESS);
-      instance->setParameter("Roughness", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("Occlusion")) {
-    if (textures_.occlusion) {
-      instance->setParameter("Occlusion", textures_.occlusion, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_OCCLUSION);
-      instance->setParameter("Occlusion", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("ORM")) {
-    if (textures_.orm) {
-      instance->setParameter("ORM", textures_.orm, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_ORM);
-      instance->setParameter("ORM", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("Emissive")) {
-    if (textures_.emissive) {
-      instance->setParameter("Emissive", textures_.emissive, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_EMISSIVE);
-      instance->setParameter("Emissive", fallback, sampler);
-    }
-  }
-  if (material->hasParameter("Reflection")) {
-    if (textures_.reflection) {
-      instance->setParameter("Reflection", textures_.reflection, sampler);
-    } else {
-      auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_USER);
-      instance->setParameter("Reflection", fallback, sampler);
-    }
-  }
+  };
+
+  TrySetTexture("BaseColor", material.color_texture, mjTEXROLE_RGB);
+  TrySetTexture("Normal", material.normal_texture, mjTEXROLE_NORMAL);
+  TrySetTexture("Metallic", material.metallic_texture, mjTEXROLE_METALLIC);
+  TrySetTexture("Roughness", material.roughness_texture, mjTEXROLE_ROUGHNESS);
+  TrySetTexture("Occlusion", material.occlusion_texture, mjTEXROLE_OCCLUSION);
+  TrySetTexture("ORM", material.orm_texture, mjTEXROLE_ORM);
+  TrySetTexture("Emissive", material.emissive_texture, mjTEXROLE_EMISSIVE);
+  TrySetTexture("Reflection", material.reflection_texture, mjTEXROLE_USER);
 }
 
 }  // namespace mujoco

src/experimental/filament/filament/material.h

@@ -17,88 +17,17 @@
 
 #include <filament/Engine.h>
 #include <filament/MaterialInstance.h>
-#include <filament/Texture.h>
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
 #include "experimental/filament/filament/object_manager.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-class Material {
- public:
-  // The different methods for rendering objects. Each mode uses a different
-  // material, but all materials "share" the same textures and parameters
-  // (unless specifically noted otherwise).
-  enum DrawMode {
-    kNormal,
-    kDepth,
-    kSegmentation,
-    kNumDrawModes,
-  };
-
-  // The textures that can be assigned to the drawable's material.
-  struct Textures {
-    const filament::Texture* color = nullptr;
-    const filament::Texture* normal = nullptr;
-    const filament::Texture* metallic = nullptr;
-    const filament::Texture* roughness = nullptr;
-    const filament::Texture* occlusion = nullptr;
-    const filament::Texture* orm = nullptr;
-    const filament::Texture* emissive = nullptr;
-    const filament::Texture* reflection = nullptr;
-  };
-
-  // The parameters that can be applied to the drawable's material.
-  struct Params {
-    filament::math::float4 color = {1, 1, 1, 1};
-    filament::math::float4 segmentation_color = {1, 1, 1, 1};
-    filament::math::float2 tex_repeat = {1, 1};
-    filament::math::float3 uv_scale = {1, 1, 1};
-    filament::math::float3 uv_offset = {0, 0, 0};
-    float specular = -1.0f;
-    float glossiness = -1.0f;
-    float metallic = -1.0f;
-    float roughness = -1.0f;
-    float emissive = -1.0f;
-    float reflectance = 0.0f;
-    bool tex_uniform = false;
-  };
-
-  Material(ObjectManager* object_mgr);
-  ~Material() noexcept;
-
-  Material(const Material&) = delete;
-  Material& operator=(const Material&) = delete;
-
-  // Assigns a material to the draw mode.
-  void SetNormalMaterialType(ObjectManager::MaterialType material_type);
-
-  // Updates the material parameters of the drawable for rendering.
-  void UpdateParams(const Params& params);
-
-  // Updates the material textures of the drawable for rendering.
-  void UpdateTextures(const Textures& textures);
-
-  // Update the reflection texture. We do this separately since the reflection
-  // texture needs to be rendered before it can be applied to the material.
-  void UpdateReflectionTexture(const filament::Texture* tex);
-
-  // Returns the material instance assigned to the draw mode.
-  filament::MaterialInstance* GetMaterialInstance(DrawMode mode) {
-    return instances_[mode];
-  }
-
- private:
-  // Updates the material instances based on the currently set parameters and
-  // textures.
-  void UpdateMaterialInstances();
-
-  ObjectManager* object_mgr_ = nullptr;
-  filament::MaterialInstance* instances_[kNumDrawModes] = {nullptr};
-  Params params_;
-  Textures textures_;
-};
+// Updates the material instance using the given parameters and texture data. In
+// some cases where a material needs a texture, but a specific texture is not
+// provided, a default texture from the ObjectManager will be used instead.
+void UpdateMaterialInstance(filament::MaterialInstance* instance,
+                            const mjrMaterial& material,
+                            ObjectManager* object_mgr);
 
 }  // namespace mujoco
 

src/experimental/filament/filament/math_util.h

@@ -1,69 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_
-
-#include <math/mat3.h>
-#include <math/mat4.h>
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-
-namespace mujoco {
-
-// Reads a float2 from an array buffer in the model/scene.
-template <typename T>
-inline filament::math::float2 ReadFloat2(const T* arr, int index = 0) {
-  const T* ptr = arr + (2 * index);
-  return filament::math::float2(ptr[0], ptr[1]);
-}
-
-// Reads a float3 from an array buffer in the model/scene.
-template <typename T>
-inline filament::math::float3 ReadFloat3(const T* arr, int index = 0) {
-  const T* ptr = arr + (3 * index);
-  return filament::math::float3(ptr[0], ptr[1], ptr[2]);
-}
-
-// Reads a float4 from an array buffer in the model/scene.
-template <typename T>
-inline filament::math::float4 ReadFloat4(const T* arr, int index = 0) {
-  const T* ptr = arr + (4 * index);
-  return filament::math::float4(ptr[0], ptr[1], ptr[2], ptr[3]);
-}
-
-// Reads a mat3 from an array buffer in the model/scene.
-template <typename T>
-inline filament::math::mat3 ReadMat3(const T* arr, int index = 0) {
-  // clang-format off
-  const T* ptr = arr + (9 * index);
-  return filament::math::mat3(ptr[0], ptr[3], ptr[6],
-                              ptr[1], ptr[4], ptr[7],
-                              ptr[2], ptr[5], ptr[8]);
-  // clang-format on
-}
-
-// Calculates a reflection matrix for a plane defined by its transform.
-filament::math::mat4 ToReflectionMatrix(const filament::math::mat4& xform);
-
-// Modifies a projection matrix so its near plane coincides with an arbitrary
-// plane defined in camera space.
-filament::math::mat4 CalculateObliqueProjection(
-    const filament::math::mat4& projection,
-    const filament::math::float4& plane);
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_

src/experimental/filament/filament/mesh.cc

@@ -0,0 +1,367 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/filament/mesh.h"
+
+#include <cfloat>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <span>
+#include <utility>
+
+#include <filament/Box.h>
+#include <filament/Engine.h>
+#include <filament/IndexBuffer.h>
+#include <filament/VertexBuffer.h>
+#include <backend/BufferDescriptor.h>
+#include <math/TVecHelpers.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+using filament::math::float3;
+using filament::math::float4;
+
+static filament::VertexAttribute GetUsage(const mjrVertexAttribute& attrib) {
+  switch (attrib.usage) {
+    case mjVERTEX_ATTRIBUTE_USAGE_POSITION:
+      return filament::VertexAttribute::POSITION;
+    case mjVERTEX_ATTRIBUTE_USAGE_NORMAL:
+      return filament::VertexAttribute::TANGENTS;
+    case mjVERTEX_ATTRIBUTE_USAGE_TANGENTS:
+      return filament::VertexAttribute::TANGENTS;
+    case mjVERTEX_ATTRIBUTE_USAGE_UV:
+      return filament::VertexAttribute::UV0;
+    case mjVERTEX_ATTRIBUTE_USAGE_COLOR:
+      return filament::VertexAttribute::COLOR;
+    default:
+      mju_error("Unsupported vertex attribute usage: %d", attrib.usage);
+      return filament::VertexAttribute::POSITION;
+  }
+}
+
+static filament::VertexBuffer::AttributeType GetType(
+    const mjrVertexAttribute& attrib) {
+  switch (attrib.type) {
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT2:
+      return filament::VertexBuffer::AttributeType::FLOAT2;
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT3:
+      return filament::VertexBuffer::AttributeType::FLOAT3;
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT4:
+      return filament::VertexBuffer::AttributeType::FLOAT4;
+    case mjVERTEX_ATTRIBUTE_TYPE_UBYTE4:
+      return filament::VertexBuffer::AttributeType::UBYTE4;
+    default:
+      mju_error("Unsupported vertex attribute type: %d", attrib.type);
+      return filament::VertexBuffer::AttributeType::FLOAT3;
+  }
+}
+
+int VertexAttributeTypeSize(const mjrVertexAttribute& attrib) {
+  switch (attrib.type) {
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT2:
+      return sizeof(float) * 2;
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT3:
+      return sizeof(float) * 3;
+    case mjVERTEX_ATTRIBUTE_TYPE_FLOAT4:
+      return sizeof(float) * 4;
+    case mjVERTEX_ATTRIBUTE_TYPE_UBYTE4:
+      return sizeof(uint8_t) * 4;
+    default:
+      mju_error("Unsupported vertex attribute type: %d", attrib.type);
+      return 0;
+  }
+}
+
+// Fills an index buffer with a basic incrementing sequence.
+template <typename T>
+int FillSequence(std::byte* buffer, std::size_t num_bytes) {
+  const T num = num_bytes / sizeof(T);
+  T* ptr = reinterpret_cast<T*>(buffer);
+  for (T i = 0; i < num; ++i) {
+    ptr[i] = i;
+  }
+  return num;
+}
+
+Mesh::Mesh(filament::Engine* engine, const mjrMeshData& data)
+    : engine_(engine), shared_state_(std::make_shared<SharedState>()) {
+  type_ = data.primitive_type == mjMESH_PRIMITIVE_TYPE_TRIANGLES
+              ? filament::RenderableManager::PrimitiveType::TRIANGLES
+              : filament::RenderableManager::PrimitiveType::LINES;
+
+  // If the user has provided a release callback, then we need to ensure we
+  // call is when filament is done with the mesh data.
+  if (data.release_callback) {
+    shared_state_->callbacks.push_back([=]() {
+      data.release_callback(data.user_data);
+    });
+  }
+
+  BuildVertexBuffer(data);
+  BuildIndexBuffer(data);
+  UpdateBounds(data);
+}
+
+Mesh::~Mesh() {
+  ReleaseResources();
+  if (index_buffer_) {
+    engine_->destroy(index_buffer_);
+  }
+  if (vertex_buffer_) {
+    engine_->destroy(vertex_buffer_);
+  }
+}
+
+void Mesh::BuildVertexBuffer(const mjrMeshData& data) {
+  if (data.nvertices == 0) {
+    mju_error("mjrMeshData has no vertices.");
+  }
+
+  // The filament BufferDescriptor callback for releasing the memory.
+  // We pass a heap-allocated shared_ptr to the shared state as the user data.
+  auto callback = +[](void* buffer, size_t size, void* user) {
+    auto* state_ptr = static_cast<std::shared_ptr<SharedState>*>(user);
+    auto state = *state_ptr;
+    delete state_ptr;
+
+    std::lock_guard<std::mutex> lock(state->mutex);
+    if (!state->called) {
+      for (const auto& cb : state->callbacks) {
+        cb();
+      }
+      state->callbacks.clear();
+      state->called = true;
+    }
+  };
+
+  // Pointers to specific attributes in the mesh data, used for additional
+  // validation and processing.
+  const mjrVertexAttribute* positions = nullptr;
+  const mjrVertexAttribute* normals = nullptr;
+  const mjrVertexAttribute* tangents = nullptr;
+  for (int i = 0; i < data.nattributes; ++i) {
+    if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_USAGE_POSITION) {
+      positions = &data.attributes[i];
+    } else if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_USAGE_NORMAL) {
+      normals = &data.attributes[i];
+    } else if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_USAGE_TANGENTS) {
+      tangents = &data.attributes[i];
+    }
+  }
+  if (!positions) {
+    mju_error("mjrMeshData has no positions.");
+  }
+  if (data.attributes[0].usage != mjVERTEX_ATTRIBUTE_USAGE_POSITION) {
+    mju_error("Positions must be the first attribute.");
+  }
+  if (normals && tangents) {
+    mju_error("mjrMeshData has both normals and tangents.");
+  }
+  if (normals && data.interleaved) {
+    // We need to build orientations from normals and so we require each
+    // attribute to be in a separate buffer.
+    mju_error("Cannot support normals with interleaved vertex attributes.");
+  }
+
+  // Build the vertex buffer.
+  filament::VertexBuffer::Builder vb_builder;
+  vb_builder.vertexCount(data.nvertices);
+
+  if (data.interleaved) {
+    // For an interleaved vertex buffer, we will create a single buffer which
+    // contains the data in the order specified by the attributes array,
+    // starting from the first attribute's payload.
+    vb_builder.bufferCount(1);
+    int total_vertex_size = 0;
+    for (int i = 0; i < data.nattributes; ++i) {
+      total_vertex_size += VertexAttributeTypeSize(data.attributes[i]);
+    }
+    const void* bytes = data.attributes[0].bytes;
+    const size_t nbytes = data.nvertices * total_vertex_size;
+
+    // We assume the buffer is tightly packed with no padding between
+    // attributes. As such, the stride is equal to the total vertex size and
+    // each offset is the sum of the sizes of the preceding attributes.
+    int offset = 0;
+    for (int i = 0; i < data.nattributes; ++i) {
+      const mjrVertexAttribute& attrib = data.attributes[i];
+      const filament::VertexAttribute usage = GetUsage(attrib);
+      filament::VertexBuffer::AttributeType type = GetType(attrib);
+      vb_builder.attribute(usage, 0, type, offset, total_vertex_size);
+      if (usage == filament::VertexAttribute::COLOR) {
+        vb_builder.normalized(usage);
+      }
+      offset += VertexAttributeTypeSize(attrib);
+      attributes_[i] = usage;
+    }
+    vertex_buffer_ = vb_builder.build(*engine_);
+    auto* user_data = new std::shared_ptr<SharedState>(shared_state_);
+    vertex_buffer_->setBufferAt(*engine_, 0, {bytes, nbytes, callback, user_data});
+  } else {
+    // For a non-interleaved vertex buffer, we assign a separate buffer to each
+    // attribute.
+    vb_builder.bufferCount(data.nattributes);
+    for (int i = 0; i < data.nattributes; ++i) {
+      const mjrVertexAttribute& attrib = data.attributes[i];
+      const filament::VertexAttribute usage = GetUsage(attrib);
+      filament::VertexBuffer::AttributeType type = GetType(attrib);
+      if (attrib.usage == mjVERTEX_ATTRIBUTE_USAGE_NORMAL) {
+        // We will replace normals with orientations.
+        type = filament::VertexBuffer::AttributeType::FLOAT4;
+      }
+      vb_builder.attribute(usage, i, type);
+      if (usage == filament::VertexAttribute::COLOR) {
+        vb_builder.normalized(usage);
+      }
+      attributes_[i] = usage;
+    }
+    num_attributes_ = data.nattributes;
+    vertex_buffer_ = vb_builder.build(*engine_);
+
+    // Assign the individual data buffers.
+    for (int i = 0; i < data.nattributes; ++i) {
+      const mjrVertexAttribute& attrib = data.attributes[i];
+      const void* bytes = attrib.bytes;
+      size_t nbytes = data.nvertices * VertexAttributeTypeSize(attrib);
+      if (attrib.usage == mjVERTEX_ATTRIBUTE_USAGE_NORMAL) {
+        // Replace normals with orientations.
+        nbytes = data.nvertices * sizeof(float4);
+        bytes = BuildOrientationsFromNormals(data.nvertices, attrib);
+      }
+      auto* user_data = new std::shared_ptr<SharedState>(shared_state_);
+      vertex_buffer_->setBufferAt(*engine_, i, {bytes, nbytes, callback, user_data});
+    }
+  }
+}
+
+void Mesh::BuildIndexBuffer(const mjrMeshData& data) {
+  if (data.nindices == 0) {
+    return;
+  }
+
+  const int element_size = data.index_type == mjINDEX_TYPE_U16
+                                ? sizeof(uint16_t)
+                                : sizeof(uint32_t);
+  const int num_bytes = data.nindices * element_size;
+
+  // If indices == 0 and nindices > 0, then the user is specifying that the
+  // vertices are provided "in order", i.e. the indices are 0, 1, 2, 3, ...
+  // In this case, we need to create the sequence of indices explicitly.
+  const void* indices = data.indices;
+  if (indices == nullptr) {
+    std::byte* sequence = new std::byte[num_bytes];
+    shared_state_->callbacks.push_back([=]() {
+      delete[] sequence;
+    });
+
+    if (data.index_type == mjINDEX_TYPE_U16) {
+      FillSequence<uint16_t>(sequence, num_bytes);
+    } else {
+      FillSequence<uint32_t>(sequence, num_bytes);
+    }
+    indices = sequence;
+  }
+
+  filament::IndexBuffer::Builder ib_builder;
+  ib_builder.indexCount(data.nindices);
+  ib_builder.bufferType(data.index_type == mjINDEX_TYPE_U16
+                            ? filament::IndexBuffer::IndexType::USHORT
+                            : filament::IndexBuffer::IndexType::UINT);
+  index_buffer_ = ib_builder.build(*engine_);
+  // We don't worry about setting a release callback here because the release
+  // callback for the vertex buffer will call release_callbacks_.
+  filament::backend::BufferDescriptor desc(indices, num_bytes);
+  index_buffer_->setBuffer(*engine_, std::move(desc));
+}
+
+float4* Mesh::BuildOrientationsFromNormals(int nvertices,
+                                           const mjrVertexAttribute& normals) {
+  float4* orientations = new float4[nvertices];
+  shared_state_->callbacks.push_back([=]() {
+    delete[] orientations;
+  });
+  const float* normals_ptr = reinterpret_cast<const float*>(normals.bytes);
+  for (int i = 0; i < nvertices; ++i) {
+    orientations[i] = CalculateOrientation(ReadFloat3(normals_ptr, i));
+  }
+  return orientations;
+}
+
+void Mesh::UpdateBounds(const mjrMeshData& data) {
+  float3 bounds_min = ReadFloat3(data.bounds_min);
+  float3 bounds_max = ReadFloat3(data.bounds_max);
+  if (bounds_min != bounds_max) {
+    bounds_.emplace().set(bounds_min, bounds_max);
+  } else if (data.compute_bounds) {
+    bounds_min = float3(FLT_MAX, FLT_MAX, FLT_MAX);
+    bounds_max = float3(-FLT_MAX, -FLT_MAX, -FLT_MAX);
+
+    if (data.attributes[0].usage != mjVERTEX_ATTRIBUTE_USAGE_POSITION) {
+      mju_error("mjrMeshData has no positions.");
+    }
+    const float* positions =
+        reinterpret_cast<const float*>(data.attributes[0].bytes);
+
+    for (int i = 0; i < data.nvertices; ++i) {
+      const float3 position = ReadFloat3(positions, i);
+      bounds_min = min(bounds_min, position);
+      bounds_max = max(bounds_max, position);
+    }
+    bounds_.emplace().set(bounds_min, bounds_max);
+  }
+}
+
+void Mesh::ReleaseResources() {
+  std::lock_guard<std::mutex> lock(shared_state_->mutex);
+  if (!shared_state_->called) {
+    for (const auto& callback : shared_state_->callbacks) {
+      callback();
+    }
+    shared_state_->callbacks.clear();
+    shared_state_->called = true;
+  }
+}
+
+filament::IndexBuffer* Mesh::GetFilamentIndexBuffer() const {
+  return index_buffer_;
+}
+
+filament::VertexBuffer* Mesh::GetFilamentVertexBuffer() const {
+  return vertex_buffer_;
+}
+
+filament::RenderableManager::PrimitiveType Mesh::GetPrimitiveType() const {
+  return type_;
+}
+
+std::span<const filament::VertexAttribute> Mesh::GetVertexAttributes() const {
+  return {attributes_.data(), attributes_.data() + num_attributes_};
+}
+
+bool Mesh::HasBounds() const {
+  return bounds_.has_value();
+}
+
+filament::Box Mesh::GetBounds() const {
+  return bounds_.value();
+}
+}  // namespace mujoco

src/experimental/filament/filament/mesh.h

@@ -0,0 +1,99 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MESH_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MESH_H_
+
+#include <array>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <span>
+#include <vector>
+
+#include <filament/Box.h>
+#include <filament/Engine.h>
+#include <filament/IndexBuffer.h>
+#include <filament/RenderableManager.h>
+#include <filament/VertexBuffer.h>
+#include <math/vec4.h>
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+// Owns a filament Vertex and Index buffer representing a geometry mesh.
+class Mesh : public mjrMesh {
+ public:
+  // Creates a Mesh from the given MeshData.
+  Mesh(filament::Engine* engine, const mjrMeshData& data);
+  ~Mesh();
+
+  Mesh(const Mesh&) = delete;
+  Mesh& operator=(const Mesh&) = delete;
+
+  // Returns the filament IndexBuffer for the mesh.
+  filament::IndexBuffer* GetFilamentIndexBuffer() const;
+
+  // Returns the filament VertexBuffer for the mesh.
+  filament::VertexBuffer* GetFilamentVertexBuffer() const;
+
+  // Returns the primitive type of the mesh.
+  filament::RenderableManager::PrimitiveType GetPrimitiveType() const;
+
+  // Returns the vertex attribute usages for the mesh.
+  std::span<const filament::VertexAttribute> GetVertexAttributes() const;
+
+  // Returns whether the mesh has bounds.
+  bool HasBounds() const;
+
+  // Returns the bounds of the mesh.
+  filament::Box GetBounds() const;
+
+  static Mesh* downcast(mjrMesh* mesh) {
+    return static_cast<Mesh*>(mesh);
+  }
+  static const Mesh* downcast(const mjrMesh* mesh) {
+    return static_cast<const Mesh*>(mesh);
+  }
+
+ private:
+  void BuildVertexBuffer(const mjrMeshData& data);
+  void BuildIndexBuffer(const mjrMeshData& data);
+  void UpdateBounds(const mjrMeshData& data);
+
+  filament::math::float4* BuildOrientationsFromNormals(
+      int nvertices, const mjrVertexAttribute& normals);
+
+  void ReleaseResources();
+
+  filament::Engine* engine_ = nullptr;
+  filament::IndexBuffer* index_buffer_ = nullptr;
+  filament::VertexBuffer* vertex_buffer_ = nullptr;
+  filament::RenderableManager::PrimitiveType type_ =
+      filament::RenderableManager::PrimitiveType::TRIANGLES;
+  std::optional<filament::Box> bounds_;
+  struct SharedState {
+    std::vector<std::function<void()>> callbacks;
+    std::mutex mutex;
+    bool called = false;
+  };
+  std::shared_ptr<SharedState> shared_state_;
+  std::array<filament::VertexAttribute, mjMAX_VERTEX_ATTRIBUTES> attributes_;
+  int num_attributes_ = 0;
+};
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MESH_H_

src/experimental/filament/filament/model_objects.cc

@@ -1,259 +0,0 @@
-// Copyright 2026 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/model_objects.h"
-
-#include <array>
-#include <utility>
-#include <vector>
-
-#include <filament/Engine.h>
-#include <filament/IndirectLight.h>
-#include <filament/Material.h>
-#include <filament/Skybox.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-#include "experimental/filament/filament/builtins.h"
-#include "experimental/filament/filament/model_util.h"
-#include "experimental/filament/filament/texture_util.h"
-
-
-namespace mujoco {
-
-ModelObjects::ModelObjects(const mjModel* model, filament::Engine* engine)
-    : model_(model), engine_(engine) {
-  const int nstack = model->vis.quality.numstacks;
-  const int nslice = model->vis.quality.numslices;
-  const int nquad = model->vis.quality.numquads;
-  shapes_[kLine] = CreateLine(engine_);
-  shapes_[kBox] = CreateBox(engine_, nquad);
-  shapes_[kLineBox] = CreateLineBox(engine_);
-  shapes_[kCone] = CreateCone(engine_, nstack, nslice);
-  shapes_[kDisk] = CreateDisk(engine_, nslice);
-  shapes_[kDome] = CreateDome(engine_, nstack / 2, nslice);
-  shapes_[kTube] = CreateTube(engine_, nstack, nslice);
-  shapes_[kPlane] = CreatePlane(engine_, nquad);
-  shapes_[kSphere] = CreateSphere(engine_, nstack, nslice);
-  shapes_[kTriangle] = CreateTriangle(engine_);
-
-  for (int i = 0; i < model_->ntex; ++i) {
-    UploadTexture(model_, i);
-  }
-  for (int i = 0; i < model_->nmesh; ++i) {
-    UploadMesh(model_, i);
-  }
-  for (int i = 0; i < model_->nhfield; ++i) {
-    UploadHeightField(model_, i);
-  }
-
-  specular_multiplier_ = ReadElement(
-      model_, "filament.phong.specular_multiplier", specular_multiplier_);
-  shininess_multiplier_ = ReadElement(
-      model_, "filament.phong.shininess_multiplier", shininess_multiplier_);
-  emissive_multiplier_ = ReadElement(
-      model_, "filament.phong.emissive_multiplier", emissive_multiplier_);
-}
-
-ModelObjects::~ModelObjects() {
-  for (auto& iter : skyboxes_) {
-    engine_->destroy(iter);
-  }
-  for (auto& iter : indirect_lights_) {
-    engine_->destroy(iter);
-  }
-  for (auto& iter : meshes_) {
-    engine_->destroy(iter.second.vertex_buffer);
-    engine_->destroy(iter.second.index_buffer);
-  }
-  for (auto& iter : shapes_) {
-    engine_->destroy(iter.vertex_buffer);
-    engine_->destroy(iter.index_buffer);
-  }
-  for (auto& iter : textures_) {
-    engine_->destroy(iter.second);
-  }
-}
-
-void ModelObjects::UploadMesh(const mjModel* model, int id) {
-  if (model != model_) {
-    mju_error("Model mismatch.");
-  }
-  if (id < 0 || id >=  model->nmesh) {
-    mju_error("Invalid mesh index %d", id);
-  }
-
-  if (auto iter = meshes_.find(id); iter != meshes_.end()) {
-    engine_->destroy(iter->second.vertex_buffer);
-    engine_->destroy(iter->second.index_buffer);
-  }
-  if (auto iter = convex_hulls_.find(id); iter != convex_hulls_.end()) {
-    engine_->destroy(iter->second.vertex_buffer);
-    engine_->destroy(iter->second.index_buffer);
-  }
-
-  FilamentBuffers& buffers = meshes_[id];
-  buffers.vertex_buffer = CreateVertexBuffer(
-      engine_, model, id, MeshType::kNormal, &buffers.bounds.emplace());
-  buffers.index_buffer =
-      CreateIndexBuffer(engine_, model, id, MeshType::kNormal);
-
-  if (model->mesh_graphadr[id] >= 0) {
-    FilamentBuffers& hull_buffers = convex_hulls_[id];
-    hull_buffers.vertex_buffer =
-        CreateVertexBuffer(engine_, model, id, MeshType::kConvexHull,
-                           &hull_buffers.bounds.emplace());
-    hull_buffers.index_buffer =
-        CreateIndexBuffer(engine_, model, id, MeshType::kConvexHull);
-  }
-}
-
-void ModelObjects::UploadTexture(const mjModel* model, int id) {
-  if (model != model_) {
-    mju_error("Model mismatch.");
-  }
-  if (id < 0 || id >= model->ntex) {
-    mju_error("Invalid texture index: %d", id);
-  }
-
-  if (auto iter = textures_.find(id); iter != textures_.end()) {
-    engine_->destroy(iter->second);
-  }
-
-  const int texture_type = model->tex_type[id];
-  if (model->tex_height[id] == 1) {
-    const mjtByte* bytes = model->tex_data + model->tex_adr[id];
-    const int num_bytes = model->tex_width[id];
-    textures_[id] =
-        CreateKtxTexture(engine_, bytes, num_bytes, spherical_harmonics_[id]);
-  } else if (texture_type == mjTEXTURE_2D) {
-    textures_[id] = CreateTexture(engine_, model, id, TextureType::kNormal2d);
-  } else if (texture_type == mjTEXTURE_CUBE) {
-    textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube);
-  } else if (texture_type == mjTEXTURE_SKYBOX) {
-    textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube);
-  } else {
-    mju_error("Unsupported: Texture type: %d", texture_type);
-  }
-}
-
-void ModelObjects::UploadHeightField(const mjModel* model, int id) {
-  if (model != model_) {
-    mju_error("Model mismatch.");
-  }
-  if (id < 0 || id >= model->nhfield) {
-    mju_error("Invalid height field index %d", id);
-  }
-
-  if (auto iter = height_fields_.find(id); iter != height_fields_.end()) {
-    engine_->destroy(iter->second.vertex_buffer);
-    engine_->destroy(iter->second.index_buffer);
-  }
-
-  FilamentBuffers& buffers = height_fields_[id];
-  buffers.vertex_buffer = CreateVertexBuffer(
-      engine_, model, id, MeshType::kHeightField, &buffers.bounds.emplace());
-  buffers.index_buffer =
-      CreateIndexBuffer(engine_, model, id, MeshType::kHeightField);
-}
-
-const FilamentBuffers* ModelObjects::GetMeshBuffer(int data_id) const {
-  // As defined by mjv_updateScene:
-  //   original mesh: mesh_id * 2
-  //   convex hull: (mesh_id * 2) + 1
-  const int mesh_id = data_id / 2;
-  if (data_id % 2 == 0) {
-    auto it = meshes_.find(mesh_id);
-    return it != meshes_.end() ? &it->second : nullptr;
-  } else {
-    auto it = convex_hulls_.find(mesh_id);
-    return it != convex_hulls_.end() ? &it->second : nullptr;
-  }
-}
-
-const FilamentBuffers* ModelObjects::GetHeightFieldBuffer(
-    int hfield_id) const {
-  auto it = height_fields_.find(hfield_id);
-  return it != height_fields_.end() ? &it->second : nullptr;
-}
-
-const FilamentBuffers* ModelObjects::GetShapeBuffer(ShapeType shape) const {
-  if (shape < 0 || shape >= kNumShapes) {
-    mju_error("Invalid shape type: %d", shape);
-  }
-  return &shapes_[shape];
-}
-
-const filament::Texture* ModelObjects::GetTexture(int tex_id) const {
-  auto it = textures_.find(tex_id);
-  return it != textures_.end() ? it->second : nullptr;
-}
-
-const filament::Texture* ModelObjects::GetTexture(int mat_id, int role) const {
-  if (mat_id < 0 || mat_id >= model_->nmat || role < 0 || role >= mjNTEXROLE) {
-    return nullptr;
-  }
-  const int tex_id = model_->mat_texid[mat_id * mjNTEXROLE + role];
-  return GetTexture(tex_id);
-}
-
-filament::IndirectLight* ModelObjects::CreateIndirectLight(int tex_id,
-                                                           float intensity) {
-  filament::Texture* texture = nullptr;
-  auto texture_iter = textures_.find(tex_id);
-  if (texture_iter != textures_.end()) {
-    texture = texture_iter->second;
-  }
-
-  SphericalHarmonics* spherical_harmonics = nullptr;
-  auto sh_iter = spherical_harmonics_.find(tex_id);
-  if (sh_iter != spherical_harmonics_.end()) {
-    spherical_harmonics = &sh_iter->second;
-  }
-
-  filament::IndirectLight::Builder builder;
-  builder.reflections(texture);
-  if (spherical_harmonics != nullptr) {
-    builder.irradiance(3, *spherical_harmonics);
-  }
-  builder.intensity(intensity);
-  // Rotate the light to match mujoco's Z-up convention.
-  builder.rotation(filament::math::mat3f::rotation(
-      filament::math::f::PI / 2, filament::math::float3{1, 0, 0}));
-  filament::IndirectLight* indirect_light = builder.build(*engine_);
-  indirect_lights_.push_back(indirect_light);
-  return indirect_light;
-}
-
-filament::Skybox* ModelObjects::CreateSkybox() {
-  filament::Texture* skybox_texture = nullptr;
-  for (auto& iter : textures_) {
-    const int texture_type = model_->tex_type[iter.first];
-    if (texture_type == mjTEXTURE_SKYBOX) {
-      skybox_texture = iter.second;
-      break;
-    }
-  }
-
-  if (skybox_texture == nullptr) {
-    return nullptr;
-  }
-
-  filament::Skybox::Builder builder;
-  builder.environment(skybox_texture);
-  filament::Skybox* skybox = builder.build(*engine_);
-  skyboxes_.push_back(skybox);
-  return skybox;
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/model_objects.h

@@ -1,100 +0,0 @@
-// Copyright 2026 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_
-
-#include <array>
-#include <unordered_map>
-#include <vector>
-
-#include <filament/Engine.h>
-#include <filament/IndirectLight.h>
-#include <filament/Skybox.h>
-#include <math/vec3.h>
-#include <mujoco/mjmodel.h>
-#include "experimental/filament/filament/buffer_util.h"
-
-namespace mujoco {
-
-// Creates and owns various filament objects based on the data in a mjrContext.
-class ModelObjects {
- public:
-  ModelObjects(const mjModel* model, filament::Engine* engine);
-  ~ModelObjects();
-
-  enum ShapeType {
-    kLine,
-    kLineBox,
-    kPlane,
-    kTriangle,
-    kBox,
-    kSphere,
-    kCone,
-    kDisk,
-    kDome,
-    kTube,
-    kNumShapes,
-  };
-
-  void UploadMesh(const mjModel* model, int id);
-
-  void UploadTexture(const mjModel* model, int id);
-
-  void UploadHeightField(const mjModel* model, int id);
-
-  // Returns the filament engine used by the ModelObjects to create filament
-  // objects.
-  filament::Engine* GetEngine() const { return engine_; }
-
-  // Returns the cached instance of a filament object created from the mjModel.
-  const FilamentBuffers* GetShapeBuffer(ShapeType shape) const;
-  const FilamentBuffers* GetMeshBuffer(int data_id) const;
-  const FilamentBuffers* GetHeightFieldBuffer(int hfield_id) const;
-  const filament::Texture* GetTexture(int tex_id) const;
-  const filament::Texture* GetTexture(int mat_id, int role) const;
-
-  filament::Skybox* CreateSkybox();
-  filament::IndirectLight* CreateIndirectLight(int tex_id, float intensity);
-
-  float GetSpecularMultiplier() const { return specular_multiplier_; }
-  float GetShininessMultiplier() const { return shininess_multiplier_; }
-  float GetEmissiveMultiplier() const { return emissive_multiplier_; }
-
-  const mjModel* GetModel() const { return model_; }
-
-  ModelObjects(const ModelObjects&) = delete;
-  ModelObjects& operator=(const ModelObjects&) = delete;
-
- private:
-  using SphericalHarmonics = filament::math::float3[9];
-
-  const mjModel* model_ = nullptr;
-  filament::Engine* engine_ = nullptr;
-  std::vector<filament::Skybox*> skyboxes_;
-  std::vector<filament::IndirectLight*> indirect_lights_;
-  std::array<FilamentBuffers, kNumShapes> shapes_;
-  std::unordered_map<int, FilamentBuffers> meshes_;
-  std::unordered_map<int, FilamentBuffers> convex_hulls_;
-  std::unordered_map<int, FilamentBuffers> height_fields_;
-  std::unordered_map<int, filament::Texture*> textures_;
-  std::unordered_map<int, SphericalHarmonics> spherical_harmonics_;
-  float specular_multiplier_ = 0.2f;
-  float shininess_multiplier_ = 0.1f;
-  float emissive_multiplier_ = 0.3f;
-};
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_

src/experimental/filament/filament/model_util.cc

@@ -1,508 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/model_util.h"
-
-#include <algorithm>
-#include <cfloat>
-#include <cstddef>
-#include <cstdint>
-#include <limits>
-
-#include <filament/Box.h>
-#include <filament/Engine.h>
-#include <filament/IndexBuffer.h>
-#include <filament/Texture.h>
-#include <filament/VertexBuffer.h>
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-#include <mujoco/mjmodel.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-#include "experimental/filament/filament/math_util.h"
-#include "experimental/filament/filament/texture_util.h"
-#include "experimental/filament/filament/vertex_util.h"
-
-namespace mujoco {
-
-using filament::math::float2;
-using filament::math::float3;
-using filament::math::float4;
-
-static bool UseFaceNormal(const float3& face_normal,
-                          const float3& mesh_normal) {
-  // clang-format off
-  return face_normal[0] * mesh_normal[0] +
-         face_normal[1] * mesh_normal[1] +
-         face_normal[2] * mesh_normal[2] < 0.8f;
-  // clang-format on
-}
-
-static void UpdateBounds(const float3& v, float3* vmin, float3* vmax) {
-  vmin->x = std::min(vmin->x, v.x);
-  vmin->y = std::min(vmin->y, v.y);
-  vmin->z = std::min(vmin->z, v.z);
-  vmax->x = std::max(vmax->x, v.x);
-  vmax->y = std::max(vmax->y, v.y);
-  vmax->z = std::max(vmax->z, v.z);
-}
-
-template <typename T>
-static void FillConvexHullBuffer(T* ptr, std::size_t num, const mjModel* model,
-                                 int meshid, float3* vmin, float3* vmax) {
-  const int numvert = model->mesh_graph[model->mesh_graphadr[meshid]];
-  const int numface = model->mesh_graph[model->mesh_graphadr[meshid] + 1];
-
-  const int vertadr = model->mesh_vertadr[meshid];
-  const float* vertices = model->mesh_vert + (3 * vertadr);
-  const int texcoordadr = model->mesh_texcoordadr[meshid];
-  const float* texcoords = model->mesh_texcoord + (2 * texcoordadr);
-
-  if (num != numface * 3) {
-    mju_error("Invalid vertex count.");
-    return;
-  }
-
-  for (int face = 0; face < numface; ++face) {
-    int j =
-        model->mesh_graphadr[meshid] + 2 + 3 * numvert + 3 * numface + 3 * face;
-
-    const float3 p1 = ReadFloat3(vertices, model->mesh_graph[j + 0]);
-    const float3 p2 = ReadFloat3(vertices, model->mesh_graph[j + 1]);
-    const float3 p3 = ReadFloat3(vertices, model->mesh_graph[j + 2]);
-    const float4 orientation = CalculateOrientation(p1, p2, p3);
-
-    UpdateBounds(p1, vmin, vmax);
-    UpdateBounds(p2, vmin, vmax);
-    UpdateBounds(p3, vmin, vmax);
-
-    ptr->position = p1;
-    ptr->orientation = orientation;
-    if constexpr (T::kHasUv) {
-      ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 0]);
-    }
-    ++ptr;
-
-    ptr->position = p2;
-    ptr->orientation = orientation;
-    if constexpr (T::kHasUv) {
-      ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 1]);
-    }
-    ++ptr;
-
-    ptr->position = p3;
-    ptr->orientation = orientation;
-    if constexpr (T::kHasUv) {
-      ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 2]);
-    }
-    ++ptr;
-  }
-}
-
-template <typename T>
-static void FillMeshBuffer(T* ptr, std::size_t num, const mjModel* model,
-                           int meshid, float3* vmin, float3* vmax) {
-  const int faceadr = model->mesh_faceadr[meshid];
-  const int facenum = model->mesh_facenum[meshid];
-  if (num != facenum * 3) {
-    mju_error("Invalid vertex count.");
-    return;
-  }
-
-  const int vertadr = model->mesh_vertadr[meshid];
-  const float* vertices = model->mesh_vert + (3 * vertadr);
-  const int normaladr = model->mesh_normaladr[meshid];
-  const float* normals = model->mesh_normal + 3 * normaladr;
-  const int texcoordadr = model->mesh_texcoordadr[meshid];
-  const float* texcoords = model->mesh_texcoord + (2 * texcoordadr);
-
-  for (int i = 0; i < facenum; ++i) {
-    const int face = 3 * (faceadr + i);
-
-    const float3 p1 = ReadFloat3(vertices, model->mesh_face[face + 0]);
-    const float3 p2 = ReadFloat3(vertices, model->mesh_face[face + 1]);
-    const float3 p3 = ReadFloat3(vertices, model->mesh_face[face + 2]);
-
-    UpdateBounds(p1, vmin, vmax);
-    UpdateBounds(p2, vmin, vmax);
-    UpdateBounds(p3, vmin, vmax);
-
-    const float3 face_normal = CalculateNormal(p1, p2, p3);
-    const float3 n1 = ReadFloat3(normals, model->mesh_facenormal[face + 0]);
-    const float3 n2 = ReadFloat3(normals, model->mesh_facenormal[face + 1]);
-    const float3 n3 = ReadFloat3(normals, model->mesh_facenormal[face + 2]);
-
-    ptr->position = p1;
-    if constexpr (T::kHasUv) {
-      ptr->orientation = CalculateOrientation(n1);
-      ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 0]);
-    } else if (UseFaceNormal(face_normal, n1)) {
-      ptr->orientation = CalculateOrientation(face_normal);
-    } else {
-      ptr->orientation = CalculateOrientation(n1);
-    }
-    ++ptr;
-
-    ptr->position = p2;
-    if constexpr (T::kHasUv) {
-      ptr->orientation = CalculateOrientation(n2);
-      ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 1]);
-    } else if (UseFaceNormal(face_normal, n2)) {
-      ptr->orientation = CalculateOrientation(face_normal);
-    } else {
-      ptr->orientation = CalculateOrientation(n2);
-    }
-    ++ptr;
-
-    ptr->position = p3;
-    if constexpr (T::kHasUv) {
-      ptr->orientation = CalculateOrientation(n3);
-      ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 2]);
-    } else if (UseFaceNormal(face_normal, n3)) {
-      ptr->orientation = CalculateOrientation(face_normal);
-    } else {
-      ptr->orientation = CalculateOrientation(n3);
-    }
-    ++ptr;
-  }
-}
-
-static void FillHeightFieldBuffer(VertexNoUv* ptr, std::size_t num,
-                                  const mjModel* model, int hfieldid,
-                                  float3* vmin, float3* vmax) {
-  int count = 0;
-  auto append_tri = [&](float3 a, float3 b, float3 c) {
-    float4 orientation = CalculateOrientation(a, b, c);
-    ptr[count].position = a;
-    ptr[count].orientation = orientation;
-    ++count;
-    ptr[count].position = b;
-    ptr[count].orientation = orientation;
-    ++count;
-    ptr[count].position = c;
-    ptr[count].orientation = orientation;
-    ++count;
-
-    UpdateBounds(a, vmin, vmax);
-    UpdateBounds(b, vmin, vmax);
-    UpdateBounds(c, vmin, vmax);
-  };
-  auto append_quad = [&](float3 a, float3 b, float3 c, float3 d) {
-    append_tri(a, b, d);
-    append_tri(d, b, c);
-  };
-
-  const float* data = model->hfield_data + model->hfield_adr[hfieldid];
-  const int nrow = model->hfield_nrow[hfieldid];
-  const int ncol = model->hfield_ncol[hfieldid];
-  const float height = 0.5f * (nrow - 1);
-  const float width = 0.5f * (ncol - 1);
-  float sz[4];
-  for (int i = 0; i < 4; ++i) {
-    sz[i] = static_cast<float>(model->hfield_size[4 * hfieldid + i]);
-  }
-
-  auto get_pos = [=](int r, int c) {
-    const float x = sz[0] * (c / width - 1.0f);
-    const float y = sz[1] * (r / height - 1.0f);
-    const float z = sz[2] * data[(r * ncol) + c];
-    return float3{x, y, z};
-  };
-
-  // For each quad defined by 4 points in the height field, we will create 4
-  // triangles by introducing a vertex in the middle of the quad.
-  //     a---b
-  //     |\ /|
-  //     | m |
-  //     |/ \|
-  //     d---c
-  for (int row = 0; row < nrow - 1; ++row) {
-    for (int col = 0; col < ncol - 1; ++col) {
-      const float3 a = get_pos(row, col);
-      const float3 b = get_pos(row, col + 1);
-      const float3 c = get_pos(row + 1, col + 1);
-      const float3 d = get_pos(row + 1, col);
-
-      const float mid_x = (a.x + b.x) * 0.5f;
-      const float mid_y = (a.y + d.y) * 0.5f;
-
-      // To determine the height of the middle vertex, we look at the heights
-      // of the opposing corners (i.e. {a, c} and {b, d}). Our goal is to avoid
-      // creating any odd bumps or valleys in the height field if possible.
-      //
-      // If one of the two opposing corners are of the same height, then we
-      // set the middle vertex such that we're effectively rendering two
-      // triangles, preventing an odd bump. Otherwise, we use the higher
-      // midpoint between two opposing corners to prevent valleys.
-      //     0---0      0---0      6---4
-      //     |\  |      |  /|      |\ /|
-      //     | 0 |      | 0 |      | 7 |
-      //     |  \|      |/  |      |/ \|
-      //     2---0      0---2      0---8
-      float mid_z = 0;
-      if (a.z == c.z && b.z != d.z) {
-        mid_z = a.z;
-      } else if (a.z != c.z && b.z == d.z) {
-        mid_z = b.z;
-      } else {
-        const float mid_z_ac = (a.z + c.z) * 0.5f;
-        const float mid_z_bd = (b.z + d.z) * 0.5f;
-        mid_z = std::max(mid_z_ac, mid_z_bd);
-      }
-
-      const float3 mid = {mid_x, mid_y, mid_z};
-      append_tri(a, b, mid);
-      append_tri(b, c, mid);
-      append_tri(c, d, mid);
-      append_tri(d, a, mid);
-    }
-  }
-  // Build the left edge.
-  for (int row = 0; row < nrow - 1; ++row) {
-    const float3 a = get_pos(row, 0);
-    const float3 b = get_pos(row + 1, 0);
-    const float3 c = {b.x, b.y, -sz[3]};
-    const float3 d = {a.x, a.y, -sz[3]};
-    append_quad(a, b, c, d);
-  }
-  // Build the right edge.
-  for (int row = 0; row < nrow - 1; ++row) {
-    const float3 a = get_pos(row + 1, ncol - 1);
-    const float3 b = get_pos(row, ncol - 1);
-    const float3 c = {b.x, b.y, -sz[3]};
-    const float3 d = {a.x, a.y, -sz[3]};
-    append_quad(a, b, c, d);
-  }
-  // Build the front edge.
-  for (int col = 0; col < ncol - 1; ++col) {
-    const float3 a = get_pos(0, col);
-    const float3 b = get_pos(0, col + 1);
-    const float3 c = {b.x, b.y, -sz[3]};
-    const float3 d = {a.x, a.y, -sz[3]};
-    append_quad(a, b, c, d);
-  }
-  // Build the back edge.
-  for (int col = 0; col < ncol - 1; ++col) {
-    const float3 a = get_pos(nrow - 1, col + 1);
-    const float3 b = get_pos(nrow - 1, col);
-    const float3 c = {b.x, b.y, -sz[3]};
-    const float3 d = {a.x, a.y, -sz[3]};
-    append_quad(a, b, c, d);
-  }
-  // Build the base. We use the visualization quality as the size rather than
-  // the height field dimensions.
-  const float base_width = (0.5f * model->vis.quality.numquads);
-  const float base_height = (0.5f * model->vis.quality.numquads);
-  for (int row = 0; row < model->vis.quality.numquads; ++row) {
-    for (int col = 0; col < model->vis.quality.numquads; ++col) {
-      const float x0 = sz[0] * ((col + 0) / base_width - 1.0f);
-      const float x1 = sz[0] * ((col + 1) / base_width - 1.0f);
-      const float y0 = sz[1] * ((row + 0) / base_height - 1.0f);
-      const float y1 = sz[1] * ((row + 1) / base_height - 1.0f);
-      append_quad({x0, y0, -sz[3]}, {x0, y1, -sz[3]}, {x1, y1, -sz[3]},
-                  {x1, y0, -sz[3]});
-    }
-  }
-  if (count != num) {
-    mju_error("Vertex count mismatch.");
-  }
-}
-
-static int CalculateHeightFieldVertexCount(const mjModel* model, int hfieldid) {
-  const int nrow = model->hfield_nrow[hfieldid];
-  const int ncol = model->hfield_ncol[hfieldid];
-
-  // For details, see the logic in FillHeightFieldBuffer for how many vertices
-  // we need. But, in general...
-
-  // We use 4 triangles (i.e. 12 vertices) per quad.
-  const int surface_count = 12 * (nrow - 1) * (ncol - 1);
-  // We use 1 quad (i.e. 6 vertices) per edge element. We double this because
-  // we have two edges per dimension (e.g. left/right and front/back).
-  const int edge_count = (12 * (nrow - 1)) + (12 * (ncol - 1));
-  // We use 1 quad (i.e. 6 vertices) per base element. We use the visualization
-  // quality as the size rather than the height field dimensions.
-  const int base_count =
-      6 * model->vis.quality.numquads * model->vis.quality.numquads;
-
-  const int total_count = surface_count + edge_count + base_count;
-  return total_count;
-}
-
-template <typename T, typename FillFn>
-static filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine,
-                                                  const mjModel* model, int id,
-                                                  int vertex_count,
-                                                  FillFn fill_fn,
-                                                  filament::Box* bounds) {
-  float3 vmin = {FLT_MAX, FLT_MAX, FLT_MAX};
-  float3 vmax = {-FLT_MAX, -FLT_MAX, -FLT_MAX};
-  filament::VertexBuffer* buffer = CreateVertexBuffer<T>(
-      engine, vertex_count, [&](std::byte* buffer, std::size_t num_bytes) {
-        auto* ptr = reinterpret_cast<T*>(buffer);
-        fill_fn(ptr, num_bytes / sizeof(T), model, id, &vmin, &vmax);
-      });
-  bounds->set(vmin, vmax);
-  return buffer;
-}
-
-filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine,
-                                           const mjModel* model, int id,
-                                           MeshType mesh_type,
-                                           filament::Box* bounds) {
-  if (id < 0) {
-    mju_error("Invalid mesh index %d", id);
-    return nullptr;
-  }
-
-  int vertex_count = 0;
-  switch (mesh_type) {
-    case MeshType::kNormal:
-      if (id >= model->nmesh) {
-        mju_error("Invalid mesh index %d", id);
-        return nullptr;
-      }
-      vertex_count = 3 * model->mesh_facenum[id];
-      break;
-    case MeshType::kConvexHull:
-      if (id >= model->nmesh) {
-        mju_error("Invalid mesh index %d", id);
-        return nullptr;
-      }
-      vertex_count = 3 * model->mesh_graph[model->mesh_graphadr[id] + 1];
-      break;
-    case MeshType::kHeightField:
-      if (id >= model->nhfield) {
-        mju_error("Invalid height field index %d", id);
-        return nullptr;
-      }
-      vertex_count = CalculateHeightFieldVertexCount(model, id);
-      break;
-  }
-
-  if (vertex_count == 0) {
-    mju_error("Vertex count is zero.");
-    return nullptr;
-  }
-
-  const bool has_texcoords = mesh_type == MeshType::kHeightField
-                                 ? false
-                                 : model->mesh_texcoordadr[id] >= 0;
-  if (has_texcoords) {
-    using VertexType = VertexWithUv;
-    switch (mesh_type) {
-      case MeshType::kNormal:
-        return CreateVertexBuffer<VertexType>(engine, model, id, vertex_count,
-                                              FillMeshBuffer<VertexType>,
-                                              bounds);
-        break;
-      case MeshType::kConvexHull:
-        return CreateVertexBuffer<VertexType>(engine, model, id, vertex_count,
-                                              FillConvexHullBuffer<VertexType>,
-                                              bounds);
-        break;
-      case MeshType::kHeightField:
-        mju_error("Height fields do not support UV coordinates.");
-        return nullptr;
-    }
-  } else {
-    using VertexType = VertexNoUv;
-    switch (mesh_type) {
-      case MeshType::kNormal:
-        return CreateVertexBuffer<VertexType>(engine, model, id, vertex_count,
-                                              FillMeshBuffer<VertexType>,
-                                              bounds);
-        break;
-      case MeshType::kConvexHull:
-        return CreateVertexBuffer<VertexType>(engine, model, id, vertex_count,
-                                              FillConvexHullBuffer<VertexType>,
-                                              bounds);
-        break;
-      case MeshType::kHeightField:
-        return CreateVertexBuffer<VertexType>(engine, model, id, vertex_count,
-                                              FillHeightFieldBuffer, bounds);
-        break;
-    }
-  }
-
-  return nullptr;
-}
-
-filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine,
-                                         const mjModel* model, int id,
-                                         MeshType mesh_type) {
-  if (id < 0) {
-    mju_error("Invalid index %d", id);
-    return nullptr;
-  }
-
-  int index_count = 0;
-  switch (mesh_type) {
-    case MeshType::kNormal:
-      if (id >= model->nmesh) {
-        mju_error("Invalid mesh index %d", id);
-        return nullptr;
-      }
-      index_count = 3 * model->mesh_facenum[id];
-      break;
-    case MeshType::kConvexHull:
-      if (id >= model->nmesh) {
-        mju_error("Invalid mesh index %d", id);
-        return nullptr;
-      }
-      index_count = 3 * model->mesh_graph[model->mesh_graphadr[id] + 1];
-      break;
-    case MeshType::kHeightField:
-      if (id >= model->nhfield) {
-        mju_error("Invalid height field index %d", id);
-        return nullptr;
-      }
-      index_count = CalculateHeightFieldVertexCount(model, id);
-      break;
-  }
-
-  if (index_count == 0) {
-    mju_error("Index count is zero.");
-    return nullptr;
-  }
-
-  if (index_count >= std::numeric_limits<uint16_t>::max()) {
-    return CreateIndexBuffer<uint32_t>(engine, index_count,
-                                       FillSequence<uint32_t>);
-  } else {
-    return CreateIndexBuffer<uint16_t>(engine, index_count,
-                                       FillSequence<uint16_t>);
-  }
-}
-
-filament::Texture* CreateTexture(filament::Engine* engine, const mjModel* model,
-                                 int id, TextureType texture_type) {
-  if (id < 0 || id >= model->ntex) {
-    mju_error("Invalid texture index %d", id);
-  }
-
-  const int width = model->tex_width[id];
-  const int height = model->tex_height[id];
-  const bool is_srgb = model->tex_colorspace[id] == mjCOLORSPACE_SRGB;
-  const int num_channels = model->tex_nchannel[id];
-  const mjtByte* data = model->tex_data + model->tex_adr[id];
-  filament::Texture* texture =
-      texture_type == TextureType::kNormal2d
-          ? Create2dTexture(engine, width, height, num_channels, data, is_srgb)
-          : CreateCubeTexture(engine, width, height, num_channels, data,
-                              is_srgb);
-  return texture;
-}
-}  // namespace mujoco

src/experimental/filament/filament/model_util.h

@@ -1,108 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_
-
-#include <string_view>
-
-#include <filament/Box.h>
-#include <filament/Engine.h>
-#include <filament/IndexBuffer.h>
-#include <filament/Texture.h>
-#include <filament/VertexBuffer.h>
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-#include <mujoco/mjmodel.h>
-#include <mujoco/mujoco.h>
-
-namespace mujoco {
-
-// The types of meshes stored in the mjModel.
-enum class MeshType {
-  kNormal,
-  kConvexHull,
-  kHeightField,
-};
-
-// The types of textures stored in the mjModel.
-enum class TextureType {
-  kNormal2d,
-  kCube,
-};
-
-// Generates a filament VertexBuffer for a given mesh in the mjModel.
-filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine,
-                                           const mjModel* model, int id,
-                                           MeshType mesh_type,
-                                           filament::Box* bounds);
-
-// Generates a filament IndexBuffer for a given mesh in the mjModel.
-filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine,
-                                         const mjModel* model, int id,
-                                         MeshType mesh_type);
-
-// Generates a filament Texture for a given 2D texture in the mjModel.
-filament::Texture* CreateTexture(filament::Engine* engine, const mjModel* model,
-                                 int id, TextureType texture_type);
-
-// Reads a value with the given name from the mjModel's data sections. The
-// default_value is returned if the named element is not found.
-template <typename T>
-T ReadElement(const mjModel* model, const char* name, T default_value = T()) {
-  constexpr bool is_string =
-      std::is_same_v<T, const char*> || std::is_same_v<T, std::string_view>;
-
-  const int type = is_string ? mjOBJ_TEXT : mjOBJ_NUMERIC;
-  const int id = mj_name2id(model, type, name);
-  if (id < 0) {
-    return default_value;
-  }
-
-  if constexpr (std::is_same_v<T, const char*>) {
-    const char* ptr = model->text_data + model->text_adr[id];
-    return ptr;
-  } else if constexpr (std::is_same_v<T, std::string_view>) {
-    const char* ptr = model->text_data + model->text_adr[id];
-    // Do not include the null terminator in the string view.
-    return std::string_view(ptr, model->text_size[id] - 1);
-  } else if constexpr (std::is_arithmetic_v<T>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    return static_cast<T>(*ptr);
-  } else if constexpr (std::is_enum_v<T>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    return static_cast<T>(static_cast<int>(*ptr));
-  } else if constexpr (std::is_same_v<T, filament::math::float2>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    if (model->numeric_size[id] != 2) mju_error("Invalid numeric size.");
-    return T{ptr[0], ptr[1]};
-  } else if constexpr (std::is_same_v<T, filament::math::float3>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    if (model->numeric_size[id] != 3) mju_error("Invalid numeric size.");
-    return T{ptr[0], ptr[1], ptr[2]};
-  } else if constexpr (std::is_same_v<T, filament::math::float4>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    if (model->numeric_size[id] != 4) mju_error("Invalid numeric size.");
-    return T{ptr[0], ptr[1], ptr[2], ptr[3]};
-  } else if constexpr (std::is_same_v<T, bool>) {
-    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
-    return static_cast<T>(*ptr != 0);
-  }
-  return default_value;
-}
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_

src/experimental/filament/filament/object_manager.cc

@@ -14,85 +14,86 @@
 
 #include "experimental/filament/filament/object_manager.h"
 
+#include <cstddef>
 #include <cstdint>
+#include <memory>
+#include <span>
 #include <string>
 #include <string_view>
+#include <utility>
 
 #include <filament/Engine.h>
 #include <filament/IndirectLight.h>
 #include <filament/Material.h>
 #include <filament/Skybox.h>
-#include <math/mat3.h>
-#include <math/scalar.h>
-#include <math/vec3.h>
+#include <filament/Texture.h>
 #include <mujoco/mujoco.h>
-#include "experimental/filament/filament/texture_util.h"
+#include "experimental/filament/filament/builtins.h"
 #include "user/user_resource.h"
 
 namespace mujoco {
-namespace {
 
-// Loads binary data from a file using mjrFilamentConfig callbacks.
-struct Asset {
-  explicit Asset(std::string_view filename) {
-    std::string path = "filament:" + std::string(filename);
+std::string ResolveFilamentAssetPath(const std::string& filename) {
+  std::string path = "filament:" + filename;
+  return path;
+}
 
-    resource = mju_openResource("", path.c_str(), nullptr, nullptr, 0);
-    size = mju_readResource(resource, const_cast<const void**>(&payload));
-  }
-
-  ~Asset() {
-    if (resource) {
-      mju_closeResource(resource);
-    }
-  }
-
-  Asset(const Asset&) = delete;
-  Asset& operator=(const Asset&) = delete;
-
-  int size = 0;
+static filament::Material* LoadMaterial(filament::Engine* engine,
+                                        std::string_view filename) {
+  const std::string path = ResolveFilamentAssetPath(std::string(filename));
+  mjResource* resource = mju_openResource("", path.c_str(), nullptr, nullptr, 0);
   void* payload = nullptr;
-  mjResource* resource = nullptr;
+  int size = mju_readResource(resource, const_cast<const void**>(&payload));
+  filament::Material::Builder material_builder;
+  material_builder.package(payload, size);
+  filament::Material* material = material_builder.build(*engine);
+  mju_closeResource(resource);
+  return material;
 };
 
-}  // namespace
-
 ObjectManager::ObjectManager(filament::Engine* engine)
     : engine_(engine) {
-  auto LoadMaterial = [this](std::string_view filename) {
-    Asset asset(filename);
-    filament::Material::Builder material_builder;
-    material_builder.package(asset.payload, asset.size);
-    return material_builder.build(*this->engine_);
-  };
-
-  materials_[kPbr] = LoadMaterial("pbr.filamat");
-  materials_[kPbrPacked] = LoadMaterial("pbr_packed.filamat");
-  materials_[kPhong2d] = LoadMaterial("phong_2d.filamat");
-  materials_[kPhong2dFade] = LoadMaterial("phong_2d_fade.filamat");
-  materials_[kPhong2dReflect] = LoadMaterial("phong_2d_reflect.filamat");
-  materials_[kPhong2dUv] = LoadMaterial("phong_2d_uv.filamat");
-  materials_[kPhong2dUvFade] = LoadMaterial("phong_2d_uv_fade.filamat");
-  materials_[kPhong2dUvReflect] = LoadMaterial("phong_2d_uv_reflect.filamat");
-  materials_[kPhongColor] = LoadMaterial("phong_color.filamat");
-  materials_[kPhongColorFade] = LoadMaterial("phong_color_fade.filamat");
-  materials_[kPhongColorReflect] = LoadMaterial("phong_color_reflect.filamat");
-  materials_[kPhongCube] = LoadMaterial("phong_cube.filamat");
-  materials_[kPhongCubeFade] = LoadMaterial("phong_cube_fade.filamat");
-  materials_[kPhongCubeReflect] = LoadMaterial("phong_cube_reflect.filamat");
-  materials_[kUnlitSegmentation] = LoadMaterial("unlit_segmentation.filamat");
-  materials_[kUnlitLine] = LoadMaterial("unlit_line.filamat");
-  materials_[kUnlitDepth] = LoadMaterial("unlit_depth.filamat");
-  materials_[kUnlitUi] = LoadMaterial("unlit_ui.filamat");
+  materials_[kPbr] = LoadMaterial(engine, "pbr.filamat");
+  materials_[kPbrPacked] = LoadMaterial(engine, "pbr_packed.filamat");
+  materials_[kPhong2d] = LoadMaterial(engine, "phong_2d.filamat");
+  materials_[kPhong2dFade] = LoadMaterial(engine, "phong_2d_fade.filamat");
+  materials_[kPhong2dReflect] = LoadMaterial(engine, "phong_2d_reflect.filamat");
+  materials_[kPhong2dUv] = LoadMaterial(engine, "phong_2d_uv.filamat");
+  materials_[kPhong2dUvFade] = LoadMaterial(engine, "phong_2d_uv_fade.filamat");
+  materials_[kPhong2dUvReflect] = LoadMaterial(engine, "phong_2d_uv_reflect.filamat");
+  materials_[kPhongColor] = LoadMaterial(engine, "phong_color.filamat");
+  materials_[kPhongColorFade] = LoadMaterial(engine, "phong_color_fade.filamat");
+  materials_[kPhongColorReflect] = LoadMaterial(engine, "phong_color_reflect.filamat");
+  materials_[kPhongCube] = LoadMaterial(engine, "phong_cube.filamat");
+  materials_[kPhongCubeFade] = LoadMaterial(engine, "phong_cube_fade.filamat");
+  materials_[kPhongCubeReflect] = LoadMaterial(engine, "phong_cube_reflect.filamat");
+  materials_[kUnlitSegmentation] = LoadMaterial(engine, "unlit_segmentation.filamat");
+  materials_[kUnlitDecor] = LoadMaterial(engine, "unlit_decor.filamat");
+  materials_[kUnlitDepth] = LoadMaterial(engine, "unlit_depth.filamat");
+  materials_[kUnlitUi] = LoadMaterial(engine, "unlit_ui.filamat");
 
   static uint8_t black_rgb[3] = {0, 0, 0};
-  fallback_black_ = Create2dTexture(engine_, 1, 1, 3, black_rgb, false);
   static uint8_t white_rgb[3] = {255, 255, 255};
-  fallback_white_ = Create2dTexture(engine_, 1, 1, 3, white_rgb, false);
   static uint8_t normal_data[3] = {128, 128, 255};
-  fallback_normal_ = Create2dTexture(engine_, 1, 1, 3, normal_data, false);
   static uint8_t orm_data[3] = {0, 255, 0};
-  fallback_orm_ = Create2dTexture(engine_, 1, 1, 3, orm_data, false);
+
+  auto CreateFallbackTexture = [this](uint8_t color[3]) {
+    filament::Texture::Builder builder;
+    builder.width(1);
+    builder.height(1);
+    builder.format(filament::Texture::InternalFormat::RGB8);
+    builder.sampler(filament::Texture::Sampler::SAMPLER_2D);
+    filament::Texture* texture = builder.build(*engine_);
+    const filament::Texture::Type type = filament::Texture::Type::UBYTE;
+    const filament::Texture::Format format = filament::Texture::Format::RGB;
+    texture->setImage(*engine_, 0, {color, 3, format, type});
+    return texture;
+  };
+
+  fallback_black_ = CreateFallbackTexture(black_rgb);
+  fallback_white_ = CreateFallbackTexture(white_rgb);
+  fallback_normal_ = CreateFallbackTexture(normal_data);
+  fallback_orm_ = CreateFallbackTexture(orm_data);
 
   fallback_textures_[mjTEXROLE_USER] = fallback_black_;
   fallback_textures_[mjTEXROLE_RGB] = fallback_white_;
@@ -102,25 +103,16 @@ ObjectManager::ObjectManager(filament::Engine* engine)
   fallback_textures_[mjTEXROLE_NORMAL] = fallback_normal_;
   fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_;
   fallback_textures_[mjTEXROLE_ORM] = fallback_orm_;
-
-  LoadFallbackIndirectLight("ibl.ktx", 1.0f);
 }
 
 ObjectManager::~ObjectManager() {
-  if (fallback_indirect_light_) {
-    engine_->destroy(fallback_indirect_light_);
-  }
-  if (fallback_indirect_light_texture_) {
-    engine_->destroy(fallback_indirect_light_texture_);
-  }
+  engine_->destroy(fallback_black_);
+  engine_->destroy(fallback_white_);
+  engine_->destroy(fallback_normal_);
+  engine_->destroy(fallback_orm_);
   for (auto& iter : materials_) {
     engine_->destroy(iter);
   }
-  // fallback_textures_ maps to these textures.
-  engine_->destroy(fallback_white_);
-  engine_->destroy(fallback_black_);
-  engine_->destroy(fallback_normal_);
-  engine_->destroy(fallback_orm_);
 }
 
 filament::Material* ObjectManager::GetMaterial(MaterialType type) const {
@@ -130,6 +122,22 @@ filament::Material* ObjectManager::GetMaterial(MaterialType type) const {
   return materials_[type];
 }
 
+Builtins* ObjectManager::GetBuiltins(int nstack, int nslice, int nquad) {
+  // Assumes nstack, nslice, and nquad are non-negative and less than 2^20.
+  std::uint64_t key = (static_cast<uint64_t>(nstack) << 20) |
+                      (static_cast<uint64_t>(nslice) << 40) |
+                      static_cast<uint64_t>(nquad);
+
+  auto iter = builtins_.find(key);
+  if (iter == builtins_.end()) {
+    auto builtins = std::make_unique<Builtins>(engine_, nstack, nslice, nquad);
+    Builtins* ptr = builtins.get();
+    builtins_[key] = std::move(builtins);
+    return ptr;
+  }
+  return iter->second.get();
+}
+
 const filament::Texture* ObjectManager::GetFallbackTexture(
     mjtTextureRole role) const {
   if (role < 0 || role >= mjNTEXROLE) {
@@ -137,43 +145,4 @@ const filament::Texture* ObjectManager::GetFallbackTexture(
   }
   return fallback_textures_[role];
 }
-
-filament::IndirectLight* ObjectManager::GetFallbackIndirectLight() {
-  return fallback_indirect_light_;
-}
-
-void ObjectManager::LoadFallbackIndirectLight(
-    std::string_view filename, float intensity) {
-  if (fallback_indirect_light_texture_ != nullptr) {
-    engine_->destroy(fallback_indirect_light_texture_);
-    fallback_indirect_light_texture_ = nullptr;
-  }
-  if (fallback_indirect_light_ != nullptr) {
-    engine_->destroy(fallback_indirect_light_);
-    fallback_indirect_light_ = nullptr;
-  }
-
-  Asset asset(filename);
-  if (asset.size == 0) {
-    return;
-  }
-
-  filament::math::float3 spherical_harmonics[9];
-  fallback_indirect_light_texture_ =
-      CreateKtxTexture(engine_, reinterpret_cast<const uint8_t*>(asset.payload),
-                       asset.size, spherical_harmonics);
-  if (fallback_indirect_light_texture_ == nullptr) {
-    return;
-  }
-
-  // Build the indirect light.
-  filament::IndirectLight::Builder builder;
-  builder.reflections(fallback_indirect_light_texture_);
-  builder.irradiance(3, spherical_harmonics);
-  builder.intensity(intensity);
-  // Rotate the light to match mujoco's Z-up convention.
-  builder.rotation(filament::math::mat3f::rotation(
-      filament::math::f::PI / 2, filament::math::float3{1, 0, 0}));
-  fallback_indirect_light_ = builder.build(*engine_);
-}
 }  // namespace mujoco

src/experimental/filament/filament/object_manager.h

@@ -16,12 +16,19 @@
 #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OBJECT_MANAGER_H_
 
 #include <array>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <span>
 #include <string_view>
+#include <unordered_map>
 
 #include <filament/Engine.h>
 #include <filament/IndirectLight.h>
 #include <filament/Skybox.h>
+#include <filament/Texture.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/filament/builtins.h"
 
 namespace mujoco {
 
@@ -31,6 +38,8 @@ class ObjectManager {
   ObjectManager(filament::Engine* engine);
   ~ObjectManager();
 
+  // The different filament::Materials that are loaded and managed by the
+  // ObjectManager.
   enum MaterialType {
     kPbr,
     kPbrPacked,
@@ -47,26 +56,25 @@ class ObjectManager {
     kPhongCubeFade,
     kPhongCubeReflect,
     kUnlitSegmentation,
+    kUnlitDecor,
     kUnlitDepth,
-    kUnlitLine,
     kUnlitUi,
     kNumMaterials,
   };
 
-  // Returns the filament Engine that owns the assets.
-  filament::Engine* GetEngine() const { return engine_; }
-
   // Returns the Material of the given type.
   filament::Material* GetMaterial(MaterialType type) const;
 
   // Returns the fallback Texture with the given role.
   const filament::Texture* GetFallbackTexture(mjtTextureRole role) const;
 
-  // Returns the fallback IndirectLight.
-  filament::IndirectLight* GetFallbackIndirectLight();
+  // Returns the built-in mesh collection with the given dimensions. For
+  // performance reasons, you should consider always using the same dimensions
+  // in order to reuse the same meshes.
+  Builtins* GetBuiltins(int nstack, int nslice, int nquad);
 
-  // Loads an indirect light from a file, setting it to the fallback.
-  void LoadFallbackIndirectLight(std::string_view filename, float intensity);
+  // Returns the filament Engine that owns the assets.
+  filament::Engine* GetEngine() const { return engine_; }
 
   ObjectManager(const ObjectManager&) = delete;
   ObjectManager& operator=(const ObjectManager&) = delete;
@@ -75,12 +83,11 @@ class ObjectManager {
   filament::Engine* engine_ = nullptr;
   std::array<filament::Material*, kNumMaterials> materials_;
   std::array<filament::Texture*, mjNTEXROLE> fallback_textures_;
+  std::unordered_map<std::uint64_t, std::unique_ptr<Builtins>> builtins_;
   filament::Texture* fallback_white_ = nullptr;
   filament::Texture* fallback_black_ = nullptr;
   filament::Texture* fallback_normal_ = nullptr;
   filament::Texture* fallback_orm_ = nullptr;
-  filament::Texture* fallback_indirect_light_texture_ = nullptr;
-  filament::IndirectLight* fallback_indirect_light_ = nullptr;
 };
 
 }  // namespace mujoco

src/experimental/filament/filament/render_target.cc

@@ -0,0 +1,141 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/filament/render_target.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <utility>
+
+#include <backend/DriverEnums.h>
+#include <backend/PixelBufferDescriptor.h>
+#include <filament/Engine.h>
+#include <filament/Renderer.h>
+#include <filament/RenderTarget.h>
+#include <filament/Texture.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+RenderTarget::RenderTarget(filament::Engine* engine,
+                           const mjrRenderTargetConfig& config)
+    : engine_(engine), config_(config) {
+  if (config_.width > 0 && config_.height > 0) {
+    Prepare(config_.width, config_.height);
+  }
+}
+
+RenderTarget::~RenderTarget() noexcept {
+  Destroy();
+}
+
+void RenderTarget::Prepare(int width, int height) {
+  if (width == width_ && height == height_) {
+    return;
+  }
+  Destroy();
+  width_ = width;
+  height_ = height;
+  if (width_ <= 0 || height_ <= 0) {
+    width_ = 0;
+    height_ = 0;
+    return;
+  }
+
+  mjrTextureConfig color_config;
+  mjr_defaultTextureConfig(&color_config);
+  Texture::InternalFlags color_flags;
+  color_config.width = width;
+  color_config.height = height;
+  color_config.sampler_type = mjTEXTURE_2D;
+  color_config.format = config_.color_format;
+  color_config.color_space = mjCOLORSPACE_LINEAR;
+  color_config.format = mjPIXEL_FORMAT_RGB8;
+  color_flags.color_attachment = true;
+  color_texture_ = std::make_unique<Texture>(engine_, color_config, color_flags);
+
+  mjrTextureConfig depth_config;
+  mjr_defaultTextureConfig(&depth_config);
+  Texture::InternalFlags depth_flags;
+  depth_config.width = width;
+  depth_config.height = height;
+  depth_config.sampler_type = mjTEXTURE_2D;
+  depth_config.format = config_.depth_format;
+  depth_config.color_space = mjCOLORSPACE_LINEAR;
+  depth_config.format = mjPIXEL_FORMAT_DEPTH32F;
+  depth_flags.depth_attachment = true;
+  depth_texture_ = std::make_unique<Texture>(engine_, depth_config, depth_flags);
+
+  filament::RenderTarget::Builder builder;
+  builder.texture(filament::RenderTarget::AttachmentPoint::COLOR,
+                  color_texture_->GetFilamentTexture());
+  builder.texture(filament::RenderTarget::AttachmentPoint::DEPTH,
+                  depth_texture_->GetFilamentTexture());
+  render_target_ = builder.build(*engine_);
+}
+
+void RenderTarget::ReadColorPixels(filament::Renderer* renderer, uint8_t* bytes,
+                                   size_t num_bytes) {
+  filament::backend::PixelDataFormat format;
+  filament::backend::PixelDataType type;
+  size_t expected_num_bytes = 0;
+  switch (config_.color_format) {
+    case mjPIXEL_FORMAT_RGB8:
+      format = filament::backend::PixelDataFormat::RGB;
+      type = filament::backend::PixelDataType::UBYTE;
+      expected_num_bytes = width_ * height_ * 3;
+      break;
+    case mjPIXEL_FORMAT_R32F:
+      format = filament::backend::PixelDataFormat::R;
+      type = filament::backend::PixelDataType::FLOAT;
+      expected_num_bytes = width_ * height_ * sizeof(float);
+      break;
+    default:
+      mju_error("Unsupported pixel format: %d", config_.color_format);
+      return;
+  }
+  if (num_bytes != expected_num_bytes) {
+    mju_error("Invalid number of bytes.");
+    return;
+  }
+
+  filament::backend::PixelBufferDescriptor desc(bytes, num_bytes, format, type);
+  renderer->readPixels(render_target_, 0, 0, width_, height_, std::move(desc));
+}
+
+void RenderTarget::Destroy() {
+  if (render_target_) {
+    engine_->destroy(render_target_);
+    render_target_ = nullptr;
+  }
+  color_texture_.reset();
+  depth_texture_.reset();
+}
+
+Texture* RenderTarget::GetColorTexture() const {
+  return color_texture_.get();
+}
+
+Texture* RenderTarget::GetDepthTexture() const {
+  return depth_texture_.get();
+}
+
+filament::RenderTarget* RenderTarget::GetFilamentRenderTarget() const {
+  return render_target_;
+}
+
+}  // namespace mujoco

src/experimental/filament/filament/render_target.h

@@ -12,63 +12,67 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_UTIL_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
 
 #include <filament/Engine.h>
 #include <filament/Texture.h>
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-// The different types of textures we can create for a render target.
-enum RenderTargetTextureType {
-  kRenderTargetNone,
-  kRenderTargetColor,
-  kRenderTargetDepth,
-  kRenderTargetDepthColor,
-  kRenderTargetReflectionColor,
-  kNumRenderTargetTextureTypes,
-};
-
 // Manages a filament RenderTarget and the textures which are bound to it.
-class RenderTargetAndTextures {
+class RenderTarget : public mjrRenderTarget {
  public:
   // Defines the types of textures to create for the color and depth
   // attachments.
-  RenderTargetAndTextures(filament::Engine* engine,
-                          RenderTargetTextureType color,
-                          RenderTargetTextureType depth);
-  ~RenderTargetAndTextures() noexcept;
+  RenderTarget(filament::Engine* engine, const mjrRenderTargetConfig& config);
+  ~RenderTarget() noexcept;
 
-  RenderTargetAndTextures(const RenderTargetAndTextures&) = delete;
-  RenderTargetAndTextures& operator=(const RenderTargetAndTextures&) = delete;
+  RenderTarget(const RenderTarget&) = delete;
+  RenderTarget& operator=(const RenderTarget&) = delete;
 
   // Creates the textures and render target if the width of height differ from
   // the last time the render target was prepared.
   void Prepare(int width, int height);
 
+  // Reads the pixels from the render target texture.
+  void ReadColorPixels(filament::Renderer* renderer, uint8_t* bytes,
+                       size_t num_bytes);
+
   // Returns the color texture.
-  filament::Texture* GetColorTexture() const { return color_texture_; }
+  Texture* GetColorTexture() const;
 
   // Returns the depth texture.
-  filament::Texture* GetDepthTexture() const { return depth_texture_; }
+  Texture* GetDepthTexture() const;
 
-  // Returns the render target.
-  filament::RenderTarget* GetRenderTarget() const { return render_target_; }
+  // Returns the underlying filament render target.
+  filament::RenderTarget* GetFilamentRenderTarget() const;
+
+  static RenderTarget* downcast(mjrRenderTarget* render_target) {
+    return static_cast<RenderTarget*>(render_target);
+  }
+  static const RenderTarget* downcast(const mjrRenderTarget* render_target) {
+    return static_cast<const RenderTarget*>(render_target);
+  }
 
  private:
   void Destroy();
 
   filament::Engine* engine_ = nullptr;
-  filament::Texture* color_texture_ = nullptr;
-  filament::Texture* depth_texture_ = nullptr;
+  mjrRenderTargetConfig config_;
   filament::RenderTarget* render_target_ = nullptr;
-  RenderTargetTextureType color_type_ = kRenderTargetNone;
-  RenderTargetTextureType depth_type_ = kRenderTargetNone;
+  std::unique_ptr<Texture> color_texture_ = nullptr;
+  std::unique_ptr<Texture> depth_texture_ = nullptr;
   int width_ = 0;
   int height_ = 0;
 };
 
 }  // namespace mujoco
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_UTIL_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDER_TARGET_H_

src/experimental/filament/filament/render_target_util.cc

@@ -1,103 +0,0 @@
-// Copyright 2026 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/render_target_util.h"
-
-#include <filament/Engine.h>
-#include <filament/RenderTarget.h>
-#include <filament/Texture.h>
-#include <mujoco/mujoco.h>
-
-namespace mujoco {
-
-static filament::Texture* CreateRenderTargetTexture(
-    filament::Engine* engine, int width, int height,
-    RenderTargetTextureType type) {
-  filament::Texture::Builder builder;
-  builder.width(width);
-  builder.height(height);
-  switch (type) {
-    case kRenderTargetColor:
-      builder.usage(filament::Texture::Usage::COLOR_ATTACHMENT |
-                    filament::Texture::Usage::BLIT_SRC);
-      builder.format(filament::Texture::InternalFormat::RGB8);
-      break;
-    case kRenderTargetDepth:
-      builder.usage(filament::Texture::Usage::DEPTH_ATTACHMENT |
-                    filament::Texture::Usage::SAMPLEABLE);
-      builder.format(filament::Texture::InternalFormat::DEPTH32F);
-      break;
-    case kRenderTargetDepthColor:
-      builder.usage(filament::Texture::Usage::COLOR_ATTACHMENT |
-                    filament::Texture::Usage::BLIT_SRC);
-      builder.format(filament::Texture::InternalFormat::R32F);
-      break;
-    case kRenderTargetReflectionColor:
-      builder.usage(filament::Texture::Usage::COLOR_ATTACHMENT |
-                    filament::Texture::Usage::BLIT_SRC |
-                    filament::Texture::Usage::SAMPLEABLE);
-      builder.format(filament::Texture::InternalFormat::RGBA8);
-      break;
-    default:
-      mju_error("Unknown type: %d", static_cast<int>(type));
-  }
-  return builder.build(*engine);
-}
-
-RenderTargetAndTextures::RenderTargetAndTextures(filament::Engine* engine,
-                                                 RenderTargetTextureType color,
-                                                 RenderTargetTextureType depth)
-    : engine_(engine), color_type_(color), depth_type_(depth) {}
-
-RenderTargetAndTextures::~RenderTargetAndTextures() noexcept {
-  Destroy();
-}
-
-void RenderTargetAndTextures::Prepare(int width, int height) {
-  if (width == width_ && height == height_) {
-    return;
-  }
-  Destroy();
-  width_ = width;
-  height_ = height;
-
-  color_texture_ =
-      CreateRenderTargetTexture(engine_, width, height, color_type_);
-  depth_texture_ =
-      CreateRenderTargetTexture(engine_, width, height, depth_type_);
-
-  filament::RenderTarget::Builder builder;
-  builder.texture(filament::RenderTarget::AttachmentPoint::COLOR,
-                  color_texture_);
-  builder.texture(filament::RenderTarget::AttachmentPoint::DEPTH,
-                  depth_texture_);
-  render_target_ = builder.build(*engine_);
-}
-
-void RenderTargetAndTextures::Destroy() {
-  if (render_target_) {
-    engine_->destroy(render_target_);
-    render_target_ = nullptr;
-  }
-  if (color_texture_) {
-    engine_->destroy(color_texture_);
-    color_texture_ = nullptr;
-  }
-  if (depth_texture_) {
-    engine_->destroy(depth_texture_);
-    depth_texture_ = nullptr;
-  }
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/renderable.cc

@@ -0,0 +1,613 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/filament/renderable.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <numbers>
+
+#include <filament/Engine.h>
+#include <filament/Material.h>
+#include <filament/RenderableManager.h>
+#include <filament/Scene.h>
+#include <filament/TransformManager.h>
+#include <math/mat4.h>
+#include <math/vec2.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <utils/EntityManager.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/filament/builtins.h"
+#include "experimental/filament/filament/material.h"
+#include "experimental/filament/filament/mesh.h"
+#include "experimental/filament/filament/object_manager.h"
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+using filament::math::float2;
+using filament::math::float3;
+using filament::math::float4;
+using filament::math::mat4f;
+
+// An arbitrary scale factor for arrows.
+static constexpr float kArrowScale = 1.f / 6.f;
+static constexpr float kArrowHeadSize = 1.75f;
+
+Renderable::Renderable(filament::Engine* engine,
+                       const mjrRenderableParams& params,
+                       ObjectManager* object_mgr)
+    : object_mgr_(object_mgr), params_(params) {
+  mjr_defaultMaterial(&material_);
+}
+
+Renderable::~Renderable() noexcept {
+  filament::Engine* engine = GetEngine();
+  utils::EntityManager& em = utils::EntityManager::get();
+
+  for (Part& part : parts_) {
+    if (assigned_scene_) {
+      assigned_scene_->remove(part.entity);
+    }
+    engine->destroy(part.entity);
+    em.destroy(part.entity);
+  }
+  for (int i = 0; i < mjNUM_DRAW_MODES; ++i) {
+    if (instances_[i] != nullptr) {
+      engine->destroy(instances_[i]);
+      instances_[i] = nullptr;
+    }
+  }
+}
+
+void Renderable::SetMesh(const Mesh* mesh, int elem_offset, int elem_count) {
+  if (mesh == nullptr) {
+    mju_error("Cannot set mesh to nullptr.");
+  }
+
+  // We use MESH, even though it could be any mesh-like geom type, e.g.
+  // heightfields, flex, skin, sdf, etc.
+  geom_type_ = mjGEOM_MESH;
+
+  filament::VertexBuffer* vertex_buffer = mesh->GetFilamentVertexBuffer();
+  if (vertex_buffer == nullptr) {
+    mju_error("Invalid (null) vertex buffer.");
+  }
+  filament::IndexBuffer* index_buffer = mesh->GetFilamentIndexBuffer();
+  if (index_buffer == nullptr) {
+    mju_error("Invalid (null) index buffer.");
+  }
+
+  if (elem_count == 0) {
+    elem_count = index_buffer->getIndexCount() - elem_offset;
+  }
+
+  if (parts_.empty()) {
+    Part& part = parts_.emplace_back();
+    part.mesh = mesh;
+    part.elem_offset = elem_offset;
+    part.elem_count = elem_count;
+    InitPartEntity(part);
+  } else if (parts_.size() == 1) {
+    Part& part = parts_[0];
+    part.mesh = mesh;
+    part.elem_offset = elem_offset;
+    part.elem_count = elem_count;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    rm.setGeometryAt(rm.getInstance(part.entity), 0,
+                     part.mesh->GetPrimitiveType(), vertex_buffer, index_buffer,
+                     part.elem_offset, part.elem_count);
+
+  } else {
+    mju_error("Cannot set mesh for renderable with multiple parts.");
+  }
+}
+
+void Renderable::InitPartEntity(Part& part) {
+  part.entity = utils::EntityManager::get().create();
+  if (part.entity.isNull()) {
+    mju_error("Failed to create entity.");
+  }
+
+  filament::VertexBuffer* vertex_buffer = part.mesh->GetFilamentVertexBuffer();
+  filament::IndexBuffer* index_buffer = part.mesh->GetFilamentIndexBuffer();
+
+  filament::RenderableManager::Builder builder(1);
+  builder.geometry(0, part.mesh->GetPrimitiveType(), vertex_buffer, index_buffer,
+                   part.elem_offset, part.elem_count);
+  if (part.mesh->HasBounds()) {
+    builder.boundingBox(part.mesh->GetBounds());
+  } else {
+    builder.culling(false);
+  }
+  if (instances_[static_cast<int>(draw_mode_)] != nullptr) {
+    builder.material(0, instances_[static_cast<int>(draw_mode_)]);
+  }
+  builder.castShadows(params_.cast_shadows);
+  builder.receiveShadows(params_.receive_shadows);
+  builder.layerMask(0xff, params_.layer_mask);
+  builder.priority(params_.priority);
+  builder.blendOrder(0, params_.blend_order);
+  builder.screenSpaceContactShadows(true);
+
+  builder.build(*GetEngine(), part.entity);
+  if (assigned_scene_) {
+    assigned_scene_->addEntity(part.entity);
+  }
+}
+
+void Renderable::SetTransform(const Trs& trs) {
+  if (parts_.empty()) {
+    transform_ = trs.ToTransform();
+    return;
+  }
+
+  filament::TransformManager& tm = GetEngine()->getTransformManager();
+  if (get_transform_fn_) {
+    for (int i = 0; i < parts_.size(); ++i) {
+      const mat4f& transform = get_transform_fn_(i, trs);
+      tm.setTransform(tm.getInstance(parts_[i].entity), transform);
+    }
+  } else {
+    for (Part& part : parts_) {
+      tm.setTransform(tm.getInstance(part.entity), trs.ToTransform());
+    }
+  }
+  transform_ = tm.getTransform(tm.getInstance(parts_[0].entity));
+}
+
+const mat4f& Renderable::GetTransform() const {
+  return transform_;
+}
+
+void Renderable::AppendMesh(const Mesh* mesh) {
+  Part& part = parts_.emplace_back();
+  part.mesh = Mesh::downcast(mesh);
+  part.elem_offset = 0;
+  part.elem_count = part.mesh->GetFilamentIndexBuffer()->getIndexCount();
+  InitPartEntity(part);
+}
+
+void Renderable::AddToScene(filament::Scene* scene) {
+  if (assigned_scene_) {
+    if (assigned_scene_ != scene) {
+      mju_error("Cannot add renderable to multiple scenes.");
+    }
+    // Entities are already added to the scene.
+    return;
+  }
+  for (Part& part : parts_) {
+    scene->addEntity(part.entity);
+  }
+  assigned_scene_ = scene;
+}
+
+void Renderable::RemoveFromScene(filament::Scene* scene) {
+  if (assigned_scene_ != scene) {
+    mju_error("Attempting to remove renderable from wrong scene.");
+  }
+  for (Part& part : parts_) {
+    scene->remove(part.entity);
+  }
+  assigned_scene_ = nullptr;
+}
+
+void Renderable::UpdateMaterial(const mjrMaterial& material) {
+  material_ = material;
+
+  AssignMaterial(mjDRAW_MODE_COLOR, GetColorMaterialType());
+  if (!material_.decor_ux) {
+    AssignMaterial(mjDRAW_MODE_DEPTH, ObjectManager::kUnlitDepth);
+    AssignMaterial(mjDRAW_MODE_SEGMENTATION, ObjectManager::kUnlitSegmentation);
+  }
+
+  for (int i = 0; i < mjNUM_DRAW_MODES; ++i) {
+    if (instances_[i]) {
+      UpdateMaterialInstance(instances_[i], material_, object_mgr_);
+    }
+  }
+  SetDrawMode(draw_mode_);
+}
+
+void Renderable::AssignMaterial(mjrDrawMode mode,
+                                ObjectManager::MaterialType material_type) {
+  const int index = static_cast<int>(mode);
+
+  filament::Material* material = object_mgr_->GetMaterial(material_type);
+  if (instances_[index]) {
+    if (instances_[index]->getMaterial() == material) {
+      // The correct material is already assigned, do nothing.
+      return;
+    } else {
+      GetEngine()->destroy(instances_[index]);
+      instances_[index] = nullptr;
+    }
+  }
+  if (material) {
+    instances_[index] = material->createInstance();
+    if (geom_type_ == mjGEOM_PLANE || geom_type_ == mjGEOM_TRIANGLE) {
+      instances_[index]->setCullingMode(
+          filament::MaterialInstance::CullingMode::NONE);
+    }
+  }
+}
+
+const mjrMaterial& Renderable::GetMaterial() const {
+  return material_;
+}
+
+void Renderable::SetDrawMode(mjrDrawMode mode) {
+  // Only SceneObjects support non-color draw modes.
+  if (material_.decor_ux) {
+    mode = mjDRAW_MODE_COLOR;
+  }
+
+  filament::MaterialInstance* instance = instances_[static_cast<int>(mode)];
+  if (instance) {
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      filament::RenderableManager::Instance ri = rm.getInstance(part.entity);
+      rm.setMaterialInstanceAt(ri, 0, instance);
+    }
+  }
+  draw_mode_ = mode;
+}
+
+std::uint8_t Renderable::SetLayerMask(std::uint8_t mask) {
+  std::uint8_t prev = params_.layer_mask;
+  if (mask != params_.layer_mask) {
+    params_.layer_mask = mask;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      rm.setLayerMask(rm.getInstance(part.entity), 0xff, params_.layer_mask);
+    }
+  }
+  return prev;
+}
+
+std::uint8_t Renderable::SetPriority(std::uint8_t priority) {
+  std::uint8_t prev = params_.priority;
+  if (priority != params_.priority) {
+    params_.priority = priority;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      rm.setPriority(rm.getInstance(part.entity), params_.priority);
+    }
+  }
+  return prev;
+}
+
+std::uint16_t Renderable::SetBlendOrder(std::uint16_t blend_order) {
+  std::uint16_t prev = params_.blend_order;
+  if (blend_order != params_.blend_order) {
+    params_.blend_order = blend_order;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      rm.setBlendOrderAt(rm.getInstance(part.entity), 0, params_.blend_order);
+    }
+  }
+  return prev;
+}
+
+void Renderable::SetCastShadows(bool cast_shadows) {
+  if (params_.cast_shadows != cast_shadows) {
+    params_.cast_shadows = cast_shadows;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+  for (Part& part : parts_) {
+        rm.setCastShadows(rm.getInstance(part.entity), params_.cast_shadows);
+    }
+  }
+}
+
+void Renderable::SetReceiveShadows(bool receive_shadows) {
+  if (params_.receive_shadows != receive_shadows) {
+    params_.receive_shadows = receive_shadows;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      rm.setReceiveShadows(rm.getInstance(part.entity), params_.receive_shadows);
+    }
+  }
+}
+
+void Renderable::SetWireframe(bool wireframe) {
+  static constexpr auto kWireframeType =
+      filament::RenderableManager::PrimitiveType::LINES;
+
+  if (wireframe != wireframe_) {
+    wireframe_ = wireframe;
+
+    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
+    for (Part& part : parts_) {
+      filament::VertexBuffer* vertex_buffer = part.mesh->GetFilamentVertexBuffer();
+      filament::IndexBuffer* index_buffer = part.mesh->GetFilamentIndexBuffer();
+      rm.setGeometryAt(rm.getInstance(part.entity), 0,
+                       wireframe_ ? kWireframeType : part.mesh->GetPrimitiveType(),
+                       vertex_buffer, index_buffer, part.elem_offset,
+                       part.elem_count);
+    }
+  }
+}
+
+ObjectManager::MaterialType Renderable::GetColorMaterialType() const {
+  if (material_.decor_ux) {
+    if (material_.color_texture) {
+      return ObjectManager::kUnlitUi;
+    } else {
+      return ObjectManager::kUnlitDecor;
+    }
+  } else if (material_.orm_texture) {
+    return ObjectManager::kPbrPacked;
+  } else if (material_.metallic_texture) {
+    return ObjectManager::kPbr;
+  } else if (material_.roughness_texture) {
+    return ObjectManager::kPbr;
+  } else if (material_.metallic >= 0) {
+    return ObjectManager::kPbr;
+  } else if (material_.roughness >= 0) {
+    return ObjectManager::kPbr;
+  }
+
+  // Check to see if we're dealing with a mesh with texture coordinates.
+  // `data_id` is the id of the mesh in model (i.e. the geom has mesh
+  // geometry) and `mesh_texcoordadr` stores the address of the mesh uvs if
+  // it has them.
+  bool has_texcoords = false;
+  const Texture* color_texture = Texture::downcast(material_.color_texture);
+  if (!parts_.empty()) {
+    const auto attribs = parts_[0].mesh->GetVertexAttributes();
+    auto it = std::find(attribs.begin(), attribs.end(),
+                        filament::VertexAttribute::UV0);
+    has_texcoords = (it != attribs.end());
+  }
+
+  if (color_texture == nullptr) {
+    if (material_.color[3] < 1.0f) {
+      return ObjectManager::kPhongColorFade;
+    } else if (material_.reflective) {
+      return ObjectManager::kPhongColorReflect;
+    } else {
+      return ObjectManager::kPhongColor;
+    }
+  } else if (color_texture->GetSamplerType() == mjTEXTURE_CUBE) {
+    if (material_.color[3] < 1.0f) {
+      return ObjectManager::kPhongCubeFade;
+    } else if (material_.reflective) {
+      return ObjectManager::kPhongCubeReflect;
+    } else {
+      return ObjectManager::kPhongCube;
+    }
+  } else if (has_texcoords) {
+    if (material_.color[3] < 1.0f) {
+      return ObjectManager::kPhong2dUvFade;
+    } else if (material_.reflective) {
+      return ObjectManager::kPhong2dUvReflect;
+    } else {
+      return ObjectManager::kPhong2dUv;
+    }
+  } else {
+    if (material_.color[3] < 1.0f) {
+      return ObjectManager::kPhong2dFade;
+    } else if (material_.reflective) {
+      return ObjectManager::kPhong2dReflect;
+    } else {
+      return ObjectManager::kPhong2d;
+    }
+  }
+}
+
+void Renderable::SetGeomMesh(mjtGeom type, int nstack, int nslice, int nquad) {
+  Builtins* builtins = object_mgr_->GetBuiltins(nstack, nslice, nquad);
+  geom_type_ = type;
+
+  switch (type) {
+    case mjGEOM_PLANE:
+      AppendMesh(builtins->Plane());
+      break;
+    case mjGEOM_SPHERE:
+      AppendMesh(builtins->Sphere());
+      break;
+    case mjGEOM_ELLIPSOID:
+      AppendMesh(builtins->Sphere());
+      break;
+    case mjGEOM_BOX:
+      AppendMesh(builtins->Box());
+      break;
+    case mjGEOM_CAPSULE:
+      // Capsules are a tube with two domes at the ends.
+      AppendMesh(builtins->Tube());
+      AppendMesh(builtins->Dome());
+      AppendMesh(builtins->Dome());
+
+      get_transform_fn_ = [](int index, const Trs& trs) {
+        // We apply an inverse scale to the domes to counteract the capsule's
+        // overall scale so that the domes remain spherical in shape.
+        const float xz_size = 0.5f * (trs.size.x + trs.size.y);
+        if (index == 0) {
+          return trs.ToTransform();
+        } else if (index == 1) {
+          // Move the first dome to the top of the capsule.
+          mat4f top = mat4f(trs.rotation, trs.translation);
+          top *= mat4f::translation(float3{0, 0, trs.size.z});
+          top *= mat4f::scaling(float3{trs.size.x, trs.size.y, xz_size});
+          return top;
+        } else if (index == 2) {
+          // Move the second dome to the bottom of the capsule and rotate it 180
+          // degrees so that it's facing the right way.
+          mat4f bottom = mat4f(trs.rotation, trs.translation);
+          bottom *= mat4f::translation(float3{0, 0, -trs.size.z});
+          bottom *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          bottom *= mat4f::scaling(float3{trs.size.x, trs.size.y, xz_size});
+          return bottom;
+        } else {
+          mju_error("Invalid index for capsule geom: %d (expected [0,2])", index);
+          return trs.ToTransform();
+        }
+      };
+      break;
+    case mjGEOM_CYLINDER:
+      // Cylinders are a tube with two disks at the ends.
+      AppendMesh(builtins->Tube());
+      AppendMesh(builtins->Disk());
+      AppendMesh(builtins->Disk());
+
+      get_transform_fn_ = [](int index, const Trs& trs) {
+        if (index == 0) {
+          return trs.ToTransform();
+        } else if (index == 1) {
+          // Move the first disk to the top of the cylinder.
+          mat4f top = mat4f(trs.rotation, trs.translation);
+          top *= mat4f::translation(float3{0, 0, trs.size.z});
+          top *= mat4f::scaling(trs.size);
+          return top;
+        } else if (index == 2) {
+          // Move the second disk to the bottom of the cylinder. Rotate the disk
+          // 180 degrees so that the normals point outwards.
+          mat4f bottom = mat4f(trs.rotation, trs.translation);
+          bottom *= mat4f::translation(float3{0, 0, -trs.size.z});
+          bottom *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          bottom *= mat4f::scaling(trs.size);
+          return bottom;
+        } else {
+          mju_error("Invalid index for cylinder geom: %d (expected [0,2])", index);
+          return trs.ToTransform();
+        }
+      };
+      break;
+    case mjGEOM_ARROW:
+      AppendMesh(builtins->Tube());
+      AppendMesh(builtins->Cone());
+      AppendMesh(builtins->Disk());
+      AppendMesh(builtins->Disk());
+
+      get_transform_fn_ = [](int index, const Trs& trs) {
+        mat4f base = mat4f(trs.rotation, trs.translation);
+        base *= mat4f::scaling(float3{1, 1, kArrowScale});
+        base *= mat4f::translation(float3{0, 0, trs.size.z});
+        if (index == 0) {
+          return base * mat4f::scaling(trs.size);
+        } else if (index == 1) {
+          mat4f top = base;
+          top *= mat4f::translation(float3{0, 0, trs.size.z});
+          top *= mat4f::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
+          return top * mat4f::scaling(trs.size);
+        } else if (index == 2) {
+          mat4f top_disk = base;
+          top_disk *= mat4f::translation(float3{0, 0, trs.size.z});
+          top_disk *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          top_disk *= mat4f::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
+          return top_disk * mat4f::scaling(trs.size);
+        } else if (index == 3) {
+          mat4f bottom = base;
+          bottom *= mat4f::translation(float3{0, 0, -trs.size.z});
+          bottom *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          return bottom * mat4f::scaling(trs.size);
+        } else {
+          mju_error("Invalid index for arrow geom: %d (expected [0,3])", index);
+          return trs.ToTransform();
+        }
+      };
+      break;
+    case mjGEOM_ARROW1:
+      AppendMesh(builtins->Tube());
+      AppendMesh(builtins->Cone());
+      AppendMesh(builtins->Disk());
+
+      get_transform_fn_ = [](int index, const Trs& trs) {
+      mat4f base = mat4f(trs.rotation, trs.translation);
+      base *= mat4f::scaling(float3{1, 1, kArrowScale});
+      base *= mat4f::translation(float3{0, 0, trs.size.z});
+        if (index == 0) {
+          return base * mat4f::scaling(trs.size);
+        } else if (index == 1) {
+          mat4f top = base;
+          top *= mat4f::translation(float3{0, 0, trs.size.z});
+          return top * mat4f::scaling(trs.size);
+        } else if (index == 2) {
+          mat4f bottom = base;
+          bottom *= mat4f::translation(float3{0, 0, -trs.size.z});
+          bottom *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          return bottom * mat4f::scaling(trs.size);
+        } else {
+          mju_error("Invalid index for arrow1 geom: %d (expected [0,2])", index);
+          return trs.ToTransform();
+        }
+      };
+      break;
+    case mjGEOM_ARROW2:
+      AppendMesh(builtins->Tube());
+      AppendMesh(builtins->Cone());
+      AppendMesh(builtins->Cone());
+      AppendMesh(builtins->Disk());
+      AppendMesh(builtins->Disk());
+
+      get_transform_fn_ = [](int index, const Trs& trs) {
+        mat4f base = mat4f(trs.rotation, trs.translation);
+        base *= mat4f::scaling(float3{1, 1, kArrowScale});
+        base *= mat4f::translation(float3{0, 0, trs.size.z});
+        if (index == 0) {
+          return base * mat4f::scaling(trs.size);
+        } else if (index == 1) {
+          mat4f top = base;
+          top *= mat4f::translation(float3{0, 0, trs.size.z});
+          top *= mat4f::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
+          return top * mat4f::scaling(trs.size);
+        } else if (index == 2) {
+          mat4f bottom = base;
+          bottom *= mat4f::translation(float3{0, 0, -trs.size.z});
+          bottom *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          bottom *= mat4f::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
+          return bottom * mat4f::scaling(trs.size);
+        } else if (index == 3) {
+          mat4f top_disk = base;
+          top_disk *= mat4f::translation(float3{0, 0, trs.size.z});
+          top_disk *= mat4f::rotation(std::numbers::pi, float3{1, 0, 0});
+          top_disk *= mat4f::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f});
+          return top_disk * mat4f::scaling(trs.size);
+        } else if (index == 4) {
+          mat4f bottom_disk = base;
+          bottom_disk *= mat4f::translation(float3{0, 0, -trs.size.z});
+          return bottom_disk * mat4f::scaling(trs.size);
+        } else {
+          mju_error("Invalid index for arrow2 geom: %d (expected [0,4])", index);
+          return trs.ToTransform();
+        }
+      };
+      break;
+    case mjGEOM_LINE:
+      AppendMesh(builtins->Line());
+      break;
+    case mjGEOM_LINEBOX:
+      AppendMesh(builtins->LineBox());
+      break;
+    case mjGEOM_TRIANGLE:
+      AppendMesh(builtins->Triangle());
+      break;
+    default:
+      mju_error("Unsupported geom type: %d", type);
+      break;
+  }
+}
+
+filament::Engine* Renderable::GetEngine() { return object_mgr_->GetEngine(); }
+
+}  // namespace mujoco

src/experimental/filament/filament/renderable.h

@@ -0,0 +1,145 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLE_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLE_H_
+
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+#include <filament/Engine.h>
+#include <filament/Scene.h>
+#include <math/mat4.h>
+#include <utils/Entity.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
+#include "experimental/filament/filament/mesh.h"
+#include "experimental/filament/filament/object_manager.h"
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+// A Renderable is effectively two things: a mesh and a material.
+//
+// The mesh describes the surface geometry of the object and the material
+// describes how that surface interacts with light (i.e. the color of each point
+// on the surface).
+class Renderable : public mjrRenderable {
+ public:
+  Renderable(filament::Engine* engine, const mjrRenderableParams& params,
+             ObjectManager* object_mgr);
+  ~Renderable() noexcept;
+
+  Renderable(const Renderable&) = delete;
+  Renderable& operator=(const Renderable&) = delete;
+
+  // Sets the mesh of this renderable. The elem_offset and elem_count parameters
+  // can be used to specify a submesh within the mesh. If elem_count is 0,
+  // assumes the entire mesh should be appended.
+  void SetMesh(const Mesh* mesh, int elem_offset = 0, int elem_count = 0);
+
+  // Sets the mesh of this renderable to a built-in mesh based on the geom type.
+  void SetGeomMesh(mjtGeom type, int nstack, int nslice, int nquad);
+
+  // Sets the transform of this renderable.
+  void SetTransform(const Trs& trs);
+
+  // Returns the current transform of this renderable.
+  const filament::math::mat4f& GetTransform() const;
+
+  // Sets the layer mask for this renderable. Layer masks can be used to
+  // show/hide groups of renderables in scenes. Returns the previous layer mask.
+  std::uint8_t SetLayerMask(std::uint8_t mask);
+
+  // Sets the draw priority this renderable. The priority determines the order
+  // in which renderables are rendered. Returns the previous priority.
+  std::uint8_t SetPriority(std::uint8_t priority);
+
+  // Sets the blend order for this renderable. This determines the order in
+  // which transparent renderables are blended together. Returns the previous
+  // blend order.
+  std::uint16_t SetBlendOrder(std::uint16_t blend_order);
+
+  // Disables this renderable from casting shadows.
+  void SetCastShadows(bool cast_shadows);
+
+  // Disables this renderable from receiving shadows.
+  void SetReceiveShadows(bool receive_shadows);
+
+  // If true, forces this renderable to use wireframe rendering.
+  void SetWireframe(bool wireframe);
+
+  // Adds this renderable to the filament Scene.
+  void AddToScene(filament::Scene* scene);
+
+  // Removes this renderable from the filament Scene.
+  void RemoveFromScene(filament::Scene* scene);
+
+  // Determines how this renderable will be drawn. See mjrDrawMode for details.
+  void SetDrawMode(mjrDrawMode mode);
+
+  // Updates the parameters and textures of the material for this renderable.
+  void UpdateMaterial(const mjrMaterial& material);
+
+  // Returns this renderable's current material.
+  const mjrMaterial& GetMaterial() const;
+
+  static Renderable* downcast(mjrRenderable* renderable) {
+    return static_cast<Renderable*>(renderable);
+  }
+  static const Renderable* downcast(const mjrRenderable* renderable) {
+    return static_cast<const Renderable*>(renderable);
+  }
+
+ private:
+  // In most cases, a Renderable will be composed of a single filament Entity.
+  // However, for some built-in geom types (e.g. capsules) we compose the
+  // renderable out of multiple Entities.
+  struct Part {
+    utils::Entity entity;
+    const Mesh* mesh = nullptr;
+    int elem_offset = 0;
+    int elem_count = 0;
+  };
+
+  // When composing a multi-part renderable, each Entity will have its own
+  // transform offset based on the transform of the Renderable itself.
+  using GetTransformFn = std::function<filament::math::mat4f(int, const Trs&)>;
+
+  void AppendMesh(const Mesh* mesh);
+  void InitPartEntity(Part& part);
+
+  ObjectManager::MaterialType GetColorMaterialType() const;
+  void AssignMaterial(mjrDrawMode mode,
+                      ObjectManager::MaterialType material_type);
+
+  filament::Engine* GetEngine();
+
+  ObjectManager* object_mgr_;
+  mjrRenderableParams params_;
+  filament::MaterialInstance* instances_[mjNUM_DRAW_MODES] = {nullptr};
+  mjtGeom geom_type_ = mjGEOM_NONE;
+  mjrMaterial material_;
+  mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR;
+  filament::Scene* assigned_scene_ = nullptr;
+  std::vector<Part> parts_;
+  filament::math::mat4f transform_;
+  GetTransformFn get_transform_fn_;
+  bool wireframe_ = false;
+};
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLE_H_

src/experimental/filament/filament/renderables.cc

@@ -1,251 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/renderables.h"
-
-#include <cstdint>
-#include <optional>
-
-#include <filament/Engine.h>
-#include <filament/RenderableManager.h>
-#include <filament/Scene.h>
-#include <utils/EntityManager.h>
-#include <mujoco/mujoco.h>
-#include "experimental/filament/filament/buffer_util.h"
-
-namespace mujoco {
-
-Renderables::Renderables(filament::Engine* engine) : engine_(engine) {}
-
-Renderables::~Renderables() noexcept {
-  while (!entities_.empty()) {
-    RemoveLast();
-  }
-}
-
-void Renderables::RemoveLast() {
-  if (entities_.empty()) {
-    return;
-  }
-
-  utils::EntityManager& em = utils::EntityManager::get();
-  utils::Entity entity = entities_.back();
-
-  if (assigned_scene_) {
-    assigned_scene_->remove(entity);
-  }
-
-  engine_->destroy(entity);
-  em.destroy(entity);
-  entities_.pop_back();
-
-  if (owned_buffers_.back().owned) {
-    engine_->destroy(owned_buffers_.back().buffers.vertex_buffer);
-    engine_->destroy(owned_buffers_.back().buffers.index_buffer);
-  }
-  owned_buffers_.pop_back();
-}
-
-void Renderables::Update(int index, const FilamentBuffers& buffers) {
-  if (index < 0 || index >= entities_.size()) {
-    mju_error("Invalid index %d for renderable.", index);
-  }
-  utils::Entity& entity = entities_[index];
-  UpdateEntity(entity, buffers);
-  UpdateBuffers(index, buffers, false);
-}
-
-void Renderables::Update(int index, FilamentBuffers&& buffers) {
-  if (index < 0 || index >= entities_.size()) {
-    mju_error("Invalid index %d for renderable.", index);
-  }
-  utils::Entity& entity = entities_[index];
-  UpdateEntity(entity, buffers);
-  UpdateBuffers(index, buffers, true);
-}
-
-void Renderables::Append(const FilamentBuffers& buffers) {
-  utils::Entity entity = CreateEntity(buffers);
-  entities_.push_back(entity);
-  owned_buffers_.push_back({.owned = false, .buffers = buffers});
-}
-
-void Renderables::Append(FilamentBuffers&& buffers) {
-  utils::Entity entity = CreateEntity(buffers);
-  entities_.push_back(entity);
-  owned_buffers_.push_back({.owned = true, .buffers = buffers});
-}
-
-utils::Entity Renderables::CreateEntity(const FilamentBuffers& buffers) {
-  if (buffers.vertex_buffer == nullptr) {
-    mju_error("Invalid (null) vertex buffer.");
-  }
-  if (buffers.index_buffer == nullptr) {
-    mju_error("Invalid (null) index buffer.");
-  }
-  utils::Entity entity = utils::EntityManager::get().create();
-  if (entity.isNull()) {
-    mju_error("Failed to create entity.");
-  }
-
-  filament::RenderableManager::Builder builder(1);
-  builder.geometry(0, buffers.type, buffers.vertex_buffer,
-                   buffers.index_buffer);
-  if (material_instance_) {
-    builder.material(0, material_instance_);
-  }
-  if (buffers.bounds.has_value()) {
-    builder.boundingBox(buffers.bounds.value());
-  } else {
-    builder.culling(false);
-  }
-  builder.castShadows(cast_shadows_);
-  builder.receiveShadows(receive_shadows_);
-  builder.layerMask(0xff, layer_mask_);
-  builder.priority(priority_);
-  builder.screenSpaceContactShadows(true);;
-
-  builder.build(*engine_, entity);
-  if (assigned_scene_) {
-    assigned_scene_->addEntity(entity);
-  }
-  return entity;
-}
-
-void Renderables::UpdateEntity(utils::Entity entity,
-                               const FilamentBuffers& buffers) {
-  if (buffers.vertex_buffer == nullptr) {
-    mju_error("Invalid (null) vertex buffer.");
-  }
-  if (buffers.index_buffer == nullptr) {
-    mju_error("Invalid (null) index buffer.");
-  }
-  filament::RenderableManager& rm = engine_->getRenderableManager();
-  rm.setGeometryAt(rm.getInstance(entity), 0, buffers.type,
-                   buffers.vertex_buffer, buffers.index_buffer, 0,
-                   buffers.index_buffer->getIndexCount());
-}
-
-void Renderables::UpdateBuffers(int index, FilamentBuffers buffers, bool owned) {
-  if (index < 0 || index >= owned_buffers_.size()) {
-    mju_error("Invalid index %d for renderable.", index);
-  }
-  if (owned_buffers_[index].owned) {
-    engine_->destroy(owned_buffers_[index].buffers.vertex_buffer);
-    engine_->destroy(owned_buffers_[index].buffers.index_buffer);
-  }
-  owned_buffers_[index].buffers = buffers;
-  owned_buffers_[index].owned = owned;
-}
-
-void Renderables::AddToScene(filament::Scene* scene) {
-  if (assigned_scene_) {
-    if (assigned_scene_ != scene) {
-      mju_error("Cannot add renderable to multiple scenes.");
-    }
-    // Entities are already added to the scene.
-    return;
-  }
-  for (utils::Entity& entity : entities_) {
-    scene->addEntity(entity);
-  }
-  assigned_scene_ = scene;
-}
-
-void Renderables::RemoveFromScene(filament::Scene* scene) {
-  if (assigned_scene_ != scene) {
-    mju_error("Attempting to remove renderable from wrong scene.");
-  }
-  for (utils::Entity& entity : entities_) {
-    scene->remove(entity);
-  }
-  assigned_scene_ = nullptr;
-}
-
-void Renderables::SetMaterialInstance(
-    filament::MaterialInstance* instance) {
-  if (instance != material_instance_) {
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (utils::Entity& entity : entities_) {
-      filament::RenderableManager::Instance ri = rm.getInstance(entity);
-      rm.setMaterialInstanceAt(ri, 0, instance);
-    }
-    material_instance_ = instance;
-  }
-}
-
-void Renderables::SetLayerMask(std::uint8_t mask) {
-  if (mask != layer_mask_) {
-    layer_mask_ = mask;
-
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (utils::Entity& entity : entities_) {
-      rm.setLayerMask(rm.getInstance(entity), 0xff, layer_mask_);
-    }
-  }
-}
-
-void Renderables::SetPriority(std::uint8_t priority) {
-  if (priority != priority_) {
-    priority_ = priority;
-
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (utils::Entity& entity : entities_) {
-      rm.setPriority(rm.getInstance(entity), priority_);
-    }
-  }
-}
-
-void Renderables::SetCastShadows(bool cast_shadows) {
-  if (cast_shadows_ != cast_shadows) {
-    cast_shadows_ = cast_shadows;
-
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (utils::Entity& entity : entities_) {
-      rm.setCastShadows(rm.getInstance(entity), cast_shadows_);
-    }
-  }
-}
-
-void Renderables::SetReceiveShadows(bool receive_shadows) {
-  if (receive_shadows_ != receive_shadows) {
-    receive_shadows_ = receive_shadows;
-
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (utils::Entity& entity : entities_) {
-      rm.setReceiveShadows(rm.getInstance(entity), receive_shadows_);
-    }
-  }
-}
-
-void Renderables::SetWireframe(bool wireframe) {
-  static constexpr auto kWireframeType =
-      filament::RenderableManager::PrimitiveType::LINES;
-
-  if (wireframe != wireframe_) {
-    wireframe_ = wireframe;
-
-    filament::RenderableManager& rm = engine_->getRenderableManager();
-    for (int i = 0; i < entities_.size(); ++i) {
-      utils::Entity& entity = entities_[i];
-      FilamentBuffers& buffers = owned_buffers_[i].buffers;
-      rm.setGeometryAt(rm.getInstance(entity), 0,
-                      wireframe_ ? kWireframeType : buffers.type,
-                      buffers.vertex_buffer, buffers.index_buffer, 0,
-                      buffers.index_buffer->getIndexCount());
-    }
-  }
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/renderables.h

@@ -1,112 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_
-
-#include <cstdint>
-#include <optional>
-#include <vector>
-
-#include <filament/Engine.h>
-#include <filament/Scene.h>
-#include <utils/Entity.h>
-#include "experimental/filament/filament/buffer_util.h"
-
-namespace mujoco {
-
-// Manages a collection of related filament Renderable Entities.
-class Renderables {
- public:
-  // Default filament values for priority and layer mask.
-  static constexpr std::uint8_t kDefaultPriority = 4;
-  static constexpr std::uint8_t kDefaultLayerMask = 0x01;
-
-  Renderables(filament::Engine* engine);
-  ~Renderables() noexcept;
-
-  Renderables(const Renderables&) = delete;
-  Renderables& operator=(const Renderables&) = delete;
-
-  // Appends a new renderable entity built from the given buffers.
-  void Append(const FilamentBuffers& buffers);
-  void Append(FilamentBuffers&& buffers);
-
-  // Updates the entity at the index with new buffers.
-  void Update(int index, const FilamentBuffers& buffers);
-  void Update(int index, FilamentBuffers&& buffers);
-
-  // Removes the last entity.
-  void RemoveLast();
-
-  // Returns the entity at the given index.
-  utils::Entity operator[](int index) { return entities_[index]; }
-
-  // Returns the owned buffers at the given index.
-  int GetNumEntities() const { return entities_.size(); }
-
-  // Hides all managed entities.
-  void SetLayerMask(std::uint8_t mask);
-
-  // Sets the priority of all managed entities.
-  void SetPriority(std::uint8_t priority);
-
-  // Disables the renderables from casting shadows.
-  void SetCastShadows(bool cast_shadows);
-
-  // Disables the renderables from receiving shadows.
-  void SetReceiveShadows(bool receive_shadows);
-
-  // If true, forces all entities to be rendered as lines.
-  void SetWireframe(bool wireframe);
-
-  // Adds all managed entities to the given filament Scene.
-  void AddToScene(filament::Scene* scene);
-
-  // Removes all managed entities from the given filament Scene.
-  void RemoveFromScene(filament::Scene* scene);
-
-  // Sets the material instance for all managed entities.
-  void SetMaterialInstance(filament::MaterialInstance* material_instance);
-
-  // Returns the filament Engine managing the entities in this collection.
-  filament::Engine* GetEngine() { return engine_; }
-
- private:
-  utils::Entity CreateEntity(const FilamentBuffers& buffers);
-  void UpdateEntity(utils::Entity entity, const FilamentBuffers& buffers);
-  void UpdateBuffers(int index, FilamentBuffers buffers, bool owned);
-
-  // Tracks whether of not the filament buffers should be destroyed by this
-  // class.
-  struct OwnedBuffers {
-    bool owned = false;
-    FilamentBuffers buffers;
-  };
-
-  filament::Engine* engine_ = nullptr;
-  filament::Scene* assigned_scene_ = nullptr;
-  filament::MaterialInstance* material_instance_ = nullptr;
-  std::vector<utils::Entity> entities_;
-  std::vector<OwnedBuffers> owned_buffers_;
-  std::uint8_t priority_ = kDefaultPriority;
-  std::uint8_t layer_mask_ = kDefaultLayerMask;
-  bool wireframe_ = false;
-  bool cast_shadows_ = true;
-  bool receive_shadows_ = true;
-};
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_

src/experimental/filament/filament/scene_view.cc

@@ -14,15 +14,14 @@
 
 #include "experimental/filament/filament/scene_view.h"
 
+#include <algorithm>
 #include <array>
 #include <cstddef>
+#include <cstdint>
 #include <memory>
-#include <optional>
 #include <string_view>
-#include <utility>
 
 #include <filament/ColorGrading.h>
-#include <filament/IndirectLight.h>
 #include <filament/LightManager.h>
 #include <filament/Material.h>
 #include <filament/Options.h>
@@ -36,40 +35,27 @@
 #include <math/mat4.h>
 #include <math/mathfwd.h>
 #include <math/scalar.h>
+#include <math/TVecHelpers.h>
 #include <math/vec3.h>
 #include <math/vec4.h>
-#include <math/TVecHelpers.h>
 #include <utils/EntityManager.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/filament_util.h"
 #include "experimental/filament/filament/color_grading_options.h"
-#include "experimental/filament/filament/drawable.h"
-#include "experimental/filament/filament/gui_view.h"
 #include "experimental/filament/filament/light.h"
-#include "experimental/filament/filament/math_util.h"
-#include "experimental/filament/filament/model_objects.h"
-#include "experimental/filament/filament/model_util.h"
-#include "experimental/filament/filament/object_manager.h"
-#include "experimental/filament/filament/render_target_util.h"
+#include "experimental/filament/filament/render_target.h"
+#include "experimental/filament/filament/renderable.h"
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
 using filament::math::float3;
 using filament::math::float4;
-using filament::math::mat3;
 using filament::math::mat4;
 
-static constexpr int kNormalIndex =
-    static_cast<int>(SceneView::DrawMode::kNormal);
-static constexpr int kDepthIndex =
-    static_cast<int>(SceneView::DrawMode::kDepth);
-static constexpr int kSegmentIndex =
-    static_cast<int>(SceneView::DrawMode::kSegmentation);
-
-static filament::Viewport ReadViewport(mjrRect rect) {
-  return filament::Viewport(rect.left, rect.bottom, rect.width, rect.height);
-}
-
-filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) {
+static filament::ColorGrading::Builder ToBuilder(
+    const ColorGradingOptions& opts) {
   return filament::ColorGrading::Builder()
       .format(opts.format)
       .dimensions(opts.dimension)
@@ -88,6 +74,27 @@ filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) {
       .curves(opts.shadow_gamma, opts.mid_point, opts.highlight_scale);
 }
 
+static void SetupCamera(const mjrCamera& cam,
+                        const filament::Viewport& viewport,
+                        filament::Camera* camera) {
+  const filament::Camera::Projection type =
+      cam.orthographic ? filament::Camera::Projection::ORTHO
+                       : filament::Camera::Projection::PERSPECTIVE;
+  const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]);
+  const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]);
+  const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]);
+  const float3 cam_at = cam_pos + cam_fwd;
+  const float aspect_ratio = (float)viewport.width / (float)viewport.height;
+  const float halfwidth =
+      cam.frustum_width
+          ? cam.frustum_width
+          : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom);
+  camera->lookAt(cam_pos, cam_at, cam_up);
+  camera->setProjection(type, cam.frustum_center - halfwidth,
+                         cam.frustum_center + halfwidth, cam.frustum_bottom,
+                         cam.frustum_top, cam.frustum_near, cam.frustum_far);
+}
+
 // Sets up the `reflection_camera`'s projection matrix so that it is a
 // reflection of the `src_camera` across the plane defined by the
 // `surface_xform`. The generated projection is an oblique projection so that
@@ -114,11 +121,8 @@ static void SetupReflectionCamera(const mat4& surface_xform,
   reflection_camera->setCustomProjection(oblique, near, far);
 }
 
-SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model)
-    : object_mgr_(object_mgr) {
-  filament::Engine* engine = object_mgr_->GetEngine();
-  model_objects_ = std::make_unique<ModelObjects>(model, engine);
-
+SceneView::SceneView(filament::Engine* engine, const mjrSceneParams& params)
+    : engine_(engine) {
   scene_ = engine->createScene();
   camera_ = engine->createCamera(utils::EntityManager::get().create());
   reflect_camera_ = engine->createCamera(utils::EntityManager::get().create());
@@ -127,6 +131,7 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model)
     view = engine->createView();
     view->setScene(scene_);
     view->setCamera(camera_);
+    view->setVisibleLayers(0xff, params.layer_mask);
   }
 
   reflect_view_ = engine->createView();
@@ -134,18 +139,279 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model)
   reflect_view_->setCamera(reflect_camera_);
   reflect_view_->setShadowingEnabled(false);
   reflect_view_->setPostProcessingEnabled(false);
+  reflect_view_->setFrontFaceWindingInverted(true);
+  reflect_view_->setVisibleLayers(0xff, params.reflection_layer_mask);
 
-  // Configure options for the normal view.
-  auto& cg = color_grading_options_;
-  cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure);
-  cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast);
-  cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance);
-  cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation);
-  cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature);
-  cg.tint = ReadElement(model, "filament.out.tint", cg.tint);
+  // Disable post processing for the depth and segmentation views to preserve
+  // the values.
+  views_[mjDRAW_MODE_DEPTH]->setPostProcessingEnabled(false);
+  views_[mjDRAW_MODE_SEGMENTATION]->setPostProcessingEnabled(false);
+
+  // Rotate the fog to align with mujoco's +Z up space.
+  auto fog = views_[mjDRAW_MODE_COLOR]->getFogEntity();
+  auto& tm = engine->getTransformManager();
+  tm.create(fog);
+  tm.setTransform(tm.getInstance(fog),
+                  mat4::rotation(filament::math::f::PI / 2, float3{-1, 0, 0}));
+
+  if (!params.enable_post_processing) {
+    DisablePostProcessing();
+  }
+  if (!params.enable_reflections) {
+    DisableReflections();
+  }
+  if (!params.enable_shadows) {
+    DisableShadows();
+  }
+}
+
+SceneView::~SceneView() {
+  if (skybox_) {
+    scene_->setSkybox(nullptr);
+    engine_->destroy(skybox_);
+  }
+  for (auto& light : lights_) {
+    light->RemoveFromScene(scene_);
+  }
+  for (auto& renderable : renderables_) {
+    renderable->RemoveFromScene(scene_);
+  }
+  lights_.clear();
+  renderables_.clear();
+  reflect_targets_.clear();
+  engine_->destroyCameraComponent(reflect_camera_->getEntity());
+  engine_->destroy(reflect_view_);
+  engine_->destroyCameraComponent(camera_->getEntity());
+  if (color_grading_) {
+    engine_->destroy(color_grading_);
+  }
+  engine_->destroy(scene_);
+  for (auto& view : views_) {
+    engine_->destroy(view);
+  }
+}
+
+void SceneView::AddToScene(Light* light) {
+  if (lights_.insert(light).second) {
+    light->AddToScene(scene_);
+  }
+}
+
+void SceneView::RemoveFromScene(Light* light) {
+  if (lights_.erase(light)) {
+    light->RemoveFromScene(scene_);
+  }
+}
+
+void SceneView::AddToScene(Renderable* renderable) {
+  if (renderables_.insert(renderable).second) {
+    renderable->AddToScene(scene_);
+    if (renderable->GetMaterial().reflective) {
+      AddReflectiveRenderable(renderable);
+    }
+  }
+}
+
+void SceneView::RemoveFromScene(Renderable* renderable) {
+  if (renderables_.erase(renderable)) {
+    auto it = std::find(reflectives_.begin(), reflectives_.end(), renderable);
+    if (it != reflectives_.end()) {
+      reflectives_.erase(it);
+    }
+    renderable->RemoveFromScene(scene_);
+  }
+}
+
+void SceneView::SetSkybox(const Texture* skybox_texture) {
+  if (skybox_) {
+    scene_->setSkybox(nullptr);
+    GetEngine()->destroy(skybox_);
+    skybox_ = nullptr;
+  }
+  if (skybox_texture) {
+    filament::Skybox::Builder builder;
+    builder.environment(skybox_texture->GetFilamentTexture());
+    skybox_ = builder.build(*GetEngine());
+    scene_->setSkybox(skybox_);
+  }
+}
+
+void SceneView::Render(filament::Renderer* renderer, const mjrRenderRequest& request) {
+  if (request.scene != this) {
+    mju_error("Invalid scene for SceneView::Render.");
+  }
+
+  filament::Viewport viewport(request.viewport.left, request.viewport.bottom,
+                              request.viewport.width, request.viewport.height);
+  for (auto& view : views_) {
+    view->setViewport(viewport);
+  }
+  reflect_view_->setViewport(viewport);
+
+  SetupCamera(request.camera, viewport, camera_);
+
+  for (auto& iter : renderables_) {
+    iter->SetDrawMode(request.draw_mode);
+  }
+
+  filament::View* view = views_[static_cast<int>(request.draw_mode)];
+  filament::MultiSampleAntiAliasingOptions options =
+      view->getMultiSampleAntiAliasingOptions();
+
+
+  RenderTarget* render_target = RenderTarget::downcast(request.target);
+  if (render_target) {
+    // We need to disable msaa in order to render to texture.
+    view->setMultiSampleAntiAliasingOptions({.enabled = false});
+  }
+
+  // Render reflection passes.
+  if (request.draw_mode == mjDRAW_MODE_COLOR && reflections_enabled_) {
+    for (size_t i = 0; i < reflectives_.size(); ++i) {
+      Renderable* renderable = reflectives_[i];
+
+      // We assume the 0th entity is the reflective entity.
+      mat4 transform(renderable->GetTransform());
+      SetupReflectionCamera(transform, camera_, reflect_camera_);
+
+      // Hide reflective surface from its own reflection pass.
+      std::uint8_t previous_layer_mask = renderable->SetLayerMask(0x00);
+
+      // Render the reflection to its render target.
+      reflect_view_->setRenderTarget(
+          reflect_targets_[i]->GetFilamentRenderTarget());
+      renderer->render(reflect_view_);
+
+      // Unhide the reflective surface.
+      renderable->SetLayerMask(previous_layer_mask);
+    }
+  }
+
+  view->setRenderTarget(render_target ? render_target->GetFilamentRenderTarget()
+                                      : nullptr);
+  renderer->render(view);
+  view->setRenderTarget(nullptr);
+
+  if (request.target) {
+    view->setMultiSampleAntiAliasingOptions(options);
+  }
+}
+
+void SceneView::AddReflectiveRenderable(Renderable* renderable) {
+  const int index = reflectives_.size();
+  reflectives_.push_back(renderable);
+
+  // Ensure we have the same number of render targets as we do reflective
+  // renderables.
+  while (reflect_targets_.size() < reflectives_.size()) {
+    mjrRenderTargetConfig config;
+    mjr_defaultRenderTargetConfig(&config);
+
+    config.color_format = mjPIXEL_FORMAT_RGBA8;
+    config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
+    reflect_targets_.push_back(std::make_unique<RenderTarget>(engine_, config));
+  }
+
+  // Prepare a render target for the reflective renderable.
+  auto viewport = reflect_view_->getViewport();
+  auto& target = reflect_targets_[index];
+  target->Prepare(viewport.width, viewport.height);
+
+  if (reflections_enabled_) {
+    mjrMaterial material = renderable->GetMaterial();
+    material.reflection_texture = target->GetColorTexture();
+    renderable->UpdateMaterial(material);
+  }
+}
+
+void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) {
+  auto tone_mapper = CreateToneMapper(opts.tone_mapper);
+  auto color_grading = ToBuilder(color_grading_options_)
+                           .toneMapper(tone_mapper.get())
+                           .build(*GetEngine());
+  views_[mjDRAW_MODE_COLOR]->setColorGrading(color_grading);
+  if (color_grading_) {
+    GetEngine()->destroy(color_grading_);
+  }
+  color_grading_ = color_grading;
+  color_grading_options_ = opts;
+}
+
+void SceneView::EnableShadows() {
+  views_[mjDRAW_MODE_COLOR]->setShadowingEnabled(true);
+}
+
+void SceneView::DisableShadows() {
+  views_[mjDRAW_MODE_COLOR]->setShadowingEnabled(false);
+}
+
+void SceneView::EnableReflections() {
+  reflections_enabled_ = true;
+
+  for (int i = 0; i < reflectives_.size(); ++i) {
+    Renderable* renderable = reflectives_[i];
+    mjrMaterial material = renderable->GetMaterial();
+    material.reflection_texture = reflect_targets_[i]->GetColorTexture();
+    renderable->UpdateMaterial(material);
+  }
+}
+
+void SceneView::DisableReflections() {
+  reflections_enabled_ = false;
+  for (Renderable* renderable : reflectives_) {
+    mjrMaterial material = renderable->GetMaterial();
+    material.reflection_texture = nullptr;
+    renderable->UpdateMaterial(material);
+  }
+}
+
+void SceneView::EnablePostProcessing() {
+  views_[mjDRAW_MODE_COLOR]->setPostProcessingEnabled(true);
+}
+
+void SceneView::DisablePostProcessing() {
+  views_[mjDRAW_MODE_COLOR]->setPostProcessingEnabled(false);
+}
+
+filament::View* SceneView::GetDefaultRenderView() {
+  return views_[mjDRAW_MODE_COLOR];
+}
+
+ColorGradingOptions SceneView::GetColorGradingOptions() const {
+  return color_grading_options_;
+}
+
+void SceneView::Configure(const mjModel* model) {
+  filament::View* view = views_[mjDRAW_MODE_COLOR];
+
+  auto cg = color_grading_options_;
+  cg.exposure = ReadElement(model, "filament.cg.exposure", cg.exposure);
+  cg.contrast = ReadElement(model, "filament.cg.contrast", cg.contrast);
+  cg.vibrance = ReadElement(model, "filament.cg.vibrance", cg.vibrance);
+  cg.saturation = ReadElement(model, "filament.cg.saturation", cg.saturation);
+  cg.temperature =
+      ReadElement(model, "filament.cg.temperature", cg.temperature);
+  cg.tint = ReadElement(model, "filament.cg.tint", cg.tint);
+  cg.gamut_mapping =
+      ReadElement(model, "filament.cg.gamut_mapping", cg.gamut_mapping);
+  cg.luminance_scaling =
+      ReadElement(model, "filament.cg.luminance_scaling", cg.luminance_scaling);
+  cg.slope = ReadElement(model, "filament.cg.slope", cg.slope);
+  cg.offset = ReadElement(model, "filament.cg.offset", cg.offset);
+  cg.power = ReadElement(model, "filament.cg.power", cg.power);
+  cg.shadow_gamma =
+      ReadElement(model, "filament.cg.shadow_gamma", cg.shadow_gamma);
+  cg.mid_point = ReadElement(model, "filament.cg.mid_point", cg.mid_point);
+  cg.highlight_scale =
+      ReadElement(model, "filament.cg.highlight_scale", cg.highlight_scale);
+  cg.shadows = ReadElement(model, "filament.cg.shadows", cg.shadows);
+  cg.midtones = ReadElement(model, "filament.cg.midtones", cg.midtones);
+  cg.highlights = ReadElement(model, "filament.cg.highlights", cg.highlights);
+  cg.tonal_ranges =
+      ReadElement(model, "filament.cg.tonal_ranges", cg.tonal_ranges);
 
   auto tone_mapping =
-      ReadElement<std::string_view>(model, "filament.out.tone_mapping");
+      ReadElement<std::string_view>(model, "filament.cg.tone_mapping");
   if (tone_mapping == "aces") {
     cg.tone_mapper = ToneMapperType::kACES;
   } else if (tone_mapping == "aces_legacy") {
@@ -159,44 +425,29 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model)
   }
   SetColorGradingOptions(cg);
 
-  auto ao = views_[kNormalIndex]->getAmbientOcclusionOptions();
+  auto ao = view->getAmbientOcclusionOptions();
   ao.enabled = ReadElement(model, "filament.ao.enabled", true);
   ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false);
   ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled);
-  ao.quality = filament::QualityLevel::ULTRA;
-  ao.lowPassFilter = filament::QualityLevel::ULTRA;
-  ao.upsampling = filament::QualityLevel::ULTRA;
-  ao.bilateralThreshold = 0.5f;
-  views_[kNormalIndex]->setAmbientOcclusionOptions(ao);
+  ao.quality =
+      ReadElement(model, "filament.ao.quality", filament::QualityLevel::ULTRA);
+  ao.lowPassFilter = ReadElement(model, "filament.ao.low_pass_filter",
+                                 filament::QualityLevel::ULTRA);
+  ao.upsampling = ReadElement(model, "filament.ao.upsampling",
+                              filament::QualityLevel::ULTRA);
+  ao.bilateralThreshold =
+      ReadElement(model, "filament.ao.bilateral_threshold", 0.5f);
+  view->setAmbientOcclusionOptions(ao);
 
-  auto msaa = views_[kNormalIndex]->getMultiSampleAntiAliasingOptions();
+  auto msaa = view->getMultiSampleAntiAliasingOptions();
   msaa.enabled = ReadElement(model, "filament.msaa.enabled", true);
-  views_[kNormalIndex]->setMultiSampleAntiAliasingOptions(msaa);
+  view->setMultiSampleAntiAliasingOptions(msaa);
 
-  default_shadow_map_size_ = ReadElement(
-      model, "filament.shadows.map_size", default_shadow_map_size_);
-  default_vsm_blur_width_ = ReadElement(
-      model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_);
-
-  auto shadow_type = views_[kNormalIndex]->getShadowType();
+  auto shadow_type = view->getShadowType();
   shadow_type = ReadElement(model, "filament.shadows.type", shadow_type);
-  views_[kNormalIndex]->setShadowType(shadow_type);
+  view->setShadowType(shadow_type);
 
-  // Disable post processing for the depth and segmentation views to preserve
-  // the values.
-  views_[kDepthIndex]->setPostProcessingEnabled(false);
-  views_[kSegmentIndex]->setPostProcessingEnabled(false);
-
-  // Rotate the fog to align with mujoco's +Z up space.
-  auto fog = views_[kNormalIndex]->getFogEntity();
-  auto& tm = engine->getTransformManager();
-  tm.create(fog);
-  auto rotation_axis = ReadElement(
-      model, "filament.fog.rotation_axis", float3{-1, 0, 0});
-  tm.setTransform(tm.getInstance(fog),
-                  mat4::rotation(filament::math::f::PI / 2, rotation_axis));
-
-  auto fog_opts = views_[kNormalIndex]->getFogOptions();
+  auto fog_opts = view->getFogOptions();
   fog_opts.enabled =
       ReadElement(model, "filament.fog.enabled", fog_opts.enabled);
   fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color);
@@ -215,339 +466,18 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model)
       model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart);
   fog_opts.inScatteringSize = ReadElement(
       model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize);
-  views_[kNormalIndex]->setFogOptions(fog_opts);
+  view->setFogOptions(fog_opts);
 
-  fallback_head_light_intensity_ =
-      ReadElement(model, "filament.fallback.head_light_intensity",
-                  fallback_head_light_intensity_);
-  fallback_scene_light_intensity_ =
-      ReadElement(model, "filament.fallback.scene_light_intensity",
-                  fallback_scene_light_intensity_);
-  fallback_environment_light_intensity_ =
-      ReadElement(model, "filament.fallback.environment_light_intensity",
-                  fallback_environment_light_intensity_);
-
-  // Create an empty/black indirect light to ensure that the skybox is oriented
-  // to respect mujoco's Z-up convention.
-  scene_->setIndirectLight(model_objects_->CreateIndirectLight(-1, 100000));
-
-  PrepareLights();
+  auto bloom = view->getBloomOptions();
+  bloom.enabled = ReadElement(model, "filament.bloom.enabled", bloom.enabled);
+  bloom.strength =
+      ReadElement(model, "filament.bloom.strength", bloom.strength);
+  bloom.dirtStrength =
+      ReadElement(model, "filament.bloom.dirt_strength", bloom.dirtStrength);
+  bloom.quality = ReadElement(model, "filament.bloom.quality", bloom.quality);
+  bloom.resolution =
+      ReadElement(model, "filament.bloom.resolution", bloom.resolution);
+  bloom.levels = ReadElement(model, "filament.bloom.levels", bloom.levels);
+  view->setBloomOptions(bloom);
 }
-
-SceneView::~SceneView() {
-  lights_.clear();
-  drawables_.clear();
-  reflect_targets_.clear();
-
-  filament::Engine* engine = object_mgr_->GetEngine();
-  engine->destroyCameraComponent(reflect_camera_->getEntity());
-  engine->destroy(reflect_view_);
-
-  engine->destroyCameraComponent(camera_->getEntity());
-  engine->destroy(views_[kNormalIndex]->getColorGrading());
-  for (auto& view : views_) {
-    engine->destroy(view);
-  }
-  engine->destroy(scene_);
-}
-
-void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode,
-                       filament::RenderTarget* target) {
-  filament::View* view = PrepareRenderView(draw_mode);
-  filament::MultiSampleAntiAliasingOptions options =
-      view->getMultiSampleAntiAliasingOptions();
-
-  if (target) {
-    // We need to disable msaa in order to render to texture.
-    view->setMultiSampleAntiAliasingOptions({.enabled = false});
-  }
-
-  // Render reflection passes.
-  if (draw_mode == DrawMode::kNormal) {
-    for (size_t i = 0; i < reflectives_.size(); ++i) {
-      Drawable* drawable = reflectives_[i];
-
-      SetupReflectionCamera(drawable->GetTransform(), camera_, reflect_camera_);
-
-      // Hide reflective surface from its own reflection pass.
-      drawable->SetLayerMask(0x00);
-
-      // Render the reflection to its render target.
-      reflect_view_->setRenderTarget(reflect_targets_[i]->GetRenderTarget());
-      renderer->render(reflect_view_);
-
-      // Unhide the reflective surface.
-      drawable->SetLayerMask(0x01);
-    }
-  }
-
-  view->setRenderTarget(target);
-  renderer->render(view);
-  view->setRenderTarget(nullptr);
-
-  if (target) {
-    view->setMultiSampleAntiAliasingOptions(options);
-  }
-}
-
-filament::View* SceneView::PrepareRenderView(DrawMode mode) {
-  for (auto& iter : drawables_) {
-    iter->SetDrawMode(mode);
-  }
-  return views_[static_cast<int>(mode)];
-}
-
-void SceneView::SetViewport(mjrRect viewport) {
-  auto filament_viewport = ReadViewport(viewport);
-  aspect_ratio_ = (float)viewport.width / (float)viewport.height;
-  for (auto& view : views_) {
-    view->setViewport(filament_viewport);
-  }
-  reflect_view_->setViewport(filament_viewport);
-}
-
-void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) {
-  filament::Engine* engine = object_mgr_->GetEngine();
-
-  auto tone_mapper = CreateToneMapper(opts.tone_mapper);
-  auto color_grading = ToBuilder(color_grading_options_)
-                           .toneMapper(tone_mapper.get())
-                           .build(*engine);
-  views_[kNormalIndex]->setColorGrading(color_grading);
-  engine->destroy(color_grading_);
-  color_grading_ = color_grading;
-  color_grading_options_ = opts;
-}
-
-void SceneView::SetEnvironmentLight(std::string_view filename,
-                                    float intensity) {
-  scene_->setIndirectLight(nullptr);
-  object_mgr_->LoadFallbackIndirectLight(filename, intensity);
-  scene_->setIndirectLight(object_mgr_->GetFallbackIndirectLight());
-}
-
-void SceneView::SetFallbackEnvironmentLight(float intensity) {
-  auto* ibl = object_mgr_->GetFallbackIndirectLight();
-  if (ibl) {
-    ibl->setIntensity(intensity);
-    scene_->setIndirectLight(ibl);
-  }
-}
-
-void SceneView::UpdateCamera(const mjvGLCamera* cameras) {
-  const mjvGLCamera cam = mjv_averageCamera(cameras, cameras + 1);
-  const filament::Camera::Projection type =
-      cam.orthographic ? filament::Camera::Projection::ORTHO
-                       : filament::Camera::Projection::PERSPECTIVE;
-  float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]);
-  float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]);
-  float3 cam_up(cam.up[0], cam.up[1], cam.up[2]);
-  float3 cam_at = cam_pos + cam_fwd;
-  camera_->lookAt(cam_pos, cam_at, cam_up);
-  float halfwidth = cam.frustum_width ? cam.frustum_width
-            : 0.5f * aspect_ratio_ * (cam.frustum_top - cam.frustum_bottom);
-  camera_->setProjection(type, cam.frustum_center - halfwidth,
-                         cam.frustum_center + halfwidth, cam.frustum_bottom,
-                         cam.frustum_top, cam.frustum_near, cam.frustum_far);
-  clip_from_world_ = camera_->getProjectionMatrix() * camera_->getViewMatrix();
-}
-
-std::optional<float3> SceneView::ClipFromWorld(const float3& pos) const{
-  const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f);
-  if (clip_pos.w == 0.0f) {
-    return std::nullopt;
-  }
-  return clip_pos.xyz / clip_pos.w;
-}
-
-void SceneView::PrepareLights() {
-  filament::Engine* engine = object_mgr_->GetEngine();
-  const mjModel* model = model_objects_->GetModel();
-  filament::Skybox* skybox = model_objects_->CreateSkybox();
-  if (skybox) {
-    scene_->setSkybox(skybox);
-  }
-
-  float total_light_intensity = 0.0f;
-
-  for (int i = 0; i < model->nlight; ++i) {
-    total_light_intensity += model->light_intensity[i];
-
-    if (model->light_type[i] == mjLIGHT_IMAGE) {
-      auto* indirect_light = model_objects_->CreateIndirectLight(
-          model->light_texid[i], model->light_intensity[i]);
-      if (indirect_light) {
-        scene_->setIndirectLight(indirect_light);
-      }
-      // Add an nullptr as a placeholder so that our indices still match.
-      lights_.emplace_back(nullptr);
-    } else {
-      Light::Params params;
-      params.color = ReadFloat3(model->light_diffuse);
-      params.type = (mjtLightType)model->light_type[i];
-      params.castshadow = model->light_castshadow[i];
-      params.bulbradius = model->light_bulbradius[i];
-      params.range = model->light_range[i];
-      params.intensity = model->light_intensity[i];
-      params.shadow_map_size = default_shadow_map_size_;
-      params.vsm_blur_width = default_vsm_blur_width_;
-      if (params.type == mjLIGHT_SPOT) {
-        params.spot_cone_angle = model->light_cutoff[i];
-      }
-
-      auto light_obj = std::make_unique<Light>(engine, params);
-#ifndef __EMSCRIPTEN__
-      // TODO(b/458045799): Re-enable when lights work on glinux and chromebook.
-      light_obj->AddToScene(scene_);
-#endif
-      lights_.emplace_back(std::move(light_obj));
-    }
-  }
-
-  // Add a placeholder (black) headlight as our last light. Going forward, we'll
-  // assume lights_.back() is always the headlight.
-  {
-    Light::Params params;
-    params.color = float3(0, 0, 0);
-    params.headlight = true;
-    params.type = mjLIGHT_DIRECTIONAL;
-    params.castshadow = 0;
-    params.intensity = 0;
-    auto light_obj = std::make_unique<Light>(engine, params);
-#ifndef __EMSCRIPTEN__
-    // TODO(b/458045799): Re-enable when lights work on glinux and chromebook.
-    light_obj->AddToScene(scene_);
-#endif
-    lights_.emplace_back(std::move(light_obj));
-  }
-
-  // There are no "physical" lights in the scene which means we're likely
-  // dealing with a "classic renderer" scene. In this case, let's add a
-  // default environment light and set the light intensity ourselves.
-  if (total_light_intensity == 0.0f) {
-    SetFallbackEnvironmentLight(fallback_environment_light_intensity_);
-    const float intensity = fallback_scene_light_intensity_ / lights_.size();
-    for (auto& light : lights_) {
-      if (light) {
-        light->SetIntensity(
-            light->IsHeadlight() ? fallback_head_light_intensity_ : intensity);
-      }
-    }
-  }
-}
-
-void SceneView::UpdateScene(const mjvScene* scene) {
-  views_[kNormalIndex]->setShadowingEnabled(scene->flags[mjRND_SHADOW]);
-
-  mjtNum hpos[3], hfwd[3];
-  float headpos[3], gazedir[3];
-  mjv_cameraInModel(hpos, hfwd, nullptr, scene);
-  mju_n2f(headpos, hpos, 3);
-  mju_n2f(gazedir, hfwd, 3);
-  UpdateCamera(scene->camera);
-
-  // Remove all drawables from previous render and prepare new ones.
-  for (auto& iter : drawables_) {
-    iter->RemoveFromScene(scene_);
-  }
-  drawables_.clear();
-  reflectives_.clear();
-  for (int i = 0; i < scene->ngeom; ++i) {
-    const mjvGeom* geom = scene->geoms + i;
-
-    if (geom->label[0] != 0) {
-      if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) {
-        DrawTextAt(geom->label, pos->x, pos->y, pos->z);
-      }
-    }
-
-    auto drawable =
-        std::make_unique<Drawable>(object_mgr_, model_objects_.get(), *geom);
-    drawable->AddToScene(scene_);
-    drawable->Update(model_objects_->GetModel(), scene, *geom);
-    if (drawable->IsReflective()) {
-      AddReflectiveDrawable(drawable.get());
-    }
-    drawables_.push_back(std::move(drawable));
-  }
-
-  bool headlight_enabled = false;
-  for (int i = 0; i < scene->nlight; ++i) {
-    const mjvLight& scene_light = scene->lights[i];
-    if (scene_light.id < 0 && scene_light.headlight) {
-      // We position the headlight slightly behind the camera to avoid some
-      // odd clipping issues.
-      headlight_enabled = true;
-      headpos[0] -= gazedir[0] * 0.05f;
-      headpos[1] -= gazedir[1] * 0.05f;
-      headpos[2] -= gazedir[2] * 0.05f;
-
-      // The headlight is always the "back" light.
-      std::unique_ptr<Light>& light = lights_.back();
-      light->SetColor(ReadFloat3(scene_light.diffuse));
-      light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir));
-      continue;
-    } else if (scene_light.id < lights_.size() - 1) {
-      std::unique_ptr<Light>& light = lights_[scene_light.id];
-      if (light) {
-        light->SetColor(ReadFloat3(scene_light.diffuse));
-        light->SetTransform(ReadFloat3(scene_light.pos),
-                            ReadFloat3(scene_light.dir));
-      }
-    } else {
-      mju_error("Unexpected light id: %d", scene_light.id);
-    }
-  }
-
-  // Enable/disable the headlight based on whether or not it's in the scene.
-  if (headlight_enabled) {
-    lights_.back()->Enable();
-  } else {
-    lights_.back()->Disable();
-  }
-}
-
-void SceneView::AddReflectiveDrawable(Drawable* drawable) {
-  const int index = reflectives_.size();
-  reflectives_.push_back(drawable);
-
-  // Ensure we have the same number of render targets as we do reflective
-  // drawables.
-  filament::Engine* engine = object_mgr_->GetEngine();
-  while (reflect_targets_.size() < reflectives_.size()) {
-    reflect_targets_.push_back(std::make_unique<RenderTargetAndTextures>(
-        engine, kRenderTargetReflectionColor, kRenderTargetDepth));
-  }
-
-  // Prepare a render target for the reflective drawable.
-  auto viewport = reflect_view_->getViewport();
-  auto& target = reflect_targets_[index];
-  target->Prepare(viewport.width, viewport.height);
-  drawable->UpdateReflectionTexture(target->GetColorTexture());
-}
-
-void SceneView::UploadMesh(const mjModel* model, int id) {
-  model_objects_->UploadMesh(model, id);
-}
-
-void SceneView::UploadTexture(const mjModel* model, int id) {
-  model_objects_->UploadTexture(model, id);
-}
-
-void SceneView::UploadHeightField(const mjModel* model, int id) {
-  model_objects_->UploadHeightField(model, id);
-}
-
-filament::Engine* SceneView::GetEngine() const {
-  return object_mgr_->GetEngine();
-}
-
-filament::View* SceneView::GetDefaultRenderView() {
-  return views_[kNormalIndex];
-}
-
-ColorGradingOptions SceneView::GetColorGradingOptions() const {
-  return color_grading_options_;
-}
-
 }  // namespace mujoco

src/experimental/filament/filament/scene_view.h

@@ -17,120 +17,106 @@
 
 #include <array>
 #include <memory>
-#include <optional>
-#include <string_view>
+#include <unordered_set>
 #include <vector>
 
 #include <filament/Camera.h>
 #include <filament/ColorGrading.h>
 #include <filament/Engine.h>
-#include <filament/IndirectLight.h>
 #include <filament/Scene.h>
 #include <filament/View.h>
-#include <math/mat4.h>
-#include <math/vec3.h>
-#include <mujoco/mjrender.h>
-#include <mujoco/mjvisualize.h>
 #include <mujoco/mujoco.h>
 #include "experimental/filament/filament/color_grading_options.h"
-#include "experimental/filament/filament/drawable.h"
 #include "experimental/filament/filament/light.h"
-#include "experimental/filament/filament/material.h"
-#include "experimental/filament/filament/model_objects.h"
-#include "experimental/filament/filament/object_manager.h"
-#include "experimental/filament/filament/render_target_util.h"
+#include "experimental/filament/filament/renderable.h"
+#include "experimental/filament/filament/render_target.h"
+#include "experimental/filament/filament/texture.h"
+#include "experimental/filament/render_context_filament.h"
 
 namespace mujoco {
 
-// Creates and owns filament Scene and View classes given a mjvScene.
+// Creates and owns the filament Scene and View (and Camera) classes.
 //
-// The filament Scene is populated with the objects (e.g. lights, geoms,
-// cameras, etc.) defined by the mjvScene. Multiple Views are created to allow
-// different rendering modes (e.g. normal, depth, segmentation, etc.)
-class SceneView {
+// The filament Scene is populated with the objects (e.g. lights, renderables,
+// skybox, etc.). It manages multiple views to support a variety of draw modes
+// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces.
+class SceneView : public mjrScene {
  public:
-  SceneView(ObjectManager* object_mgr, const mjModel* model);
+  SceneView(filament::Engine* engine, const mjrSceneParams& params);
   ~SceneView();
 
-  // Updates all views to render into the given viewport.
-  void SetViewport(mjrRect viewport);
-
-  // Updates the color grading options for the main render view.
-  void SetColorGradingOptions(const ColorGradingOptions& opts);
-
-  // Updates the environment light using the KTX image at the given path.
-  void SetEnvironmentLight(std::string_view filename, float intensity);
-
-  // Updates the environment light to the fallback light
-  void SetFallbackEnvironmentLight(float intensity);
-
-  // Updates the Entities in the filament Scene to match the current mjvScene
-  // state.
-  void UpdateScene(const mjvScene* scene);
-
-  using DrawMode = Material::DrawMode;
-
-  void Render(filament::Renderer* renderer, DrawMode draw_mode,
-              filament::RenderTarget* target = nullptr);
-
-  void UploadMesh(const mjModel* model, int id);
-  void UploadTexture(const mjModel* model, int id);
-  void UploadHeightField(const mjModel* model, int id);
-
-  // Accessors.
-  filament::Engine* GetEngine() const;
-  filament::View* GetDefaultRenderView();
-  ColorGradingOptions GetColorGradingOptions() const;
-
   SceneView(const SceneView&) = delete;
   SceneView& operator=(const SceneView&) = delete;
 
+  // Adds/removes entities from the scene.
+  void AddToScene(Light* light);
+  void RemoveFromScene(Light* light);
+  void AddToScene(Renderable* renderable);
+  void RemoveFromScene(Renderable* renderable);
+  void SetSkybox(const Texture* skybox_texture);
+
+  void Render(filament::Renderer* renderer, const mjrRenderRequest& request);
+
+  // Returns the filament Engine managing the scene.
+  filament::Engine* GetEngine() const { return engine_; }
+
+  // Enables/disables shadows for the default render view.
+  void EnableShadows();
+  void DisableShadows();
+
+  // Enables/disables reflections for the default render view.
+  void EnableReflections();
+  void DisableReflections();
+
+  // Enables/disables post processing for the default render view.
+  void EnablePostProcessing();
+  void DisablePostProcessing();
+
+  // Returns the underlying filament View that is used for normal rendering.
+  // Callers can update rendering settings (e.g. post processing) directly.
+  filament::View* GetDefaultRenderView();
+
+  // Helpers for managing the color grading options for the default render view.
+  ColorGradingOptions GetColorGradingOptions() const;
+  void SetColorGradingOptions(const ColorGradingOptions& opts);
+
+  // Reads filament-specific settings from the mjModel and configures the
+  // scene view accordingly.
+  void Configure(const mjModel* model);
+
+  static SceneView* downcast(mjrScene* scene) {
+    return static_cast<SceneView*>(scene);
+  }
+  static const SceneView* downcast(const mjrScene* scene) {
+    return static_cast<const SceneView*>(scene);
+  }
+
  private:
-  // Prepares and returns the filament View for the given draw mode.
-  filament::View* PrepareRenderView(DrawMode mode);
+  // Marks a renderable as reflective. Reflective renderables have to be
+  // rendered in their own passes to create the reflective texture.
+  void AddReflectiveRenderable(Renderable* renderable);
 
-  void UpdateCamera(const mjvGLCamera* cameras);
-
-  void PrepareLights();
-
-  // Registers the given drawable as a reflective surface.
-  void AddReflectiveDrawable(Drawable* drawable);
-
-  // Converts a point in world space to clip space, eg. in the range [-1,-1, 0]
-  // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera.
-  std::optional<filament::math::float3> ClipFromWorld(
-      const filament::math::float3& pos) const;
-
-  ObjectManager* object_mgr_ = nullptr;
+  filament::Engine* engine_ = nullptr;
   filament::Scene* scene_ = nullptr;
   filament::Camera* camera_ = nullptr;
   filament::ColorGrading* color_grading_ = nullptr;
-  std::vector<std::unique_ptr<Light>> lights_;
-  std::vector<std::unique_ptr<Drawable>> drawables_;
-  std::unique_ptr<ModelObjects> model_objects_;
-  std::array<filament::View*, DrawMode::kNumDrawModes> views_;
-  filament::math::mat4 clip_from_world_;
   ColorGradingOptions color_grading_options_;
-  DrawMode active_mode_ = DrawMode::kNumDrawModes;
-  float aspect_ratio_ = 1.0f;
-  int default_shadow_map_size_ = 2048;
-  float default_vsm_blur_width_ = 0.0f;
-  float fallback_head_light_intensity_ = 0.f;
-  float fallback_scene_light_intensity_ = 80'000.f;
-  float fallback_environment_light_intensity_ = 5'000.f;
+  std::array<filament::View*, mjNUM_DRAW_MODES> views_;
+
+  // Scene objects.
+  std::unordered_set<Light*> lights_;
+  std::unordered_set<Renderable*> renderables_;
+  filament::Skybox* skybox_ = nullptr;
 
   // Custom view and camera for reflective surfaces.
   filament::View* reflect_view_ = nullptr;
   filament::Camera* reflect_camera_ = nullptr;
 
-  // The list of drawables that are reflective.
-  std::vector<Drawable*> reflectives_;
-
-  // Each reflective drawable has its own render target which is used to render
-  // the reflected image.
-  std::vector<std::unique_ptr<RenderTargetAndTextures>> reflect_targets_;
+  // The list of reflective renderables and their corresponding render targets.
+  bool reflections_enabled_ = true;
+  std::vector<Renderable*> reflectives_;
+  std::vector<std::unique_ptr<RenderTarget>> reflect_targets_;
 };
-
 }  // namespace mujoco
 
 #endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_

src/experimental/filament/filament/texture.cc

@@ -0,0 +1,243 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/filament/filament/texture.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <utility>
+
+#include <filament/Engine.h>
+#include <filament/Texture.h>
+#include <image/Ktx1Bundle.h>
+#include <ktxreader/Ktx1Reader.h>
+#include <mujoco/mujoco.h>
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+static constexpr int kNumFacesPerCube = 6;
+
+static bool IsCompressed(const mjrTextureConfig& config) {
+  return config.format == mjPIXEL_FORMAT_KTX;
+}
+
+static bool IsCubeMap(const mjrTextureConfig& config) {
+  return config.sampler_type == mjTEXTURE_CUBE ||
+         config.sampler_type == mjTEXTURE_SKYBOX;
+}
+
+static int GetFaceHeight(const mjrTextureConfig& config) {
+  int face_height = config.height;
+  if (config.width != config.height) {
+    if (config.width * kNumFacesPerCube != config.height) {
+      mju_error("Cube maps must contain 6 square images.");
+    }
+    face_height = config.height / kNumFacesPerCube;
+  }
+  if (config.width != face_height) {
+    mju_error("Cube map faces must be square.");
+  }
+  return face_height;
+}
+
+static int GetNumChannels(const mjrTextureConfig& config) {
+  switch (config.format) {
+    case mjPIXEL_FORMAT_R8:
+      return 1;
+    case mjPIXEL_FORMAT_RGB8:
+      return 3;
+    case mjPIXEL_FORMAT_RGBA8:
+      return 4;
+    default:
+      mju_error("Unsupported format: %d", (int)config.format);
+      return 0;
+  }
+}
+
+static filament::Texture::Format GetTextureFormat(const mjrTextureConfig& config) {
+  switch (config.format) {
+    case mjPIXEL_FORMAT_R8:
+      return filament::Texture::Format::R;
+    case mjPIXEL_FORMAT_RGB8:
+      return filament::Texture::Format::RGB;
+    case mjPIXEL_FORMAT_RGBA8:
+      return filament::Texture::Format::RGBA;
+    default:
+      mju_error("Unsupported format: %d", (int)config.format);
+      return filament::Texture::Format::UNUSED;
+    }
+}
+
+static filament::Texture::InternalFormat GetTextureInternalFormat(
+    const mjrTextureConfig& config) {
+  if (config.color_space == mjCOLORSPACE_SRGB) {
+    switch (config.format) {
+      case mjPIXEL_FORMAT_RGB8:
+        return filament::Texture::InternalFormat::SRGB8;
+      case mjPIXEL_FORMAT_RGBA8:
+        return filament::Texture::InternalFormat::SRGB8_A8;
+      default:
+        mju_error("Unsupported format: %d", (int)config.format);
+        return filament::Texture::InternalFormat::UNUSED;
+    }
+  } else {
+    switch (config.format) {
+      case mjPIXEL_FORMAT_R8:
+        return filament::Texture::InternalFormat::R8;
+      case mjPIXEL_FORMAT_RGB8:
+        return filament::Texture::InternalFormat::RGB8;
+      case mjPIXEL_FORMAT_RGBA8:
+        return filament::Texture::InternalFormat::RGBA8;
+      case mjPIXEL_FORMAT_R32F:
+        return filament::Texture::InternalFormat::R32F;
+      case mjPIXEL_FORMAT_DEPTH32F:
+        return filament::Texture::InternalFormat::DEPTH32F;
+      default:
+        mju_error("Unsupported format: %d", (int)config.format);
+        return filament::Texture::InternalFormat::UNUSED;
+    }
+  }
+}
+
+Texture::Texture(filament::Engine* engine, const mjrTextureConfig& config,
+                 InternalFlags flags)
+    : engine_(engine), config_(config) {
+  if (IsCompressed(config_)) {
+    // We defer creation of compressed textures until Upload() is called. In
+    // the meantime, we don't really know anything about the texture (e.g.
+    // width, height, etc.).
+    return;
+  }
+
+  filament::Texture::Builder builder;
+  builder.width(config_.width);
+  builder.height(config_.height);
+  builder.format(GetTextureInternalFormat(config_));
+
+  if (IsCubeMap(config_)) {
+    if (config_.format != mjPIXEL_FORMAT_RGB8) {
+      mju_error("Only support RGB cubemaps.");
+      return;
+    }
+    builder.height(GetFaceHeight(config_));
+    builder.sampler(filament::Texture::Sampler::SAMPLER_CUBEMAP);
+  } else {
+    builder.sampler(filament::Texture::Sampler::SAMPLER_2D);
+  }
+
+  filament::Texture::Usage usage = filament::Texture::Usage::DEFAULT;
+  if (flags.color_attachment) {
+    usage |= filament::Texture::Usage::COLOR_ATTACHMENT;
+    usage |= filament::Texture::Usage::BLIT_SRC;
+  } else if (flags.depth_attachment) {
+    usage |= filament::Texture::Usage::DEPTH_ATTACHMENT;
+    usage |= filament::Texture::Usage::BLIT_SRC;
+  } else if (config_.color_space != mjCOLORSPACE_SRGB) {
+    usage |= filament::Texture::Usage::GEN_MIPMAPPABLE;
+  }
+  builder.usage(usage);
+
+  texture_ = builder.build(*engine_);
+}
+
+Texture::~Texture() {
+  ReleaseData();
+  if (texture_) {
+    engine_->destroy(texture_);
+  }
+}
+
+void Texture::Upload(const mjrTextureData& data) {
+  user_data_ = data.user_data;
+  release_callback_ = data.release_callback;
+
+  if (data.bytes == nullptr || data.nbytes == 0) {
+    ReleaseData();
+    return;
+  }
+
+  if (config_.format == mjPIXEL_FORMAT_KTX) {
+    image::Ktx1Bundle* bundle = new image::Ktx1Bundle(
+        reinterpret_cast<const uint8_t*>(data.bytes), data.nbytes);
+    has_spherical_harmonics_ = true;
+    bundle->getSphericalHarmonics(spherical_harmonics_);
+    const bool is_srgb = false;
+    texture_ = ktxreader::Ktx1Reader::createTexture(engine_, bundle, is_srgb);
+    config_.width = texture_->getWidth();
+    config_.height = texture_->getHeight();
+    ReleaseData();
+    return;
+  }
+
+  const int num_channels = GetNumChannels(config_);
+  const filament::Texture::Type type = filament::Texture::Type::UBYTE;
+  const filament::Texture::Format format = GetTextureFormat(config_);
+
+  if (!IsCubeMap(config_)) {
+    if (config_.width * config_.height * num_channels != data.nbytes) {
+      mju_error("Texture size does not match data size.");
+    }
+
+    auto callback = +[](void* buffer, size_t size, void* user) {
+      reinterpret_cast<Texture*>(user)->ReleaseData();
+    };
+    filament::Texture::PixelBufferDescriptor desc(data.bytes, data.nbytes,
+                                                  format, type, callback, this);
+    texture_->setImage(*engine_, 0, std::move(desc));
+  } else {
+    const int face_size = config_.width * GetFaceHeight(config_) * num_channels;
+    const int num_bytes = face_size * kNumFacesPerCube;
+    filament::Texture::FaceOffsets offsets(face_size);
+
+    if (config_.width == config_.height) {
+      uint8_t* copy = new uint8_t[num_bytes];
+      auto release_callback = +[](void* buffer, size_t size, void* user) {
+        delete [] reinterpret_cast<uint8_t*>(buffer);
+      };
+      for (int i = 0; i < kNumFacesPerCube; ++i) {
+        std::memcpy(copy + (i * face_size), data.bytes, face_size);
+      }
+      filament::Texture::PixelBufferDescriptor desc(copy, num_bytes, format,
+                                                    type, release_callback);
+      texture_->setImage(*engine_, 0, std::move(desc), offsets);
+      ReleaseData();
+    } else {
+      if (num_bytes != data.nbytes) {
+        mju_error("Texture size does not match data size.");
+      }
+      auto callback = +[](void* buffer, size_t size, void* user) {
+        reinterpret_cast<Texture*>(user)->ReleaseData();
+      };
+      filament::Texture::PixelBufferDescriptor desc(
+          data.bytes, data.nbytes, format, type, callback, this);
+      texture_->setImage(*engine_, 0, std::move(desc), offsets);
+    }
+  }
+
+  if (config_.color_space != mjCOLORSPACE_SRGB) {
+    texture_->generateMipmaps(*engine_);
+  }
+}
+
+void Texture::ReleaseData() {
+  if (release_callback_) {
+    release_callback_(user_data_);
+    release_callback_ = nullptr;
+    user_data_ = nullptr;
+  }
+}
+
+}  // namespace mujoco

src/experimental/filament/filament/texture.h

@@ -0,0 +1,87 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_H_
+
+#include <filament/Engine.h>
+#include <filament/Texture.h>
+#include <math/vec3.h>
+#include "experimental/filament/render_context_filament.h"
+
+// Functions for creating filament textures.
+namespace mujoco {
+
+// Wrapper around a filament::Texture.
+class Texture : public mjrTexture {
+ public:
+  // Flags for internal use.
+  struct InternalFlags {
+    InternalFlags() : color_attachment(false), depth_attachment(false) {}
+    bool color_attachment;
+    bool depth_attachment;
+  };
+
+  // Creates a texture with the given data.
+  Texture(filament::Engine* engine, const mjrTextureConfig& config,
+          InternalFlags flags = InternalFlags());
+
+  ~Texture();
+
+  Texture(const Texture&) = delete;
+  Texture& operator=(const Texture&) = delete;
+
+  // Uploads the given data to the texture.
+  void Upload(const mjrTextureData& data);
+
+  // Returns the width of the texture.
+  int GetWidth() const { return config_.width; }
+
+  // Returns the height of the texture.
+  int GetHeight() const { return config_.height; }
+
+  // Returns the target of the texture.
+  mjrSamplerType GetSamplerType() const { return config_.sampler_type; }
+
+  // Returns the underlying filament texture.
+  filament::Texture* GetFilamentTexture() const { return texture_; }
+
+  // Returns any spherical harmonics data associated with the texture.
+  using SphericalHarmonics = filament::math::float3[9];
+  const SphericalHarmonics* GetSphericalHarmonics() const {
+    return has_spherical_harmonics_ ? &spherical_harmonics_ : nullptr;
+  }
+
+  static Texture* downcast(mjrTexture* texture) {
+    return static_cast<Texture*>(texture);
+  }
+  static const Texture* downcast(const mjrTexture* texture) {
+    return static_cast<const Texture*>(texture);
+  }
+
+ private:
+  void ReleaseData();
+
+  filament::Engine* engine_ = nullptr;
+  filament::Texture* texture_ = nullptr;
+  mjrTextureConfig config_;
+  SphericalHarmonics spherical_harmonics_;
+  bool has_spherical_harmonics_ = false;
+
+  void* user_data_ = nullptr;
+  void (*release_callback_)(void* user_data) = nullptr;
+};
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_H_

src/experimental/filament/filament/texture_util.cc

@@ -1,181 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/texture_util.h"
-
-#include <cstddef>
-#include <cstdint>
-#include <cstring>
-#include <utility>
-
-#include <filament/Engine.h>
-#include <filament/Texture.h>
-#include <image/Ktx1Bundle.h>
-#include <ktxreader/Ktx1Reader.h>
-#include <math/vec3.h>
-#include <mujoco/mujoco.h>
-
-
-namespace mujoco {
-
-static filament::Texture::Format GetTextureFormat(int num_channels) {
-  switch (num_channels) {
-    case 1:
-      return filament::Texture::Format::R;
-    case 3:
-      return filament::Texture::Format::RGB;
-    case 4:
-      return filament::Texture::Format::RGBA;
-    default:
-      mju_error("Unsupported number of channels: %d", num_channels);
-      return filament::Texture::Format::UNUSED;
-    }
-}
-
-static filament::Texture::InternalFormat GetTextureInternalFormat(
-    int num_channels, bool is_srgb) {
-  if (is_srgb) {
-    switch (num_channels) {
-      case 3:
-        return filament::Texture::InternalFormat::SRGB8;
-      case 4:
-        return filament::Texture::InternalFormat::SRGB8_A8;
-      default:
-        mju_error("Unsupported number of channels: %d", num_channels);
-        return filament::Texture::InternalFormat::UNUSED;
-    }
-  } else {
-    switch (num_channels) {
-      case 1:
-        return filament::Texture::InternalFormat::R8;
-      case 3:
-        return filament::Texture::InternalFormat::RGB8;
-      case 4:
-        return filament::Texture::InternalFormat::RGBA8;
-      default:
-        mju_error("Unsupported number of channels: %d", num_channels);
-        return filament::Texture::InternalFormat::UNUSED;
-    }
-  }
-}
-
-filament::Texture* Create2dTexture(filament::Engine* engine, int width,
-                                   int height, int num_channels,
-                                   const uint8_t* data, bool is_srgb) {
-  if (num_channels != 1 && num_channels != 3 && num_channels != 4) {
-    mju_error("Unsupported number of channels: %d", num_channels);
-    return nullptr;
-  }
-
-  filament::Texture::Builder builder;
-  builder.width(width);
-  builder.height(height);
-  builder.format(GetTextureInternalFormat(num_channels, is_srgb));
-  builder.sampler(filament::Texture::Sampler::SAMPLER_2D);
-  if (!is_srgb) {
-    builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE |
-                  filament::Texture::Usage::SAMPLEABLE |
-                  filament::Texture::Usage::UPLOADABLE);
-  }
-  filament::Texture* texture = builder.build(*engine);
-
-  if (data) {
-    const size_t num_bytes = width * height * sizeof(uint8_t) * num_channels;
-    const filament::Texture::Format format = GetTextureFormat(num_channels);
-    texture->setImage(
-        *engine, 0,
-        filament::Texture::PixelBufferDescriptor(
-            data, num_bytes, format, filament::Texture::Type::UBYTE));
-    if (!is_srgb) {
-      texture->generateMipmaps(*engine);
-    }
-  }
-  return texture;
-}
-
-filament::Texture* CreateCubeTexture(filament::Engine* engine, int width,
-                                     int height, int num_channels,
-                                     const uint8_t* data, bool is_srgb) {
-  if (num_channels != 3) {
-    mju_error("Only support RGB cubemaps.");
-    return nullptr;
-  }
-
-  const int kNumFacesPerCube = 6;
-
-  int face_height = height;
-  if (width != height) {
-    if (width * kNumFacesPerCube != height) {
-        mju_error("Cube maps must contain 6 square images.");
-    }
-    face_height = height / kNumFacesPerCube;
-  }
-  if (width != face_height) {
-    mju_error("Cube map faces must be square.");
-  }
-
-  filament::Texture::Builder builder;
-  builder.width(width);
-  builder.height(face_height);
-  builder.format(GetTextureInternalFormat(num_channels, is_srgb));
-  builder.sampler(filament::Texture::Sampler::SAMPLER_CUBEMAP);
-  if (!is_srgb) {
-    builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE |
-                  filament::Texture::Usage::SAMPLEABLE |
-                  filament::Texture::Usage::UPLOADABLE);
-  }
-  filament::Texture* texture = builder.build(*engine);
-
-  const int face_size = width * face_height * num_channels;
-  const int num_bytes = face_size * kNumFacesPerCube;
-
-  uint8_t* buffer = new uint8_t[num_bytes];
-  auto callback = +[](void* buffer, size_t size, void* user) {
-    delete [] reinterpret_cast<uint8_t*>(buffer);
-  };
-
-  filament::Texture::FaceOffsets offsets(face_size);
-  if (width == height) {
-    // Copy the image to all the faces.
-    for (int i = 0; i < kNumFacesPerCube; ++i) {
-      std::memcpy(buffer + (i * face_size), data, face_size);
-    }
-  } else {
-    // Use the cubemap as is.
-    std::memcpy(buffer, data, num_bytes);
-  }
-
-  if (data) {
-    filament::Texture::PixelBufferDescriptor desc(
-                          buffer, num_bytes, filament::Texture::Format::RGB,
-                          filament::Texture::Type::UBYTE, callback);
-    texture->setImage(*engine, 0, std::move(desc), offsets);
-    if (!is_srgb) {
-      texture->generateMipmaps(*engine);
-    }
-  }
-  return texture;
-}
-
-filament::Texture* CreateKtxTexture(
-    filament::Engine* engine, const uint8_t* data, int size,
-    filament::math::float3* spherical_harmonics_out) {
-  image::Ktx1Bundle* bundle = new image::Ktx1Bundle(data, size);
-  if (spherical_harmonics_out) {
-    bundle->getSphericalHarmonics(spherical_harmonics_out);
-  }
-  const bool is_srgb = false;
-  return ktxreader::Ktx1Reader::createTexture(engine, bundle, is_srgb);
-}
-}  // namespace mujoco

src/experimental/filament/filament/texture_util.h

@@ -1,44 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_
-
-#include <cstdint>
-
-#include <filament/Engine.h>
-#include <filament/Texture.h>
-#include <math/vec3.h>
-
-// Functions for creating filament textures.
-namespace mujoco {
-
-// Creates a filament Texture for the given 2D texture.
-filament::Texture* Create2dTexture(filament::Engine* engine, int width,
-                                   int height, int num_channels,
-                                   const uint8_t* data, bool is_srgb);
-
-// Creates a filament Texture for the given cube texture.
-filament::Texture* CreateCubeTexture(filament::Engine* engine, int width,
-                                     int height, int num_channels,
-                                     const uint8_t* data, bool is_srgb);
-
-// Creates a filament Texture for the given KTX payload.
-filament::Texture* CreateKtxTexture(
-    filament::Engine* engine, const uint8_t* data, int size,
-    filament::math::float3* spherical_harmonics_out);
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_

src/experimental/filament/filament/vertex_util.cc

@@ -1,65 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "experimental/filament/filament/vertex_util.h"
-
-#include <limits>
-
-#include <math/TVecHelpers.h>
-#include <math/mat3.h>
-#include <math/quat.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-
-namespace mujoco {
-
-using filament::math::float3;
-using filament::math::float4;
-using filament::math::mat3f;
-using filament::math::quatf;
-
-float4 CalculateOrientation(const float3& normal) {
-  float3 tangent;
-  float3 bitangent;
-  if (normal.y < -1.0f + std::numeric_limits<float>::epsilon()) {
-    // Handle the singularity.
-    tangent = float3{-1.0f, 0.0f, 0.0f};
-    bitangent = float3{0.0f, 0.0f, -1.0f};
-  } else {
-    const float a = 1.0f / (1.0f + normal.y);
-    const float b = -normal.z * normal.x * a;
-    tangent = float3(b, -normal.z, 1.0f - normal.z * normal.z * a);
-    bitangent = float3(1.0f - normal.x * normal.x * a, -normal.x, b);
-  }
-  quatf orientation = mat3f::packTangentFrame({tangent, bitangent, normal});
-  return float4(orientation.xyz, orientation.w);
-}
-
-float3 CalculateNormal(
-    const filament::math::float3& p1,
-    const filament::math::float3& p2,
-    const filament::math::float3& p3) {
-  const float3 v12 = p2 - p1;
-  const float3 v13 = p3 - p1;
-  return normalize(cross(v12, v13));
-}
-
-float4 CalculateOrientation(
-    const filament::math::float3& p1,
-    const filament::math::float3& p2,
-    const filament::math::float3& p3) {
-  return CalculateOrientation(CalculateNormal(p1, p2, p3));
-}
-
-}  // namespace mujoco

src/experimental/filament/filament/vertex_util.h

@@ -1,95 +0,0 @@
-// Copyright 2025 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_
-
-#include <math/vec2.h>
-#include <math/vec3.h>
-#include <math/vec4.h>
-
-namespace mujoco {
-
-// Calculates the normal of a triangle given its three vertices.
-filament::math::float3 CalculateNormal(
-    const filament::math::float3& p1,
-    const filament::math::float3& p2,
-    const filament::math::float3& p3);
-
-// Calculates the orientation of a vertex given just its normal.
-filament::math::float4 CalculateOrientation(
-    const filament::math::float3& normal);
-
-// Calculates the orientation of a triangle given its three vertices.
-filament::math::float4 CalculateOrientation(
-    const filament::math::float3& p1,
-    const filament::math::float3& p2,
-    const filament::math::float3& p3);
-
-// A standard vertex with no UV coordinates.
-struct VertexNoUv {
-  VertexNoUv() = default;
-  VertexNoUv(filament::math::float3 position,
-             filament::math::float4 orientation)
-      : position(position), orientation(orientation) {}
-
-  filament::math::float3 position;
-  filament::math::float4 orientation;
-
-  static constexpr bool kHasPosition = true;
-  static constexpr bool kHasPosition2d = false;
-  static constexpr bool kHasOrientation = true;
-  static constexpr bool kHasUv = false;
-  static constexpr bool kHasColor = false;
-};
-
-// A standard vertex with UV coordinates.
-struct VertexWithUv {
-  VertexWithUv() = default;
-  VertexWithUv(filament::math::float3 position,
-               filament::math::float4 orientation, filament::math::float2 uv)
-      : position(position), orientation(orientation), uv(uv) {}
-
-  filament::math::float3 position;
-  filament::math::float4 orientation;
-  filament::math::float2 uv;
-
-  static constexpr bool kHasPosition = true;
-  static constexpr bool kHasPosition2d = false;
-  static constexpr bool kHasOrientation = true;
-  static constexpr bool kHasUv = true;
-  static constexpr bool kHasColor = false;
-};
-
-// A vertex for rendering GUI elements.
-struct GuiVertex {
-  GuiVertex() = default;
-  GuiVertex(filament::math::float2 position, filament::math::float2 uv,
-            filament::math::ubyte4 color)
-      : position(position), uv(uv), color(color) {}
-
-  filament::math::float2 position;
-  filament::math::float2 uv;
-  filament::math::ubyte4 color;
-
-  static constexpr bool kHasPosition = false;
-  static constexpr bool kHasPosition2d = true;
-  static constexpr bool kHasOrientation = false;
-  static constexpr bool kHasUv = true;
-  static constexpr bool kHasColor = true;
-};
-
-}  // namespace mujoco
-
-#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_

src/experimental/filament/filament_util.cc

@@ -12,9 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/filament/filament/math_util.h"
+#include "experimental/filament/filament_util.h"
+#include <limits>
 
+#include <math/mat3.h>
 #include <math/mat4.h>
+#include <math/quat.h>
 #include <math/vec3.h>
 #include <math/vec4.h>
 #include <math/TVecHelpers.h>
@@ -23,7 +26,9 @@ namespace mujoco {
 
 using filament::math::float3;
 using filament::math::float4;
+using filament::math::mat3f;
 using filament::math::mat4;
+using filament::math::quatf;
 
 mat4 ToReflectionMatrix(const mat4& xform) {
   const float3 normal = xform[2].xyz;
@@ -81,4 +86,38 @@ mat4 CalculateObliqueProjection(const mat4& projection, const float4& plane) {
   return res;
 }
 
+
+float4 CalculateOrientation(const float3& normal) {
+  float3 tangent;
+  float3 bitangent;
+  if (normal.y < -1.0f + std::numeric_limits<float>::epsilon()) {
+    // Handle the singularity.
+    tangent = float3{-1.0f, 0.0f, 0.0f};
+    bitangent = float3{0.0f, 0.0f, -1.0f};
+  } else {
+    const float a = 1.0f / (1.0f + normal.y);
+    const float b = -normal.z * normal.x * a;
+    tangent = float3(b, -normal.z, 1.0f - normal.z * normal.z * a);
+    bitangent = float3(1.0f - normal.x * normal.x * a, -normal.x, b);
+  }
+  quatf orientation = mat3f::packTangentFrame({tangent, bitangent, normal});
+  return float4(orientation.xyz, orientation.w);
+}
+
+float3 CalculateNormal(
+    const filament::math::float3& p1,
+    const filament::math::float3& p2,
+    const filament::math::float3& p3) {
+  const float3 v12 = p2 - p1;
+  const float3 v13 = p3 - p1;
+  return normalize(cross(v12, v13));
+}
+
+float4 CalculateOrientation(
+    const filament::math::float3& p1,
+    const filament::math::float3& p2,
+    const filament::math::float3& p3) {
+  return CalculateOrientation(CalculateNormal(p1, p2, p3));
+}
+
 }  // namespace mujoco

src/experimental/filament/filament_util.h

@@ -0,0 +1,148 @@
+// Copyright 2025 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_UTIL_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_UTIL_H_
+
+#include <math/mat3.h>
+#include <math/mat4.h>
+#include <math/vec2.h>
+#include <math/vec3.h>
+#include <math/vec4.h>
+#include <mujoco/mjmodel.h>
+#include <mujoco/mujoco.h>
+
+namespace mujoco {
+
+// Reads a float2 from an array buffer in the model/scene.
+template <typename T>
+inline filament::math::float2 ReadFloat2(const T* arr, int index = 0) {
+  const T* ptr = arr + (2 * index);
+  return filament::math::float2(ptr[0], ptr[1]);
+}
+
+// Reads a float3 from an array buffer in the model/scene.
+template <typename T>
+inline filament::math::float3 ReadFloat3(const T* arr, int index = 0) {
+  const T* ptr = arr + (3 * index);
+  return filament::math::float3(ptr[0], ptr[1], ptr[2]);
+}
+
+// Reads a float4 from an array buffer in the model/scene.
+template <typename T>
+inline filament::math::float4 ReadFloat4(const T* arr, int index = 0) {
+  const T* ptr = arr + (4 * index);
+  return filament::math::float4(ptr[0], ptr[1], ptr[2], ptr[3]);
+}
+
+// Reads a mat3 from an array buffer in the model/scene.
+template <typename T>
+inline filament::math::mat3f ReadMat3(const T* arr, int index = 0) {
+  // clang-format off
+  const T* ptr = arr + (9 * index);
+  return filament::math::mat3f(ptr[0], ptr[3], ptr[6],
+                               ptr[1], ptr[4], ptr[7],
+                               ptr[2], ptr[5], ptr[8]);
+  // clang-format on
+}
+
+// A tuple of translation, rotation, and size.
+struct Trs {
+  filament::math::float3 translation{0.0f, 0.0f, 0.0f};
+  filament::math::mat3f rotation;
+  // Note: this is _slightly_ different than scale. For example, for capsules,
+  // the size determines the length of the tube and the radius of the domes,
+  // but the shape remains a capsule.
+  filament::math::float3 size{1.0f, 1.0f, 1.0f};
+
+  // Converts the TRS to a transform matrix.
+  filament::math::mat4f ToTransform() const {
+    return filament::math::mat4f(rotation, translation) *
+           filament::math::mat4f::scaling(size);
+  }
+};
+
+// Calculates a reflection matrix for a plane defined by its transform.
+filament::math::mat4 ToReflectionMatrix(const filament::math::mat4& xform);
+
+// Modifies a projection matrix so its near plane coincides with an arbitrary
+// plane defined in camera space.
+filament::math::mat4 CalculateObliqueProjection(
+    const filament::math::mat4& projection,
+    const filament::math::float4& plane);
+
+// Calculates the normal of a triangle given its three vertices.
+filament::math::float3 CalculateNormal(
+    const filament::math::float3& p1,
+    const filament::math::float3& p2,
+    const filament::math::float3& p3);
+
+// Calculates the orientation of a vertex given just its normal.
+filament::math::float4 CalculateOrientation(
+    const filament::math::float3& normal);
+
+// Calculates the orientation of a triangle given its three vertices.
+filament::math::float4 CalculateOrientation(
+    const filament::math::float3& p1,
+    const filament::math::float3& p2,
+    const filament::math::float3& p3);
+
+// Reads a value with the given name from the mjModel's data sections. The
+// default_value is returned if the named element is not found.
+template <typename T>
+T ReadElement(const mjModel* model, const char* name, T default_value = T()) {
+  constexpr bool is_string =
+      std::is_same_v<T, const char*> || std::is_same_v<T, std::string_view>;
+
+  const int type = is_string ? mjOBJ_TEXT : mjOBJ_NUMERIC;
+  const int id = mj_name2id(model, type, name);
+  if (id < 0) {
+    return default_value;
+  }
+
+  if constexpr (std::is_same_v<T, const char*>) {
+    const char* ptr = model->text_data + model->text_adr[id];
+    return ptr;
+  } else if constexpr (std::is_same_v<T, std::string_view>) {
+    const char* ptr = model->text_data + model->text_adr[id];
+    // Do not include the null terminator in the string view.
+    return std::string_view(ptr, model->text_size[id] - 1);
+  } else if constexpr (std::is_arithmetic_v<T>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    return static_cast<T>(*ptr);
+  } else if constexpr (std::is_enum_v<T>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    return static_cast<T>(static_cast<int>(*ptr));
+  } else if constexpr (std::is_same_v<T, filament::math::float2>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    if (model->numeric_size[id] != 2) mju_error("Invalid numeric size.");
+    return T{ptr[0], ptr[1]};
+  } else if constexpr (std::is_same_v<T, filament::math::float3>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    if (model->numeric_size[id] != 3) mju_error("Invalid numeric size.");
+    return T{ptr[0], ptr[1], ptr[2]};
+  } else if constexpr (std::is_same_v<T, filament::math::float4>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    if (model->numeric_size[id] != 4) mju_error("Invalid numeric size.");
+    return T{ptr[0], ptr[1], ptr[2], ptr[3]};
+  } else if constexpr (std::is_same_v<T, bool>) {
+    const mjtNum* ptr = model->numeric_data + model->numeric_adr[id];
+    return static_cast<T>(*ptr != 0);
+  }
+  return default_value;
+}
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_UTIL_H_

src/experimental/filament/mjr_compat.cc

@@ -36,7 +36,7 @@ void mjr_freeContext(mjrContext* con) {
   mjrf_freeContext(con);
 }
 void mjr_render(mjrRect viewport, mjvScene* scn, const mjrContext* con) {
-  mjrf_render(viewport, scn, con);
+  mjrf_renderScene(viewport, scn, con);
 }
 void mjr_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) {
   mjrf_uploadMesh(m, con, meshid);

src/experimental/filament/render_context_filament.cc

@@ -14,20 +14,30 @@
 
 #include "experimental/filament/render_context_filament.h"
 
+#include <array>
 #include <cstdint>
 #include <cstring>
 
+#include <math/mat3.h>
+#include <math/vec3.h>
 #include <mujoco/mjmodel.h>
 #include <mujoco/mjrender.h>
 #include <mujoco/mjvisualize.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/compat/mjr_filament_renderer.h"
 #include "experimental/filament/filament/filament_context.h"
+#include "experimental/filament/filament/light.h"
+#include "experimental/filament/filament/mesh.h"
+#include "experimental/filament/filament/render_target.h"
+#include "experimental/filament/filament/renderable.h"
+#include "experimental/filament/filament/scene_view.h"
+#include "experimental/filament/filament/texture.h"
 
 
 #if defined(TLS_FILAMENT_CONTEXT)
-static thread_local mujoco::FilamentContext* g_filament_context = nullptr;
+static thread_local mujoco::MjrFilamentRenderer* g_filament_context = nullptr;
 #else
-static mujoco::FilamentContext* g_filament_context = nullptr;
+static mujoco::MjrFilamentRenderer* g_filament_context = nullptr;
 #endif
 
 static void CheckFilamentContext() {
@@ -36,20 +46,329 @@ static void CheckFilamentContext() {
   }
 }
 
+template <int N>
+static void setf(float (&arr)[N], const std::array<float, N>& values) {
+  for (int i = 0; i < N; ++i) {
+    arr[i] = values[i];
+  }
+}
+
+
 extern "C" {
 
 void mjrf_defaultFilamentConfig(mjrFilamentConfig* config) {
   memset(config, 0, sizeof(mjrFilamentConfig));
 }
 
+void mjr_defaultTextureData(mjrTextureData* data) {
+  memset(data, 0, sizeof(mjrTextureData));
+}
+
+void mjr_defaultTextureConfig(mjrTextureConfig* config) {
+  memset(config, 0, sizeof(mjrTextureConfig));
+}
+
+void mjr_defaultMeshData(mjrMeshData* data) {
+  memset(data, 0, sizeof(mjrMeshData));
+}
+
+void mjr_defaultSceneParams(mjrSceneParams* params) {
+  memset(params, 0, sizeof(mjrSceneParams));
+  params->enable_post_processing = true;
+  params->enable_reflections = true;
+  params->enable_shadows = true;
+  params->layer_mask = 0xff;
+  params->reflection_layer_mask = 0xff;
+}
+
+void mjr_defaultLightParams(mjrLightParams* params) {
+  memset(params, 0, sizeof(mjrLightParams));
+  params->type = mjLIGHT_POINT;
+  params->texture = nullptr;
+  params->color[0] = 0;
+  params->color[1] = 0;
+  params->color[2] = 0;
+  params->intensity = 0.0f;
+  params->cast_shadows = true;
+  params->range = 10.0f;
+  params->spot_cone_angle = 180.f;
+  params->bulb_radius = 0.0f;
+  params->shadow_map_size = 2048;
+  params->vsm_blur_width = 0.0f;
+}
+
+void mjr_defaultMaterial(mjrMaterial* material) {
+  memset(material, 0, sizeof(mjrMaterial));
+  setf(material->color, {1.f, 1.f, 1.f, 1.f});
+  setf(material->segmentation_color, {1, 1, 1, 1});
+  setf(material->uv_scale, {1, 1, 1});
+  material->emissive = -1.0f;
+  material->specular = -1.0f;
+  material->glossiness = -1.0f;
+  material->metallic = -1.0f;
+  material->roughness = -1.0f;
+}
+
+void mjr_defaultRenderableParams(mjrRenderableParams* params) {
+  memset(params, 0, sizeof(mjrRenderableParams));
+  params->cast_shadows = true;
+  params->receive_shadows = true;
+  params->layer_mask = 0x01;
+  params->priority = 4;
+  params->blend_order = 0;
+}
+
+void mjr_defaultRenderTargetConfig(mjrRenderTargetConfig* config) {
+  memset(config, 0, sizeof(mjrRenderTargetConfig));
+  config->color_format = mjPIXEL_FORMAT_RGBA8;
+  config->depth_format = mjPIXEL_FORMAT_DEPTH32F;
+}
+
+void mjr_defaultRenderRequest(mjrRenderRequest* request) {
+  memset(request, 0, sizeof(mjrRenderRequest));
+}
+
+void mjr_defaultReadPixelsRequest(mjrReadPixelsRequest* request) {
+  memset(request, 0, sizeof(mjrReadPixelsRequest));
+}
+
+void mjr_defaultFrameStats(mjrFrameStats* stats) {
+  memset(stats, 0, sizeof(mjrFrameStats));
+}
+
+mjrfContext* mjrf_createContext(const mjrFilamentConfig* config) {
+  return new mujoco::FilamentContext(config);
+}
+
+void mjrf_destroyContext(mjrfContext* ctx) {
+  delete mujoco::FilamentContext::downcast(ctx);
+}
+
+mjrTexture* mjrf_createTexture(mjrfContext* ctx,
+                               const mjrTextureConfig* config) {
+  return new mujoco::Texture(
+      mujoco::FilamentContext::downcast(ctx)->GetEngine(), *config);
+}
+
+void mjrf_destroyTexture(mjrTexture* texture) {
+  delete mujoco::Texture::downcast(texture);
+}
+
+mjrMesh* mjrf_createMesh(mjrfContext* ctx, const mjrMeshData* data) {
+  return new mujoco::Mesh(mujoco::FilamentContext::downcast(ctx)->GetEngine(),
+                          *data);
+}
+
+void mjrf_destroyMesh(mjrMesh* mesh) { delete mujoco::Mesh::downcast(mesh); }
+
+mjrScene* mjrf_createScene(mjrfContext* ctx, const mjrSceneParams* params) {
+  return new mujoco::SceneView(
+      mujoco::FilamentContext::downcast(ctx)->GetEngine(), *params);
+}
+
+void mjrf_destroyScene(mjrScene* scene) {
+  delete mujoco::SceneView::downcast(scene);
+}
+
+mjrLight* mjrf_createLight(mjrfContext* ctx, const mjrLightParams* params) {
+  return new mujoco::Light(mujoco::FilamentContext::downcast(ctx)->GetEngine(),
+                           *params);
+}
+
+void mjrf_destroyLight(mjrLight* light) {
+  delete mujoco::Light::downcast(light);
+}
+
+mjrRenderable* mjrf_createRenderable(mjrfContext* ctx,
+                                     const mjrRenderableParams* params) {
+  return new mujoco::Renderable(
+      mujoco::FilamentContext::downcast(ctx)->GetEngine(), *params,
+      mujoco::FilamentContext::downcast(ctx)->GetObjectManager());
+}
+
+void mjrf_destroyRenderable(mjrRenderable* renderable) {
+  delete mujoco::Renderable::downcast(renderable);
+}
+
+mjrRenderTarget* mjrf_createRenderTarget(mjrfContext* ctx,
+                                         const mjrRenderTargetConfig* config) {
+  return new mujoco::RenderTarget(
+      mujoco::FilamentContext::downcast(ctx)->GetEngine(), *config);
+}
+
+void mjrf_destroyRenderTarget(mjrRenderTarget* render_target) {
+  delete mujoco::RenderTarget::downcast(render_target);
+}
+
+void mjrf_setTextureData(mjrTexture* texture, const mjrTextureData* data) {
+  mujoco::Texture::downcast(texture)->Upload(*data);
+}
+
+int mjrf_getTextureWidth(const mjrTexture* texture) {
+  return mujoco::Texture::downcast(texture)->GetWidth();
+}
+
+int mjrf_getTextureHeight(const mjrTexture* texture) {
+  return mujoco::Texture::downcast(texture)->GetHeight();
+}
+
+mjrSamplerType mjrf_getSamplerType(const mjrTexture* texture) {
+  return mujoco::Texture::downcast(texture)->GetSamplerType();
+}
+
+void mjrf_setLightEnabled(mjrLight* light, mjtByte enabled) {
+  if (enabled) {
+    mujoco::Light::downcast(light)->Enable();
+  } else {
+    mujoco::Light::downcast(light)->Disable();
+  }
+}
+
+void mjrf_setLightIntensity(mjrLight* light, float intensity) {
+  mujoco::Light::downcast(light)->SetIntensity(intensity);
+}
+
+void mjrf_setLightColor(mjrLight* light, const float color[3]) {
+  mujoco::Light::downcast(light)->SetColor({color[0], color[1], color[2]});
+}
+
+void mjrf_setLightTransform(mjrLight* light, const float position[3],
+                            const float direction[3]) {
+  mujoco::Light::downcast(light)->SetTransform(
+      {position[0], position[1], position[2]},
+      {direction[0], direction[1], direction[2]});
+}
+
+mjrLightType mjrf_getLightType(const mjrLight* light) {
+  return mujoco::Light::downcast(light)->GetType();
+}
+
+void mjrf_setRenderableMesh(mjrRenderable* renderable, const mjrMesh* mesh,
+                            int elem_offset, int elem_count) {
+  mujoco::Renderable::downcast(renderable)
+      ->SetMesh(mujoco::Mesh::downcast(mesh), elem_offset, elem_count);
+}
+
+void mjrf_setRenderableGeomMesh(mjrRenderable* renderable, mjtGeom type,
+                                int nstack, int nslice, int nquad) {
+  mujoco::Renderable::downcast(renderable)->SetGeomMesh(type, nstack, nslice,
+                                                        nquad);
+}
+
+void mjrf_setRenderableMaterial(mjrRenderable* renderable,
+                                const mjrMaterial* material) {
+  mujoco::Renderable::downcast(renderable)->UpdateMaterial(*material);
+}
+
+void mjrf_setRenderableTransform(mjrRenderable* renderable,
+                                 const float position[3],
+                                 const float rotation[9], const float size[3]) {
+  const filament::math::float3 fposition{position[0], position[1], position[2]};
+  const filament::math::float3 fsize{size[0], size[1], size[2]};
+  const filament::math::mat3f frotation{rotation[0], rotation[3], rotation[6],
+                                        rotation[1], rotation[4], rotation[7],
+                                        rotation[2], rotation[5], rotation[8]};
+  mujoco::Renderable::downcast(renderable)
+      ->SetTransform({fposition, frotation, fsize});
+}
+
+void mjrf_setRenderableLayerMask(mjrRenderable* renderable,
+                                 uint8_t layer_mask) {
+  mujoco::Renderable::downcast(renderable)->SetLayerMask(layer_mask);
+}
+
+void mjrf_setRenderableWireframe(mjrRenderable* renderable, mjtByte wireframe) {
+  mujoco::Renderable::downcast(renderable)->SetWireframe(wireframe);
+}
+
+void mjrf_setRenderableCastShadows(mjrRenderable* renderable,
+                                   mjtByte cast_shadows) {
+  mujoco::Renderable::downcast(renderable)->SetCastShadows(cast_shadows);
+}
+
+void mjrf_setRenderableReceiveShadows(mjrRenderable* renderable,
+                                      mjtByte receive_shadows) {
+  mujoco::Renderable::downcast(renderable)->SetReceiveShadows(receive_shadows);
+}
+
+void mjrf_addLightToScene(mjrScene* scene, mjrLight* light) {
+  mujoco::SceneView::downcast(scene)->AddToScene(
+      mujoco::Light::downcast(light));
+}
+
+void mjrf_removeLightFromScene(mjrScene* scene, mjrLight* light) {
+  mujoco::SceneView::downcast(scene)->RemoveFromScene(
+      mujoco::Light::downcast(light));
+}
+
+void mjrf_addRenderableToScene(mjrScene* scene, mjrRenderable* renderable) {
+  mujoco::SceneView::downcast(scene)->AddToScene(
+      mujoco::Renderable::downcast(renderable));
+}
+
+void mjrf_removeRenderableFromScene(mjrScene* scene,
+                                    mjrRenderable* renderable) {
+  mujoco::SceneView::downcast(scene)->RemoveFromScene(
+      mujoco::Renderable::downcast(renderable));
+}
+
+void mjrf_setSceneSkybox(mjrScene* scene, const mjrTexture* texture) {
+  mujoco::SceneView::downcast(scene)->SetSkybox(
+      mujoco::Texture::downcast(texture));
+}
+
+void mjrf_setSceneShadowsEnabled(mjrScene* scene, mjtByte enabled) {
+  if (enabled) {
+    mujoco::SceneView::downcast(scene)->EnableShadows();
+  } else {
+    mujoco::SceneView::downcast(scene)->DisableShadows();
+  }
+}
+
+void mjrf_setSceneReflectionsEnabled(mjrScene* scene, mjtByte enabled) {
+  if (enabled) {
+    mujoco::SceneView::downcast(scene)->EnableReflections();
+  } else {
+    mujoco::SceneView::downcast(scene)->DisableReflections();
+  }
+}
+
+void mjrf_configureSceneFromModel(mjrScene* scene, const mjModel* model) {
+  mujoco::SceneView::downcast(scene)->Configure(model);
+}
+
+mjrFrameHandle mjrf_render(mjrfContext* ctx, const mjrRenderRequest* req,
+                           int nreq, const mjrReadPixelsRequest* read_req,
+                           int nread_req) {
+  return mujoco::FilamentContext::downcast(ctx)->Render(
+      {req, static_cast<size_t>(nreq)},
+      {read_req, static_cast<size_t>(nread_req)});
+}
+
+void mjrf_waitForFrame(mjrfContext* ctx, mjrFrameHandle frame) {
+  mujoco::FilamentContext::downcast(ctx)->WaitForFrame(frame);
+}
+
+void mjrf_setClearColor(mjrfContext* ctx, const float color[3]) {
+  mujoco::FilamentContext::downcast(ctx)->SetClearColor(
+      {color[0], color[1], color[2], 1.0f});
+}
+
+void mjrf_getFrameStats(mjrfContext* ctx, mjrFrameHandle frame,
+                        mjrFrameStats* stats_out) {
+  mujoco::FilamentContext::downcast(ctx)->GetFrameStats(frame, stats_out);
+}
+
+// Legacy API, to be deprecated.
+
 void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con,
-                             const mjrFilamentConfig* config) {
+                              const mjrFilamentConfig* config) {
   // TODO: Support multiple contexts and multiple threads. For now, we'll just
   // assume a single, global context.
   if (g_filament_context != nullptr) {
     mju_error("Context already exists!");
   }
-  g_filament_context = new mujoco::FilamentContext(config);
+  g_filament_context = new mujoco::MjrFilamentRenderer(config);
   g_filament_context->Init(m);
 }
 
@@ -75,7 +394,7 @@ void mjrf_freeContext(mjrContext* con) {
   mjrf_defaultContext(con);
 }
 
-void mjrf_render(mjrRect viewport, mjvScene* scn, const mjrContext* con) {
+void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con) {
   CheckFilamentContext();
   g_filament_context->Render(viewport, scn);
 }

src/experimental/filament/render_context_filament.h

@@ -29,14 +29,81 @@ extern "C" {
 // IMPORTANT: This API should still be considered experimental and is likely
 // change frequently.
 
-typedef enum mjtGraphicsApi_ {  // backend graphics API to use
-  mjGFX_DEFAULT        = 0,     // default based on platform
-  mjGFX_OPENGL,                 // OpenGL (desktop)
-  mjGFX_VULKAN                  // Vulkan
-} mjtGraphicsApi;
+// This library provides a C API for the filament rendering library
+// (https://github.com/google/filament) that is designed to work with the
+// MuJoCo library for visualizing simulations.
+//
+// The filament renderer is a real-time physically based rendering (PBR) engine
+// developed by Google. It is designed to be as small as possible and as
+// efficient as possible, while still providing high-quality results. It works
+// across all major platforms (Linux, Windows, macOS, Android, iOS, Web) and
+// supports OpenGL, Vulkan, and Metal.
+//
+// For the purposes of this API, we assume the reader has a basic understanding
+// of rendering concepts (e.g. textures, vertices, cameras, framebuffers, etc.).
+// We will also highlight some of the key differences between this renderer and
+// the legacy/classic MuJoCo (mjr) renderer.
+//
+// ## API Overview
+//
+// There are seven key components: Context, Texture, Mesh, Scene, Light,
+// Renderable, and RenderTarget. We'll describe these in detail further below.
+//
+// Each object is created using a `create` function and destroyed using a
+// `destroy` function, e.g. `mjrf_createTexture` and `mjrf_destroyTexture`.
+// The `create` functions accept a pointer to a configuration struct (e.g.
+// `mjrTextureConfig`) which describes the parameters for the object to be
+// created. Each of these structs has a corresponding `default` function (e.g.
+// `mjr_defaultTextureConfig`) which can be used to initialize the struct to
+// default values. Default values are assumed to be 0/NULL unless otherwise
+// specified.
+//
+// For now, we'll just define opaque handles for each of our components.
+struct mjrfContext {};
+struct mjrTexture {};
+struct mjrMesh {};
+struct mjrScene {};
+struct mjrLight {};
+struct mjrRenderable {};
+struct mjrRenderTarget {};
 
+
+// ## Rendering Context (mjrfContext)
+//
+// The Context is the main entry point for the library. It manages all the
+// core filament objects that are responsible for the rendering of an image.
+//
+// Filament uses a separate thread for doing the actual rendering. However,
+// despite that, this API is not thread-safe; calls are expected to be made
+// from a single thread. Also, due to the asynchronous nature of filament,
+// some APIs provide handles or callbacks to signal when an operation is
+// complete. (Note: for WASM builds, filament does not use a separate thread.)
+//
+// All other objects (e.g. Textures, Meshes, Scenes, etc.) need a Context in
+// order to be created. Otherwise, the main function to use with the Context is
+// `mjrf_render()` which does the actual rendering.
+//
+// There are two key differences between the mjrfContext and the classic
+// mjrContext. Firstly, the filament context will manage the underlying graphics
+// context itself. This means users do not need to initialize EGL or similar
+// libraries beforehand. Secondly, the filament context is independent of a
+// MuJoCo model. That means you can use a single mjrfContext to render images
+// for multiple models.
+
+// Underlying graphics API library to use for the Context.
+typedef enum mjrGraphicsApi_ {
+  // Default, based on current platform.
+  mjGRAPHICS_API_DEFAULT = 0,
+  // OpenGL (desktop), GLES (mobile), WebGL (web)
+  mjGRAPHICS_API_OPENGL,
+  // Vulkan
+  mjGRAPHICS_API_VULKAN,
+} mjrGraphicsApi;
+
+// Configuration parameters for the filament rendering context.
 struct mjrFilamentConfig {
-  // The native window handle into which we can render directly.
+  // The native window handle into which we can render directly. If nullptr,
+  // rendering will be done to an offscreen framebuffer.
   void* native_window;
 
   // The initial width and height of the offscreen framebuffer.
@@ -44,12 +111,651 @@ struct mjrFilamentConfig {
   int height;
 
   // The backend graphics API to use.
-  int graphics_api;
+  mjrGraphicsApi graphics_api;
 
   // Use software rendering even if the platform supports hardware rendering.
-  bool force_software_rendering;
+  mjtByte force_software_rendering;
 };
 
+// Initializes the mjrFilamentConfig to default values.
+void mjrf_defaultFilamentConfig(mjrFilamentConfig* config);
+
+// Creates a filament rendering context.
+mjrfContext* mjrf_createContext(const mjrFilamentConfig* config);
+
+// Destroys the filament rendering context.
+void mjrf_destroyContext(mjrfContext* ctx);
+
+// Describes the look/intention of the final rendered image.
+typedef enum mjrDrawMode_ {
+  // Render the scene with "normal" colors and lighting.
+  mjDRAW_MODE_COLOR,
+  // Render the scene as a grayscale depth map.
+  mjDRAW_MODE_DEPTH,
+  // Render each object with a unique, uniform (flat) color regardless of
+  // lighting and texture.
+  mjDRAW_MODE_SEGMENTATION,
+} mjrDrawMode;
+
+enum { mjNUM_DRAW_MODES = 3 };  // Number of modes in `mjrDrawMode`.
+
+// Parameters describing the camera to use for rendering an image.
+typedef mjvGLCamera mjrCamera;
+
+// Describes a single rendering operation; used by `mjrf_render()`.
+struct mjrRenderRequest {
+  // The scene to render.
+  mjrScene* scene;
+
+  // The camera from which to render the scene.
+  mjrCamera camera;
+
+  // The method (e.g. Color, Depth, Segmentation, etc.) to use for rendering.
+  mjrDrawMode draw_mode;
+
+  // The viewport into which to render the image.
+  mjrRect viewport;
+
+  // The render target into which to render the image. If nullptr, the image
+  // will be rendered to the window (as previously configured in
+  // mjrFilamentConfig::native_window).
+  mjrRenderTarget* target;
+};
+
+// Initializes the mjrRenderRequest to default values.
+void mjr_defaultRenderRequest(mjrRenderRequest* request);
+
+// Information needed to read pixels; used by `mjrf_render()`.
+struct mjrReadPixelsRequest {
+  // The render target from which to read the image pixels.
+  mjrRenderTarget* target;
+
+  // The buffer into which the read pixels will be written.
+  void* output;
+
+  // The number of bytes in the output buffer. This should match the size of
+  // the render target texture.
+  mjtSize num_bytes;
+
+  // Callback when the read pixels operation is complete. This function can
+  // optionally be used to free the output buffer if needed.
+  void (*read_completed_callback)(void* user_data);
+
+  // User data to pass to the completion callback.
+  void* user_data;
+};
+
+// Initializes the mjrReadPixelsRequest to default values.
+void mjr_defaultReadPixelsRequest(mjrReadPixelsRequest* request);
+
+// Because rendering is asynchronous, each render request is assigned a
+// unique Handle which can be used to query the status of the request. The
+// Handle can also be used to block until the request is completed.
+typedef std::uint64_t mjrFrameHandle;
+
+// Submits the given requests for rendering. Because rendering may happen
+// asynchronously, we have to submit both the render and read requests in the
+// same call. This function is also when any callbacks will be triggered,
+// though there is no guarantee on when exactly that will be done.
+//
+// Multiple requests and reads can be submitted in a single call. These
+// requests will be processed in order, so some care must be taken. Firstly,
+// requests should be grouped by target. Next, the combined area of the
+// viewports for all requests for a given target must be contained within the
+// dimensions of the target itself.
+mjrFrameHandle mjrf_render(mjrfContext* ctx, const mjrRenderRequest* req,
+                           int nreq, const mjrReadPixelsRequest* read_req,
+                           int nread_req);
+
+// Waits for all rendering operations to complete for the given frame handle,
+// triggering any callbacks as needed.
+void mjrf_waitForFrame(mjrfContext* ctx, mjrFrameHandle frame);
+
+// Sets the clear color for the renderer.
+void mjrf_setClearColor(mjrfContext* ctx, const float color[3]);
+
+// Information about a single frame of rendering.
+struct mjrFrameStats {
+  // The frame rate of the renderer, in frames per second.
+  double frame_rate;
+};
+
+// Initializes the mjrFrameStats to default values.
+void mjr_defaultFrameStats(mjrFrameStats* stats);
+
+// Returns the stats for the given frame but updating the given `stats_out`.
+void mjrf_getFrameStats(mjrfContext* ctx, mjrFrameHandle frame,
+                        mjrFrameStats* stats_out);
+
+// ## Textures (mjrTexture)
+//
+// A texture is a 2D or 3D (cubemap) image that adds visual detail to a rendered
+// model, such as color or bumpiness, without increasing geometric complexity.
+//
+// For textures intended to be used for image-based lights (see `mjrLight`
+// below), you should use filament's `cmgen` tool to generate a KTX image from
+// your source image. This tool will calculate additional data (i.e. the
+// spherical harmonics) and encode that information into the KTX file.
+
+// Pixel formats for textures.
+typedef enum mjrPixelFormat_ {
+  mjPIXEL_FORMAT_UNKNOWN = 0,
+  mjPIXEL_FORMAT_R8,
+  mjPIXEL_FORMAT_RGB8,
+  mjPIXEL_FORMAT_RGBA8,
+  mjPIXEL_FORMAT_R32F,
+  mjPIXEL_FORMAT_DEPTH32F,
+  mjPIXEL_FORMAT_KTX,
+} mjrPixelFormat;
+
+// Type of texture.
+typedef mjtTexture mjrSamplerType;
+
+// Type of color space encoding.
+typedef mjtColorSpace mjrColorSpace;
+
+// Defines the basic properties of a texture.
+struct mjrTextureConfig {
+  // The width of the texture. For compressed textures (e.g. KTX), this is the
+  // number of bytes in the compressed data.
+  int width;
+
+  // The height of the texture. For compressed textures (e.g. KTX), this should
+  // be 0.
+  int height;
+
+  // How the texture will be interpreted by the renderer (e.g. 2D, cube, etc.).
+  mjrSamplerType sampler_type;
+
+  // The format of the pixels in the texture (e.g. RGB8, RGBA8, KTX, etc.)
+  mjrPixelFormat format;
+
+  // The color space of the texture (e.g. LINEAR, sRGB, etc.)
+  mjrColorSpace color_space;
+};
+
+// Initializes the mjrTextureConfig to default values.
+void mjr_defaultTextureConfig(mjrTextureConfig* config);
+
+// Creates a texture with the given configuration. Note that the texture will
+// not be created on the GPU until `mjrf_setTextureData()` is called.
+mjrTexture* mjrf_createTexture(mjrfContext* ctx, const mjrTextureConfig* config);
+
+// Destroys the texture.
+void mjrf_destroyTexture(mjrTexture* texture);
+
+// The binary data for a texture.
+struct mjrTextureData {
+  // Pointer to the data. If null, an empty texture will be created.
+  const void* bytes;
+
+  // The number of bytes in the image data.
+  mjtSize nbytes;
+
+  // Because rendering may be multithreaded, we cannot make assumptions about
+  // when the image data will finish uploading to the GPU. As such, we will use
+  // this callback to notify callers when it is safe to free the image data.
+  void (*release_callback)(void* user_data);
+
+  // User data to pass to the release callback.
+  void* user_data;
+};
+
+// Initializes the mjrTextureData to default values.
+void mjr_defaultTextureData(mjrTextureData* data);
+
+// Uploads the given texture data to the texture.
+void mjrf_setTextureData(mjrTexture* texture, const mjrTextureData* data);
+
+// Returns the width of the texture.
+int mjrf_getTextureWidth(const mjrTexture* texture);
+
+// Returns the height of the texture.
+int mjrf_getTextureHeight(const mjrTexture* texture);
+
+// Returns the target type of the texture.
+mjrSamplerType mjrf_getSamplerType(const mjrTexture* texture);
+
+// ## Meshes (mjrMesh)
+//
+// A mesh describes the surface geometry of an object to be rendered. It is
+// defined as a collection of vertices (i.e. a VertexBuffer), a set of indices
+// (i.e. an IndexBuffer) that describes the order in which the vertices should
+// be processed, and a primitive type that defined how the vertices are to be
+// interpreted (e.g. triangles, lines, etc.) when rendering the surface.
+//
+// Filament does not directly support normals. Instead, it encodes the normal,
+// tangen, and bitangent into a 4-component quaternion describing the
+// "orientation" of the vertex. Ideally, you should preprocess your assets
+// to generate this data offline, but we will compute it on the fly if needed
+// (at a performance cost).
+//
+// We also suggest precomputing the bounds of the mesh, otherwise we will also
+// compute it on the fly.
+
+// The usage/purpose of an attribute of a vertex.
+typedef enum mjrVertexAttributeUsage_ {
+  mjVERTEX_ATTRIBUTE_USAGE_POSITION = 0,
+  mjVERTEX_ATTRIBUTE_USAGE_NORMAL,
+  mjVERTEX_ATTRIBUTE_USAGE_TANGENTS,
+  mjVERTEX_ATTRIBUTE_USAGE_UV,
+  mjVERTEX_ATTRIBUTE_USAGE_COLOR,
+} mjrVertexAttributeUsage;
+
+// The data format of an attribute of a vertex.
+typedef enum mjrVertexAttributeType_ {
+  mjVERTEX_ATTRIBUTE_TYPE_FLOAT2 = 0,
+  mjVERTEX_ATTRIBUTE_TYPE_FLOAT3,
+  mjVERTEX_ATTRIBUTE_TYPE_FLOAT4,
+  mjVERTEX_ATTRIBUTE_TYPE_UBYTE4,
+} mjrVertexAttributeType;
+
+// The type of data stored in an index buffer.
+typedef enum mjrIndexType_ {
+  mjINDEX_TYPE_U16 = 0,
+  mjINDEX_TYPE_U32,
+} mjrIndexType;
+
+// The type of primitive to be drawn by vertex data.
+typedef enum mjrMeshPrimitiveType_ {
+  mjMESH_PRIMITIVE_TYPE_TRIANGLES = 0,
+  mjMESH_PRIMITIVE_TYPE_LINES,
+} mjrMeshPrimitiveType;
+
+// Information about a single attribute of a vertex.
+struct mjrVertexAttribute {
+  // The data for the attribute.
+  const void* bytes;
+
+  // The usage/purpose of the attribute.
+  mjrVertexAttributeUsage usage;
+
+  // The data format of the attribute.
+  mjrVertexAttributeType type;
+};
+
+// Maximum number of vertex attributes in a mesh.
+enum { mjMAX_VERTEX_ATTRIBUTES = 16 };
+
+// The binary contents of a mesh.
+struct mjrMeshData {
+  // The number of vertices in the mesh. Each of the vertex arrays below is
+  // assumed to have this number of elements.
+  mjtSize nvertices;
+
+  // The number of attributes for each vertex in the mesh.
+  int nattributes;
+
+  // Information about each attribute of a vertex in the mesh. See `interleaved`
+  // for more details.
+  mjrVertexAttribute attributes[mjMAX_VERTEX_ATTRIBUTES];
+
+  // Whether the vertex attributes are interleaved or not.
+  //
+  // If true, assumes that the attributes are packed in the order specified in
+  // the attributes array, with no padding in-between. Additionally, the
+  // `data` pointer for each attribute is assumed to point to the first element
+  // of that type.
+  //
+  // If false, assume each attribute is stored in a separate array as defined
+  // by the `data` field of the attribute.
+  mjtByte interleaved;
+
+  // The number of indices in the mesh. The indices array is assumed to have
+  // this number of elements.
+  mjtSize nindices;
+
+  // The indices of the mesh, stored as either ushort or uint depending on the
+  // index type.
+  const void* indices;
+
+  // The type of data stored in the indices array.
+  mjrIndexType index_type;
+
+  // The type of primitive to be drawn by vertex data.
+  mjrMeshPrimitiveType primitive_type;
+
+  // Whether to compute the bounds of the mesh using the vertex positions.
+  mjtByte compute_bounds;
+
+  // The bounds of the mesh. If bounds_min == bounds_max, then we assume that
+  // that the bounds are not set (i.e. the bounds is empty).
+  float bounds_min[3];
+  float bounds_max[3];
+
+  // Because rendering may be multithreaded, we cannot make assumptions about
+  // when the mesh data will finish uploading to the GPU. As such, we will use
+  // this callback to notify callers when it is safe to free the mesh data.
+  void (*release_callback)(void* user_data);
+
+  // User data to pass to the release callback.
+  void* user_data;
+};
+
+// Initializes the mjrMeshData to default values.
+void mjr_defaultMeshData(mjrMeshData* data);
+
+// Creates a mesh with the given data.
+mjrMesh* mjrf_createMesh(mjrfContext* ctx, const mjrMeshData* data);
+
+// Destroys the mesh.
+void mjrf_destroyMesh(mjrMesh* mesh);
+
+// ## Scenes (mjrScene)
+//
+// A scene is a collection of entities (Lights and Renderables) that defines
+// what is to be rendered. It also specifies the various effects that are to be
+// applied to the rendering (e.g. shadows, reflections, post-processing, etc.)
+
+// Configuration parameters for a Scene.
+struct mjrSceneParams {
+  // Whether or not to enable post processing; enabled by default.
+  mjtByte enable_post_processing;
+
+  // Whether or not to enable reflections; enabled by default.
+  mjtByte enable_reflections;
+
+  // Whether or not to enable shadows; enabled by default.
+  mjtByte enable_shadows;
+
+  // This mask, in conjunction with the layer mask in the Renderable, determines
+  // which Renderables to render within the Scene.
+  uint8_t layer_mask;
+
+  // The layer mask to use for reflections.
+  uint8_t reflection_layer_mask;
+};
+
+// Initializes the mjrSceneParams to default values.
+void mjr_defaultSceneParams(mjrSceneParams* params);
+
+// Creates a scene with the given parameters.
+mjrScene* mjrf_createScene(mjrfContext* ctx, const mjrSceneParams* params);
+
+// Destroys the scene.
+void mjrf_destroyScene(mjrScene* scene);
+
+// Adds a light to the scene.
+void mjrf_addLightToScene(mjrScene* scene, mjrLight* light);
+
+// Removes the light from the scene.
+void mjrf_removeLightFromScene(mjrScene* scene, mjrLight* light);
+
+// Adds a renderable to the scene.
+void mjrf_addRenderableToScene(mjrScene* scene, mjrRenderable* renderable);
+
+// Removes the renderable from the scene.
+void mjrf_removeRenderableFromScene(mjrScene* scene, mjrRenderable* renderable);
+
+// Sets the skybox (cube texture) for the scene.
+void mjrf_setSceneSkybox(mjrScene* scene, const mjrTexture* texture);
+
+// Enables (or disables) shadows in the scene.
+void mjrf_setSceneShadowsEnabled(mjrScene* scene, mjtByte enabled);
+
+// Enables (or disables) reflections in the scene.
+void mjrf_setSceneReflectionsEnabled(mjrScene* scene, mjtByte enabled);
+
+// Configures the scene based on the parameters in an mjModel.
+void mjrf_configureSceneFromModel(mjrScene* scene, const mjModel* model);
+
+// ## Lights (mjrLight)
+//
+// A light is a source of illumination in the scene. (Without lights, a scene
+// will be completely black.) There are several different types of lights such
+// as directional, spot, point, and image lights.
+//
+// The primary light in a scene is the image light (also sometimes known as the
+// environment light). This is a light that "surrounds" the entire scene and
+// is defined as a 3D texture. Each "pixel" of the cubemap is interpreted as the
+// color of projected into the scene from a particular direction.
+//
+// Directional lights are the next most common type of light and is usually
+// used to simulate the sun; a uniformly colored light that is emitted in a
+// single direction.
+//
+// Filament only supports a single image and directional light. You can define
+// as many point or spot lights as you want. Each light source (except image
+// based lights) may or may not cast shadows. Each shadow-casting light incurs a
+// performance cost.
+
+// The type of light (spot, directional, image, etc.).
+typedef mjtLightType mjrLightType;
+
+// Configuration parameters for a light.
+struct mjrLightParams {
+  // The type of light (e.g. spot, point, directional, etc.)
+  mjrLightType type;
+  // The texture to use for image lights.
+  const mjrTexture* texture;
+  // The color of the light.
+  float color[3];
+  // The intensity of the light, in candela.
+  float intensity;
+  // Whether or not the light casts shadows.
+  mjtByte cast_shadows;
+  // The range/distance in which the light is effective, in meters.
+  float range;
+  // The angle of the spot light cone, in degrees.
+  float spot_cone_angle;
+  // The radius of the bulb used for soft shadows.
+  float bulb_radius;
+  // The size of the shadow map.
+  int shadow_map_size;
+  // Blur width for EL VSM.
+  float vsm_blur_width;
+};
+
+// Initializes the mjrLightParams to default values.
+void mjr_defaultLightParams(mjrLightParams* params);
+
+// Creates a light for the filament renderer.
+mjrLight* mjrf_createLight(mjrfContext* ctx, const mjrLightParams* params);
+
+// Destroys the light.
+void mjrf_destroyLight(mjrLight* light);
+
+// Enables or disables the light.
+void mjrf_setLightEnabled(mjrLight* light, mjtByte enabled);
+
+// Sets the intensity of the light, in candela.
+void mjrf_setLightIntensity(mjrLight* light, float intensity);
+
+// Sets the RGB color of the light.
+void mjrf_setLightColor(mjrLight* light, const float color[3]);
+
+// Sets the position and direction of the light.
+void mjrf_setLightTransform(mjrLight* light, const float position[3],
+                            const float direction[3]);
+
+// Returns the type of the light.
+mjrLightType mjrf_getLightType(const mjrLight* light);
+
+// ## Renderables (mjrRenderable)
+//
+// A renderable is a single drawable object in the scene. It is defined as a
+// combination of a mesh (i.e. surface geometry) and a material (i.e. surface
+// appearance and properties).
+//
+// In terms of materials, there are three lighting models currently supported:
+//
+// 1. Metallic-roughness (PBR): this is the preferred model for rendering
+//    models based standard metallic-roughness workflows.
+// 2. Specular-glossiness (non-PBR): this is a legacy model designed to be
+//    compatible with classic mjr renderer, though it is not 100% identical.
+// 3. Unlit: this model ignores lighting and used for rendering UX or decorative
+//    elements like contact forces and labels.
+//
+// Which lighting model is used is determined by the mjrMaterial properties.
+
+// The material to be applied to a renderable.
+struct mjrMaterial {
+  // The color of the object. Defaults to white.
+  float color[4];
+
+  // The color to use for segmentation rendering. Defaults to white.
+  float segmentation_color[4];
+
+  // Applies an addition scale to the UV coordinates of the object. Defaults to
+  // (1, 1, 1).
+  float uv_scale[3];
+
+  // Applies an offset to the UV coordinates of the object. Defaults to (0, 0,
+  // 0).
+  float uv_offset[3];
+
+  // Applies a scissor test to the object.
+  float scissor[4];
+
+  // Factors for PBR metallic-roughness materials.
+  float metallic;
+  float roughness;
+
+  // Factors for (non-PBR) specular-glossiness materials.
+  float specular;
+  float glossiness;
+
+  // The emissive (glow) factor of the object.
+  float emissive;
+
+  // Whether or not the object is a reflective surface. Only applies to planes.
+  mjtByte reflective;
+  // The blend factor to use for reflective surfaces. A value of 1.0 means that
+  // the surface is fully reflective (i.e. a mirror).
+  float reflectance;
+
+  // If true, does not apply any lighting to the object. Assumes the object is
+  // used for UX or decorative elements like contact forces and labels.
+  mjtByte decor_ux;
+
+  // The texture containing the base color of the object.
+  const mjrTexture* color_texture;
+
+  // The normal map of the object.
+  const mjrTexture* normal_texture;
+
+  // The metallic map of the object.
+  const mjrTexture* metallic_texture;
+
+  // The roughness map of the object.
+  const mjrTexture* roughness_texture;
+
+  // The occlusion map of the object.
+  const mjrTexture* occlusion_texture;
+
+  // A texture containing the occlusion, roughness, and metallic maps packed
+  // into the R, G, B channels, respectively.
+  const mjrTexture* orm_texture;
+
+  // An emissive texture for the object.
+  const mjrTexture* emissive_texture;
+
+  // The reflection texture to use for the object. For internal use only.
+  const mjrTexture* reflection_texture;
+};
+
+// Initializes the mjrMaterial to default values.
+void mjr_defaultMaterial(mjrMaterial* material);
+
+// Configuration parameters for a Renderable.
+struct mjrRenderableParams {
+  // Whether or not the Renderable casts shadows.
+  mjtByte cast_shadows;
+  // Whether or not the Renderable receives shadows.
+  mjtByte receive_shadows;
+  // The layers to which the Renderable belongs. This mask is used in
+  // conjunction with the layer mask in the Scene to determine which
+  // Renderables to render. Defaults to 0xff.
+  uint8_t layer_mask;
+  // Controls the order in which the Renderable is drawn relative to other
+  // Renderables; defaults to 4.
+  uint8_t priority;
+  // Similar to priority, but provides finer-grained control for Renderables
+  // with transparency; defaults to 0.
+  uint16_t blend_order;
+};
+
+// Initializes the mjrRenderableParams to default values.
+void mjr_defaultRenderableParams(mjrRenderableParams* params);
+
+// Creates a renderable with the given parameters.
+mjrRenderable* mjrf_createRenderable(mjrfContext* ctx,
+                                     const mjrRenderableParams* params);
+
+// Destroys the renderable.
+void mjrf_destroyRenderable(mjrRenderable* renderable);
+
+// Sets the mesh of the renderable.
+void mjrf_setRenderableMesh(mjrRenderable* renderable, const mjrMesh* mesh,
+                            int elem_offset, int elem_count);
+
+// Sets the mesh of the renderable to a built-in mesh based on the geom type.
+// Note: using the same parameters (nstack, nslice, nquad) will have better
+// performance as the internal mesh data can be shared across renderables.
+void mjrf_setRenderableGeomMesh(mjrRenderable* renderable, mjtGeom type,
+                                int nstack, int nslice, int nquad);
+
+// Sets the material properties and textures of the renderable.
+void mjrf_setRenderableMaterial(mjrRenderable* renderable,
+                                const mjrMaterial* material);
+
+// Sets the transform (position, rotation, and size) of the renderable. Note
+// that `size` is not the same as `scale`. For example, the z-size of a capsule
+// only scales the tubular-portion of its geometry, but not the spherical caps.
+void mjrf_setRenderableTransform(mjrRenderable* renderable,
+                                 const float position[3],
+                                 const float rotation[9], const float size[3]);
+
+// Sets whether the renderable casts shadows or not.
+void mjrf_setRenderableCastShadows(mjrRenderable* renderable,
+                                   mjtByte cast_shadows);
+
+// Sets whether the renderable receives shadows or not.
+void mjrf_setRenderableReceiveShadows(mjrRenderable* renderable,
+                                      mjtByte receive_shadows);
+
+// Forces the renderable to be rendered using lines.
+void mjrf_setRenderableWireframe(mjrRenderable* renderable, mjtByte wireframe);
+
+// Sets the layer mask of the renderable. See mjrRenderableParams for details.
+void mjrf_setRenderableLayerMask(mjrRenderable* renderable, uint8_t layer_mask);
+
+// ## Render Targets (mjrRenderTarget)
+//
+// A render target is a memory buffer that holds the results of a rendering
+// operation. (This is an alternative to rendering directly to the screen.)
+// See mjrf_render for more details.
+
+// Defines the basic properties of a render target.
+struct mjrRenderTargetConfig {
+  // The width of the render target.
+  int width;
+  // The height of the render target.
+  int height;
+  // The format of the color buffer in the render target.
+  mjrPixelFormat color_format;
+  // The format of the depth buffer in the render target.
+  mjrPixelFormat depth_format;
+};
+
+// Initializes the RenderTargetConfig to default values.
+void mjr_defaultRenderTargetConfig(mjrRenderTargetConfig* config);
+
+// Creates a render target for the filament renderer.
+mjrRenderTarget* mjrf_createRenderTarget(mjrfContext* ctx,
+                                         const mjrRenderTargetConfig* config);
+
+// Destroys the render target.
+void mjrf_destroyRenderTarget(mjrRenderTarget* render_target);
+
+// ## Debug-only functions.
+
+// Draws an ImGui editor for the given scene, exposing filament-specific
+// settings.
+void mjrf_DEBUG_drawImguiEditor(mjrScene* scene);
+
+// Legacy API, to be deprecated.
+
 void mjrf_defaultFilamentConfig(mjrFilamentConfig* config);
 
 void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con,
@@ -61,7 +767,7 @@ void mjrf_makeContext(const mjModel* m, mjrContext* con, int fontscale);
 
 void mjrf_freeContext(mjrContext* con);
 
-void mjrf_render(mjrRect viewport, mjvScene* scn, const mjrContext* con);
+void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con);
 
 void mjrf_uploadMesh(const mjModel* m, const mjrContext* con, int meshid);
 

src/experimental/filament/render_context_filament_cpp.h

@@ -0,0 +1,74 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_CPP_H_
+#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_CPP_H_
+
+#include <memory>
+#include <string>
+
+#include "experimental/filament/render_context_filament.h"
+
+namespace mujoco {
+
+// A unique pointer to a mujoco object.
+template <typename T>
+using UniquePtr = std::unique_ptr<T, void (*)(T*)>;
+
+inline UniquePtr<mjrfContext> CreateContext(const mjrFilamentConfig& config) {
+  mjrfContext* context = mjrf_createContext(&config);
+  return UniquePtr<mjrfContext>(context, mjrf_destroyContext);
+}
+
+inline UniquePtr<mjrTexture> CreateTexture(mjrfContext* ctx,
+                                           const mjrTextureConfig& config) {
+  mjrTexture* texture = mjrf_createTexture(ctx, &config);
+  return UniquePtr<mjrTexture>(texture, mjrf_destroyTexture);
+}
+
+inline UniquePtr<mjrMesh> CreateMesh(mjrfContext* ctx,
+                                     const mjrMeshData& data) {
+  mjrMesh* mesh = mjrf_createMesh(ctx, &data);
+  return UniquePtr<mjrMesh>(mesh, mjrf_destroyMesh);
+}
+
+inline UniquePtr<mjrScene> CreateScene(mjrfContext* ctx,
+                                       const mjrSceneParams& params) {
+  mjrScene* scene = mjrf_createScene(ctx, &params);
+  return UniquePtr<mjrScene>(scene, mjrf_destroyScene);
+}
+
+inline UniquePtr<mjrLight> CreateLight(mjrfContext* ctx,
+                                       const mjrLightParams& params) {
+  mjrLight* light = mjrf_createLight(ctx, &params);
+  return UniquePtr<mjrLight>(light, mjrf_destroyLight);
+}
+
+inline UniquePtr<mjrRenderable> CreateRenderable(
+    mjrfContext* ctx, const mjrRenderableParams& params) {
+  mjrRenderable* renderable = mjrf_createRenderable(ctx, &params);
+  return UniquePtr<mjrRenderable>(renderable, mjrf_destroyRenderable);
+}
+
+inline UniquePtr<mjrRenderTarget> CreateRenderTarget(
+    mjrfContext* ctx, const mjrRenderTargetConfig& config) {
+  mjrRenderTarget* render_target = mjrf_createRenderTarget(ctx, &config);
+  return UniquePtr<mjrRenderTarget>(render_target, mjrf_destroyRenderTarget);
+}
+
+std::string ResolveFilamentAssetPath(const std::string& filename);
+
+}  // namespace mujoco
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_CPP_H_

src/experimental/mjz/CMakeLists.txt

@@ -1,37 +0,0 @@
-# Copyright 2026 DeepMind Technologies Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cmake_minimum_required(VERSION 3.16)
-
-set(MUJOCO_MJZ_TARGET_NAME mujoco_mjz)
-
-add_library(${MUJOCO_MJZ_TARGET_NAME} STATIC)
-
-target_include_directories(${MUJOCO_MJZ_TARGET_NAME} PRIVATE
-    ../..
-)
-
-target_sources(${MUJOCO_MJZ_TARGET_NAME}
-    PUBLIC
-        mjz_decoder.cc
-)
-
-include(third_party_deps/miniz)
-
-target_link_libraries(${MUJOCO_MJZ_TARGET_NAME}
-    miniz
-    mujoco::mujoco
-)
-
-add_library(mujoco::mjz ALIAS ${MUJOCO_MJZ_TARGET_NAME})

src/experimental/mjz/mjz_decoder.cc

@@ -1,214 +0,0 @@
-// Copyright 2026 DeepMind Technologies Limited
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <cstdarg>
-#include <cstdio>
-#include <cstring>
-#include <filesystem>
-#include <string>
-#include <string_view>
-#include <span>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include <miniz_zip.h>
-#include <mujoco/mjspec.h>
-#include <mujoco/mujoco.h>
-#include "user/user_resource.h"
-
-static void mjPRINTFLIKE(3, 4)
-    SetError(char* error, int error_sz, const char* format, ...) {
-  if (error) {
-    va_list args;
-    va_start(args, format);
-    std::vsnprintf(error, error_sz, format, args);
-    va_end(args);
-  }
-}
-
-// A mjpResourceProvider that reads files from a zip archive.
-//
-// The zip archive itself is provided as a byte buffer in the constructor. This
-// class can then be used to lazily read the contents of the individual files
-// from within the archive as needed.
-class ZipArchiveProvider : public mjpResourceProvider {
- public:
-  ZipArchiveProvider(std::string name, const void* buffer, int nbuffer,
-                     char* error, int error_sz)
-      : name_(std::move(name)), buffer_((char*)buffer, (char*)buffer + nbuffer) {
-    mjp_defaultResourceProvider(this);
-
-    std::memset(&archive_, 0, sizeof(archive_));
-    if (!mz_zip_reader_init_mem(&archive_, buffer_.data(), nbuffer, 0)) {
-      SetError(error, error_sz, "Zip error: invalid zip archive");
-      return;
-    }
-
-    // Create an index of the files in the archive.
-    const int num_files = mz_zip_reader_get_num_files(&archive_);
-    for (int i = 0; i < num_files; ++i) {
-      mz_zip_archive_file_stat stat;
-      if (!mz_zip_reader_file_stat(&archive_, i, &stat)) {
-        SetError(error, error_sz, "Zip error: failed to stat item %d.", i);
-        files_.clear();
-        return;
-      }
-      const int size = static_cast<int>(stat.m_uncomp_size);
-      if (size == 0) {
-        continue;
-      }
-      files_[stat.m_filename] = FileInfo{i, size, {}};
-    }
-
-    // Look for the root XML model in the archive. First look for an XML file
-    // with the same name as the archive itself. Failing that, look for an XML
-    // file within a subdirectory with the same name as the archive.
-    const std::filesystem::path path(name_);
-    root_model_ = (path / path.stem()).string() + ".xml";
-    if (!Contains(root_model_)) {
-      root_model_ = (path / path.stem() / path.stem()).string() + ".xml";
-      if (!Contains(root_model_)) {
-        SetError(error, error_sz, "Zip error: no root XML file found.");
-        return;
-      }
-    }
-
-    // Setup mjpResourceProvider callbacks.
-    mount = [](mjResource* resource) {
-      return 0;
-    };
-    unmount = [](mjResource* resource) {
-      ZipArchiveProvider* self = (ZipArchiveProvider*)resource->provider;
-      delete self;
-      return 0;
-    };
-    open = [](mjResource* resource) {
-      ZipArchiveProvider* self = (ZipArchiveProvider*)resource->provider;
-      const bool found = self->Contains(resource->name);
-      return found ? 1 : 0;
-    };
-    read = [](mjResource* resource, const void** buffer) {
-      ZipArchiveProvider* self = (ZipArchiveProvider*)resource->provider;
-      std::span<char> bytes = self->Read(resource->name);
-      *buffer = bytes.data();
-      return static_cast<int>(bytes.size());
-    };
-    close = [](mjResource* resource) {
-      // no-op
-    };
-  }
-
-  ~ZipArchiveProvider() {
-    mz_zip_reader_end(&archive_);
-  }
-
-  ZipArchiveProvider(const ZipArchiveProvider&) = delete;
-  ZipArchiveProvider& operator=(const ZipArchiveProvider&) = delete;
-
-  // Returns the path to the root XML model in the archive.
-  std::string GetRootModelPath() const {
-    return root_model_;
-  }
-
-  // Returns true if the archive contains a file with the given name/path.
-  bool Contains(std::string_view name) const {
-    const std::string_view filename = name.substr(name_.size() + 1);
-    return files_.find(filename.data()) != files_.end();
-  }
-
-  // Reads the contents of the file with the given name/path. The contents are
-  // cached internally so that subsequent reads for the same file do not need to
-  // re-read the file from the archive.
-  std::span<char> Read(const std::string& name) {
-    const std::string filename = name.substr(name_.size() + 1);
-    auto it = files_.find(filename);
-    if (it == files_.end()) {
-      return {};
-    }
-
-    FileInfo& info = it->second;
-
-    // Lazily read and store the file contents from the archive.
-    if (info.contents.empty()) {
-      info.contents.resize(info.size);
-      if (!mz_zip_reader_extract_to_mem(&archive_, info.index,
-                                        info.contents.data(), info.size, 0)) {
-        return {};
-      }
-    }
-    return info.contents;
-  }
-
- private:
-  struct FileInfo {
-    // Index of the file in the archive.
-    int index = 0;
-
-    // Size (in bytes) of the uncompressed file
-    int size = 0;
-
-    // Contents of the uncompressed file.
-    std::vector<char> contents;
-  };
-
-  std::string name_;
-  std::string root_model_;
-  mz_zip_archive archive_;
-  std::vector<char> buffer_;
-  std::unordered_map<std::string, FileInfo> files_;
-};
-
-static mjSpec* ParseZipBuffer(const void* buffer, int nbuffer, const char* name,
-                              mjVFS* vfs, char* error, int error_sz) {
-  if (error) {
-    error[0] = 0;
-  }
-
-  ZipArchiveProvider* provider =
-      new ZipArchiveProvider(name, buffer, nbuffer, error, error_sz);
-  if (error && error[0]) {
-    return nullptr;
-  }
-
-  const int status = mj_mountVFS(vfs, name, provider);
-  if (status != 0) {
-    SetError(error, error_sz, "Failed to mount zip archive: %s", name);
-    return nullptr;
-  }
-
-  const std::string root = provider->GetRootModelPath();
-  return mj_parseXML(root.c_str(), vfs, error, error_sz);
-}
-
-mjPLUGIN_LIB_INIT {
-  mjpDecoder decoder;
-  decoder.content_type = "application/zip";
-  decoder.extension = ".mjz|.zip";
-  decoder.can_decode = +[](const mjResource* resource) {
-    const char* ext = strrchr(resource->name, '.');
-    return ext ? (!strcmp(ext, ".mjz") || !strcmp(ext, ".zip")) : 0;
-  };
-  decoder.decode = +[](mjResource* resource, const mjVFS* vfs) -> mjSpec* {
-    const char* buffer = nullptr;
-    const int size = mju_readResource(resource, (const void**)&buffer);
-    if (size <= 0) {
-      return nullptr;
-    }
-    char error[1024];
-    return ParseZipBuffer(buffer, size, resource->name, const_cast<mjVFS*>(vfs),
-                          error, sizeof(error));
-  };
-  mjp_registerDecoder(&decoder);
-}

src/experimental/platform/CMakeLists.txt

@@ -20,67 +20,67 @@ add_library(${MUJOCO_PLATFORM_TARGET_NAME} STATIC)
 
 target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
     PUBLIC
-        egl_utils.cc
-        egl_utils.h
-        enum_utils.h
-        file_dialog.h
-        graphics_mode.cc
-        graphics_mode.h
-        gui.cc
-        gui.h
-        gui_spec.cc
-        gui_spec.h
         helpers.cc
         helpers.h
-        imgui_widgets.cc
-        imgui_widgets.h
-        interaction.cc
-        interaction.h
-        model_holder.cc
-        model_holder.h
-        picture_gui.h
-        picture_gui.cc
-        plugin.cc
-        plugin.h
-        renderer.cc
-        renderer.h
-        sim_history.cc
-        sim_history.h
-        sim_profiler.cc
-        sim_profiler.h
-        spec_editor.cc
-        spec_editor.h
-        step_control.cc
-        step_control.h
-        window.cc
-        window.h
+        hal/egl_utils.cc
+        hal/egl_utils.h
+        hal/graphics_mode.cc
+        hal/graphics_mode.h
+        hal/renderer.cc
+        hal/renderer.h
+        hal/window.cc
+        hal/window.h
+        sim/model_holder.cc
+        sim/model_holder.h
+        sim/sim_history.cc
+        sim/sim_history.h
+        sim/sim_profiler.cc
+        sim/sim_profiler.h
+        sim/step_control.cc
+        sim/step_control.h
+        ux/enum_utils.h
+        ux/file_dialog.h
+        ux/gui.cc
+        ux/gui.h
+        ux/gui_spec.cc
+        ux/gui_spec.h
+        ux/imgui_widgets.cc
+        ux/imgui_widgets.h
+        ux/interaction.cc
+        ux/interaction.h
+        ux/picture_gui.h
+        ux/picture_gui.cc
+        ux/plugin.cc
+        ux/plugin.h
+        ux/spec_editor.cc
+        ux/spec_editor.h
 )
 
 if(NOT WIN32)
   target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
       PUBLIC
-          object_launcher_plugin.cc
+          ux/object_launcher_plugin.cc
   )
 endif()
 
 if(APPLE)
-    set_source_files_properties(window_osx.mm PROPERTIES
+    set_source_files_properties(hal/window_osx.mm PROPERTIES
         COMPILE_FLAGS "-x objective-c++")
     target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC
-        window_osx.mm
-        file_dialog_cocoa.mm
+        hal/window_osx.mm
+        ux/file_dialog_cocoa.mm
     )
 elseif(UNIX AND NOT APPLE)
     target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC
-        file_dialog_zenity.cc
+        ux/file_dialog_zenity.cc
     )
 elseif(WIN32)
     target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC
-        file_dialog_win.cc
+        ux/file_dialog_win.cc
     )
 else()
     target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC
-        file_dialog.cc
+        ux/file_dialog.cc
     )
 endif()
 

src/experimental/platform/hal/egl_utils.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/egl_utils.h"
+#include "experimental/platform/hal/egl_utils.h"
 
 #include <memory>
 

src/experimental/platform/hal/egl_utils.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_EGL_UTILS_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_EGL_UTILS_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_EGL_UTILS_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_EGL_UTILS_H_
 
 #include <memory>
 
@@ -25,4 +25,4 @@ std::shared_ptr<void> CreateEglContext();
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_EGL_UTILS_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_EGL_UTILS_H_

src/experimental/platform/hal/graphics_mode.cc

@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/graphics_mode.h"
+#include "experimental/platform/hal/graphics_mode.h"
 
 #include <string_view>
+
 #include <mujoco/mujoco.h>
 
 namespace mujoco::platform {
@@ -27,6 +28,7 @@ bool IsClassic(GraphicsMode gfx_mode) {
 bool IsFilament(GraphicsMode gfx_mode) {
   return gfx_mode == GraphicsMode::FilamentOpenGl ||
          gfx_mode == GraphicsMode::FilamentVulkan ||
+         gfx_mode == GraphicsMode::FilamentVulkanSoftware ||
          gfx_mode == GraphicsMode::FilamentWebGl ||
          gfx_mode == GraphicsMode::FilamentOpenGlHeadless ||
          gfx_mode == GraphicsMode::FilamentOpenGlSoftware;
@@ -41,7 +43,8 @@ bool IsOpenGl(GraphicsMode gfx_mode) {
 }
 
 bool IsVulkan(GraphicsMode gfx_mode) {
-  return gfx_mode == GraphicsMode::FilamentVulkan;
+  return gfx_mode == GraphicsMode::FilamentVulkan ||
+         gfx_mode == GraphicsMode::FilamentVulkanSoftware;
 }
 
 bool IsWebGl(GraphicsMode gfx_mode) {
@@ -55,7 +58,8 @@ bool IsHeadless(GraphicsMode gfx_mode) {
 }
 
 bool IsSoftware(GraphicsMode gfx_mode) {
-  return gfx_mode == GraphicsMode::FilamentOpenGlSoftware;
+  return gfx_mode == GraphicsMode::FilamentOpenGlSoftware ||
+         gfx_mode == GraphicsMode::FilamentVulkanSoftware;
 }
 
 GraphicsMode GraphicsModeFromString(std::string_view str,
@@ -68,6 +72,8 @@ GraphicsMode GraphicsModeFromString(std::string_view str,
     return GraphicsMode::FilamentOpenGl;
   } else if (str == "vulkan") {
     return GraphicsMode::FilamentVulkan;
+  } else if (str == "vulkan_software") {
+    return GraphicsMode::FilamentVulkanSoftware;
   } else if (str == "webgl") {
     return GraphicsMode::FilamentWebGl;
   } else if (str == "opengl_headless") {

src/experimental/platform/hal/graphics_mode.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_GRAPHICS_MODE_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_GRAPHICS_MODE_H_
 
 #include <string_view>
 
@@ -33,6 +33,9 @@ enum class GraphicsMode {
   // The Filament-based renderer running on Vulkan.
   FilamentVulkan,
 
+  // The Filament-based renderer running on Vulkan using software rendering.
+  FilamentVulkanSoftware,
+
   // The Filament-based renderer running on WebGL.
   FilamentWebGl,
 
@@ -56,4 +59,4 @@ GraphicsMode GraphicsModeFromString(std::string_view str,
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_GRAPHICS_MODE_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_GRAPHICS_MODE_H_

src/experimental/platform/hal/renderer.cc

@@ -12,23 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/renderer.h"
+#include "experimental/platform/hal/renderer.h"
 
 #include <chrono>
 #include <cstddef>
 #include <span>
 #include <utility>
 
-#include <mujoco/mujoco.h>
-
-#include <imgui.h>
 #include <backends/imgui_impl_opengl3.h>
+#include <imgui.h>
+#include <mujoco/mujoco.h>
 #if !defined(__EMSCRIPTEN__) && !defined(__APPLE__)
-#include "experimental/platform/egl_utils.h"
+#include "experimental/platform/hal/egl_utils.h"
 #endif
 #include "experimental/filament/render_context_filament.h"
-#include "experimental/platform/graphics_mode.h"
-#include "experimental/platform/plugin.h"
+#include "experimental/platform/hal/graphics_mode.h"
+#include "experimental/platform/ux/plugin.h"
 
 namespace mujoco::platform {
 
@@ -47,9 +46,9 @@ Renderer::Renderer(void* native_window, GraphicsMode gfx)
     : native_window_(native_window), gfx_(gfx) {
   if (IsClassic(gfx_)) {
     if (native_window == nullptr) {
-      #if !defined(__EMSCRIPTEN__) && !defined(__APPLE__)
-        graphics_api_context_ = CreateEglContext();
-      #endif
+#if !defined(__EMSCRIPTEN__) && !defined(__APPLE__)
+      graphics_api_context_ = CreateEglContext();
+#endif
     }
     if (ImGui::GetCurrentContext()) {
       ImGui_ImplOpenGL3_Init();
@@ -91,11 +90,12 @@ void Renderer::Init(const mjModel* model) {
       render_config.width = model->vis.global.offwidth;
       render_config.height = model->vis.global.offheight;
       render_config.force_software_rendering = IsSoftware(gfx_);
-      render_config.graphics_api =
-          IsOpenGl(gfx_) || IsWebGl(gfx_) ? mjGFX_OPENGL : mjGFX_VULKAN;
+      render_config.graphics_api = IsOpenGl(gfx_) || IsWebGl(gfx_)
+                                       ? mjGRAPHICS_API_OPENGL
+                                       : mjGRAPHICS_API_VULKAN;
       mjrf_makeFilamentContext(model, &render_context_, &render_config);
       render_ = [&](mjrRect rect, mjvScene* scene) {
-        mjrf_render(rect, scene, &render_context_);
+        mjrf_renderScene(rect, scene, &render_context_);
       };
       set_buffer_ = [&](int framebuffer) {
         mjrf_setBuffer(framebuffer, &render_context_);
@@ -147,8 +147,7 @@ void Renderer::Render(const mjModel* model, mjData* data,
     vis_option = &default_opt;
   }
 
-  mjv_updateScene(model, data, vis_option, perturb, camera, mjCAT_ALL,
-                  &scene_);
+  mjv_updateScene(model, data, vis_option, perturb, camera, mjCAT_ALL, &scene_);
 
   const bool render_to_texture = !pixels.empty();
   if (render_to_texture) {
@@ -215,8 +214,8 @@ int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
     return 0;
   } else {
     return mjrf_uploadGuiImage(texture_id,
-                              reinterpret_cast<const unsigned char*>(pixels),
-                              width, height, bpp, &render_context_);
+                               reinterpret_cast<const unsigned char*>(pixels),
+                               width, height, bpp, &render_context_);
   }
 }
 
@@ -240,7 +239,7 @@ void Renderer::UpdateFps() {
 
 }  // namespace mujoco::platform
 
-mjPLUGIN_LIB_INIT {
+mjPLUGIN_LIB_INIT(renderer) {
   mujoco::platform::GuiPlugin plugin;
   plugin.name = "Filament";
   plugin.update = [](mujoco::platform::GuiPlugin* self) {

src/experimental/platform/hal/renderer.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_RENDERER_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_RENDERER_H_
 
 #include <chrono>
 #include <cstddef>
@@ -23,7 +23,7 @@
 #include <span>
 
 #include <mujoco/mujoco.h>
-#include "experimental/platform/graphics_mode.h"
+#include "experimental/platform/hal/graphics_mode.h"
 
 namespace mujoco::platform {
 
@@ -113,4 +113,4 @@ class Renderer {
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RENDERER_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_RENDERER_H_

src/experimental/platform/hal/renderer_test.cc

@@ -0,0 +1,127 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "experimental/platform/hal/renderer.h"
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "third_party/mujoco/google/gfx/opengl_dynamic_loader.h"
+#include <mujoco/mujoco.h>
+#include "experimental/platform/hal/graphics_mode.h"
+#include "experimental/platform/sim/model_holder.h"
+
+#include "testing/base/public/gunit.h"
+
+namespace mujoco::platform {
+namespace {
+
+class RendererTest : public ::testing::Test {
+ public:
+  void SetUp() {
+    mjSpec* spec = mj_makeSpec();
+    // Set a clear color.
+    const double clear_color[] = {1.0, 1.0, 1.0, 1.0};
+    mjsNumeric* numeric = mjs_addNumeric(spec);
+    mjs_setName(numeric->element, "filament.clearColor");
+    numeric->size = 4;
+    mjs_setDouble(numeric->data, clear_color, numeric->size);
+    holder_ = ModelHolder::FromSpec(spec);
+    holder_->model()->vis.global.offwidth = width_;
+    holder_->model()->vis.global.offheight = height_;
+  }
+
+  void Test();
+
+  int width_ = 2;
+  int height_ = 2;
+  std::unique_ptr<ModelHolder> holder_;
+};
+
+TEST_F(RendererTest, OpengGlSoftware) {
+  Renderer renderer(nullptr, GraphicsMode::FilamentOpenGlSoftware);
+  renderer.Init(holder_->model());
+
+  gl::DriverType driver_type = gl::GetLoadedDriverType();
+  ASSERT_EQ(driver_type, gl::DriverType::kOsMesa);
+
+  std::vector<std::byte> pixels(width_ * height_ * 3);
+  renderer.Render(holder_->model(), holder_->data(), nullptr, nullptr, nullptr,
+              width_, height_, pixels);
+  // We set the clear color to white, but we don't know the exact color due to
+  // post processing, but it should definitely not be black.
+  for (int i = 0; i < pixels.size(); i += 3) {
+    EXPECT_NE((int)pixels[i + 0], 0);
+    EXPECT_NE((int)pixels[i + 1], 0);
+    EXPECT_NE((int)pixels[i + 2], 0);
+  }
+}
+
+TEST_F(RendererTest, OpengGlHeadless) {
+  Renderer renderer(nullptr, GraphicsMode::FilamentOpenGlHeadless);
+  renderer.Init(holder_->model());
+
+  gl::DriverType driver_type = gl::GetLoadedDriverType();
+  #if TEST_HAS_GPU
+  ASSERT_EQ(driver_type, gl::DriverType::kEgl);
+  #else
+  ASSERT_EQ(driver_type, gl::DriverType::kOsMesa);
+  #endif
+
+  std::vector<std::byte> pixels(width_ * height_ * 3);
+  renderer.Render(holder_->model(), holder_->data(), nullptr, nullptr, nullptr,
+              width_, height_, pixels);
+  // We set the clear color to white, but we don't know the exact color due to
+  // post processing, but it should definitely not be black.
+  for (int i = 0; i < pixels.size(); i += 3) {
+    EXPECT_NE((int)pixels[i + 0], 0);
+    EXPECT_NE((int)pixels[i + 1], 0);
+    EXPECT_NE((int)pixels[i + 2], 0);
+  }
+}
+
+TEST_F(RendererTest, VulkanSoftware) {
+  Renderer renderer(nullptr, GraphicsMode::FilamentVulkanSoftware);
+  renderer.Init(holder_->model());
+
+  std::vector<std::byte> pixels(width_ * height_ * 3);
+  renderer.Render(holder_->model(), holder_->data(), nullptr, nullptr, nullptr,
+              width_, height_, pixels);
+  // We set the clear color to white, but we don't know the exact color due to
+  // post processing, but it should definitely not be black.
+  for (int i = 0; i < pixels.size(); i += 3) {
+    EXPECT_NE((int)pixels[i + 0], 0);
+    EXPECT_NE((int)pixels[i + 1], 0);
+    EXPECT_NE((int)pixels[i + 2], 0);
+  }
+}
+
+TEST_F(RendererTest, VulkanHeadless) {
+  Renderer renderer(nullptr, GraphicsMode::FilamentVulkan);
+  renderer.Init(holder_->model());
+
+  std::vector<std::byte> pixels(width_ * height_ * 3);
+  renderer.Render(holder_->model(), holder_->data(), nullptr, nullptr, nullptr,
+              width_, height_, pixels);
+  // We set the clear color to white, but we don't know the exact color due to
+  // post processing, but it should definitely not be black.
+  for (int i = 0; i < pixels.size(); i += 3) {
+    EXPECT_NE((int)pixels[i + 0], 0);
+    EXPECT_NE((int)pixels[i + 1], 0);
+    EXPECT_NE((int)pixels[i + 2], 0);
+  }
+}
+
+}  // namespace
+}  // namespace mujoco::platform

src/experimental/platform/hal/window.cc

@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/window.h"
+#include "experimental/platform/hal/window.h"
 
 #include <algorithm>
 #include <cstddef>
-#include <span>
 #include <string>
 #include <string_view>
 
@@ -32,10 +31,9 @@
 #include <backends/imgui_impl_sdl2.h>
 #include <imgui.h>
 #include <mujoco/mujoco.h>
-#include "experimental/platform/graphics_mode.h"
+#include "experimental/platform/hal/graphics_mode.h"
 #include "user/user_resource.h"
 
-
 // Because X11/Xlib.h defines Status.
 #ifdef Status
 #undef Status
@@ -47,8 +45,8 @@ extern void* GetNativeWindowOsx(void* window);
 
 namespace mujoco::platform {
 
-static void InitImGui(SDL_Window* window, float content_scale, bool load_fonts,
-                      bool build_fonts) {
+static void InitImGui(SDL_Window* window, float content_scale,
+                      bool load_fonts) {
   ImGui::CreateContext();
 
   ImGuiIO& io = ImGui::GetIO();
@@ -85,10 +83,6 @@ static void InitImGui(SDL_Window* window, float content_scale, bool load_fonts,
     constexpr ImWchar icon_ranges[] = {0xf000, 0xf3ff, 0x000};
     io.Fonts->AddFontFromMemoryTTF(data, size, 14.f, &icon_cfg, icon_ranges);
 
-    if (build_fonts) {
-      io.Fonts->Build();
-    }
-
     // Note: we purposefully do not "close" the font resources as ImGui may
     // need them again to resize fonts.
   }
@@ -125,17 +119,18 @@ Window::Window(std::string_view title, int width, int height, Config config)
     mju_error("Unsupported window config: %d", config_.gfx_mode);
   }
 
-  const float content_scale = ImGui_ImplSDL2_GetContentScaleForDisplay(0);
-  sdl_window_ = SDL_CreateWindow(title.data(), SDL_WINDOWPOS_UNDEFINED,
-                                 SDL_WINDOWPOS_UNDEFINED, width,
-                                 height, window_flags);
+  const float content_scale =
+      std::max(1.0f, ImGui_ImplSDL2_GetContentScaleForDisplay(0));
+  width_ = width * content_scale;
+  height_ = height * content_scale;
+  sdl_window_ =
+      SDL_CreateWindow(title.data(), SDL_WINDOWPOS_UNDEFINED,
+                       SDL_WINDOWPOS_UNDEFINED, width_, height_, window_flags);
   if (!sdl_window_) {
     mju_error("Error creating window: %s", SDL_GetError());
   }
 
-  InitImGui(sdl_window_, content_scale, config.load_fonts,
-            (config_.gfx_mode != GraphicsMode::ClassicOpenGl &&
-             config_.gfx_mode != GraphicsMode::ClassicOpenGlHeadless));
+  InitImGui(sdl_window_, content_scale, config.load_fonts);
 
   // Filament (except WebGL) manages its own swap chain including when to swap.
   // In all other cases, we'll use SDL to manage the swap chain.
@@ -156,18 +151,18 @@ Window::Window(std::string_view title, int width, int height, Config config)
     SDL_VERSION(&wmi.version);
     SDL_GetWindowWMInfo(sdl_window_, &wmi);
 
-    #if defined(__linux__)
-      native_window_ = reinterpret_cast<void*>(wmi.info.x11.window);
-    #elif defined(__WIN32__)
-      native_window_ = reinterpret_cast<void*>(wmi.info.win.window);
-    #elif defined(__APPLE__)
-      native_window_ =
-          GetNativeWindowOsx(reinterpret_cast<void*>(wmi.info.cocoa.window));
-    #endif
+#if defined(__linux__)
+    native_window_ = reinterpret_cast<void*>(wmi.info.x11.window);
+#elif defined(__WIN32__)
+    native_window_ = reinterpret_cast<void*>(wmi.info.win.window);
+#elif defined(__APPLE__)
+    native_window_ =
+        GetNativeWindowOsx(reinterpret_cast<void*>(wmi.info.cocoa.window));
+#endif
   }
 
-  int drawable_width = width;
-  int drawable_height = height;
+  int drawable_width = width_;
+  int drawable_height = height_;
   SDL_GL_GetDrawableSize(sdl_window_, &drawable_width, &drawable_height);
   scale_ = (float)drawable_width / (float)width_;
 }
@@ -182,6 +177,18 @@ void Window::SetTitle(std::string_view title) {
   SDL_SetWindowTitle(sdl_window_, title.data());
 }
 
+void Window::Resize(int width, int height) {
+  SDL_SetWindowSize(sdl_window_, width, height);
+  SDL_SetWindowPosition(sdl_window_, SDL_WINDOWPOS_CENTERED,
+                        SDL_WINDOWPOS_CENTERED);
+
+  SDL_GetWindowSize(sdl_window_, &width_, &height_);
+  int drawable_width = width_;
+  int drawable_height = height_;
+  SDL_GL_GetDrawableSize(sdl_window_, &drawable_width, &drawable_height);
+  scale_ = (float)drawable_width / (float)width_;
+}
+
 void Window::DisableWindowResizing() {
   SDL_SetWindowResizable(sdl_window_, SDL_FALSE);
 }
@@ -244,8 +251,7 @@ void Window::Present(std::span<const std::byte> pixels) {
     SDL_Surface* surface = SDL_GetWindowSurface(sdl_window_);
     const unsigned char* src =
         reinterpret_cast<const unsigned char*>(pixels.data());
-    unsigned char* dst =
-        static_cast<unsigned char*>(surface->pixels);
+    unsigned char* dst = static_cast<unsigned char*>(surface->pixels);
 
     for (int i = 0; i < height_; ++i) {
       for (int j = 0; j < width_; ++j) {
@@ -260,14 +266,12 @@ void Window::Present(std::span<const std::byte> pixels) {
     }
 
     SDL_RenderPresent(sdl_renderer_);
-  } else if (config_.gfx_mode != GraphicsMode::FilamentVulkan
-     && config_.gfx_mode != GraphicsMode::FilamentOpenGl) {
+  } else if (config_.gfx_mode != GraphicsMode::FilamentVulkan &&
+             config_.gfx_mode != GraphicsMode::FilamentOpenGl) {
     SDL_GL_SwapWindow(sdl_window_);
   }
 }
 
-GraphicsMode Window::GetGraphicsMode() const {
-  return config_.gfx_mode;
-}
+GraphicsMode Window::GetGraphicsMode() const { return config_.gfx_mode; }
 
 }  // namespace mujoco::platform

src/experimental/platform/hal/window.h

@@ -12,17 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_WINDOW_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_WINDOW_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_WINDOW_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_WINDOW_H_
 
 #include <cstddef>
 #include <span>
 #include <string>
 #include <string_view>
 
-#include <SDL_video.h>
 #include <SDL_render.h>
-#include "experimental/platform/graphics_mode.h"
+#include <SDL_video.h>
+#include "experimental/platform/hal/graphics_mode.h"
 
 namespace mujoco::platform {
 
@@ -79,6 +79,9 @@ class Window {
   // Returns the graphics configuration of the window.
   GraphicsMode GetGraphicsMode() const;
 
+  // Resizes the window to the given width and height.
+  void Resize(int width, int height);
+
   // Enables window resizing.
   void EnableWindowResizing();
 
@@ -104,4 +107,4 @@ class Window {
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_WINDOW_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_HAL_WINDOW_H_

src/experimental/platform/helpers.cc

@@ -14,6 +14,7 @@
 
 #include "experimental/platform/helpers.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstdint>
 #include <cstdio>
@@ -23,6 +24,7 @@
 #include <ios>
 #include <iterator>
 #include <string>
+#include <vector>
 
 #include "webp/encode.h"
 #include "webp/types.h"
@@ -77,8 +79,13 @@ std::string ResolveFile(const std::string& filename,
       return resolved;
     }
 
+    std::vector<std::filesystem::path> entries;
     for (const auto& it : std::filesystem::recursive_directory_iterator(path)) {
-      resolved = CheckPathForFile(it.path(), filename);
+      entries.push_back(it.path());
+    }
+    std::sort(entries.begin(), entries.end());
+    for (const auto& entry : entries) {
+      resolved = CheckPathForFile(entry, filename);
       if (!resolved.empty()) {
         return resolved;
       }

src/experimental/platform/sim/model_holder.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/model_holder.h"
+#include "experimental/platform/sim/model_holder.h"
 
 #include <cstddef>
 #include <cstring>

src/experimental/platform/sim/model_holder.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_MODEL_HOLDER_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_MODEL_HOLDER_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_MODEL_HOLDER_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_MODEL_HOLDER_H_
 
 #include <memory>
 #include <cstddef>
@@ -81,4 +81,4 @@ class ModelHolder {
 };
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_MODEL_HOLDER_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_MODEL_HOLDER_H_

src/experimental/platform/sim/sim_history.cc

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/sim_history.h"
+#include "experimental/platform/sim/sim_history.h"
 
 #include <algorithm>
 #include <climits>
 #include <span>
+
 #include <mujoco/mujoco.h>
 
 namespace mujoco::platform {

src/experimental/platform/sim/sim_history.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_HISTORY_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_HISTORY_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_HISTORY_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_HISTORY_H_
 
 #include <span>
 #include <vector>
@@ -91,4 +91,4 @@ class SimHistory {
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_HISTORY_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_HISTORY_H_

src/experimental/platform/sim/sim_profiler.cc

@@ -12,17 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/sim_profiler.h"
+#include "experimental/platform/sim/sim_profiler.h"
 
-#include <mujoco/mujoco.h>
 #include <imgui.h>
 #include <implot.h>
+#include <mujoco/mujoco.h>
+#include "experimental/platform/ux/imgui_widgets.h"
 
 namespace mujoco::platform {
 
-SimProfiler::SimProfiler() {
-  Clear();
-}
+SimProfiler::SimProfiler() { Clear(); }
 
 void SimProfiler::Clear() {
   constexpr int kProfilerMaxFrames = 200;
@@ -88,21 +87,21 @@ void SimProfiler::Update(const mjModel* model, const mjData* data) {
   // Solver diagnostics.
   mjtNum sqrt_nnz = 0;
   int solver_niter = 0;
-  const int nisland = data->nefc ? mjMAX(1, mjMIN(data->nisland, mjNISLAND)) : 0;
-  for (int island=0; island < nisland; island++) {
+  const int nisland =
+      data->nefc ? mjMAX(1, mjMIN(data->nisland, mjNISLAND)) : 0;
+  for (int island = 0; island < nisland; island++) {
     sqrt_nnz += data->solver_nnz[island];
     solver_niter += data->solver_niter[island];
   }
   sqrt_nnz = mju_sqrt(sqrt_nnz);
 
   dim_dof_.erase(dim_dof_.begin());
-  int nv = (model->opt.enableflags & mjENBL_SLEEP) ? data->nv_awake
-                                                     : model->nv;
+  int nv = (model->opt.enableflags & mjENBL_SLEEP) ? data->nv_awake : model->nv;
   dim_dof_.push_back(nv);
 
   dim_body_.erase(dim_body_.begin());
   int nbody = (model->opt.enableflags & mjENBL_SLEEP) ? data->nbody_awake
-                                                        : model->nbody;
+                                                      : model->nbody;
   dim_body_.push_back(nbody);
 
   dim_constraint_.erase(dim_constraint_.begin());
@@ -115,16 +114,17 @@ void SimProfiler::Update(const mjModel* model, const mjData* data) {
   dim_contact_.push_back(data->ncon);
 
   dim_iteration_.erase(dim_iteration_.begin());
-  dim_iteration_.push_back(static_cast<float>(solver_niter) / nisland);
+  dim_iteration_.push_back(static_cast<float>(solver_niter) /
+                           mjMAX(1, nisland));
 }
 
-
-void SimProfiler::CpuTimeGraph() {
-  if (ImPlot::BeginPlot("CPU Time", ImVec2(-1, 0), ImPlotFlags_NoMouseText)) {
+void SimProfiler::CpuTimeGraph(ImVec2 plot_size) {
+  ImPlotFlags flags =
+      ImPlot_SetupPlotFlags(plot_size) | ImPlotFlags_NoMouseText;
+  if (ImPlot::BeginPlot("CPU msec vs frame", plot_size, flags)) {
     ImPlot::PushStyleVar(ImPlotStyleVar_LineWeight, 2.0f);
-    ImPlot::SetupAxis(ImAxis_X1, "frame", ImPlotAxisFlags_AutoFit);
-    ImPlot::SetupAxis(ImAxis_Y1, "msec", ImPlotAxisFlags_AutoFit);
-    ImPlot::SetupAxisFormat(ImAxis_Y1, "%.2f");
+    ImPlot_SetupTimeAxis(plot_size, "");
+    ImPlot_SetupValueAxis(plot_size, "", "%.2f");
     ImPlot::SetupLegend(ImPlotLocation_NorthEast);
     ImPlot::SetupFinish();
 
@@ -143,12 +143,13 @@ void SimProfiler::CpuTimeGraph() {
   }
 }
 
-void SimProfiler::DimensionsGraph() {
-  if (ImPlot::BeginPlot("Dimensions", ImVec2(-1, 0), ImPlotFlags_NoMouseText)) {
+void SimProfiler::DimensionsGraph(ImVec2 plot_size) {
+  ImPlotFlags flags =
+      ImPlot_SetupPlotFlags(plot_size) | ImPlotFlags_NoMouseText;
+  if (ImPlot::BeginPlot("Dimensions vs frame", plot_size, flags)) {
     ImPlot::PushStyleVar(ImPlotStyleVar_LineWeight, 2.0f);
-    ImPlot::SetupAxis(ImAxis_X1, "frame", ImPlotAxisFlags_AutoFit);
-    ImPlot::SetupAxis(ImAxis_Y1, "count", ImPlotAxisFlags_AutoFit);
-    ImPlot::SetupAxisFormat(ImAxis_Y1, "%.0f");
+    ImPlot_SetupTimeAxis(plot_size, "");
+    ImPlot_SetupValueAxis(plot_size, "", "%.0f");
     ImPlot::SetupLegend(ImPlotLocation_NorthEast);
     ImPlot::SetupFinish();
 

src/experimental/platform/sim/sim_profiler.h

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_PROFILER_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_PROFILER_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_PROFILER_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_PROFILER_H_
 
 #include <vector>
 
+#include <imgui.h>
 #include <mujoco/mujoco.h>
 
 namespace mujoco::platform {
@@ -33,8 +34,8 @@ class SimProfiler {
   void Update(const mjModel* model, const mjData* data);
 
   // Displays the profiling data using ImPlot.
-  void CpuTimeGraph();
-  void DimensionsGraph();
+  void CpuTimeGraph(ImVec2 plot_size = ImVec2(-1, 0));
+  void DimensionsGraph(ImVec2 plot_size = ImVec2(-1, 0));
 
  private:
   std::vector<float> cpu_total_;
@@ -52,4 +53,4 @@ class SimProfiler {
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_PROFILER_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_SIM_PROFILER_H_

src/experimental/platform/sim/step_control.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "experimental/platform/step_control.h"
+#include "experimental/platform/sim/step_control.h"
 
 #include <algorithm>
 #include <chrono>
@@ -83,10 +83,14 @@ void StepControl::SetNoiseParameters(float ctrl_noise_scale,
   ctrl_noise_rate_ = ctrl_noise_rate;
 }
 
-void StepControl::SetPauseState(PauseState state, mjModel* m) {
+void StepControl::SetPauseState(PauseState state) {
   pause_state_ = state;
 }
 
+StepControl::PauseState StepControl::GetPauseState() const {
+  return pause_state_;
+}
+
 StepControl::Status StepControl::Advance(mjModel* m, mjData* d) {
   if (!m) {
     return Status::kOk;
@@ -184,7 +188,7 @@ StepControl::Status StepControl::Advance(mjModel* m, mjData* d) {
       for (mjtWarning w : kDivergedWarnings) {
         if (d->warning[w].number > 0) {
           // Stop stepping if the simulation diverged.
-          pause_state_ = PauseState::kNormalPaused;
+          SetPauseState(PauseState::kNormalPaused);
           return Status::kDiverged;
         }
       }

src/experimental/platform/sim/step_control.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_STEP_CONTROL_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_STEP_CONTROL_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_STEP_CONTROL_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_STEP_CONTROL_H_
 
 #include <chrono>
 #include <string>
@@ -70,11 +70,10 @@ class StepControl {
   enum class PauseState { kUnpaused, kNormalPaused, kViscousPaused };
 
   // Sets the pause state of the simulation.
-  // m must be non-null for viscous pausing.
-  void SetPauseState(PauseState state, mjModel* m = nullptr);
+  void SetPauseState(PauseState state);
 
   // Gets the current pause state of the simulation.
-  PauseState GetPauseState() const { return pause_state_; }
+  PauseState GetPauseState() const;
 
   // If the simulation is paused, will perform a single step on the next
   // Advance() call.
@@ -123,4 +122,4 @@ class StepControl {
 
 }  // namespace mujoco::platform
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_STEP_CONTROL_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_SIM_STEP_CONTROL_H_

src/experimental/platform/ux/enum_utils.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_ENUM_UTILS_H_
-#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_ENUM_UTILS_H_
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_UX_ENUM_UTILS_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_UX_ENUM_UTILS_H_
 
 #include <array>
 #include <bit>
@@ -167,4 +167,4 @@ constexpr std::string_view enum_to_string(E value) {
 
 }  // namespace mujoco::platform::enum_utils
 
-#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_ENUM_UTILS_H_
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_UX_ENUM_UTILS_H_