From b083ae487e5523e5b0216c90490acb3f8c5d2793 Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Tue, 9 Jun 2026 08:31:11 -0700 Subject: [PATCH] Introduce outline render passes and shader. Use the jump flood algorithm to render an outline around selected objects. This requires using custom post-processing shaders and render passes which is managed by a new "Outliner" class. PiperOrigin-RevId: 929212147 Change-Id: Ie6f475c7c677df7b0d4fc030a8e9520ff9dd03af --- src/experimental/filament/CMakeLists.txt | 5 + .../filament/assets/outline_composite.mat | 82 ++++++ .../filament/assets/outline_flatten.mat | 54 ++++ .../filament/assets/outline_jumpflood.mat | 110 ++++++++ .../filament/compat/scene_geom_util.cc | 6 + .../filament/filament/material_manager.cc | 4 +- .../filament/filament/object_manager.cc | 39 ++- .../filament/filament/object_manager.h | 12 +- .../filament/filament/outliner.cc | 246 ++++++++++++++++++ src/experimental/filament/filament/outliner.h | 87 +++++++ .../filament/filament/renderable.cc | 22 +- .../filament/filament/renderable.h | 3 + .../filament/filament/scene_view.cc | 66 ++++- .../filament/filament/scene_view.h | 14 +- .../filament/render_context_filament.cc | 3 +- src/experimental/studio/live.js | 3 + 16 files changed, 726 insertions(+), 30 deletions(-) create mode 100644 src/experimental/filament/assets/outline_composite.mat create mode 100644 src/experimental/filament/assets/outline_flatten.mat create mode 100644 src/experimental/filament/assets/outline_jumpflood.mat create mode 100644 src/experimental/filament/filament/outliner.cc create mode 100644 src/experimental/filament/filament/outliner.h diff --git a/src/experimental/filament/CMakeLists.txt b/src/experimental/filament/CMakeLists.txt index 0a514f7f..7c8a6224 100644 --- a/src/experimental/filament/CMakeLists.txt +++ b/src/experimental/filament/CMakeLists.txt @@ -43,6 +43,8 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME} filament/mesh.h filament/object_manager.cc filament/object_manager.h + filament/outliner.cc + filament/outliner.h filament/reflection_manager.cc filament/reflection_manager.h filament/render_target.cc @@ -117,6 +119,9 @@ set(MATERIAL_FILES phong_cube_fade.mat phong_cube.mat phong_cube_reflect.mat + outline_composite.mat + outline_flatten.mat + outline_jumpflood.mat unlit_decor.mat unlit_depth.mat unlit_segmentation.mat diff --git a/src/experimental/filament/assets/outline_composite.mat b/src/experimental/filament/assets/outline_composite.mat new file mode 100644 index 00000000..8d42edbe --- /dev/null +++ b/src/experimental/filament/assets/outline_composite.mat @@ -0,0 +1,82 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This material uses the results of the Jump Flooding Algorithm to draw an +// outline around an object. The outline is drawn by checking the distance to +// the nearest edge point (as computed by the algorithm) and, if within the +// desired width, setting the color of the pixel to the desired outline color. + +material { + name : outline_composite, + blending : transparent, + culling : none, + depthWrite : false, + depthCulling : false, + parameters : [ + { type : sampler2d, name : source }, + { type : float4, name : color }, + { type : float, name : width } + ], + variables : [ + vertex + ], + domain : postprocess +} + +vertex { + void postProcessVertex(inout PostProcessVertexInputs postProcess) { + postProcess.vertex.xy = uvToRenderTargetUV(postProcess.normalizedUV); + } +} + +fragment { + vec2 unpack(vec4 value) { + float x = value.r + value.g / 255.0; + float y = value.b + value.a / 255.0; + // Reverse scale and bias + return (vec2(x, y) - 0.001) / 0.999; + } + + void postProcess(inout PostProcessInputs postProcess) { + vec4 tex = texture(materialParams_source, variable_vertex.xy); + if (tex == vec4(0.0)) { + discard; + } + + vec2 edge = unpack(tex); + + // Calculate distance to the nearest edge in pixel units. + vec2 size = vec2(textureSize(materialParams_source, 0)); + vec2 diff = (edge - variable_vertex.xy) * size; + float dist = length(diff); + + // Exclude the interior of the selected object itself. + if (dist <= 0.5) { + discard; + } + + // Only draw outline if the pixel is within 'width' of an edge. + float width = materialParams.width; + if (dist >= width) { + discard; + } + + // Inner edge anti-aliasing (smooths transition from object to outline). + // Outer edge anti-aliasing (fades at the outline boundary). + float inner_alpha = smoothstep(0.0, 1.0, dist); + float outer_alpha = smoothstep(width, width - 1.0, dist); + float alpha = outer_alpha * inner_alpha; + postProcess.color = mix(vec4(0), materialParams.color, alpha * materialParams.color.a); + } +} diff --git a/src/experimental/filament/assets/outline_flatten.mat b/src/experimental/filament/assets/outline_flatten.mat new file mode 100644 index 00000000..3fa4a529 --- /dev/null +++ b/src/experimental/filament/assets/outline_flatten.mat @@ -0,0 +1,54 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// This material writes the the screen-space position of each fragment as the +// output color of that fragment. This will provide us with the initial data for +// the Jump Flood Algorithm. See "outline_jumpflood.mat" for more details. +material { + name : outline_flatten, + shadingModel : unlit, + culling : none, + depthWrite : false +} + +fragment { + // Encode a vec2 value with the range [0-1] into a 32-bit (RGBA) value. + // We do this by packing each coordinate into a 16-bit value. Values are + // adjusted to ensure [0x0000] is reserved for "untouched" pixels rather + // than the pixel at the position (0, 0). + vec4 pack(vec2 value) { + // Scale and bias the value to avoid (0,0,0,0). + vec2 adjusted = value * 0.999 + 0.001; + + // Pack float [0, 1] into two 8-bit channels. + float val_x = adjusted.x * 255.0; + float r_x = floor(val_x) / 255.0; + float g_x = fract(val_x); + + float val_y = adjusted.y * 255.0; + float r_y = floor(val_y) / 255.0; + float g_y = fract(val_y); + + return vec4(r_x, g_x, r_y, g_y); + } + + void material(inout MaterialInputs material) { + prepareMaterial(material); + + // Get the screen-space position of the fragment, normalized to [0-1]. + vec2 uv = gl_FragCoord.xy * frameUniforms.resolution.zw; + material.baseColor = pack(uv); + } +} diff --git a/src/experimental/filament/assets/outline_jumpflood.mat b/src/experimental/filament/assets/outline_jumpflood.mat new file mode 100644 index 00000000..e513bc5c --- /dev/null +++ b/src/experimental/filament/assets/outline_jumpflood.mat @@ -0,0 +1,110 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This material implements a single pass of the Jump Flooding Algorithm (JFA). +// This algorithm is used to efficiently compute a distance transform where each +// pixel stores the UV coordinates of its closest edge point. In each pass, +// pixels look at their neighbors at a distance 'step' to see if they have a +// closer edge point. +material { + name : outline_jumpflood, + culling : none, + depthWrite : false, + depthCulling : false, + parameters : [ + { type : sampler2d, name : source }, + { type : float, name : step } + ], + variables : [ + vertex + ], + domain : postprocess +} + +vertex { + void postProcessVertex(inout PostProcessVertexInputs postProcess) { + postProcess.vertex.xy = uvToRenderTargetUV(postProcess.normalizedUV); + } +} + +fragment { + // Encode a vec2 value with the range [0-1] into a 32-bit (RGBA) value. + // We do this by packing each coordinate into a 16-bit value. Values are + // adjusted to ensure [0x0000] is reserved for "untouched" pixels rather + // than the pixel at the position (0, 0). + vec4 pack(vec2 value) { + // Scale and bias the value to avoid (0,0,0,0). + vec2 adjusted = value * 0.999 + 0.001; + + // Pack float [0, 1] into two 8-bit channels. + float val_x = adjusted.x * 255.0; + float r_x = floor(val_x) / 255.0; + float g_x = fract(val_x); + + float val_y = adjusted.y * 255.0; + float r_y = floor(val_y) / 255.0; + float g_y = fract(val_y); + + return vec4(r_x, g_x, r_y, g_y); + } + + vec2 unpack(vec4 value) { + float x = value.r + value.g / 255.0; + float y = value.b + value.a / 255.0; + // Reverse scale and bias + return (vec2(x, y) - 0.001) / 0.999; + } + + vec3 jump(vec3 best, vec2 offset) { + vec2 uv = variable_vertex.xy + offset; + if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) { + return best; + } + + vec4 tex = texture(materialParams_source, uv); + if (tex == vec4(0.0)) { + return best; + } + + vec2 edge = unpack(tex); + vec2 diff = (edge - variable_vertex.xy); + float dist_sq = dot(diff, diff); + if (dist_sq < best.z) { + best = vec3(edge.x, edge.y, dist_sq); + } + return best; + } + + void postProcess(inout PostProcessInputs postProcess) { + vec2 size = vec2(textureSize(materialParams_source, 0)); + vec2 dist = materialParams.step / size; + + vec3 best = vec3(-1.0, -1.0, 1e20); + best = jump(best, dist * vec2(+1, +1)); + best = jump(best, dist * vec2(+1, 0)); + best = jump(best, dist * vec2(+1, -1)); + best = jump(best, dist * vec2( 0, +1)); + best = jump(best, dist * vec2( 0, 0)); + best = jump(best, dist * vec2( 0, -1)); + best = jump(best, dist * vec2(-1, +1)); + best = jump(best, dist * vec2(-1, 0)); + best = jump(best, dist * vec2(-1, -1)); + + if (best.x >= 0.0) { + postProcess.color = pack(best.xy); + } else { + postProcess.color = vec4(0.0); + } + } +} diff --git a/src/experimental/filament/compat/scene_geom_util.cc b/src/experimental/filament/compat/scene_geom_util.cc index 945a0137..b9e17703 100644 --- a/src/experimental/filament/compat/scene_geom_util.cc +++ b/src/experimental/filament/compat/scene_geom_util.cc @@ -185,6 +185,12 @@ static void UpdateGeomMaterial(mjrfRenderable* renderable, const mjvGeom& geom, material.segmentation_id = geom.segid; + // Assume an emissive object is a selected object. + if (geom.emission > 0 && geom.emission == model->vis.global.glow) { + material.selected = true; + material.emissive = 0.0f; + } + // UvScale only applies to objects that don't have explicit UV coordinates // in their vertex buffer. Instead, we set the UV coordinate to be the same // as the vertex position. diff --git a/src/experimental/filament/filament/material_manager.cc b/src/experimental/filament/filament/material_manager.cc index 48a2a249..a1a83b6a 100644 --- a/src/experimental/filament/filament/material_manager.cc +++ b/src/experimental/filament/filament/material_manager.cc @@ -193,7 +193,9 @@ MaterialManager::MaterialKey MaterialManager::PrepareMaterialInstance( const mjrfMaterial& material, mjrDrawMode draw_mode, mjtGeom geom_type, const Mesh* mesh) { ObjectManager::MaterialType type; - if (draw_mode == mjDRAW_MODE_DEPTH) { + if (material.selected) { + type = ObjectManager::kOutlineFlatten; + } else if (draw_mode == mjDRAW_MODE_DEPTH) { type = ObjectManager::kUnlitDepth; } else if (draw_mode == mjDRAW_MODE_SEGMENTATION_BY_ID) { type = ObjectManager::kUnlitSegmentation; diff --git a/src/experimental/filament/filament/object_manager.cc b/src/experimental/filament/filament/object_manager.cc index 47630bb6..2f7cffa6 100644 --- a/src/experimental/filament/filament/object_manager.cc +++ b/src/experimental/filament/filament/object_manager.cc @@ -14,10 +14,8 @@ #include "experimental/filament/filament/object_manager.h" -#include #include #include -#include #include #include #include @@ -27,6 +25,7 @@ #include #include #include +#include #include #include "experimental/filament/filament/builtins.h" #include "user/user_resource.h" @@ -73,6 +72,9 @@ ObjectManager::ObjectManager(filament::Engine* engine) materials_[kUnlitDecor] = LoadMaterial(engine, "unlit_decor.filamat"); materials_[kUnlitDepth] = LoadMaterial(engine, "unlit_depth.filamat"); materials_[kUnlitUi] = LoadMaterial(engine, "unlit_ui.filamat"); + materials_[kOutlineComposite] = LoadMaterial(engine, "outline_composite.filamat"); + materials_[kOutlineFlatten] = LoadMaterial(engine, "outline_flatten.filamat"); + materials_[kOutlineJumpFlood] = LoadMaterial(engine, "outline_jumpflood.filamat"); static uint8_t black_rgb[3] = {0, 0, 0}; static uint8_t white_rgb[3] = {255, 255, 255}; @@ -106,9 +108,13 @@ ObjectManager::ObjectManager(filament::Engine* engine) fallback_textures_[mjTEXROLE_NORMAL] = fallback_normal_; fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_; fallback_textures_[mjTEXROLE_ORM] = fallback_orm_; + + CreateQuadBuffers(); } ObjectManager::~ObjectManager() { + engine_->destroy(quad_vb_); + engine_->destroy(quad_ib_); engine_->destroy(fallback_black_); engine_->destroy(fallback_white_); engine_->destroy(fallback_normal_); @@ -148,4 +154,33 @@ const filament::Texture* ObjectManager::GetFallbackTexture( } return fallback_textures_[role]; } + +void ObjectManager::CreateQuadBuffers() { + // Define a single triangle that completely covers the viewport. (A single + // large triangle is more efficient than a two-triangle quad because it + // avoids the diagonal edge where pixels might be rasterized twice.) + static const filament::math::float2 kVertices[3] = { + {-1.0f, -1.0f}, + { 3.0f, -1.0f}, + {-1.0f, 3.0f} + }; + + static const uint16_t kIndices[3] = {0, 1, 2}; + + filament::VertexBuffer::Builder vb_builder; + vb_builder.bufferCount(1); + vb_builder.vertexCount(3); + vb_builder.attribute(filament::VertexAttribute::POSITION, 0, + filament::VertexBuffer::AttributeType::FLOAT2, 0, + sizeof(filament::math::float2)); + quad_vb_ = vb_builder.build(*engine_); + quad_vb_->setBufferAt(*engine_, 0, {kVertices, sizeof(kVertices)}); + + filament::IndexBuffer::Builder ib_builder; + ib_builder.indexCount(3); + ib_builder.bufferType(filament::IndexBuffer::IndexType::USHORT); + quad_ib_ = ib_builder.build(*engine_); + quad_ib_->setBuffer(*engine_, {kIndices, sizeof(kIndices)}); +} + } // namespace mujoco diff --git a/src/experimental/filament/filament/object_manager.h b/src/experimental/filament/filament/object_manager.h index 05ce49d5..022e6f0e 100644 --- a/src/experimental/filament/filament/object_manager.h +++ b/src/experimental/filament/filament/object_manager.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -61,6 +60,9 @@ class ObjectManager { kUnlitDecor, kUnlitDepth, kUnlitUi, + kOutlineComposite, + kOutlineFlatten, + kOutlineJumpFlood, kNumMaterials, }; @@ -70,6 +72,10 @@ class ObjectManager { // Returns the fallback Texture with the given role. const filament::Texture* GetFallbackTexture(mjtTextureRole role) const; + // Returns the buffers for creating a full-screen quad. + filament::VertexBuffer* GetQuadVertexBuffer() const { return quad_vb_; } + filament::IndexBuffer* GetQuadIndexBuffer() const { return quad_ib_; } + // Returns the built-in mesh collection with the given dimensions. For // performance reasons, you should consider always using the same dimensions // in order to reuse the same meshes. @@ -82,6 +88,8 @@ class ObjectManager { ObjectManager& operator=(const ObjectManager&) = delete; private: + void CreateQuadBuffers(); + filament::Engine* engine_ = nullptr; std::array materials_; std::array fallback_textures_; @@ -90,6 +98,8 @@ class ObjectManager { filament::Texture* fallback_black_ = nullptr; filament::Texture* fallback_normal_ = nullptr; filament::Texture* fallback_orm_ = nullptr; + filament::VertexBuffer* quad_vb_ = nullptr; + filament::IndexBuffer* quad_ib_ = nullptr; }; } // namespace mujoco diff --git a/src/experimental/filament/filament/outliner.cc b/src/experimental/filament/filament/outliner.cc new file mode 100644 index 00000000..7534a0b0 --- /dev/null +++ b/src/experimental/filament/filament/outliner.cc @@ -0,0 +1,246 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/outliner.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/texture.h" +#include "experimental/filament/render_context_filament.h" + +namespace mujoco { + +Outliner::Outliner(ObjectManager* object_mgr, uint8_t layer_mask, + filament::math::float4 color, float thickness) + : object_mgr_(object_mgr), + engine_(object_mgr->GetEngine()), + layer_mask_(layer_mask), + color_(color), + thickness_(thickness) {} + +Outliner::~Outliner() { + Reset(); +} + +void Outliner::Prepare(int width, int height) { + if (width == width_ && height == height_) { + return; + } + Reset(); + width_ = width; + height_ = height; + if (width_ <= 0 || height_ <= 0) { + return; + } + + auto& em = utils::EntityManager::get(); + + // Sets up a pass with a full-screen quad that renders with a given material. + auto setup_fullscreen = [&](int pass, ObjectManager::MaterialType type) { + filament::Material* material = object_mgr_->GetMaterial(type); + filament::VertexBuffer* quad_vb = object_mgr_->GetQuadVertexBuffer(); + filament::IndexBuffer* quad_ib = object_mgr_->GetQuadIndexBuffer(); + const filament::RenderableManager::PrimitiveType primitive_type = + filament::RenderableManager::PrimitiveType::TRIANGLES; + + material_instances_[pass] = material->createInstance(); + quads_[pass] = em.create(); + filament::RenderableManager::Builder(1) + .geometry(0, primitive_type, quad_vb, quad_ib) + .material(0, material_instances_[pass]) + .culling(false) + .receiveShadows(false) + .castShadows(false) + .build(*engine_, quads_[pass]); + scenes_[pass] = engine_->createScene(); + scenes_[pass]->addEntity(quads_[pass]); + views_[pass]->setScene(scenes_[pass]); + }; + + // Sets up a pass to use the given render target as the source and/or output. + auto bind = [&](int pass, int src, int out) { + // Set the `source` texture as the color texture from a render target. + if (src >= 0) { + const filament::TextureSampler sampler( + filament::TextureSampler::MinFilter::NEAREST, + filament::TextureSampler::MagFilter::NEAREST); + material_instances_[pass]->setParameter( + "source", targets_[src]->GetColorTexture()->GetFilamentTexture(), + sampler); + } + // Set the output of the view to the given render target. + if (out >= 0) { + views_[pass]->setRenderTarget(targets_[out]->GetFilamentRenderTarget()); + } + }; + + // Set up two render targets. We will alternate between the two targets to + // allow for chaining passes together. + for (int i = 0; i < 2; ++i) { + mjrfRenderTargetConfig config; + mjrf_defaultRenderTargetConfig(&config); + config.color_format = mjPIXEL_FORMAT_RGBA8; + config.depth_format = mjPIXEL_FORMAT_DEPTH32F; + config.width = width; + config.height = height; + targets_[i] = std::make_unique(engine_, config); + targets_[i]->Prepare(width, height); + } + + // Setup orthographic camera for full-screen quad rendering. + camera_ = engine_->createCamera(em.create()); + camera_->setProjection(filament::Camera::Projection::ORTHO, -1.0, 1.0, -1.0, + 1.0, -1.0, 1.0); + camera_->lookAt({0, 0, 1}, {0, 0, 0}, {0, 1, 0}); + + // Setup all the views. + for (auto& view : views_) { + view = engine_->createView(); + view->setCamera(camera_); + view->setViewport({0, 0, (uint32_t)width, (uint32_t)height}); + view->setPostProcessingEnabled(false); + view->setShadowingEnabled(false); + view->setMultiSampleAntiAliasingOptions({.enabled = false}); + } + + // In the first pass, we will render a given scene, but only render the + // objects marked as outlines. We assume that the objects have already been + // assigned the kOutlineFlatten material. + views_[kPassFlatten]->setVisibleLayers(0xff, layer_mask_); + + // All subsequent passes are full-screen post-processing passes. + setup_fullscreen(kPassJumpFlood1, ObjectManager::kOutlineJumpFlood); + setup_fullscreen(kPassJumpFlood2, ObjectManager::kOutlineJumpFlood); + setup_fullscreen(kPassJumpFlood3, ObjectManager::kOutlineJumpFlood); + setup_fullscreen(kPassJumpFlood4, ObjectManager::kOutlineJumpFlood); + setup_fullscreen(kPassJumpFlood5, ObjectManager::kOutlineJumpFlood); + setup_fullscreen(kPassDrawOutline, ObjectManager::kOutlineComposite); + + // Chain the passes together such that the output of a pass is the input to + // the next pass. The first pass has no input (we are just rendering the + // selected objects) and the last pass has no output (we are just rendering + // the outline to the externally provided target). + bind(kPassFlatten, -1, 0); + bind(kPassJumpFlood1, 0, 1); + bind(kPassJumpFlood2, 1, 0); + bind(kPassJumpFlood3, 0, 1); + bind(kPassJumpFlood4, 1, 0); + bind(kPassJumpFlood5, 0, 1); + bind(kPassDrawOutline, 1, -1); + + // Bind the parameters for each pass. For the jump flood passes, the step + // parameter determines how far to propagate the outline in each pass. + material_instances_[kPassJumpFlood1]->setParameter("step", 16.0f); + material_instances_[kPassJumpFlood2]->setParameter("step", 8.0f); + material_instances_[kPassJumpFlood3]->setParameter("step", 4.0f); + material_instances_[kPassJumpFlood4]->setParameter("step", 2.0f); + material_instances_[kPassJumpFlood5]->setParameter("step", 1.0f); + + // The final pass renders the actual outline onto a render target. + material_instances_[kPassDrawOutline]->setParameter("color", color_); + material_instances_[kPassDrawOutline]->setParameter("width", thickness_); + + // Commit all the material instances to the engine. + for (auto& material_instance : material_instances_) { + if (material_instance) { + material_instance->commit(*engine_); + } + } +} + +void Outliner::Reset() { + auto& em = utils::EntityManager::get(); + + for (auto& scene : scenes_) { + if (scene) { + engine_->destroy(scene); + scene = nullptr; + } + } + for (auto& quad : quads_) { + if (quad) { + engine_->destroy(quad); + em.destroy(quad); + quad = utils::Entity(); + } + } + for (auto& material_instance : material_instances_) { + if (material_instance) { + engine_->destroy(material_instance); + material_instance = nullptr; + } + } + for (auto& view : views_) { + if (view) { + engine_->destroy(view); + view = nullptr; + } + } + if (camera_) { + utils::Entity entity = camera_->getEntity(); + engine_->destroyCameraComponent(entity); + em.destroy(entity); + camera_ = nullptr; + } + for (auto& target : targets_) { + target.reset(); + } + width_ = 0; + height_ = 0; +} + +void Outliner::Render(filament::Renderer* renderer, + filament::View* view, + filament::RenderTarget* render_target) { + filament::Viewport viewport = view->getViewport(); + Prepare(viewport.width, viewport.height); + + for (auto& view : views_) { + view->setViewport(viewport); + } + + // Re-render the view's scene to create the flattened selection mask. + auto prev_clear_opts = renderer->getClearOptions(); + renderer->setClearOptions({.clearColor = {0, 0, 0, 0}, .clear = true}); + views_[kPassFlatten]->setScene(view->getScene()); + views_[kPassFlatten]->setCamera(&view->getCamera()); + renderer->render(views_[kPassFlatten]); + renderer->setClearOptions(prev_clear_opts); + + // Run the jump flood steps to expand the selection mask + for (int i = 0; i < kNumJumpFloodPasses; ++i) { + renderer->render(views_[kPassJumpFlood1 + i]); + } + + // Render the final pass as an outline onto the provided render target. + views_[kPassDrawOutline]->setRenderTarget(render_target); + renderer->render(views_[kPassDrawOutline]); + views_[kPassDrawOutline]->setRenderTarget(nullptr); +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/outliner.h b/src/experimental/filament/filament/outliner.h new file mode 100644 index 00000000..43338df6 --- /dev/null +++ b/src/experimental/filament/filament/outliner.h @@ -0,0 +1,87 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OUTLINER_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OUTLINER_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/render_target.h" + +namespace mujoco { + +// Renders an outline of selected objects. +// +// This class uses the "jump flood" algorithm to create an outline of selected +// objects. +class Outliner { + public: + Outliner(ObjectManager* object_mgr, uint8_t layer_mask, + filament::math::float4 color, float thickness); + ~Outliner(); + + Outliner(const Outliner&) = delete; + Outliner& operator=(const Outliner&) = delete; + + // Renders the outline of the selected objects in the given view. The outline + // will be rendered on top of the provided render target. This assumes that + // any objects to be outlined have been assigned an `OutlineFlatten` material + // instance. + void Render(filament::Renderer* renderer, filament::View* view, + filament::RenderTarget* render_target); + + private: + void Prepare(int width, int height); + void Reset(); + + enum Pass { + kPassFlatten, + kPassJumpFlood1, + kPassJumpFlood2, + kPassJumpFlood3, + kPassJumpFlood4, + kPassJumpFlood5, + kPassDrawOutline, + kNumPasses, + + kNumJumpFloodPasses = kPassJumpFlood5 - kPassJumpFlood1 + 1, + }; + + ObjectManager* object_mgr_ = nullptr; + filament::Engine* engine_ = nullptr; + uint8_t layer_mask_ = 0xff; + filament::math::float4 color_ = {1.0f, 1.0f, 1.0f, 1.0f}; + float thickness_ = 2.5f; + + int width_ = 0; + int height_ = 0; + + filament::Camera* camera_ = nullptr; + std::unique_ptr targets_[2]; + + filament::View* views_[kNumPasses] = {}; + filament::Scene* scenes_[kNumPasses] = {}; + utils::Entity quads_[kNumPasses] = {}; + filament::MaterialInstance* material_instances_[kNumPasses] = {}; +}; +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OUTLINER_H_ diff --git a/src/experimental/filament/filament/renderable.cc b/src/experimental/filament/filament/renderable.cc index 44186482..3473f2ce 100644 --- a/src/experimental/filament/filament/renderable.cc +++ b/src/experimental/filament/filament/renderable.cc @@ -225,6 +225,9 @@ void Renderable::UpdateMaterial(const mjrfMaterial& material) { if (material.decor_ux) { layer_mask_ = kLayerMask_Decor; } + if (material.selected) { + layer_mask_ |= kLayerMask_Outline; + } SetLayerMask(layer_mask_); material_ = material; } @@ -244,6 +247,7 @@ void Renderable::Prepare(std::span requests, for (const mjrfRenderRequest* request : requests) { DrawState draw_state; mjrfMaterial material = material_; + material.selected = false; draw_state.wireframe = (request->draw_mode == mjDRAW_MODE_WIREFRAME); if (material.decor_ux) { @@ -374,23 +378,23 @@ void Renderable::BindMaterialInstance(const mjrfRenderRequest& request) { SetCastShadows(state.cast_shadows); SetReceiveShadows(state.receive_shadows); SetWireframe(state.wireframe); + SetMaterialInstance(state.material_key); + curr_state_ = state; + draw_queue_.pop_front(); +} - if (state.material_key != curr_state_.material_key) { +MaterialManager::MaterialKey Renderable::SetMaterialInstance(MaterialManager::MaterialKey key) { + MaterialManager::MaterialKey prev = curr_state_.material_key; + if (key != curr_state_.material_key) { filament::MaterialInstance* instance = - material_mgr_->GetInstance(state.material_key); - if (!instance) { - mju_error("Failed to get material instance."); - return; - } + material_mgr_->GetInstance(key); filament::RenderableManager& rm = GetEngine()->getRenderableManager(); for (Part& part : parts_) { filament::RenderableManager::Instance ri = rm.getInstance(part.entity); rm.setMaterialInstanceAt(ri, 0, instance); } } - - curr_state_ = state; - draw_queue_.pop_front(); + return prev; } std::uint8_t Renderable::SetLayerMask(std::uint8_t mask) { diff --git a/src/experimental/filament/filament/renderable.h b/src/experimental/filament/filament/renderable.h index d082c224..be56e482 100644 --- a/src/experimental/filament/filament/renderable.h +++ b/src/experimental/filament/filament/renderable.h @@ -38,6 +38,7 @@ namespace mujoco { enum LayerMask : uint8_t { kLayerMask_Object = 0x01 << 1, kLayerMask_Decor = 0x01 << 2, + kLayerMask_Outline = 0x01 << 3, kLayerMask_All = 0xff, kLayerMask_None = 0x00, }; @@ -113,6 +114,8 @@ class Renderable : public mjrfRenderable { // Binds the material instance for the given render request. void BindMaterialInstance(const mjrfRenderRequest& request); + MaterialManager::MaterialKey SetMaterialInstance(MaterialManager::MaterialKey key); + static Renderable* downcast(mjrfRenderable* renderable) { return static_cast(renderable); } diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index f97341e9..87c3afba 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -38,10 +39,13 @@ #include #include #include +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/outliner.h" #include "experimental/filament/filament/reflection_manager.h" #include "experimental/filament/filament_util.h" #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/material_manager.h" #include "experimental/filament/filament/render_target.h" #include "experimental/filament/filament/renderable.h" #include "experimental/filament/filament/texture.h" @@ -120,9 +124,11 @@ static void SetupReflectionCamera(const mat4& surface_xform, reflection_camera->setCustomProjection(oblique, near, far); } -SceneView::SceneView(filament::Engine* engine, const mjrfSceneParams& params) - : engine_(engine) { - reflection_mgr_ = std::make_unique(engine_); +SceneView::SceneView(ObjectManager* object_mgr, MaterialManager* material_mgr, + const mjrfSceneParams& params) + : object_mgr_(object_mgr), material_mgr_(material_mgr) { + filament::Engine* engine = object_mgr_->GetEngine(); + reflection_mgr_ = std::make_unique(engine); scene_ = engine->createScene(); camera_ = engine->createCamera(utils::EntityManager::get().create()); @@ -157,9 +163,12 @@ SceneView::SceneView(filament::Engine* engine, const mjrfSceneParams& params) } SceneView::~SceneView() { + filament::Engine* engine = object_mgr_->GetEngine(); + + outliner_.reset(); if (skybox_) { scene_->setSkybox(nullptr); - engine_->destroy(skybox_); + engine->destroy(skybox_); } for (auto& light : lights_) { light->RemoveFromScene(scene_); @@ -169,15 +178,15 @@ SceneView::~SceneView() { } lights_.clear(); renderables_.clear(); - engine_->destroyCameraComponent(reflect_camera_->getEntity()); - engine_->destroy(reflect_view_); - engine_->destroyCameraComponent(camera_->getEntity()); + engine->destroyCameraComponent(reflect_camera_->getEntity()); + engine->destroy(reflect_view_); + engine->destroyCameraComponent(camera_->getEntity()); if (color_grading_) { - engine_->destroy(color_grading_); + engine->destroy(color_grading_); } - engine_->destroy(scene_); - engine_->destroy(depth_segment_view_); - engine_->destroy(main_view_); + engine->destroy(scene_); + engine->destroy(depth_segment_view_); + engine->destroy(main_view_); } void SceneView::AddToScene(Light* light) { @@ -225,9 +234,21 @@ void SceneView::PrepareToRender(std::span requests) { } } + bool has_selected = false; reflection_mgr_->ClearRenderables(); for (Renderable* renderable : renderables_) { renderable->Prepare(requests, reflection_mgr_.get()); + if (renderable->GetMaterial().selected) { + has_selected = true; + } + } + + if (has_selected) { + mjrfMaterial material; + mjrf_defaultMaterial(&material); + material.selected = true; + outline_material_key_ = material_mgr_->PrepareMaterialInstance( + material, mjDRAW_MODE_DEFAULT, mjGEOM_NONE, nullptr); } } @@ -259,8 +280,13 @@ void SceneView::Render(filament::Renderer* renderer, const mjrfRenderRequest& re SetupCamera(request.camera, viewport, camera_); + std::vector selected_renderables; for (auto& iter : renderables_) { iter->BindMaterialInstance(request); + if (iter->GetMaterial().selected && + request.draw_mode == mjDRAW_MODE_DEFAULT) { + selected_renderables.push_back(iter); + } } for (int i = 0; i < reflection_mgr_->GetNumRenderables(); ++i) { @@ -285,11 +311,25 @@ void SceneView::Render(filament::Renderer* renderer, const mjrfRenderRequest& re renderable->SetLayerMask(prev_layer_mask); } - view->setRenderTarget(render_target ? render_target->GetFilamentRenderTarget() - : nullptr); + filament::RenderTarget* filament_render_target = + render_target ? render_target->GetFilamentRenderTarget() : nullptr; + view->setRenderTarget(filament_render_target); renderer->render(view); view->setRenderTarget(nullptr); + if (!selected_renderables.empty()) { + if (!outliner_) { + outliner_ = + std::make_unique(object_mgr_, kLayerMask_Outline, + float4{0.9f, 0.9f, 0.2f, 1.0f}, 3.5f); + } + + for (Renderable* renderable : selected_renderables) { + renderable->SetMaterialInstance(outline_material_key_); + } + outliner_->Render(renderer, view, filament_render_target); + } + if (request.target) { view->setMultiSampleAntiAliasingOptions(options); } diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index e4822a35..b72082c1 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -27,6 +27,9 @@ #include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/material_manager.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/outliner.h" #include "experimental/filament/filament/renderable.h" #include "experimental/filament/filament/reflection_manager.h" #include "experimental/filament/filament/texture.h" @@ -41,7 +44,8 @@ namespace mujoco { // (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. class SceneView : public mjrfScene { public: - SceneView(filament::Engine* engine, const mjrfSceneParams& params); + SceneView(ObjectManager* object_mgr, MaterialManager* material_mgr, + const mjrfSceneParams& params); ~SceneView(); SceneView(const SceneView&) = delete; @@ -63,7 +67,7 @@ class SceneView : public mjrfScene { void Render(filament::Renderer* renderer, const mjrfRenderRequest& request); // Returns the filament Engine managing the scene. - filament::Engine* GetEngine() const { return engine_; } + filament::Engine* GetEngine() const { return object_mgr_->GetEngine(); } // Returns the underlying filament View that is used for normal rendering. // Callers can update rendering settings (e.g. post processing) directly. @@ -85,7 +89,8 @@ class SceneView : public mjrfScene { } private: - filament::Engine* engine_ = nullptr; + ObjectManager* object_mgr_ = nullptr; + MaterialManager* material_mgr_ = nullptr; filament::Scene* scene_ = nullptr; filament::Camera* camera_ = nullptr; filament::ColorGrading* color_grading_ = nullptr; @@ -101,6 +106,9 @@ class SceneView : public mjrfScene { // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr; filament::Camera* reflect_camera_ = nullptr; + + MaterialManager::MaterialKey outline_material_key_; + std::unique_ptr outliner_; std::unique_ptr reflection_mgr_; }; } // namespace mujoco diff --git a/src/experimental/filament/render_context_filament.cc b/src/experimental/filament/render_context_filament.cc index 4826b161..c21d190c 100644 --- a/src/experimental/filament/render_context_filament.cc +++ b/src/experimental/filament/render_context_filament.cc @@ -143,7 +143,8 @@ void mjrf_destroyMesh(mjrfMesh* mesh) { delete mujoco::Mesh::downcast(mesh); } mjrfScene* mjrf_createScene(mjrfContext* ctx, const mjrfSceneParams* params) { return new mujoco::SceneView( - mujoco::FilamentContext::downcast(ctx)->GetEngine(), *params); + mujoco::FilamentContext::downcast(ctx)->GetObjectManager(), + mujoco::FilamentContext::downcast(ctx)->GetMaterialManager(), *params); } void mjrf_destroyScene(mjrfScene* scene) { diff --git a/src/experimental/studio/live.js b/src/experimental/studio/live.js index e5f51fff..0a33def7 100644 --- a/src/experimental/studio/live.js +++ b/src/experimental/studio/live.js @@ -159,6 +159,9 @@ var Module = { "assets/phong_cube_fade.filamat", "assets/phong_cube.filamat", "assets/phong_cube_reflect.filamat", + "assets/outline_composite.filamat", + "assets/outline_flatten.filamat", + "assets/outline_jumpflood.filamat", "assets/unlit_decor.filamat", "assets/unlit_depth.filamat", "assets/unlit_segmentation.filamat",