Make the selection outline occlusion-aware in the filament renderer.

PiperOrigin-RevId: 956489408
Change-Id: I948493587eb154cac6e0fc31d5df987967405ff8

src/render/filament/assets/outline_composite.mat

@@ -16,6 +16,8 @@
 // outline around an object. The outline is drawn by checking the distance to
 // the nearest edge point (as computed by the algorithm) and, if within the
 // desired width, setting the color of the pixel to the desired outline color.
+// The outline is dimmed where it is occluded, i.e. where the scene contains
+// geometry that is closer than the selected object at the nearest edge point.
 
 material {
     name : outline_composite,
@@ -25,8 +27,11 @@ material {
     depthCulling : false,
     parameters : [
         { type : sampler2d, name : source },
+        { type : sampler2d, name : scene_depth, precision : high, filterable : false },
+        { type : sampler2d, name : selection_depth, precision : high, filterable : false },
         { type : float4, name : color },
-        { type : float, name : width }
+        { type : float, name : width },
+        { type : float, name : dim }
     ],
     variables : [
         vertex
@@ -77,6 +82,18 @@ fragment {
     float inner_alpha = smoothstep(0.0, 1.0, dist);
     float outer_alpha = smoothstep(width, width - 1.0, dist);
     float alpha = outer_alpha * inner_alpha;
+
+    // Dim the outline where it is occluded. The depth buffer is reversed
+    // (1 = near, 0 = far), so the outline is occluded if the scene depth at
+    // this pixel is greater than the depth of the selected object at the
+    // nearest edge point. The epsilon avoids spurious dimming where the two
+    // depths are nearly equal, e.g. at contact points.
+    highp float scene_z = texture(materialParams_scene_depth, variable_vertex.xy).r;
+    highp float edge_z = texture(materialParams_selection_depth, edge).r;
+    if (scene_z > edge_z * 1.001 + 1e-6) {
+      alpha *= materialParams.dim;
+    }
+
     postProcess.color = mix(vec4(0), materialParams.color, alpha * materialParams.color.a);
   }
 }

src/render/filament/assets/outline_flatten.mat

@@ -16,11 +16,15 @@
 // This material writes the the screen-space position of each fragment as the
 // output color of that fragment. This will provide us with the initial data for
 // the Jump Flood Algorithm. See "outline_jumpflood.mat" for more details.
+//
+// Depth is also written so that the render target's depth attachment holds the
+// depth of the selected objects; "outline_composite.mat" samples it to dim
+// occluded portions of the outline.
 material {
     name : outline_flatten,
     shadingModel : unlit,
     culling : none,
-    depthWrite : false
+    depthWrite : true
 }
 
 fragment {

src/render/filament/core/outliner.cc

@@ -37,12 +37,15 @@
 namespace mujoco {
 
 Outliner::Outliner(ObjectManager* object_mgr, uint8_t layer_mask,
-                   filament::math::float4 color, float thickness)
+                   uint8_t scene_layer_mask, filament::math::float4 color,
+                   float thickness, float occlusion_dim)
     : object_mgr_(object_mgr),
       engine_(object_mgr->GetEngine()),
       layer_mask_(layer_mask),
+      scene_layer_mask_(scene_layer_mask),
       color_(color),
-      thickness_(thickness) {}
+      thickness_(thickness),
+      occlusion_dim_(occlusion_dim) {}
 
 Outliner::~Outliner() { Reset(); }
 
@@ -98,9 +101,10 @@ void Outliner::Prepare(int width, int height) {
     }
   };
 
-  // Set up two render targets. We will alternate between the two targets to
+  // Set up the render targets: one for the scene depth, one for the flattened
-  // allow for chaining passes together.
+  // selection mask and its depth, and two ping-pong targets for chaining the
-  for (int i = 0; i < 2; ++i) {
+  // jump flood passes together.
+  for (int i = 0; i < kNumTargets; ++i) {
     mjrfRenderTargetConfig config;
     mjrf_defaultRenderTargetConfig(&config);
     config.color_format = mjPIXEL_FORMAT_RGBA8;
@@ -127,9 +131,11 @@ void Outliner::Prepare(int width, int height) {
     view->setMultiSampleAntiAliasingOptions({.enabled = false});
   }
 
-  // In the first pass, we will render a given scene, but only render the
+  // In the first pass, we render the scene layers that can occlude the
-  // objects marked as outlines. We assume that the objects have already been
+  // outline; only the resulting depth is used. In the second pass, we render
-  // assigned the kOutlineFlatten material.
+  // the given scene again, but only the objects marked as outlines. We assume
+  // that the objects have already been assigned the kOutlineFlatten material.
+  views_[kPassSceneDepth]->setVisibleLayers(0xff, scene_layer_mask_);
   views_[kPassFlatten]->setVisibleLayers(0xff, layer_mask_);
 
   // All subsequent passes are full-screen post-processing passes.
@@ -141,16 +147,19 @@ void Outliner::Prepare(int width, int height) {
   setup_fullscreen(kPassDrawOutline, ObjectManager::kOutlineComposite);
 
   // Chain the passes together such that the output of a pass is the input to
-  // the next pass. The first pass has no input (we are just rendering the
+  // the next pass. The scene passes have no input (we are just rendering the
-  // selected objects) and the last pass has no output (we are just rendering
+  // scene) and the last pass has no output (we are just rendering the outline
-  // the outline to the externally provided target).
+  // to the externally provided target). The flatten pass has its own target
-  bind(kPassFlatten, -1, 0);
+  // (rather than a ping-pong target) so that its depth attachment survives the
-  bind(kPassJumpFlood1, 0, 1);
+  // jump flood passes and can be sampled by the composite pass.
-  bind(kPassJumpFlood2, 1, 0);
+  bind(kPassSceneDepth, -1, kTargetSceneDepth);
-  bind(kPassJumpFlood3, 0, 1);
+  bind(kPassFlatten, -1, kTargetFlatten);
-  bind(kPassJumpFlood4, 1, 0);
+  bind(kPassJumpFlood1, kTargetFlatten, kTargetPing);
-  bind(kPassJumpFlood5, 0, 1);
+  bind(kPassJumpFlood2, kTargetPing, kTargetPong);
-  bind(kPassDrawOutline, 1, -1);
+  bind(kPassJumpFlood3, kTargetPong, kTargetPing);
+  bind(kPassJumpFlood4, kTargetPing, kTargetPong);
+  bind(kPassJumpFlood5, kTargetPong, kTargetPing);
+  bind(kPassDrawOutline, kTargetPing, -1);
 
   // Bind the parameters for each pass. For the jump flood passes, the step
   // parameter determines how far to propagate the outline in each pass.
@@ -160,9 +169,22 @@ void Outliner::Prepare(int width, int height) {
   material_instances_[kPassJumpFlood4]->setParameter("step", 2.0f);
   material_instances_[kPassJumpFlood5]->setParameter("step", 1.0f);
 
-  // The final pass renders the actual outline onto a render target.
+  // The final pass renders the actual outline onto a render target. It samples
+  // the scene depth and the selection depth to dim occluded outline pixels.
+  const filament::TextureSampler depth_sampler(
+      filament::TextureSampler::MinFilter::NEAREST,
+      filament::TextureSampler::MagFilter::NEAREST);
   material_instances_[kPassDrawOutline]->setParameter("color", color_);
   material_instances_[kPassDrawOutline]->setParameter("width", thickness_);
+  material_instances_[kPassDrawOutline]->setParameter("dim", occlusion_dim_);
+  material_instances_[kPassDrawOutline]->setParameter(
+      "scene_depth",
+      targets_[kTargetSceneDepth]->GetDepthTexture()->GetFilamentTexture(),
+      depth_sampler);
+  material_instances_[kPassDrawOutline]->setParameter(
+      "selection_depth",
+      targets_[kTargetFlatten]->GetDepthTexture()->GetFilamentTexture(),
+      depth_sampler);
 
   // Commit all the material instances to the engine.
   for (auto& material_instance : material_instances_) {
@@ -222,9 +244,13 @@ void Outliner::Render(filament::Renderer* renderer, filament::View* view,
     view->setViewport(viewport);
   }
 
-  // Re-render the view's scene to create the flattened selection mask.
+  // Re-render the view's scene to capture the scene depth (used for occlusion
+  // dimming) and to create the flattened selection mask.
   auto prev_clear_opts = renderer->getClearOptions();
   renderer->setClearOptions({.clearColor = {0, 0, 0, 0}, .clear = true});
+  views_[kPassSceneDepth]->setScene(view->getScene());
+  views_[kPassSceneDepth]->setCamera(&view->getCamera());
+  renderer->render(views_[kPassSceneDepth]);
   views_[kPassFlatten]->setScene(view->getScene());
   views_[kPassFlatten]->setCamera(&view->getCamera());
   renderer->render(views_[kPassFlatten]);

src/render/filament/core/outliner.h

@@ -31,11 +31,13 @@ namespace mujoco {
 // Renders an outline of selected objects.
 //
 // This class uses the "jump flood" algorithm to create an outline of selected
-// objects.
+// objects. The outline is dimmed by `occlusion_dim` where the selected objects
+// are occluded by scene geometry on the `scene_layer_mask` layers.
 class Outliner {
  public:
   Outliner(ObjectManager* object_mgr, uint8_t layer_mask,
-           filament::math::float4 color, float thickness);
+           uint8_t scene_layer_mask, filament::math::float4 color,
+           float thickness, float occlusion_dim);
   ~Outliner();
 
   Outliner(const Outliner&) = delete;
@@ -53,6 +55,7 @@ class Outliner {
   void Reset();
 
   enum Pass {
+    kPassSceneDepth,
     kPassFlatten,
     kPassJumpFlood1,
     kPassJumpFlood2,
@@ -65,17 +68,27 @@ class Outliner {
     kNumJumpFloodPasses = kPassJumpFlood5 - kPassJumpFlood1 + 1,
   };
 
+  enum Target {
+    kTargetSceneDepth,  // depth of the scene, used for occlusion dimming
+    kTargetFlatten,     // selection mask (color) and selection depth (depth)
+    kTargetPing,        // jump flood ping-pong buffer
+    kTargetPong,        // jump flood ping-pong buffer
+    kNumTargets,
+  };
+
   ObjectManager* object_mgr_ = nullptr;
   filament::Engine* engine_ = nullptr;
   uint8_t layer_mask_ = 0xff;
+  uint8_t scene_layer_mask_ = 0xff;
   filament::math::float4 color_ = {1.0f, 1.0f, 1.0f, 1.0f};
   float thickness_ = 2.5f;
+  float occlusion_dim_ = 1.0f;
 
   int width_ = 0;
   int height_ = 0;
 
   filament::Camera* camera_ = nullptr;
-  std::unique_ptr<RenderTarget> targets_[2];
+  std::unique_ptr<RenderTarget> targets_[kNumTargets];
 
   filament::View* views_[kNumPasses] = {};
   filament::Scene* scenes_[kNumPasses] = {};

src/render/filament/core/scene_view.cc

@@ -321,9 +321,9 @@ void SceneView::Render(filament::Renderer* renderer,
 
   if (!selected_renderables.empty()) {
     if (!outliner_) {
-      outliner_ =
+      outliner_ = std::make_unique<Outliner>(
-          std::make_unique<Outliner>(object_mgr_, kLayerMask_Outline,
+          object_mgr_, kLayerMask_Outline, kLayerMask_Object,
-                                     float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f);
+          float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f, 0.25f);
     }
 
     for (Renderable* renderable : selected_renderables) {