Remove layer mask from public API.

Instead, use materials to determine the layer mask of an object.

This hides the details/complexities of how layers are used
to render different passes from the user and will allow the
renderer to do more advanced rendering passes.

PiperOrigin-RevId: 928682525
Change-Id: I1f53dc8d12d2e574de6528977f55e79f896c6e42

src/experimental/filament/compat/scene_bridge.cc

@@ -43,8 +43,6 @@ SceneBridge::SceneBridge(mjrfContext* ctx, const mjModel* model)
     : ctx_(ctx) {
   mjrfSceneParams params;
   mjrf_defaultSceneParams(&params);
-  params.layer_mask = mjCAT_ALL;
-  params.reflection_layer_mask = mjCAT_DYNAMIC | mjCAT_STATIC;
   scene_ = CreateScene(ctx_, params);
   model_objects_ = std::make_unique<ModelObjects>(model, ctx_);
 

src/experimental/filament/compat/scene_geom_util.cc

@@ -268,7 +268,6 @@ UniquePtr<mjrfRenderable> CreateGeomRenderable(
     const mjtByte render_flags[mjNRNDFLAG]) {
   mjrfRenderableParams params;
   mjrf_defaultRenderableParams(&params);
-  params.layer_mask = geom.category;
   auto renderable = CreateRenderable(ctx, params);
   PrepareGeomMeshes(renderable.get(), geom, model_objs);
   UpdateGeomMaterial(renderable.get(), geom, model_objs, render_flags);

src/experimental/filament/filament/renderable.cc

@@ -136,8 +136,6 @@ void Renderable::InitPartEntity(Part& part) {
   }
   builder.castShadows(params_.cast_shadows);
   builder.receiveShadows(params_.receive_shadows);
-  builder.layerMask(0xff, params_.layer_mask);
-  builder.priority(params_.priority);
   builder.blendOrder(0, params_.blend_order);
   builder.screenSpaceContactShadows(true);
 
@@ -223,6 +221,11 @@ void Renderable::RemoveFromScene(filament::Scene* scene) {
 }
 
 void Renderable::UpdateMaterial(const mjrfMaterial& material) {
+  uint8_t layer_mask_ = kLayerMask_Object;
+  if (material.decor_ux) {
+    layer_mask_ = kLayerMask_Decor;
+  }
+  SetLayerMask(layer_mask_);
   material_ = material;
 }
 
@@ -390,26 +393,13 @@ void Renderable::BindMaterialInstance(const mjrfRenderRequest& request) {
 }
 
 std::uint8_t Renderable::SetLayerMask(std::uint8_t mask) {
-  std::uint8_t prev = params_.layer_mask;
-  if (mask != params_.layer_mask) {
-    params_.layer_mask = mask;
+  std::uint8_t prev = layer_mask_;
+  if (mask != layer_mask_) {
+    layer_mask_ = mask;
 
     filament::RenderableManager& rm = GetEngine()->getRenderableManager();
     for (Part& part : parts_) {
-      rm.setLayerMask(rm.getInstance(part.entity), 0xff, params_.layer_mask);
-    }
-  }
-  return prev;
-}
-
-std::uint8_t Renderable::SetPriority(std::uint8_t priority) {
-  std::uint8_t prev = params_.priority;
-  if (priority != params_.priority) {
-    params_.priority = priority;
-
-    filament::RenderableManager& rm = GetEngine()->getRenderableManager();
-    for (Part& part : parts_) {
-      rm.setPriority(rm.getInstance(part.entity), params_.priority);
+      rm.setLayerMask(rm.getInstance(part.entity), 0xff, layer_mask_);
     }
   }
   return prev;

src/experimental/filament/filament/renderable.h

@@ -35,6 +35,13 @@
 
 namespace mujoco {
 
+enum LayerMask : uint8_t {
+  kLayerMask_Object  = 0x01 << 1,
+  kLayerMask_Decor   = 0x01 << 2,
+  kLayerMask_All     = 0xff,
+  kLayerMask_None    = 0x00,
+};
+
 // A Renderable is effectively two things: a mesh and a material.
 //
 // The mesh describes the surface geometry of the object and the material
@@ -73,10 +80,6 @@ class Renderable : public mjrfRenderable {
   // show/hide groups of renderables in scenes. Returns the previous layer mask.
   std::uint8_t SetLayerMask(std::uint8_t mask);
 
-  // Sets the draw priority this renderable. The priority determines the order
-  // in which renderables are rendered. Returns the previous priority.
-  std::uint8_t SetPriority(std::uint8_t priority);
-
   // Sets the blend order for this renderable. This determines the order in
   // which transparent renderables are blended together. Returns the previous
   // blend order.
@@ -173,6 +176,7 @@ class Renderable : public mjrfRenderable {
   filament::math::mat4f transform_;
   GetTransformFn get_transform_fn_;
   Trs trs_;
+  uint8_t layer_mask_ = 0x00;
   bool infinite_plane_ = false;
 };
 

src/experimental/filament/filament/scene_view.cc

@@ -131,12 +131,12 @@ SceneView::SceneView(filament::Engine* engine, const mjrfSceneParams& params)
   main_view_ = engine->createView();
   main_view_->setScene(scene_);
   main_view_->setCamera(camera_);
-  main_view_->setVisibleLayers(0xff, params.layer_mask);
+  main_view_->setVisibleLayers(0xff, kLayerMask_All);
 
   depth_segment_view_ = engine->createView();
   depth_segment_view_->setScene(scene_);
   depth_segment_view_->setCamera(camera_);
-  depth_segment_view_->setVisibleLayers(0xff, params.layer_mask);
+  depth_segment_view_->setVisibleLayers(0xff, kLayerMask_Object);
   depth_segment_view_->setPostProcessingEnabled(false);
 
   reflect_view_ = engine->createView();
@@ -145,7 +145,7 @@ SceneView::SceneView(filament::Engine* engine, const mjrfSceneParams& params)
   reflect_view_->setShadowingEnabled(false);
   reflect_view_->setPostProcessingEnabled(false);
   reflect_view_->setFrontFaceWindingInverted(true);
-  reflect_view_->setVisibleLayers(0xff, params.reflection_layer_mask);
+  reflect_view_->setVisibleLayers(0xff, kLayerMask_Object);
   reflect_view_->setMultiSampleAntiAliasingOptions({.enabled = false});
 
   // Rotate the fog to align with mujoco's +Z up space.
@@ -270,7 +270,7 @@ void SceneView::Render(filament::Renderer* renderer, const mjrfRenderRequest& re
     SetupReflectionCamera(transform, camera_, reflect_camera_);
 
     // Hide reflective surface from its own reflection pass.
-    std::uint8_t previous_layer_mask = renderable->SetLayerMask(0x00);
+    std::uint8_t prev_layer_mask = renderable->SetLayerMask(kLayerMask_None);
 
     // Render the reflection to its render target.
     viewport.left = 0;
@@ -282,7 +282,7 @@ void SceneView::Render(filament::Renderer* renderer, const mjrfRenderRequest& re
     reflect_view_->setRenderTarget(nullptr);
 
     // Unhide the reflective surface.
-    renderable->SetLayerMask(previous_layer_mask);
+    renderable->SetLayerMask(prev_layer_mask);
   }
 
   view->setRenderTarget(render_target ? render_target->GetFilamentRenderTarget()

src/experimental/filament/render_context_filament.cc

@@ -57,8 +57,6 @@ void mjrf_defaultMeshData(mjrfMeshData* data) {
 
 void mjrf_defaultSceneParams(mjrfSceneParams* params) {
   memset(params, 0, sizeof(mjrfSceneParams));
-  params->layer_mask = 0xff;
-  params->reflection_layer_mask = 0xff;
 }
 
 void mjrf_defaultLightParams(mjrfLightParams* params) {
@@ -94,8 +92,6 @@ void mjrf_defaultRenderableParams(mjrfRenderableParams* params) {
   memset(params, 0, sizeof(mjrfRenderableParams));
   params->cast_shadows = true;
   params->receive_shadows = true;
-  params->layer_mask = 0x01;
-  params->priority = 4;
   params->blend_order = 0;
 }
 

src/experimental/filament/render_context_filament.h

@@ -467,12 +467,6 @@ void mjrf_destroyMesh(mjrfMesh* mesh);
 
 // Configuration parameters for a Scene.
 struct mjrfSceneParams {
-  // This mask, in conjunction with the layer mask in the Renderable, determines
-  // which Renderables to render within the Scene.
-  uint8_t layer_mask;
-
-  // The layer mask to use for reflections.
-  uint8_t reflection_layer_mask;
 };
 
 // Initializes the mjrfSceneParams to default values.
@@ -687,15 +681,6 @@ struct mjrfRenderableParams {
   // Whether or not the Renderable receives shadows.
   mjtBool receive_shadows;
 
-  // The layers to which the Renderable belongs. This mask is used in
-  // conjunction with the layer mask in the Scene to determine which
-  // Renderables to render. Defaults to 0xff.
-  uint8_t layer_mask;
-
-  // Controls the order in which the Renderable is drawn relative to other
-  // Renderables; defaults to 4.
-  uint8_t priority;
-
   // Similar to priority, but provides finer-grained control for Renderables
   // with transparency; defaults to 0.
   uint16_t blend_order;