Allow Light objects to be image based lights.

PiperOrigin-RevId: 899488765
Change-Id: I3b56fc6499404bb011f9361c03dba23b736ad4bc

src/experimental/filament/filament/light.cc

@@ -17,30 +17,50 @@
 #include <numbers>
 
 #include <filament/Engine.h>
+#include <filament/IndirectLight.h>
 #include <filament/LightManager.h>
 #include <filament/Scene.h>
+#include <math/mat3.h>
 #include <math/vec3.h>
 #include <utils/Entity.h>
 #include <utils/EntityManager.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/filament/texture.h"
 
 namespace mujoco {
 
+using filament::math::float3;
+using filament::math::mat3f;
+
 Light::Light(filament::Engine* engine, const Params& params)
     : engine_(engine), params_(params) {
+  // Filament treats image-based lights (IBLs) as separate objects (i.e.
+  // filament::IndirectLight) and so we need to handle IBLs specially.
+  if (params.type == mjLIGHT_IMAGE) {
+    filament::IndirectLight::Builder builder;
+    if (params.texture) {
+      // Allow null textures for fallback lights.
+      builder.reflections(params.texture->GetFilamentTexture());
+      const Texture::SphericalHarmonics* spherical_harmonics =
+          params.texture->GetSphericalHarmonics();
+      if (spherical_harmonics != nullptr) {
+        builder.irradiance(3, *spherical_harmonics);
+      }
+    }
+    builder.intensity(params.intensity);
+    // Rotate the light to match mujoco's Z-up convention.
+    builder.rotation(mat3f::rotation(std::numbers::pi / 2, float3{1, 0, 0}));
+    ibl_ = builder.build(*engine_);
+    return;
+  }
+
   filament::LightManager::Type type;
   switch (params.type) {
     case mjLIGHT_SPOT:
       type = filament::LightManager::Type::FOCUSED_SPOT;
       break;
     case mjLIGHT_DIRECTIONAL:
-      // We break with the spec here slightly and use a spot light for the head
-      // light instead of a directional params. This is because filament only
-      // supports a single directional light, and we'd rather allow a scene
-      // light to be that directional params. It's also a bit odd for a
-      // directional light to move with the camera.
-      type = params.headlight ? filament::LightManager::Type::FOCUSED_SPOT
-                             : filament::LightManager::Type::DIRECTIONAL;
+      type = filament::LightManager::Type::DIRECTIONAL;
       break;
     case mjLIGHT_POINT:
       type = filament::LightManager::Type::POINT;
@@ -55,12 +75,8 @@ Light::Light(filament::Engine* engine, const Params& params)
   builder.intensityCandela(params.intensity);
   builder.castShadows(params.castshadow);
   if (type == filament::LightManager::Type::FOCUSED_SPOT) {
-    if (params.headlight) {
-      builder.spotLightCone(0, std::numbers::pi / 2.0f);
-    } else {
-      builder.spotLightCone(0,
-                            params.spot_cone_angle * std::numbers::pi / 180.0f);
-    }
+    builder.spotLightCone(0,
+                          params.spot_cone_angle * std::numbers::pi / 180.0f);
   }
   if (type != filament::LightManager::Type::DIRECTIONAL) {
     builder.falloff(params.range);
@@ -86,52 +102,86 @@ Light::Light(filament::Engine* engine, const Params& params)
 }
 
 Light::~Light() noexcept {
-  utils::EntityManager& em = utils::EntityManager::get();
-  if (!entity_.isNull()) {
-    engine_->destroy(entity_);
-    em.destroy(entity_);
+  if (ibl_) {
+    engine_->destroy(ibl_);
+  } else {
+    utils::EntityManager& em = utils::EntityManager::get();
+    if (!entity_.isNull()) {
+      engine_->destroy(entity_);
+      em.destroy(entity_);
+    }
   }
 }
 
-void Light::AddToScene(filament::Scene* scene) { scene->addEntity(entity_); }
+void Light::AddToScene(filament::Scene* scene) {
+  if (ibl_) {
+    scene->setIndirectLight(ibl_);
+  } else {
+    scene->addEntity(entity_);
+  }
+}
 
-void Light::RemoveFromScene(filament::Scene* scene) { scene->remove(entity_); }
+void Light::RemoveFromScene(filament::Scene* scene) {
+  if (ibl_) {
+    scene->setIndirectLight(nullptr);
+  } else {
+    scene->remove(entity_);
+  }
+}
 
 void Light::SetTransform(filament::math::float3 position,
                          filament::math::float3 direction) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setPosition(li, position);
-  lm.setDirection(li, direction);
+  if (!ibl_) {
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setPosition(li, position);
+    lm.setDirection(li, direction);
+  }
 }
 
 void Light::SetColor(const filament::math::float3& color) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setColor(li, color);
+  if (!ibl_) {
+    params_.color = color;
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setColor(li, color);
+  }
 }
 
 void Light::SetIntensity(float intensity) {
-  filament::LightManager& lm = engine_->getLightManager();
-  const filament::LightManager::Instance li = lm.getInstance(entity_);
-  lm.setIntensityCandela(li, intensity);
+  params_.intensity = intensity;
+  if (ibl_) {
+    ibl_->setIntensity(intensity);
+  } else {
+    filament::LightManager& lm = engine_->getLightManager();
+    const filament::LightManager::Instance li = lm.getInstance(entity_);
+    lm.setIntensityCandela(li, intensity);
+  }
 }
 
 void Light::Enable() {
   if (!enabled_) {
     enabled_ = true;
-    filament::LightManager& lm = engine_->getLightManager();
-    const filament::LightManager::Instance li = lm.getInstance(entity_);
-    lm.setLightChannel(li, 0, enabled_);
+    if (ibl_) {
+      ibl_->setIntensity(params_.intensity);
+    } else {
+      filament::LightManager& lm = engine_->getLightManager();
+      const filament::LightManager::Instance li = lm.getInstance(entity_);
+      lm.setLightChannel(li, 0, enabled_);
+    }
   }
 }
 
 void Light::Disable() {
   if (enabled_) {
     enabled_ = false;
-    filament::LightManager& lm = engine_->getLightManager();
-    const filament::LightManager::Instance li = lm.getInstance(entity_);
-    lm.setLightChannel(li, 0, enabled_);
+    if (ibl_) {
+      ibl_->setIntensity(0.f);
+    } else {
+      filament::LightManager& lm = engine_->getLightManager();
+      const filament::LightManager::Instance li = lm.getInstance(entity_);
+      lm.setLightChannel(li, 0, enabled_);
+    }
   }
 }
 

src/experimental/filament/filament/light.h

@@ -20,6 +20,7 @@
 #include <math/vec3.h>
 #include <utils/Entity.h>
 #include <mujoco/mujoco.h>
+#include "experimental/filament/filament/texture.h"
 
 namespace mujoco {
 
@@ -30,6 +31,8 @@ class Light {
   struct Params {
     // The type of light (e.g. spot, point, directional, etc.)
     mjtLightType type;
+    // The texture to use for image lights.
+    const Texture* texture = nullptr;
     // The color of the light.
     filament::math::float3 color = {0, 0, 0};
     // The intensity of the light, in candela.
@@ -46,8 +49,6 @@ class Light {
     int shadow_map_size = 2048;
     // Blur width for EL VSM.
     float vsm_blur_width = 0.0f;
-    // Whether or not the light is a headlight.
-    bool headlight = false;
   };
 
   Light(filament::Engine* engine, const Params& params);
@@ -72,15 +73,16 @@ class Light {
   // Sets the intensity of the light in candela.
   void SetIntensity(float intensity);
 
+  // Returns the type of the light.
+  mjtLightType GetType() const { return params_.type; }
+
   // Enables/disables the light in the scene.
   void Enable();
   void Disable();
 
-  // Returns true if the light is a headlight.
-  bool IsHeadlight() const { return params_.headlight; }
-
  private:
   filament::Engine* engine_ = nullptr;
+  filament::IndirectLight* ibl_ = nullptr;
   utils::Entity entity_;
   bool enabled_ = true;
   Params params_;

src/experimental/filament/filament/object_manager.cc

@@ -127,14 +127,10 @@ ObjectManager::ObjectManager(filament::Engine* engine)
   fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_.get();
   fallback_textures_[mjTEXROLE_ORM] = fallback_orm_.get();
 
-  LoadFallbackIndirectLight("ibl.ktx", 1.0f);
+  LoadFallbackIndirectLight("ibl.ktx");
 }
 
 ObjectManager::~ObjectManager() {
-  if (fallback_indirect_light_) {
-    engine_->destroy(fallback_indirect_light_);
-  }
-  fallback_indirect_light_texture_.reset();
   for (auto& iter : materials_) {
     engine_->destroy(iter);
   }
@@ -155,17 +151,12 @@ const Texture* ObjectManager::GetFallbackTexture(
   return fallback_textures_[role];
 }
 
-filament::IndirectLight* ObjectManager::GetFallbackIndirectLight() {
-  return fallback_indirect_light_;
+const Texture* ObjectManager::GetFallbackIndirectLightTexture() {
+  return fallback_indirect_light_texture_.get();
 }
 
-void ObjectManager::LoadFallbackIndirectLight(
-    std::string_view filename, float intensity) {
+void ObjectManager::LoadFallbackIndirectLight(std::string_view filename) {
   fallback_indirect_light_texture_.reset();
-  if (fallback_indirect_light_ != nullptr) {
-    engine_->destroy(fallback_indirect_light_);
-    fallback_indirect_light_ = nullptr;
-  }
 
   Asset* asset = new Asset(filename);
   auto release_asset = +[](void* user_data) {
@@ -194,23 +185,5 @@ void ObjectManager::LoadFallbackIndirectLight(
   payload.user_data = asset;
 
   fallback_indirect_light_texture_->Upload(payload);
-  if (fallback_indirect_light_texture_ == nullptr) {
-    return;
-  }
-
-  const Texture::SphericalHarmonics* spherical_harmonics =
-      fallback_indirect_light_texture_->GetSphericalHarmonics();
-
-  // Build the indirect light.
-  filament::IndirectLight::Builder builder;
-  builder.reflections(fallback_indirect_light_texture_->GetFilamentTexture());
-  if (spherical_harmonics) {
-    builder.irradiance(3, *spherical_harmonics);
-  }
-  builder.intensity(intensity);
-  // Rotate the light to match mujoco's Z-up convention.
-  builder.rotation(filament::math::mat3f::rotation(
-      filament::math::f::PI / 2, filament::math::float3{1, 0, 0}));
-  fallback_indirect_light_ = builder.build(*engine_);
 }
 }  // namespace mujoco

src/experimental/filament/filament/object_manager.h

@@ -65,10 +65,10 @@ class ObjectManager {
   const Texture* GetFallbackTexture(mjtTextureRole role) const;
 
   // Returns the fallback IndirectLight.
-  filament::IndirectLight* GetFallbackIndirectLight();
+  const Texture* GetFallbackIndirectLightTexture();
 
   // Loads an indirect light from a file, setting it to the fallback.
-  void LoadFallbackIndirectLight(std::string_view filename, float intensity);
+  void LoadFallbackIndirectLight(std::string_view filename);
 
   ObjectManager(const ObjectManager&) = delete;
   ObjectManager& operator=(const ObjectManager&) = delete;
@@ -81,8 +81,7 @@ class ObjectManager {
   std::unique_ptr<Texture> fallback_black_ = nullptr;
   std::unique_ptr<Texture> fallback_normal_ = nullptr;
   std::unique_ptr<Texture> fallback_orm_ = nullptr;
-  std::unique_ptr<Texture> fallback_indirect_light_texture_ = nullptr;
-  filament::IndirectLight* fallback_indirect_light_ = nullptr;
+  std::unique_ptr<Texture> fallback_indirect_light_texture_;
 };
 
 }  // namespace mujoco

src/experimental/filament/filament/scene_bridge.cc

@@ -148,13 +148,6 @@ SceneBridge::SceneBridge(ObjectManager* object_mgr, const mjModel* model,
   fallback_textures_.orm = object_mgr_->GetFallbackTexture(mjTEXROLE_ORM);
   fallback_textures_.emissive = object_mgr_->GetFallbackTexture(mjTEXROLE_EMISSIVE);
   fallback_textures_.reflection = object_mgr_->GetFallbackTexture(mjTEXROLE_USER);
-
-  // Create an empty/black indirect light to ensure that the skybox is oriented
-  // to respect mujoco's Z-up convention.
-  filament::IndirectLight* empty_ibl =
-      model_objects_->CreateIndirectLight(-1, 100000);
-  scene_view_->AddToScene(empty_ibl);
-
   PrepareLights();
 }
 
@@ -172,11 +165,26 @@ SceneBridge::~SceneBridge() {
 
 void SceneBridge::SetEnvironmentLight(std::string_view filename,
                                     float intensity) {
-  filament::IndirectLight* ibl = nullptr;
-  scene_view_->AddToScene(ibl);
-  object_mgr_->LoadFallbackIndirectLight(filename, intensity);
-  ibl = object_mgr_->GetFallbackIndirectLight();
-  scene_view_->AddToScene(ibl);
+  for (auto& light : lights_) {
+    if (light->GetType() == mjLIGHT_IMAGE) {
+      scene_view_->RemoveFromScene(light.get());
+      light.reset();
+      break;
+    }
+  }
+  if (fallback_ibl_) {
+    scene_view_->RemoveFromScene(fallback_ibl_.get());
+    fallback_ibl_.reset();
+  }
+
+  object_mgr_->LoadFallbackIndirectLight(filename);
+
+  Light::Params params;
+  params.type = mjLIGHT_IMAGE;
+  params.texture = object_mgr_->GetFallbackIndirectLightTexture();
+  params.intensity = intensity;
+  fallback_ibl_ = std::make_unique<Light>(object_mgr_->GetEngine(), params);
+  scene_view_->AddToScene(fallback_ibl_.get());
 }
 
 std::optional<float3> SceneBridge::ClipFromWorld(const float3& pos) const{
@@ -190,24 +198,21 @@ std::optional<float3> SceneBridge::ClipFromWorld(const float3& pos) const{
 void SceneBridge::PrepareLights() {
   filament::Engine* engine = object_mgr_->GetEngine();
   const mjModel* model = model_objects_->GetModel();
-  filament::Skybox* skybox = model_objects_->CreateSkybox();
-  if (skybox) {
-    scene_view_->AddToScene(skybox);
-  }
 
+  bool has_image_based_light = false;
   float total_light_intensity = 0.0f;
-
   for (int i = 0; i < model->nlight; ++i) {
     total_light_intensity += model->light_intensity[i];
 
     if (model->light_type[i] == mjLIGHT_IMAGE) {
-      auto* indirect_light = model_objects_->CreateIndirectLight(
-          model->light_texid[i], model->light_intensity[i]);
-      if (indirect_light) {
-        scene_view_->AddToScene(indirect_light);
-      }
-      // Add an nullptr as a placeholder so that our indices still match.
-      lights_.emplace_back(nullptr);
+      Light::Params params;
+      params.type = mjLIGHT_IMAGE;
+      params.texture = model_objects_->GetTexture(model->light_texid[i]);
+      params.intensity = model->light_intensity[i];
+      auto light_obj = std::make_unique<Light>(engine, params);
+      scene_view_->AddToScene(light_obj.get());
+      lights_.emplace_back(std::move(light_obj));
+      has_image_based_light = true;
     } else {
       Light::Params params;
       params.color = ReadFloat3(model->light_diffuse);
@@ -236,10 +241,15 @@ void SceneBridge::PrepareLights() {
   {
     Light::Params params;
     params.color = float3(0, 0, 0);
-    params.headlight = true;
-    params.type = mjLIGHT_DIRECTIONAL;
+    // We break with the spec here slightly and use a spot light for the head
+    // light instead of a directional params. This is because filament only
+    // supports a single directional light, and we'd rather allow a scene
+    // light to be that directional params. It's also a bit odd for a
+    // directional light to move with the camera.
+    params.type = mjLIGHT_SPOT;
     params.castshadow = 0;
-    params.intensity = 0;
+    params.intensity = 0.0f;
+    params.spot_cone_angle = 90.0f;
     auto light_obj = std::make_unique<Light>(engine, params);
 #ifndef __EMSCRIPTEN__
     // TODO(b/458045799): Re-enable when lights work on glinux and chromebook.
@@ -248,23 +258,44 @@ void SceneBridge::PrepareLights() {
     lights_.emplace_back(std::move(light_obj));
   }
 
+  if (!has_image_based_light && total_light_intensity > 0.0f) {
+    // Create a black indirect light to ensure that the skybox is
+    // oriented to respect mujoco's Z-up convention.
+    filament::Engine* engine = object_mgr_->GetEngine();
+    Light::Params params;
+    params.type = mjLIGHT_IMAGE;
+    params.intensity = 10.0f;
+    fallback_ibl_ = std::make_unique<Light>(engine, params);
+    scene_view_->AddToScene(fallback_ibl_.get());
+  }
+
   // There are no "physical" lights in the scene which means we're likely
   // dealing with a "classic renderer" scene. In this case, let's add a
   // default environment light and set the light intensity ourselves.
   if (total_light_intensity == 0.0f) {
-    auto* ibl = object_mgr_->GetFallbackIndirectLight();
-    if (ibl) {
-      ibl->setIntensity(fallback_environment_light_intensity_);
-      scene_view_->AddToScene(ibl);
-    }
+    // Create a fallback environment light.
+    Light::Params params;
+    params.type = mjLIGHT_IMAGE;
+    params.texture = object_mgr_->GetFallbackIndirectLightTexture();
+    params.intensity = fallback_environment_light_intensity_;
+    fallback_ibl_ = std::make_unique<Light>(engine, params);
+    scene_view_->AddToScene(fallback_ibl_.get());
+
+    // Distribute the fallback scene light intensity among the lights.
     const float intensity = fallback_scene_light_intensity_ / lights_.size();
     for (auto& light : lights_) {
       if (light) {
-        light->SetIntensity(
-            light->IsHeadlight() ? fallback_head_light_intensity_ : intensity);
+        const bool is_headlight = (light == lights_.back());
+        light->SetIntensity(is_headlight ? fallback_head_light_intensity_
+                                         : intensity);
       }
     }
   }
+
+  filament::Skybox* skybox = model_objects_->CreateSkybox();
+  if (skybox) {
+    scene_view_->AddToScene(skybox);
+  }
 }
 
 filament::math::mat4 CalculateClipFromWorld(const mjrRect& viewport,

src/experimental/filament/filament/scene_bridge.h

@@ -71,6 +71,7 @@ class SceneBridge {
   SceneView* scene_view_ = nullptr;
   ObjectManager* object_mgr_ = nullptr;
   std::unique_ptr<ModelObjects> model_objects_;
+  std::unique_ptr<Light> fallback_ibl_;
   std::vector<std::unique_ptr<Light>> lights_;
   std::vector<std::unique_ptr<Renderable>> renderables_;
   filament::math::mat4 clip_from_world_;

src/experimental/filament/filament/scene_view.cc

@@ -20,7 +20,6 @@
 #include <memory>
 
 #include <filament/ColorGrading.h>
-#include <filament/IndirectLight.h>
 #include <filament/LightManager.h>
 #include <filament/Material.h>
 #include <filament/Options.h>
@@ -222,18 +221,6 @@ void SceneView::RemoveFromScene(filament::Skybox* skybox) {
   }
 }
 
-void SceneView::AddToScene(filament::IndirectLight* indirect_light) {
-  indirect_light_ = indirect_light;
-  scene_->setIndirectLight(indirect_light);
-}
-
-void SceneView::RemoveFromScene(filament::IndirectLight* indirect_light) {
-  if (indirect_light_ == indirect_light) {
-    indirect_light_ = nullptr;
-    scene_->setIndirectLight(nullptr);
-  }
-}
-
 void SceneView::Render(filament::Renderer* renderer,
                        const RenderRequest& request) {
   filament::Viewport viewport(request.viewport.left, request.viewport.bottom,

src/experimental/filament/filament/scene_view.h

@@ -51,8 +51,6 @@ class SceneView {
   void RemoveFromScene(Renderable* renderable);
   void AddToScene(filament::Skybox* skybox);
   void RemoveFromScene(filament::Skybox* skybox);
-  void AddToScene(filament::IndirectLight* indirect_light);
-  void RemoveFromScene(filament::IndirectLight* indirect_light);
 
   // Parameters for rendering the scene.
   using DrawMode = Material::DrawMode;
@@ -101,7 +99,6 @@ class SceneView {
   std::unordered_set<Light*> lights_;
   std::unordered_set<Renderable*> renderables_;
   filament::Skybox* skybox_ = nullptr;
-  filament::IndirectLight* indirect_light_ = nullptr;
 
   // Custom view and camera for reflective surfaces.
   filament::View* reflect_view_ = nullptr;