From aef0589442098a7cfe95d2826ae39cfc087aa08e Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Tue, 14 Apr 2026 03:48:41 -0700 Subject: [PATCH] Allow Light objects to be image based lights. PiperOrigin-RevId: 899488765 Change-Id: I3b56fc6499404bb011f9361c03dba23b736ad4bc --- src/experimental/filament/filament/light.cc | 120 +++++++++++++----- src/experimental/filament/filament/light.h | 12 +- .../filament/filament/object_manager.cc | 35 +---- .../filament/filament/object_manager.h | 7 +- .../filament/filament/scene_bridge.cc | 99 ++++++++++----- .../filament/filament/scene_bridge.h | 1 + .../filament/filament/scene_view.cc | 13 -- .../filament/filament/scene_view.h | 3 - 8 files changed, 165 insertions(+), 125 deletions(-) diff --git a/src/experimental/filament/filament/light.cc b/src/experimental/filament/filament/light.cc index cbc68092..d00f6097 100644 --- a/src/experimental/filament/filament/light.cc +++ b/src/experimental/filament/filament/light.cc @@ -17,30 +17,50 @@ #include #include +#include #include #include +#include #include #include #include #include +#include "experimental/filament/filament/texture.h" namespace mujoco { +using filament::math::float3; +using filament::math::mat3f; + Light::Light(filament::Engine* engine, const Params& params) : engine_(engine), params_(params) { + // Filament treats image-based lights (IBLs) as separate objects (i.e. + // filament::IndirectLight) and so we need to handle IBLs specially. + if (params.type == mjLIGHT_IMAGE) { + filament::IndirectLight::Builder builder; + if (params.texture) { + // Allow null textures for fallback lights. + builder.reflections(params.texture->GetFilamentTexture()); + const Texture::SphericalHarmonics* spherical_harmonics = + params.texture->GetSphericalHarmonics(); + if (spherical_harmonics != nullptr) { + builder.irradiance(3, *spherical_harmonics); + } + } + builder.intensity(params.intensity); + // Rotate the light to match mujoco's Z-up convention. + builder.rotation(mat3f::rotation(std::numbers::pi / 2, float3{1, 0, 0})); + ibl_ = builder.build(*engine_); + return; + } + filament::LightManager::Type type; switch (params.type) { case mjLIGHT_SPOT: type = filament::LightManager::Type::FOCUSED_SPOT; break; case mjLIGHT_DIRECTIONAL: - // We break with the spec here slightly and use a spot light for the head - // light instead of a directional params. This is because filament only - // supports a single directional light, and we'd rather allow a scene - // light to be that directional params. It's also a bit odd for a - // directional light to move with the camera. - type = params.headlight ? filament::LightManager::Type::FOCUSED_SPOT - : filament::LightManager::Type::DIRECTIONAL; + type = filament::LightManager::Type::DIRECTIONAL; break; case mjLIGHT_POINT: type = filament::LightManager::Type::POINT; @@ -55,12 +75,8 @@ Light::Light(filament::Engine* engine, const Params& params) builder.intensityCandela(params.intensity); builder.castShadows(params.castshadow); if (type == filament::LightManager::Type::FOCUSED_SPOT) { - if (params.headlight) { - builder.spotLightCone(0, std::numbers::pi / 2.0f); - } else { - builder.spotLightCone(0, - params.spot_cone_angle * std::numbers::pi / 180.0f); - } + builder.spotLightCone(0, + params.spot_cone_angle * std::numbers::pi / 180.0f); } if (type != filament::LightManager::Type::DIRECTIONAL) { builder.falloff(params.range); @@ -86,52 +102,86 @@ Light::Light(filament::Engine* engine, const Params& params) } Light::~Light() noexcept { - utils::EntityManager& em = utils::EntityManager::get(); - if (!entity_.isNull()) { - engine_->destroy(entity_); - em.destroy(entity_); + if (ibl_) { + engine_->destroy(ibl_); + } else { + utils::EntityManager& em = utils::EntityManager::get(); + if (!entity_.isNull()) { + engine_->destroy(entity_); + em.destroy(entity_); + } } } -void Light::AddToScene(filament::Scene* scene) { scene->addEntity(entity_); } +void Light::AddToScene(filament::Scene* scene) { + if (ibl_) { + scene->setIndirectLight(ibl_); + } else { + scene->addEntity(entity_); + } +} -void Light::RemoveFromScene(filament::Scene* scene) { scene->remove(entity_); } +void Light::RemoveFromScene(filament::Scene* scene) { + if (ibl_) { + scene->setIndirectLight(nullptr); + } else { + scene->remove(entity_); + } +} void Light::SetTransform(filament::math::float3 position, filament::math::float3 direction) { - filament::LightManager& lm = engine_->getLightManager(); - const filament::LightManager::Instance li = lm.getInstance(entity_); - lm.setPosition(li, position); - lm.setDirection(li, direction); + if (!ibl_) { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setPosition(li, position); + lm.setDirection(li, direction); + } } void Light::SetColor(const filament::math::float3& color) { - filament::LightManager& lm = engine_->getLightManager(); - const filament::LightManager::Instance li = lm.getInstance(entity_); - lm.setColor(li, color); + if (!ibl_) { + params_.color = color; + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setColor(li, color); + } } void Light::SetIntensity(float intensity) { - filament::LightManager& lm = engine_->getLightManager(); - const filament::LightManager::Instance li = lm.getInstance(entity_); - lm.setIntensityCandela(li, intensity); + params_.intensity = intensity; + if (ibl_) { + ibl_->setIntensity(intensity); + } else { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setIntensityCandela(li, intensity); + } } void Light::Enable() { if (!enabled_) { enabled_ = true; - filament::LightManager& lm = engine_->getLightManager(); - const filament::LightManager::Instance li = lm.getInstance(entity_); - lm.setLightChannel(li, 0, enabled_); + if (ibl_) { + ibl_->setIntensity(params_.intensity); + } else { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setLightChannel(li, 0, enabled_); + } } } void Light::Disable() { if (enabled_) { enabled_ = false; - filament::LightManager& lm = engine_->getLightManager(); - const filament::LightManager::Instance li = lm.getInstance(entity_); - lm.setLightChannel(li, 0, enabled_); + if (ibl_) { + ibl_->setIntensity(0.f); + } else { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setLightChannel(li, 0, enabled_); + } } } diff --git a/src/experimental/filament/filament/light.h b/src/experimental/filament/filament/light.h index 858b0ef8..93974972 100644 --- a/src/experimental/filament/filament/light.h +++ b/src/experimental/filament/filament/light.h @@ -20,6 +20,7 @@ #include #include #include +#include "experimental/filament/filament/texture.h" namespace mujoco { @@ -30,6 +31,8 @@ class Light { struct Params { // The type of light (e.g. spot, point, directional, etc.) mjtLightType type; + // The texture to use for image lights. + const Texture* texture = nullptr; // The color of the light. filament::math::float3 color = {0, 0, 0}; // The intensity of the light, in candela. @@ -46,8 +49,6 @@ class Light { int shadow_map_size = 2048; // Blur width for EL VSM. float vsm_blur_width = 0.0f; - // Whether or not the light is a headlight. - bool headlight = false; }; Light(filament::Engine* engine, const Params& params); @@ -72,15 +73,16 @@ class Light { // Sets the intensity of the light in candela. void SetIntensity(float intensity); + // Returns the type of the light. + mjtLightType GetType() const { return params_.type; } + // Enables/disables the light in the scene. void Enable(); void Disable(); - // Returns true if the light is a headlight. - bool IsHeadlight() const { return params_.headlight; } - private: filament::Engine* engine_ = nullptr; + filament::IndirectLight* ibl_ = nullptr; utils::Entity entity_; bool enabled_ = true; Params params_; diff --git a/src/experimental/filament/filament/object_manager.cc b/src/experimental/filament/filament/object_manager.cc index 80679284..03e636ee 100644 --- a/src/experimental/filament/filament/object_manager.cc +++ b/src/experimental/filament/filament/object_manager.cc @@ -127,14 +127,10 @@ ObjectManager::ObjectManager(filament::Engine* engine) fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_.get(); fallback_textures_[mjTEXROLE_ORM] = fallback_orm_.get(); - LoadFallbackIndirectLight("ibl.ktx", 1.0f); + LoadFallbackIndirectLight("ibl.ktx"); } ObjectManager::~ObjectManager() { - if (fallback_indirect_light_) { - engine_->destroy(fallback_indirect_light_); - } - fallback_indirect_light_texture_.reset(); for (auto& iter : materials_) { engine_->destroy(iter); } @@ -155,17 +151,12 @@ const Texture* ObjectManager::GetFallbackTexture( return fallback_textures_[role]; } -filament::IndirectLight* ObjectManager::GetFallbackIndirectLight() { - return fallback_indirect_light_; +const Texture* ObjectManager::GetFallbackIndirectLightTexture() { + return fallback_indirect_light_texture_.get(); } -void ObjectManager::LoadFallbackIndirectLight( - std::string_view filename, float intensity) { +void ObjectManager::LoadFallbackIndirectLight(std::string_view filename) { fallback_indirect_light_texture_.reset(); - if (fallback_indirect_light_ != nullptr) { - engine_->destroy(fallback_indirect_light_); - fallback_indirect_light_ = nullptr; - } Asset* asset = new Asset(filename); auto release_asset = +[](void* user_data) { @@ -194,23 +185,5 @@ void ObjectManager::LoadFallbackIndirectLight( payload.user_data = asset; fallback_indirect_light_texture_->Upload(payload); - if (fallback_indirect_light_texture_ == nullptr) { - return; - } - - const Texture::SphericalHarmonics* spherical_harmonics = - fallback_indirect_light_texture_->GetSphericalHarmonics(); - - // Build the indirect light. - filament::IndirectLight::Builder builder; - builder.reflections(fallback_indirect_light_texture_->GetFilamentTexture()); - if (spherical_harmonics) { - builder.irradiance(3, *spherical_harmonics); - } - builder.intensity(intensity); - // Rotate the light to match mujoco's Z-up convention. - builder.rotation(filament::math::mat3f::rotation( - filament::math::f::PI / 2, filament::math::float3{1, 0, 0})); - fallback_indirect_light_ = builder.build(*engine_); } } // namespace mujoco diff --git a/src/experimental/filament/filament/object_manager.h b/src/experimental/filament/filament/object_manager.h index 52320347..83d40e74 100644 --- a/src/experimental/filament/filament/object_manager.h +++ b/src/experimental/filament/filament/object_manager.h @@ -65,10 +65,10 @@ class ObjectManager { const Texture* GetFallbackTexture(mjtTextureRole role) const; // Returns the fallback IndirectLight. - filament::IndirectLight* GetFallbackIndirectLight(); + const Texture* GetFallbackIndirectLightTexture(); // Loads an indirect light from a file, setting it to the fallback. - void LoadFallbackIndirectLight(std::string_view filename, float intensity); + void LoadFallbackIndirectLight(std::string_view filename); ObjectManager(const ObjectManager&) = delete; ObjectManager& operator=(const ObjectManager&) = delete; @@ -81,8 +81,7 @@ class ObjectManager { std::unique_ptr fallback_black_ = nullptr; std::unique_ptr fallback_normal_ = nullptr; std::unique_ptr fallback_orm_ = nullptr; - std::unique_ptr fallback_indirect_light_texture_ = nullptr; - filament::IndirectLight* fallback_indirect_light_ = nullptr; + std::unique_ptr fallback_indirect_light_texture_; }; } // namespace mujoco diff --git a/src/experimental/filament/filament/scene_bridge.cc b/src/experimental/filament/filament/scene_bridge.cc index 8ac95d60..4a9c17bd 100644 --- a/src/experimental/filament/filament/scene_bridge.cc +++ b/src/experimental/filament/filament/scene_bridge.cc @@ -148,13 +148,6 @@ SceneBridge::SceneBridge(ObjectManager* object_mgr, const mjModel* model, fallback_textures_.orm = object_mgr_->GetFallbackTexture(mjTEXROLE_ORM); fallback_textures_.emissive = object_mgr_->GetFallbackTexture(mjTEXROLE_EMISSIVE); fallback_textures_.reflection = object_mgr_->GetFallbackTexture(mjTEXROLE_USER); - - // Create an empty/black indirect light to ensure that the skybox is oriented - // to respect mujoco's Z-up convention. - filament::IndirectLight* empty_ibl = - model_objects_->CreateIndirectLight(-1, 100000); - scene_view_->AddToScene(empty_ibl); - PrepareLights(); } @@ -172,11 +165,26 @@ SceneBridge::~SceneBridge() { void SceneBridge::SetEnvironmentLight(std::string_view filename, float intensity) { - filament::IndirectLight* ibl = nullptr; - scene_view_->AddToScene(ibl); - object_mgr_->LoadFallbackIndirectLight(filename, intensity); - ibl = object_mgr_->GetFallbackIndirectLight(); - scene_view_->AddToScene(ibl); + for (auto& light : lights_) { + if (light->GetType() == mjLIGHT_IMAGE) { + scene_view_->RemoveFromScene(light.get()); + light.reset(); + break; + } + } + if (fallback_ibl_) { + scene_view_->RemoveFromScene(fallback_ibl_.get()); + fallback_ibl_.reset(); + } + + object_mgr_->LoadFallbackIndirectLight(filename); + + Light::Params params; + params.type = mjLIGHT_IMAGE; + params.texture = object_mgr_->GetFallbackIndirectLightTexture(); + params.intensity = intensity; + fallback_ibl_ = std::make_unique(object_mgr_->GetEngine(), params); + scene_view_->AddToScene(fallback_ibl_.get()); } std::optional SceneBridge::ClipFromWorld(const float3& pos) const{ @@ -190,24 +198,21 @@ std::optional SceneBridge::ClipFromWorld(const float3& pos) const{ void SceneBridge::PrepareLights() { filament::Engine* engine = object_mgr_->GetEngine(); const mjModel* model = model_objects_->GetModel(); - filament::Skybox* skybox = model_objects_->CreateSkybox(); - if (skybox) { - scene_view_->AddToScene(skybox); - } + bool has_image_based_light = false; float total_light_intensity = 0.0f; - for (int i = 0; i < model->nlight; ++i) { total_light_intensity += model->light_intensity[i]; if (model->light_type[i] == mjLIGHT_IMAGE) { - auto* indirect_light = model_objects_->CreateIndirectLight( - model->light_texid[i], model->light_intensity[i]); - if (indirect_light) { - scene_view_->AddToScene(indirect_light); - } - // Add an nullptr as a placeholder so that our indices still match. - lights_.emplace_back(nullptr); + Light::Params params; + params.type = mjLIGHT_IMAGE; + params.texture = model_objects_->GetTexture(model->light_texid[i]); + params.intensity = model->light_intensity[i]; + auto light_obj = std::make_unique(engine, params); + scene_view_->AddToScene(light_obj.get()); + lights_.emplace_back(std::move(light_obj)); + has_image_based_light = true; } else { Light::Params params; params.color = ReadFloat3(model->light_diffuse); @@ -236,10 +241,15 @@ void SceneBridge::PrepareLights() { { Light::Params params; params.color = float3(0, 0, 0); - params.headlight = true; - params.type = mjLIGHT_DIRECTIONAL; + // We break with the spec here slightly and use a spot light for the head + // light instead of a directional params. This is because filament only + // supports a single directional light, and we'd rather allow a scene + // light to be that directional params. It's also a bit odd for a + // directional light to move with the camera. + params.type = mjLIGHT_SPOT; params.castshadow = 0; - params.intensity = 0; + params.intensity = 0.0f; + params.spot_cone_angle = 90.0f; auto light_obj = std::make_unique(engine, params); #ifndef __EMSCRIPTEN__ // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. @@ -248,23 +258,44 @@ void SceneBridge::PrepareLights() { lights_.emplace_back(std::move(light_obj)); } + if (!has_image_based_light && total_light_intensity > 0.0f) { + // Create a black indirect light to ensure that the skybox is + // oriented to respect mujoco's Z-up convention. + filament::Engine* engine = object_mgr_->GetEngine(); + Light::Params params; + params.type = mjLIGHT_IMAGE; + params.intensity = 10.0f; + fallback_ibl_ = std::make_unique(engine, params); + scene_view_->AddToScene(fallback_ibl_.get()); + } + // There are no "physical" lights in the scene which means we're likely // dealing with a "classic renderer" scene. In this case, let's add a // default environment light and set the light intensity ourselves. if (total_light_intensity == 0.0f) { - auto* ibl = object_mgr_->GetFallbackIndirectLight(); - if (ibl) { - ibl->setIntensity(fallback_environment_light_intensity_); - scene_view_->AddToScene(ibl); - } + // Create a fallback environment light. + Light::Params params; + params.type = mjLIGHT_IMAGE; + params.texture = object_mgr_->GetFallbackIndirectLightTexture(); + params.intensity = fallback_environment_light_intensity_; + fallback_ibl_ = std::make_unique(engine, params); + scene_view_->AddToScene(fallback_ibl_.get()); + + // Distribute the fallback scene light intensity among the lights. const float intensity = fallback_scene_light_intensity_ / lights_.size(); for (auto& light : lights_) { if (light) { - light->SetIntensity( - light->IsHeadlight() ? fallback_head_light_intensity_ : intensity); + const bool is_headlight = (light == lights_.back()); + light->SetIntensity(is_headlight ? fallback_head_light_intensity_ + : intensity); } } } + + filament::Skybox* skybox = model_objects_->CreateSkybox(); + if (skybox) { + scene_view_->AddToScene(skybox); + } } filament::math::mat4 CalculateClipFromWorld(const mjrRect& viewport, diff --git a/src/experimental/filament/filament/scene_bridge.h b/src/experimental/filament/filament/scene_bridge.h index e6fdd765..8e847953 100644 --- a/src/experimental/filament/filament/scene_bridge.h +++ b/src/experimental/filament/filament/scene_bridge.h @@ -71,6 +71,7 @@ class SceneBridge { SceneView* scene_view_ = nullptr; ObjectManager* object_mgr_ = nullptr; std::unique_ptr model_objects_; + std::unique_ptr fallback_ibl_; std::vector> lights_; std::vector> renderables_; filament::math::mat4 clip_from_world_; diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index e8c20c30..607cab2f 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -222,18 +221,6 @@ void SceneView::RemoveFromScene(filament::Skybox* skybox) { } } -void SceneView::AddToScene(filament::IndirectLight* indirect_light) { - indirect_light_ = indirect_light; - scene_->setIndirectLight(indirect_light); -} - -void SceneView::RemoveFromScene(filament::IndirectLight* indirect_light) { - if (indirect_light_ == indirect_light) { - indirect_light_ = nullptr; - scene_->setIndirectLight(nullptr); - } -} - void SceneView::Render(filament::Renderer* renderer, const RenderRequest& request) { filament::Viewport viewport(request.viewport.left, request.viewport.bottom, diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index fff29dd7..d7f02021 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -51,8 +51,6 @@ class SceneView { void RemoveFromScene(Renderable* renderable); void AddToScene(filament::Skybox* skybox); void RemoveFromScene(filament::Skybox* skybox); - void AddToScene(filament::IndirectLight* indirect_light); - void RemoveFromScene(filament::IndirectLight* indirect_light); // Parameters for rendering the scene. using DrawMode = Material::DrawMode; @@ -101,7 +99,6 @@ class SceneView { std::unordered_set lights_; std::unordered_set renderables_; filament::Skybox* skybox_ = nullptr; - filament::IndirectLight* indirect_light_ = nullptr; // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr;