From edc807895f065d454b6c99214edff3dceeba7945 Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Thu, 9 Apr 2026 03:32:17 -0700 Subject: [PATCH] Split SceneView into two parts: SceneView and SceneBridge. SceneView owns the filament Scene, View, and Camera objects. is responsible for rendering the Scene (to which Entities have been added). SceneBridge takes the mjvScene as an input and updates the corresponding light and drawable Entities in the SceneView. PiperOrigin-RevId: 897007401 Change-Id: I3ec245f5bd48c3654672b11a55fb011f113454cc --- src/experimental/filament/CMakeLists.txt | 2 + .../filament/filament/filament_context.cc | 51 +- .../filament/filament/filament_context.h | 3 + .../filament/filament/imgui_editor.cc | 10 +- .../filament/filament/imgui_editor.h | 4 +- .../filament/filament/scene_bridge.cc | 370 +++++++++++++ .../filament/filament/scene_bridge.h | 85 +++ .../filament/filament/scene_view.cc | 492 +++++------------- .../filament/filament/scene_view.h | 112 ++-- 9 files changed, 682 insertions(+), 447 deletions(-) create mode 100644 src/experimental/filament/filament/scene_bridge.cc create mode 100644 src/experimental/filament/filament/scene_bridge.h diff --git a/src/experimental/filament/CMakeLists.txt b/src/experimental/filament/CMakeLists.txt index 367f2725..a495863b 100644 --- a/src/experimental/filament/CMakeLists.txt +++ b/src/experimental/filament/CMakeLists.txt @@ -54,6 +54,8 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME} filament/render_target.h filament/renderables.cc filament/renderables.h + filament/scene_bridge.cc + filament/scene_bridge.h filament/scene_view.cc filament/scene_view.h filament/texture.cc diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc index b960eb49..cbc79c24 100644 --- a/src/experimental/filament/filament/filament_context.cc +++ b/src/experimental/filament/filament/filament_context.cc @@ -44,6 +44,7 @@ #include "experimental/filament/filament/model_util.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" #include "experimental/filament/render_context_filament.h" @@ -82,6 +83,7 @@ FilamentContext::FilamentContext(const mjrFilamentConfig* config) FilamentContext::~FilamentContext() { DestroyRenderTargets(); gui_view_.reset(); + scene_bridge_.reset(); scene_view_.reset(); object_manager_.reset(); engine_->destroy(renderer_); @@ -91,7 +93,9 @@ FilamentContext::~FilamentContext() { } void FilamentContext::Init(const mjModel* model) { - scene_view_ = std::make_unique(object_manager_.get(), model); + scene_view_ = std::make_unique(engine_); + scene_bridge_ = std::make_unique(object_manager_.get(), model, + scene_view_.get()); gui_view_ = std::make_unique( engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi)); @@ -120,8 +124,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { window_height_ = viewport.height; } - scene_view_->SetViewport(viewport); - scene_view_->UpdateScene(scene); + scene_bridge_->Update(viewport, scene); // Update the UX renderable entity after processing the scene in case there // are any elements in the scene which generate UX draw calls (e.g. labels). if (gui_view_ && gui_swap_chain_target_ == scene_swap_chain_target_) { @@ -137,6 +140,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { } else if (scene->flags[mjRND_DEPTH]) { last_render_mode_ = SceneView::DrawMode::kDepth; } + last_camera_ = mjv_averageCamera(scene->camera, scene->camera + 1); // Render the frame if we're not rendering to a texture. if (scene_swap_chain_target_ == kWindowSwapChain) { @@ -146,7 +150,11 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { } if (renderer_->beginFrame(window_swap_chain_)) { - scene_view_->Render(renderer_, last_render_mode_); + SceneView::RenderRequest request; + request.draw_mode = last_render_mode_; + request.viewport = viewport; + request.camera = last_camera_; + scene_view_->Render(renderer_, request); if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) { gui_view_->Render(renderer_); @@ -240,7 +248,12 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, if (rgb) { if (renderer_->beginFrame(offscreen_swap_chain_)) { - scene_view_->Render(renderer_, last_render_mode_, color_target_.get()); + SceneView::RenderRequest request; + request.draw_mode = last_render_mode_; + request.viewport = viewport; + request.target = color_target_.get(); + request.camera = last_camera_; + scene_view_->Render(renderer_, request); // Render the GUI to the texture as well if requested. if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) { @@ -256,8 +269,12 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, if (depth) { if (renderer_->beginFrame(offscreen_swap_chain_)) { - scene_view_->Render(renderer_, SceneView::DrawMode::kDepth, - depth_target_.get()); + SceneView::RenderRequest request; + request.draw_mode = SceneView::DrawMode::kDepth; + request.viewport = viewport; + request.target = depth_target_.get(); + request.camera = last_camera_; + scene_view_->Render(renderer_, request); const size_t num_bytes = viewport.width * viewport.height * sizeof(float); ReadDepthPixels(renderer_, depth_target_.get(), viewport, depth, @@ -276,24 +293,24 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, } void FilamentContext::UploadMesh(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadMesh(model, id); + scene_bridge_->UploadMesh(model, id); } void FilamentContext::UploadTexture(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadTexture(model, id); + scene_bridge_->UploadTexture(model, id); } void FilamentContext::UploadHeightField(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadHeightField(model, id); + scene_bridge_->UploadHeightField(model, id); } uintptr_t FilamentContext::UploadGuiImage(uintptr_t tex_id, @@ -315,6 +332,6 @@ double FilamentContext::GetFrameRate() const { return 1.0e9 / static_cast(ns); } -void FilamentContext::UpdateGui() { DrawGui(scene_view_.get()); } +void FilamentContext::UpdateGui() { DrawGui(scene_bridge_.get()); } } // namespace mujoco diff --git a/src/experimental/filament/filament/filament_context.h b/src/experimental/filament/filament/filament_context.h index 8de07824..77c22a88 100644 --- a/src/experimental/filament/filament/filament_context.h +++ b/src/experimental/filament/filament/filament_context.h @@ -27,6 +27,7 @@ #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/render_context_filament.h" @@ -80,12 +81,14 @@ class FilamentContext { std::unique_ptr platform_; SceneView::DrawMode last_render_mode_ = SceneView::DrawMode::kNormal; + mjvGLCamera last_camera_; SwapChainType scene_swap_chain_target_ = kWindowSwapChain; SwapChainType gui_swap_chain_target_ = kWindowSwapChain; std::unique_ptr color_target_; std::unique_ptr depth_target_; std::unique_ptr object_manager_; std::unique_ptr scene_view_; + std::unique_ptr scene_bridge_; std::unique_ptr gui_view_; int window_width_ = 0; int window_height_ = 0; diff --git a/src/experimental/filament/filament/imgui_editor.cc b/src/experimental/filament/filament/imgui_editor.cc index 6b67072e..bfc773f4 100644 --- a/src/experimental/filament/filament/imgui_editor.cc +++ b/src/experimental/filament/filament/imgui_editor.cc @@ -35,6 +35,7 @@ #include #include #include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" namespace mujoco { @@ -560,7 +561,7 @@ void DrawCameraGui(SceneView* scene_view) { Ui("Direction", &direction); } -void DrawIndirectLightGui(SceneView* scene_view) { +void DrawIndirectLightGui(SceneBridge* scene_bridge, SceneView* scene_view) { filament::View* view = scene_view->GetDefaultRenderView(); auto ibl = view->getScene()->getIndirectLight(); @@ -575,7 +576,7 @@ void DrawIndirectLightGui(SceneView* scene_view) { static char filename[256]; ImGui::InputText("Filename", filename, sizeof(filename)); if (ImGui::Button("Load")) { - scene_view->SetEnvironmentLight(filename, intensity); + scene_bridge->SetEnvironmentLight(filename, intensity); } } @@ -649,7 +650,8 @@ void DrawLightGui(filament::LightManager& lm, } } -void DrawGui(SceneView* scene_view) { +void DrawGui(SceneBridge* scene_bridge) { + SceneView* scene_view = scene_bridge->GetSceneView(); filament::View* view = scene_view->GetDefaultRenderView(); filament::Engine* engine = scene_view->GetEngine(); filament::LightManager& lm = engine->getLightManager(); @@ -716,7 +718,7 @@ void DrawGui(SceneView* scene_view) { } if (ImGui::TreeNodeEx("Lights")) { if (ImGui::TreeNodeEx("Indirect (Image-based) Light")) { - DrawIndirectLightGui(scene_view); + DrawIndirectLightGui(scene_bridge, scene_view); ImGui::TreePop(); } view->getScene()->forEach([&](utils::Entity entity) { diff --git a/src/experimental/filament/filament/imgui_editor.h b/src/experimental/filament/filament/imgui_editor.h index b8f45357..f073fe45 100644 --- a/src/experimental/filament/filament/imgui_editor.h +++ b/src/experimental/filament/filament/imgui_editor.h @@ -15,12 +15,12 @@ #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ -#include "experimental/filament/filament/scene_view.h" +#include "experimental/filament/filament/scene_bridge.h" namespace mujoco { // Generates a ImGui Window for the given scene views. -void DrawGui(SceneView* scene_views); +void DrawGui(SceneBridge* scene_bridge); } // namespace mujoco diff --git a/src/experimental/filament/filament/scene_bridge.cc b/src/experimental/filament/filament/scene_bridge.cc new file mode 100644 index 00000000..6b445035 --- /dev/null +++ b/src/experimental/filament/filament/scene_bridge.cc @@ -0,0 +1,370 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/scene_bridge.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/gui_view.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/model_objects.h" +#include "experimental/filament/filament/model_util.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +using filament::math::float3; +using filament::math::float4; +using filament::math::mat3; +using filament::math::mat4; + +SceneBridge::SceneBridge(ObjectManager* object_mgr, const mjModel* model, + SceneView* scene_view) + : scene_view_(scene_view), object_mgr_(object_mgr) { + model_objects_ = + std::make_unique(model, object_mgr_->GetEngine()); + + // Configure options for the normal view. + auto cg = scene_view_->GetColorGradingOptions(); + cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure); + cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast); + cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance); + cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation); + cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature); + cg.tint = ReadElement(model, "filament.out.tint", cg.tint); + + auto tone_mapping = + ReadElement(model, "filament.out.tone_mapping"); + if (tone_mapping == "aces") { + cg.tone_mapper = ToneMapperType::kACES; + } else if (tone_mapping == "aces_legacy") { + cg.tone_mapper = ToneMapperType::kACESLegacy; + } else if (tone_mapping == "filmic") { + cg.tone_mapper = ToneMapperType::kFilmic; + } else if (tone_mapping == "linear") { + cg.tone_mapper = ToneMapperType::kLinear; + } else if (tone_mapping == "pbr_neutral") { + cg.tone_mapper = ToneMapperType::kPBRNeutral; + } + scene_view_->SetColorGradingOptions(cg); + + filament::View* fview = scene_view_->GetDefaultRenderView(); + auto ao = fview->getAmbientOcclusionOptions(); + ao.enabled = ReadElement(model, "filament.ao.enabled", true); + ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false); + ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled); + ao.quality = filament::QualityLevel::ULTRA; + ao.lowPassFilter = filament::QualityLevel::ULTRA; + ao.upsampling = filament::QualityLevel::ULTRA; + ao.bilateralThreshold = 0.5f; + fview->setAmbientOcclusionOptions(ao); + + auto msaa = fview->getMultiSampleAntiAliasingOptions(); + msaa.enabled = ReadElement(model, "filament.msaa.enabled", true); + fview->setMultiSampleAntiAliasingOptions(msaa); + + default_shadow_map_size_ = ReadElement( + model, "filament.shadows.map_size", default_shadow_map_size_); + default_vsm_blur_width_ = ReadElement( + model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); + + auto shadow_type = fview->getShadowType(); + shadow_type = ReadElement(model, "filament.shadows.type", shadow_type); + fview->setShadowType(shadow_type); + + auto fog_opts = fview->getFogOptions(); + fog_opts.enabled = + ReadElement(model, "filament.fog.enabled", fog_opts.enabled); + fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color); + fog_opts.distance = ReadElement( + model, "filament.fog.distance", fog_opts.distance); + fog_opts.density = ReadElement( + model, "filament.fog.density", fog_opts.density); + fog_opts.cutOffDistance = ReadElement( + model, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); + fog_opts.maximumOpacity = ReadElement( + model, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); + fog_opts.height = ReadElement(model, "filament.fog.height", fog_opts.height); + fog_opts.heightFalloff = ReadElement( + model, "filament.fog.heightFalloff", fog_opts.heightFalloff); + fog_opts.inScatteringStart = ReadElement( + model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); + fog_opts.inScatteringSize = ReadElement( + model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); + fview->setFogOptions(fog_opts); + + fallback_head_light_intensity_ = + ReadElement(model, "filament.fallback.head_light_intensity", + fallback_head_light_intensity_); + fallback_scene_light_intensity_ = + ReadElement(model, "filament.fallback.scene_light_intensity", + fallback_scene_light_intensity_); + fallback_environment_light_intensity_ = + ReadElement(model, "filament.fallback.environment_light_intensity", + fallback_environment_light_intensity_); + + // Create an empty/black indirect light to ensure that the skybox is oriented + // to respect mujoco's Z-up convention. + filament::IndirectLight* empty_ibl = + model_objects_->CreateIndirectLight(-1, 100000); + scene_view_->AddToScene(empty_ibl); + + PrepareLights(); +} + +SceneBridge::~SceneBridge() { + for (auto& iter : lights_) { + scene_view_->RemoveFromScene(iter.get()); + } + lights_.clear(); + + for (auto& iter : drawables_) { + scene_view_->RemoveFromScene(iter.get()); + } + drawables_.clear(); +} + +void SceneBridge::SetEnvironmentLight(std::string_view filename, + float intensity) { + filament::IndirectLight* ibl = nullptr; + scene_view_->AddToScene(ibl); + object_mgr_->LoadFallbackIndirectLight(filename, intensity); + ibl = object_mgr_->GetFallbackIndirectLight(); + scene_view_->AddToScene(ibl); +} + +std::optional SceneBridge::ClipFromWorld(const float3& pos) const{ + const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f); + if (clip_pos.w == 0.0f) { + return std::nullopt; + } + return clip_pos.xyz / clip_pos.w; +} + +void SceneBridge::PrepareLights() { + filament::Engine* engine = object_mgr_->GetEngine(); + const mjModel* model = model_objects_->GetModel(); + filament::Skybox* skybox = model_objects_->CreateSkybox(); + if (skybox) { + scene_view_->AddToScene(skybox); + } + + float total_light_intensity = 0.0f; + + for (int i = 0; i < model->nlight; ++i) { + total_light_intensity += model->light_intensity[i]; + + if (model->light_type[i] == mjLIGHT_IMAGE) { + auto* indirect_light = model_objects_->CreateIndirectLight( + model->light_texid[i], model->light_intensity[i]); + if (indirect_light) { + scene_view_->AddToScene(indirect_light); + } + // Add an nullptr as a placeholder so that our indices still match. + lights_.emplace_back(nullptr); + } else { + Light::Params params; + params.color = ReadFloat3(model->light_diffuse); + params.type = (mjtLightType)model->light_type[i]; + params.castshadow = model->light_castshadow[i]; + params.bulbradius = model->light_bulbradius[i]; + params.range = model->light_range[i]; + params.intensity = model->light_intensity[i]; + params.shadow_map_size = default_shadow_map_size_; + params.vsm_blur_width = default_vsm_blur_width_; + if (params.type == mjLIGHT_SPOT) { + params.spot_cone_angle = model->light_cutoff[i]; + } + + auto light_obj = std::make_unique(engine, params); +#ifndef __EMSCRIPTEN__ + // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. + scene_view_->AddToScene(light_obj.get()); +#endif + lights_.emplace_back(std::move(light_obj)); + } + } + + // Add a placeholder (black) headlight as our last light. Going forward, we'll + // assume lights_.back() is always the headlight. + { + Light::Params params; + params.color = float3(0, 0, 0); + params.headlight = true; + params.type = mjLIGHT_DIRECTIONAL; + params.castshadow = 0; + params.intensity = 0; + auto light_obj = std::make_unique(engine, params); +#ifndef __EMSCRIPTEN__ + // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. + scene_view_->AddToScene(light_obj.get()); +#endif + lights_.emplace_back(std::move(light_obj)); + } + + // There are no "physical" lights in the scene which means we're likely + // dealing with a "classic renderer" scene. In this case, let's add a + // default environment light and set the light intensity ourselves. + if (total_light_intensity == 0.0f) { + auto* ibl = object_mgr_->GetFallbackIndirectLight(); + if (ibl) { + ibl->setIntensity(fallback_environment_light_intensity_); + scene_view_->AddToScene(ibl); + } + const float intensity = fallback_scene_light_intensity_ / lights_.size(); + for (auto& light : lights_) { + if (light) { + light->SetIntensity( + light->IsHeadlight() ? fallback_head_light_intensity_ : intensity); + } + } + } +} + +filament::math::mat4 CalculateClipFromWorld(const mjrRect& viewport, + const mjvGLCamera& cam) { + const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); + const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); + const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); + const float3 cam_at = cam_pos + cam_fwd; + const float aspect_ratio = (float)viewport.width / (float)viewport.height; + const float halfwidth = + cam.frustum_width + ? cam.frustum_width + : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom); + const float left = cam.frustum_center - halfwidth; + const float right = cam.frustum_center + halfwidth; + + mat4 projection; + if (cam.orthographic) { + projection = mat4::ortho(left, right, cam.frustum_bottom, cam.frustum_top, + cam.frustum_near, cam.frustum_far); + } else { + projection = mat4::frustum(left, right, cam.frustum_bottom, cam.frustum_top, + cam.frustum_near, cam.frustum_far); + projection[2][2] = -1.0f; + projection[3][2] = -2.0f * cam.frustum_near; + } + mat4 look_at = mat4::lookAt(cam_pos, cam_at, cam_up); + return projection * inverse(look_at); +} + +void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) { + filament::View* view = scene_view_->GetDefaultRenderView(); + view->setShadowingEnabled(scene->flags[mjRND_SHADOW] ? true : false); + + mjtNum hpos[3], hfwd[3]; + float headpos[3], gazedir[3]; + mjv_cameraInModel(hpos, hfwd, nullptr, scene); + mju_n2f(headpos, hpos, 3); + mju_n2f(gazedir, hfwd, 3); + + const mjvGLCamera gl_camera = + mjv_averageCamera(scene->camera, scene->camera + 1); + clip_from_world_ = CalculateClipFromWorld(viewport, gl_camera); + + // Remove all drawables from previous render and prepare new ones. + for (auto& iter : drawables_) { + scene_view_->RemoveFromScene(iter.get()); + } + drawables_.clear(); + for (int i = 0; i < scene->ngeom; ++i) { + const mjvGeom* geom = scene->geoms + i; + + if (geom->label[0] != 0) { + if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) { + DrawTextAt(geom->label, pos->x, pos->y, pos->z); + } + } + + auto drawable = + std::make_unique(object_mgr_, model_objects_.get(), *geom); + drawable->Update(model_objects_->GetModel(), scene, *geom); + scene_view_->AddToScene(drawable.get()); + drawables_.push_back(std::move(drawable)); + } + + bool headlight_enabled = false; + for (int i = 0; i < scene->nlight; ++i) { + const mjvLight& scene_light = scene->lights[i]; + if (scene_light.id < 0 && scene_light.headlight) { + // We position the headlight slightly behind the camera to avoid some + // odd clipping issues. + headlight_enabled = true; + headpos[0] -= gazedir[0] * 0.05f; + headpos[1] -= gazedir[1] * 0.05f; + headpos[2] -= gazedir[2] * 0.05f; + + // The headlight is always the "back" light. + std::unique_ptr& light = lights_.back(); + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir)); + continue; + } else if (scene_light.id < lights_.size() - 1) { + std::unique_ptr& light = lights_[scene_light.id]; + if (light) { + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(scene_light.pos), + ReadFloat3(scene_light.dir)); + } + } else { + mju_error("Unexpected light id: %d", scene_light.id); + } + } + + // Enable/disable the headlight based on whether or not it's in the scene. + if (headlight_enabled) { + lights_.back()->Enable(); + } else { + lights_.back()->Disable(); + } +} + +void SceneBridge::UploadMesh(const mjModel* model, int id) { + model_objects_->UploadMesh(model, id); +} + +void SceneBridge::UploadTexture(const mjModel* model, int id) { + model_objects_->UploadTexture(model, id); +} + +void SceneBridge::UploadHeightField(const mjModel* model, int id) { + model_objects_->UploadHeightField(model, id); +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/scene_bridge.h b/src/experimental/filament/filament/scene_bridge.h new file mode 100644 index 00000000..a1e56796 --- /dev/null +++ b/src/experimental/filament/filament/scene_bridge.h @@ -0,0 +1,85 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/model_objects.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +// Manages all mjModel data and updates a SceneView using an mjvScene. +class SceneBridge { + public: + SceneBridge(ObjectManager* object_mgr, const mjModel* model, + SceneView* scene_view); + ~SceneBridge(); + + // Updates the environment light using the KTX image at the given path. + void SetEnvironmentLight(std::string_view filename, float intensity); + + // Updates the environment light to the fallback light + void SetFallbackEnvironmentLight(float intensity); + + // Updates the Entities in the filament Scene to match the current mjvScene + // state. + void Update(const mjrRect& viewport, const mjvScene* scene); + + // Creates the filament objects from the mjModel. + void UploadMesh(const mjModel* model, int id); + void UploadTexture(const mjModel* model, int id); + void UploadHeightField(const mjModel* model, int id); + + SceneView* GetSceneView() const { return scene_view_; } + + SceneBridge(const SceneBridge&) = delete; + SceneBridge& operator=(const SceneBridge&) = delete; + + private: + void PrepareLights(); + + // Converts a point in world space to clip space, eg. in the range [-1,-1, 0] + // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera. + std::optional ClipFromWorld( + const filament::math::float3& pos) const; + + SceneView* scene_view_ = nullptr; + ObjectManager* object_mgr_ = nullptr; + std::unique_ptr model_objects_; + std::vector> lights_; + std::vector> drawables_; + filament::math::mat4 clip_from_world_; + int default_shadow_map_size_ = 2048; + float default_vsm_blur_width_ = 0.0f; + float fallback_head_light_intensity_ = 0.f; + float fallback_scene_light_intensity_ = 80'000.f; + float fallback_environment_light_intensity_ = 5'000.f; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index 4f6f1ceb..3cd7f929 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -14,12 +14,10 @@ #include "experimental/filament/filament/scene_view.h" +#include #include #include #include -#include -#include -#include #include #include @@ -43,12 +41,9 @@ #include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/drawable.h" -#include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/material.h" #include "experimental/filament/filament/math_util.h" -#include "experimental/filament/filament/model_objects.h" -#include "experimental/filament/filament/model_util.h" -#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" #include "experimental/filament/filament/texture.h" @@ -56,21 +51,17 @@ namespace mujoco { using filament::math::float3; using filament::math::float4; -using filament::math::mat3; using filament::math::mat4; static constexpr int kNormalIndex = - static_cast(SceneView::DrawMode::kNormal); + static_cast(Material::DrawMode::kNormal); static constexpr int kDepthIndex = - static_cast(SceneView::DrawMode::kDepth); + static_cast(Material::DrawMode::kDepth); static constexpr int kSegmentIndex = - static_cast(SceneView::DrawMode::kSegmentation); + static_cast(Material::DrawMode::kSegmentation); -static filament::Viewport ReadViewport(mjrRect rect) { - return filament::Viewport(rect.left, rect.bottom, rect.width, rect.height); -} - -filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) { +static filament::ColorGrading::Builder ToBuilder( + const ColorGradingOptions& opts) { return filament::ColorGrading::Builder() .format(opts.format) .dimensions(opts.dimension) @@ -89,6 +80,27 @@ filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) { .curves(opts.shadow_gamma, opts.mid_point, opts.highlight_scale); } +static void SetupCamera(const mjvGLCamera& cam, + const filament::Viewport& viewport, + filament::Camera* camera) { + const filament::Camera::Projection type = + cam.orthographic ? filament::Camera::Projection::ORTHO + : filament::Camera::Projection::PERSPECTIVE; + const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); + const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); + const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); + const float3 cam_at = cam_pos + cam_fwd; + const float aspect_ratio = (float)viewport.width / (float)viewport.height; + const float halfwidth = + cam.frustum_width + ? cam.frustum_width + : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom); + camera->lookAt(cam_pos, cam_at, cam_up); + camera->setProjection(type, cam.frustum_center - halfwidth, + cam.frustum_center + halfwidth, cam.frustum_bottom, + cam.frustum_top, cam.frustum_near, cam.frustum_far); +} + // Sets up the `reflection_camera`'s projection matrix so that it is a // reflection of the `src_camera` across the plane defined by the // `surface_xform`. The generated projection is an oblique projection so that @@ -115,11 +127,7 @@ static void SetupReflectionCamera(const mat4& surface_xform, reflection_camera->setCustomProjection(oblique, near, far); } -SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) - : object_mgr_(object_mgr) { - filament::Engine* engine = object_mgr_->GetEngine(); - model_objects_ = std::make_unique(model, engine); - +SceneView::SceneView(filament::Engine* engine) : engine_(engine) { scene_ = engine->createScene(); camera_ = engine->createCamera(utils::EntityManager::get().create()); reflect_camera_ = engine->createCamera(utils::EntityManager::get().create()); @@ -136,53 +144,6 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) reflect_view_->setShadowingEnabled(false); reflect_view_->setPostProcessingEnabled(false); - // Configure options for the normal view. - auto& cg = color_grading_options_; - cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure); - cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast); - cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance); - cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation); - cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature); - cg.tint = ReadElement(model, "filament.out.tint", cg.tint); - - auto tone_mapping = - ReadElement(model, "filament.out.tone_mapping"); - if (tone_mapping == "aces") { - cg.tone_mapper = ToneMapperType::kACES; - } else if (tone_mapping == "aces_legacy") { - cg.tone_mapper = ToneMapperType::kACESLegacy; - } else if (tone_mapping == "filmic") { - cg.tone_mapper = ToneMapperType::kFilmic; - } else if (tone_mapping == "linear") { - cg.tone_mapper = ToneMapperType::kLinear; - } else if (tone_mapping == "pbr_neutral") { - cg.tone_mapper = ToneMapperType::kPBRNeutral; - } - SetColorGradingOptions(cg); - - auto ao = views_[kNormalIndex]->getAmbientOcclusionOptions(); - ao.enabled = ReadElement(model, "filament.ao.enabled", true); - ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false); - ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled); - ao.quality = filament::QualityLevel::ULTRA; - ao.lowPassFilter = filament::QualityLevel::ULTRA; - ao.upsampling = filament::QualityLevel::ULTRA; - ao.bilateralThreshold = 0.5f; - views_[kNormalIndex]->setAmbientOcclusionOptions(ao); - - auto msaa = views_[kNormalIndex]->getMultiSampleAntiAliasingOptions(); - msaa.enabled = ReadElement(model, "filament.msaa.enabled", true); - views_[kNormalIndex]->setMultiSampleAntiAliasingOptions(msaa); - - default_shadow_map_size_ = ReadElement( - model, "filament.shadows.map_size", default_shadow_map_size_); - default_vsm_blur_width_ = ReadElement( - model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); - - auto shadow_type = views_[kNormalIndex]->getShadowType(); - shadow_type = ReadElement(model, "filament.shadows.type", shadow_type); - views_[kNormalIndex]->setShadowType(shadow_type); - // Disable post processing for the depth and segmentation views to preserve // the values. views_[kDepthIndex]->setPostProcessingEnabled(false); @@ -192,79 +153,115 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) auto fog = views_[kNormalIndex]->getFogEntity(); auto& tm = engine->getTransformManager(); tm.create(fog); - auto rotation_axis = ReadElement( - model, "filament.fog.rotation_axis", float3{-1, 0, 0}); tm.setTransform(tm.getInstance(fog), - mat4::rotation(filament::math::f::PI / 2, rotation_axis)); - - auto fog_opts = views_[kNormalIndex]->getFogOptions(); - fog_opts.enabled = - ReadElement(model, "filament.fog.enabled", fog_opts.enabled); - fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color); - fog_opts.distance = ReadElement( - model, "filament.fog.distance", fog_opts.distance); - fog_opts.density = ReadElement( - model, "filament.fog.density", fog_opts.density); - fog_opts.cutOffDistance = ReadElement( - model, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); - fog_opts.maximumOpacity = ReadElement( - model, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); - fog_opts.height = ReadElement(model, "filament.fog.height", fog_opts.height); - fog_opts.heightFalloff = ReadElement( - model, "filament.fog.heightFalloff", fog_opts.heightFalloff); - fog_opts.inScatteringStart = ReadElement( - model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); - fog_opts.inScatteringSize = ReadElement( - model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); - views_[kNormalIndex]->setFogOptions(fog_opts); - - fallback_head_light_intensity_ = - ReadElement(model, "filament.fallback.head_light_intensity", - fallback_head_light_intensity_); - fallback_scene_light_intensity_ = - ReadElement(model, "filament.fallback.scene_light_intensity", - fallback_scene_light_intensity_); - fallback_environment_light_intensity_ = - ReadElement(model, "filament.fallback.environment_light_intensity", - fallback_environment_light_intensity_); - - // Create an empty/black indirect light to ensure that the skybox is oriented - // to respect mujoco's Z-up convention. - scene_->setIndirectLight(model_objects_->CreateIndirectLight(-1, 100000)); - - PrepareLights(); + mat4::rotation(filament::math::f::PI / 2, float3{-1, 0, 0})); } SceneView::~SceneView() { + for (auto& light : lights_) { + light->RemoveFromScene(scene_); + } + for (auto& drawable : drawables_) { + drawable->RemoveFromScene(scene_); + } lights_.clear(); drawables_.clear(); reflect_targets_.clear(); - - filament::Engine* engine = object_mgr_->GetEngine(); - engine->destroyCameraComponent(reflect_camera_->getEntity()); - engine->destroy(reflect_view_); - - engine->destroyCameraComponent(camera_->getEntity()); - engine->destroy(views_[kNormalIndex]->getColorGrading()); - for (auto& view : views_) { - engine->destroy(view); + engine_->destroyCameraComponent(reflect_camera_->getEntity()); + engine_->destroy(reflect_view_); + engine_->destroyCameraComponent(camera_->getEntity()); + if (color_grading_) { + engine_->destroy(color_grading_); + } + engine_->destroy(scene_); + for (auto& view : views_) { + engine_->destroy(view); } - engine->destroy(scene_); } -void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode, - RenderTarget* target) { - filament::View* view = PrepareRenderView(draw_mode); +void SceneView::AddToScene(Light* light) { + if (lights_.insert(light).second) { + light->AddToScene(scene_); + } +} + +void SceneView::RemoveFromScene(Light* light) { + if (lights_.erase(light)) { + light->RemoveFromScene(scene_); + } +} + +void SceneView::AddToScene(Drawable* drawable) { + if (drawables_.insert(drawable).second) { + drawable->AddToScene(scene_); + if (drawable->IsReflective()) { + AddReflectiveDrawable(drawable); + } + } +} + +void SceneView::RemoveFromScene(Drawable* drawable) { + if (drawables_.erase(drawable)) { + auto it = std::find(reflectives_.begin(), reflectives_.end(), drawable); + if (it != reflectives_.end()) { + reflectives_.erase(it); + } + drawable->RemoveFromScene(scene_); + } +} + +void SceneView::AddToScene(filament::Skybox* skybox) { + skybox_ = skybox; + scene_->setSkybox(skybox); +} + +void SceneView::RemoveFromScene(filament::Skybox* skybox) { + if (skybox_ == skybox) { + skybox_ = nullptr; + scene_->setSkybox(nullptr); + } +} + +void SceneView::AddToScene(filament::IndirectLight* indirect_light) { + indirect_light_ = indirect_light; + scene_->setIndirectLight(indirect_light); +} + +void SceneView::RemoveFromScene(filament::IndirectLight* indirect_light) { + if (indirect_light_ == indirect_light) { + indirect_light_ = nullptr; + scene_->setIndirectLight(nullptr); + } +} + +void SceneView::Render(filament::Renderer* renderer, + const RenderRequest& request) { + filament::Viewport viewport(request.viewport.left, request.viewport.bottom, + request.viewport.width, request.viewport.height); + for (auto& view : views_) { + view->setViewport(viewport); + } + reflect_view_->setViewport(viewport); + + SetupCamera(request.camera, viewport, camera_); + + for (auto& iter : drawables_) { + iter->SetDrawMode(request.draw_mode); + } + + filament::View* view = views_[static_cast(request.draw_mode)]; filament::MultiSampleAntiAliasingOptions options = view->getMultiSampleAntiAliasingOptions(); - if (target) { + filament::RenderTarget* render_target = + request.target ? request.target->GetFilamentRenderTarget() : nullptr; + if (render_target) { // We need to disable msaa in order to render to texture. view->setMultiSampleAntiAliasingOptions({.enabled = false}); } // Render reflection passes. - if (draw_mode == DrawMode::kNormal) { + if (request.draw_mode == DrawMode::kNormal) { for (size_t i = 0; i < reflectives_.size(); ++i) { Drawable* drawable = reflectives_[i]; @@ -283,242 +280,24 @@ void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode, } } - view->setRenderTarget(target ? target->GetFilamentRenderTarget() : nullptr); + view->setRenderTarget(render_target); renderer->render(view); view->setRenderTarget(nullptr); - if (target) { + if (request.target) { view->setMultiSampleAntiAliasingOptions(options); } } -filament::View* SceneView::PrepareRenderView(DrawMode mode) { - for (auto& iter : drawables_) { - iter->SetDrawMode(mode); - } - return views_[static_cast(mode)]; -} - -void SceneView::SetViewport(mjrRect viewport) { - auto filament_viewport = ReadViewport(viewport); - aspect_ratio_ = (float)viewport.width / (float)viewport.height; - for (auto& view : views_) { - view->setViewport(filament_viewport); - } - reflect_view_->setViewport(filament_viewport); -} - -void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { - filament::Engine* engine = object_mgr_->GetEngine(); - - auto tone_mapper = CreateToneMapper(opts.tone_mapper); - auto color_grading = ToBuilder(color_grading_options_) - .toneMapper(tone_mapper.get()) - .build(*engine); - views_[kNormalIndex]->setColorGrading(color_grading); - engine->destroy(color_grading_); - color_grading_ = color_grading; - color_grading_options_ = opts; -} - -void SceneView::SetEnvironmentLight(std::string_view filename, - float intensity) { - scene_->setIndirectLight(nullptr); - object_mgr_->LoadFallbackIndirectLight(filename, intensity); - scene_->setIndirectLight(object_mgr_->GetFallbackIndirectLight()); -} - -void SceneView::SetFallbackEnvironmentLight(float intensity) { - auto* ibl = object_mgr_->GetFallbackIndirectLight(); - if (ibl) { - ibl->setIntensity(intensity); - scene_->setIndirectLight(ibl); - } -} - -void SceneView::UpdateCamera(const mjvGLCamera* cameras) { - const mjvGLCamera cam = mjv_averageCamera(cameras, cameras + 1); - const filament::Camera::Projection type = - cam.orthographic ? filament::Camera::Projection::ORTHO - : filament::Camera::Projection::PERSPECTIVE; - float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); - float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); - float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); - float3 cam_at = cam_pos + cam_fwd; - camera_->lookAt(cam_pos, cam_at, cam_up); - float halfwidth = cam.frustum_width ? cam.frustum_width - : 0.5f * aspect_ratio_ * (cam.frustum_top - cam.frustum_bottom); - camera_->setProjection(type, cam.frustum_center - halfwidth, - cam.frustum_center + halfwidth, cam.frustum_bottom, - cam.frustum_top, cam.frustum_near, cam.frustum_far); - clip_from_world_ = camera_->getProjectionMatrix() * camera_->getViewMatrix(); -} - -std::optional SceneView::ClipFromWorld(const float3& pos) const{ - const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f); - if (clip_pos.w == 0.0f) { - return std::nullopt; - } - return clip_pos.xyz / clip_pos.w; -} - -void SceneView::PrepareLights() { - filament::Engine* engine = object_mgr_->GetEngine(); - const mjModel* model = model_objects_->GetModel(); - filament::Skybox* skybox = model_objects_->CreateSkybox(); - if (skybox) { - scene_->setSkybox(skybox); - } - - float total_light_intensity = 0.0f; - - for (int i = 0; i < model->nlight; ++i) { - total_light_intensity += model->light_intensity[i]; - - if (model->light_type[i] == mjLIGHT_IMAGE) { - auto* indirect_light = model_objects_->CreateIndirectLight( - model->light_texid[i], model->light_intensity[i]); - if (indirect_light) { - scene_->setIndirectLight(indirect_light); - } - // Add an nullptr as a placeholder so that our indices still match. - lights_.emplace_back(nullptr); - } else { - Light::Params params; - params.color = ReadFloat3(model->light_diffuse); - params.type = (mjtLightType)model->light_type[i]; - params.castshadow = model->light_castshadow[i]; - params.bulbradius = model->light_bulbradius[i]; - params.range = model->light_range[i]; - params.intensity = model->light_intensity[i]; - params.shadow_map_size = default_shadow_map_size_; - params.vsm_blur_width = default_vsm_blur_width_; - if (params.type == mjLIGHT_SPOT) { - params.spot_cone_angle = model->light_cutoff[i]; - } - - auto light_obj = std::make_unique(engine, params); -#ifndef __EMSCRIPTEN__ - // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. - light_obj->AddToScene(scene_); -#endif - lights_.emplace_back(std::move(light_obj)); - } - } - - // Add a placeholder (black) headlight as our last light. Going forward, we'll - // assume lights_.back() is always the headlight. - { - Light::Params params; - params.color = float3(0, 0, 0); - params.headlight = true; - params.type = mjLIGHT_DIRECTIONAL; - params.castshadow = 0; - params.intensity = 0; - auto light_obj = std::make_unique(engine, params); -#ifndef __EMSCRIPTEN__ - // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. - light_obj->AddToScene(scene_); -#endif - lights_.emplace_back(std::move(light_obj)); - } - - // There are no "physical" lights in the scene which means we're likely - // dealing with a "classic renderer" scene. In this case, let's add a - // default environment light and set the light intensity ourselves. - if (total_light_intensity == 0.0f) { - SetFallbackEnvironmentLight(fallback_environment_light_intensity_); - const float intensity = fallback_scene_light_intensity_ / lights_.size(); - for (auto& light : lights_) { - if (light) { - light->SetIntensity( - light->IsHeadlight() ? fallback_head_light_intensity_ : intensity); - } - } - } -} - -void SceneView::UpdateScene(const mjvScene* scene) { - views_[kNormalIndex]->setShadowingEnabled(scene->flags[mjRND_SHADOW]); - - mjtNum hpos[3], hfwd[3]; - float headpos[3], gazedir[3]; - mjv_cameraInModel(hpos, hfwd, nullptr, scene); - mju_n2f(headpos, hpos, 3); - mju_n2f(gazedir, hfwd, 3); - UpdateCamera(scene->camera); - - // Remove all drawables from previous render and prepare new ones. - for (auto& iter : drawables_) { - iter->RemoveFromScene(scene_); - } - drawables_.clear(); - reflectives_.clear(); - for (int i = 0; i < scene->ngeom; ++i) { - const mjvGeom* geom = scene->geoms + i; - - if (geom->label[0] != 0) { - if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) { - DrawTextAt(geom->label, pos->x, pos->y, pos->z); - } - } - - auto drawable = - std::make_unique(object_mgr_, model_objects_.get(), *geom); - drawable->AddToScene(scene_); - drawable->Update(model_objects_->GetModel(), scene, *geom); - if (drawable->IsReflective()) { - AddReflectiveDrawable(drawable.get()); - } - drawables_.push_back(std::move(drawable)); - } - - bool headlight_enabled = false; - for (int i = 0; i < scene->nlight; ++i) { - const mjvLight& scene_light = scene->lights[i]; - if (scene_light.id < 0 && scene_light.headlight) { - // We position the headlight slightly behind the camera to avoid some - // odd clipping issues. - headlight_enabled = true; - headpos[0] -= gazedir[0] * 0.05f; - headpos[1] -= gazedir[1] * 0.05f; - headpos[2] -= gazedir[2] * 0.05f; - - // The headlight is always the "back" light. - std::unique_ptr& light = lights_.back(); - light->SetColor(ReadFloat3(scene_light.diffuse)); - light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir)); - continue; - } else if (scene_light.id < lights_.size() - 1) { - std::unique_ptr& light = lights_[scene_light.id]; - if (light) { - light->SetColor(ReadFloat3(scene_light.diffuse)); - light->SetTransform(ReadFloat3(scene_light.pos), - ReadFloat3(scene_light.dir)); - } - } else { - mju_error("Unexpected light id: %d", scene_light.id); - } - } - - // Enable/disable the headlight based on whether or not it's in the scene. - if (headlight_enabled) { - lights_.back()->Enable(); - } else { - lights_.back()->Disable(); - } -} - void SceneView::AddReflectiveDrawable(Drawable* drawable) { const int index = reflectives_.size(); reflectives_.push_back(drawable); // Ensure we have the same number of render targets as we do reflective // drawables. - filament::Engine* engine = object_mgr_->GetEngine(); while (reflect_targets_.size() < reflectives_.size()) { reflect_targets_.push_back(std::make_unique( - engine, RenderTargetTextureType::kReflectionColor, + engine_, RenderTargetTextureType::kReflectionColor, RenderTargetTextureType::kDepth)); } @@ -529,20 +308,17 @@ void SceneView::AddReflectiveDrawable(Drawable* drawable) { drawable->UpdateReflectionTexture(target->GetColorTexture()); } -void SceneView::UploadMesh(const mjModel* model, int id) { - model_objects_->UploadMesh(model, id); -} - -void SceneView::UploadTexture(const mjModel* model, int id) { - model_objects_->UploadTexture(model, id); -} - -void SceneView::UploadHeightField(const mjModel* model, int id) { - model_objects_->UploadHeightField(model, id); -} - -filament::Engine* SceneView::GetEngine() const { - return object_mgr_->GetEngine(); +void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { + auto tone_mapper = CreateToneMapper(opts.tone_mapper); + auto color_grading = ToBuilder(color_grading_options_) + .toneMapper(tone_mapper.get()) + .build(*engine_); + views_[kNormalIndex]->setColorGrading(color_grading); + if (color_grading_) { + engine_->destroy(color_grading_); + } + color_grading_ = color_grading; + color_grading_options_ = opts; } filament::View* SceneView::GetDefaultRenderView() { diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index 8fb686e1..a7d7c3aa 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -17,120 +17,100 @@ #include #include -#include -#include +#include #include #include #include #include -#include #include #include -#include -#include -#include -#include #include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/drawable.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/material.h" -#include "experimental/filament/filament/model_objects.h" -#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" namespace mujoco { -// Creates and owns filament Scene and View classes given a mjvScene. +// Creates and owns the filament Scene and View (and Camera) classes. // -// The filament Scene is populated with the objects (e.g. lights, geoms, -// cameras, etc.) defined by the mjvScene. Multiple Views are created to allow -// different rendering modes (e.g. normal, depth, segmentation, etc.) +// The filament Scene is populated with the objects (e.g. lights, renderables, +// skybox, etc.). It manages multiple views to support a variety of draw modes +// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. class SceneView { public: - SceneView(ObjectManager* object_mgr, const mjModel* model); + SceneView(filament::Engine* engine); ~SceneView(); - // Updates all views to render into the given viewport. - void SetViewport(mjrRect viewport); - - // Updates the color grading options for the main render view. - void SetColorGradingOptions(const ColorGradingOptions& opts); - - // Updates the environment light using the KTX image at the given path. - void SetEnvironmentLight(std::string_view filename, float intensity); - - // Updates the environment light to the fallback light - void SetFallbackEnvironmentLight(float intensity); - - // Updates the Entities in the filament Scene to match the current mjvScene - // state. - void UpdateScene(const mjvScene* scene); + // Adds/removes entities from the scene. + void AddToScene(Light* light); + void RemoveFromScene(Light* light); + void AddToScene(Drawable* drawable); + void RemoveFromScene(Drawable* drawable); + void AddToScene(filament::Skybox* skybox); + void RemoveFromScene(filament::Skybox* skybox); + void AddToScene(filament::IndirectLight* indirect_light); + void RemoveFromScene(filament::IndirectLight* indirect_light); + // Parameters for rendering the scene. using DrawMode = Material::DrawMode; + struct RenderRequest { + // The draw mode (e.g. normal, depth, segmentation) to render. + DrawMode draw_mode = DrawMode::kNormal; + // The target viewport for the rendered image. + mjrRect viewport; + // The camera from which to render the scene. + mjvGLCamera camera; + // An optional render target into which the scene will be rendered. + RenderTarget* target = nullptr; + }; - void Render(filament::Renderer* renderer, DrawMode draw_mode, - RenderTarget* target = nullptr); + // Renders the scene. + void Render(filament::Renderer* renderer, const RenderRequest& request); - void UploadMesh(const mjModel* model, int id); - void UploadTexture(const mjModel* model, int id); - void UploadHeightField(const mjModel* model, int id); + // Returns the filament Engine managing the scene. + filament::Engine* GetEngine() const { return engine_; } - // Accessors. - filament::Engine* GetEngine() const; + // Returns the underlying filament View that is used for normal rendering. + // Callers can update rendering settings (e.g. post processing) directly. filament::View* GetDefaultRenderView(); + + // Helpers for managing the color grading options for the default render view. ColorGradingOptions GetColorGradingOptions() const; + void SetColorGradingOptions(const ColorGradingOptions& opts); SceneView(const SceneView&) = delete; SceneView& operator=(const SceneView&) = delete; private: - // Prepares and returns the filament View for the given draw mode. - filament::View* PrepareRenderView(DrawMode mode); - - void UpdateCamera(const mjvGLCamera* cameras); - - void PrepareLights(); - - // Registers the given drawable as a reflective surface. + // Marks a drawable as reflective. Reflective drawables have to be rendered + // in their own passes to create the reflective texture. void AddReflectiveDrawable(Drawable* drawable); - // Converts a point in world space to clip space, eg. in the range [-1,-1, 0] - // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera. - std::optional ClipFromWorld( - const filament::math::float3& pos) const; - - ObjectManager* object_mgr_ = nullptr; + filament::Engine* engine_ = nullptr; filament::Scene* scene_ = nullptr; filament::Camera* camera_ = nullptr; filament::ColorGrading* color_grading_ = nullptr; - std::vector> lights_; - std::vector> drawables_; - std::unique_ptr model_objects_; - std::array views_; - filament::math::mat4 clip_from_world_; ColorGradingOptions color_grading_options_; + std::array views_; DrawMode active_mode_ = DrawMode::kNumDrawModes; - float aspect_ratio_ = 1.0f; - int default_shadow_map_size_ = 2048; - float default_vsm_blur_width_ = 0.0f; - float fallback_head_light_intensity_ = 0.f; - float fallback_scene_light_intensity_ = 80'000.f; - float fallback_environment_light_intensity_ = 5'000.f; + + // Scene objects. + std::unordered_set lights_; + std::unordered_set drawables_; + filament::Skybox* skybox_ = nullptr; + filament::IndirectLight* indirect_light_ = nullptr; // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr; filament::Camera* reflect_camera_ = nullptr; - // The list of drawables that are reflective. + // The list of reflective drawables and their corresponding render targets. std::vector reflectives_; - - // Each reflective drawable has its own render target which is used to render - // the reflected image. std::vector> reflect_targets_; }; - } // namespace mujoco #endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_