diff --git a/src/experimental/filament/CMakeLists.txt b/src/experimental/filament/CMakeLists.txt index 367f2725..a495863b 100644 --- a/src/experimental/filament/CMakeLists.txt +++ b/src/experimental/filament/CMakeLists.txt @@ -54,6 +54,8 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME} filament/render_target.h filament/renderables.cc filament/renderables.h + filament/scene_bridge.cc + filament/scene_bridge.h filament/scene_view.cc filament/scene_view.h filament/texture.cc diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc index b960eb49..cbc79c24 100644 --- a/src/experimental/filament/filament/filament_context.cc +++ b/src/experimental/filament/filament/filament_context.cc @@ -44,6 +44,7 @@ #include "experimental/filament/filament/model_util.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" #include "experimental/filament/render_context_filament.h" @@ -82,6 +83,7 @@ FilamentContext::FilamentContext(const mjrFilamentConfig* config) FilamentContext::~FilamentContext() { DestroyRenderTargets(); gui_view_.reset(); + scene_bridge_.reset(); scene_view_.reset(); object_manager_.reset(); engine_->destroy(renderer_); @@ -91,7 +93,9 @@ FilamentContext::~FilamentContext() { } void FilamentContext::Init(const mjModel* model) { - scene_view_ = std::make_unique(object_manager_.get(), model); + scene_view_ = std::make_unique(engine_); + scene_bridge_ = std::make_unique(object_manager_.get(), model, + scene_view_.get()); gui_view_ = std::make_unique( engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi)); @@ -120,8 +124,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { window_height_ = viewport.height; } - scene_view_->SetViewport(viewport); - scene_view_->UpdateScene(scene); + scene_bridge_->Update(viewport, scene); // Update the UX renderable entity after processing the scene in case there // are any elements in the scene which generate UX draw calls (e.g. labels). if (gui_view_ && gui_swap_chain_target_ == scene_swap_chain_target_) { @@ -137,6 +140,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { } else if (scene->flags[mjRND_DEPTH]) { last_render_mode_ = SceneView::DrawMode::kDepth; } + last_camera_ = mjv_averageCamera(scene->camera, scene->camera + 1); // Render the frame if we're not rendering to a texture. if (scene_swap_chain_target_ == kWindowSwapChain) { @@ -146,7 +150,11 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { } if (renderer_->beginFrame(window_swap_chain_)) { - scene_view_->Render(renderer_, last_render_mode_); + SceneView::RenderRequest request; + request.draw_mode = last_render_mode_; + request.viewport = viewport; + request.camera = last_camera_; + scene_view_->Render(renderer_, request); if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) { gui_view_->Render(renderer_); @@ -240,7 +248,12 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, if (rgb) { if (renderer_->beginFrame(offscreen_swap_chain_)) { - scene_view_->Render(renderer_, last_render_mode_, color_target_.get()); + SceneView::RenderRequest request; + request.draw_mode = last_render_mode_; + request.viewport = viewport; + request.target = color_target_.get(); + request.camera = last_camera_; + scene_view_->Render(renderer_, request); // Render the GUI to the texture as well if requested. if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) { @@ -256,8 +269,12 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, if (depth) { if (renderer_->beginFrame(offscreen_swap_chain_)) { - scene_view_->Render(renderer_, SceneView::DrawMode::kDepth, - depth_target_.get()); + SceneView::RenderRequest request; + request.draw_mode = SceneView::DrawMode::kDepth; + request.viewport = viewport; + request.target = depth_target_.get(); + request.camera = last_camera_; + scene_view_->Render(renderer_, request); const size_t num_bytes = viewport.width * viewport.height * sizeof(float); ReadDepthPixels(renderer_, depth_target_.get(), viewport, depth, @@ -276,24 +293,24 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, } void FilamentContext::UploadMesh(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadMesh(model, id); + scene_bridge_->UploadMesh(model, id); } void FilamentContext::UploadTexture(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadTexture(model, id); + scene_bridge_->UploadTexture(model, id); } void FilamentContext::UploadHeightField(const mjModel* model, int id) { - if (!scene_view_) { - mju_error("SceneView is not initialized."); + if (!scene_bridge_) { + mju_error("SceneBridge is not initialized."); } - scene_view_->UploadHeightField(model, id); + scene_bridge_->UploadHeightField(model, id); } uintptr_t FilamentContext::UploadGuiImage(uintptr_t tex_id, @@ -315,6 +332,6 @@ double FilamentContext::GetFrameRate() const { return 1.0e9 / static_cast(ns); } -void FilamentContext::UpdateGui() { DrawGui(scene_view_.get()); } +void FilamentContext::UpdateGui() { DrawGui(scene_bridge_.get()); } } // namespace mujoco diff --git a/src/experimental/filament/filament/filament_context.h b/src/experimental/filament/filament/filament_context.h index 8de07824..77c22a88 100644 --- a/src/experimental/filament/filament/filament_context.h +++ b/src/experimental/filament/filament/filament_context.h @@ -27,6 +27,7 @@ #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/render_context_filament.h" @@ -80,12 +81,14 @@ class FilamentContext { std::unique_ptr platform_; SceneView::DrawMode last_render_mode_ = SceneView::DrawMode::kNormal; + mjvGLCamera last_camera_; SwapChainType scene_swap_chain_target_ = kWindowSwapChain; SwapChainType gui_swap_chain_target_ = kWindowSwapChain; std::unique_ptr color_target_; std::unique_ptr depth_target_; std::unique_ptr object_manager_; std::unique_ptr scene_view_; + std::unique_ptr scene_bridge_; std::unique_ptr gui_view_; int window_width_ = 0; int window_height_ = 0; diff --git a/src/experimental/filament/filament/imgui_editor.cc b/src/experimental/filament/filament/imgui_editor.cc index 6b67072e..bfc773f4 100644 --- a/src/experimental/filament/filament/imgui_editor.cc +++ b/src/experimental/filament/filament/imgui_editor.cc @@ -35,6 +35,7 @@ #include #include #include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" namespace mujoco { @@ -560,7 +561,7 @@ void DrawCameraGui(SceneView* scene_view) { Ui("Direction", &direction); } -void DrawIndirectLightGui(SceneView* scene_view) { +void DrawIndirectLightGui(SceneBridge* scene_bridge, SceneView* scene_view) { filament::View* view = scene_view->GetDefaultRenderView(); auto ibl = view->getScene()->getIndirectLight(); @@ -575,7 +576,7 @@ void DrawIndirectLightGui(SceneView* scene_view) { static char filename[256]; ImGui::InputText("Filename", filename, sizeof(filename)); if (ImGui::Button("Load")) { - scene_view->SetEnvironmentLight(filename, intensity); + scene_bridge->SetEnvironmentLight(filename, intensity); } } @@ -649,7 +650,8 @@ void DrawLightGui(filament::LightManager& lm, } } -void DrawGui(SceneView* scene_view) { +void DrawGui(SceneBridge* scene_bridge) { + SceneView* scene_view = scene_bridge->GetSceneView(); filament::View* view = scene_view->GetDefaultRenderView(); filament::Engine* engine = scene_view->GetEngine(); filament::LightManager& lm = engine->getLightManager(); @@ -716,7 +718,7 @@ void DrawGui(SceneView* scene_view) { } if (ImGui::TreeNodeEx("Lights")) { if (ImGui::TreeNodeEx("Indirect (Image-based) Light")) { - DrawIndirectLightGui(scene_view); + DrawIndirectLightGui(scene_bridge, scene_view); ImGui::TreePop(); } view->getScene()->forEach([&](utils::Entity entity) { diff --git a/src/experimental/filament/filament/imgui_editor.h b/src/experimental/filament/filament/imgui_editor.h index b8f45357..f073fe45 100644 --- a/src/experimental/filament/filament/imgui_editor.h +++ b/src/experimental/filament/filament/imgui_editor.h @@ -15,12 +15,12 @@ #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ -#include "experimental/filament/filament/scene_view.h" +#include "experimental/filament/filament/scene_bridge.h" namespace mujoco { // Generates a ImGui Window for the given scene views. -void DrawGui(SceneView* scene_views); +void DrawGui(SceneBridge* scene_bridge); } // namespace mujoco diff --git a/src/experimental/filament/filament/scene_bridge.cc b/src/experimental/filament/filament/scene_bridge.cc new file mode 100644 index 00000000..6b445035 --- /dev/null +++ b/src/experimental/filament/filament/scene_bridge.cc @@ -0,0 +1,370 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/scene_bridge.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/gui_view.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/model_objects.h" +#include "experimental/filament/filament/model_util.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +using filament::math::float3; +using filament::math::float4; +using filament::math::mat3; +using filament::math::mat4; + +SceneBridge::SceneBridge(ObjectManager* object_mgr, const mjModel* model, + SceneView* scene_view) + : scene_view_(scene_view), object_mgr_(object_mgr) { + model_objects_ = + std::make_unique(model, object_mgr_->GetEngine()); + + // Configure options for the normal view. + auto cg = scene_view_->GetColorGradingOptions(); + cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure); + cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast); + cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance); + cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation); + cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature); + cg.tint = ReadElement(model, "filament.out.tint", cg.tint); + + auto tone_mapping = + ReadElement(model, "filament.out.tone_mapping"); + if (tone_mapping == "aces") { + cg.tone_mapper = ToneMapperType::kACES; + } else if (tone_mapping == "aces_legacy") { + cg.tone_mapper = ToneMapperType::kACESLegacy; + } else if (tone_mapping == "filmic") { + cg.tone_mapper = ToneMapperType::kFilmic; + } else if (tone_mapping == "linear") { + cg.tone_mapper = ToneMapperType::kLinear; + } else if (tone_mapping == "pbr_neutral") { + cg.tone_mapper = ToneMapperType::kPBRNeutral; + } + scene_view_->SetColorGradingOptions(cg); + + filament::View* fview = scene_view_->GetDefaultRenderView(); + auto ao = fview->getAmbientOcclusionOptions(); + ao.enabled = ReadElement(model, "filament.ao.enabled", true); + ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false); + ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled); + ao.quality = filament::QualityLevel::ULTRA; + ao.lowPassFilter = filament::QualityLevel::ULTRA; + ao.upsampling = filament::QualityLevel::ULTRA; + ao.bilateralThreshold = 0.5f; + fview->setAmbientOcclusionOptions(ao); + + auto msaa = fview->getMultiSampleAntiAliasingOptions(); + msaa.enabled = ReadElement(model, "filament.msaa.enabled", true); + fview->setMultiSampleAntiAliasingOptions(msaa); + + default_shadow_map_size_ = ReadElement( + model, "filament.shadows.map_size", default_shadow_map_size_); + default_vsm_blur_width_ = ReadElement( + model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); + + auto shadow_type = fview->getShadowType(); + shadow_type = ReadElement(model, "filament.shadows.type", shadow_type); + fview->setShadowType(shadow_type); + + auto fog_opts = fview->getFogOptions(); + fog_opts.enabled = + ReadElement(model, "filament.fog.enabled", fog_opts.enabled); + fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color); + fog_opts.distance = ReadElement( + model, "filament.fog.distance", fog_opts.distance); + fog_opts.density = ReadElement( + model, "filament.fog.density", fog_opts.density); + fog_opts.cutOffDistance = ReadElement( + model, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); + fog_opts.maximumOpacity = ReadElement( + model, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); + fog_opts.height = ReadElement(model, "filament.fog.height", fog_opts.height); + fog_opts.heightFalloff = ReadElement( + model, "filament.fog.heightFalloff", fog_opts.heightFalloff); + fog_opts.inScatteringStart = ReadElement( + model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); + fog_opts.inScatteringSize = ReadElement( + model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); + fview->setFogOptions(fog_opts); + + fallback_head_light_intensity_ = + ReadElement(model, "filament.fallback.head_light_intensity", + fallback_head_light_intensity_); + fallback_scene_light_intensity_ = + ReadElement(model, "filament.fallback.scene_light_intensity", + fallback_scene_light_intensity_); + fallback_environment_light_intensity_ = + ReadElement(model, "filament.fallback.environment_light_intensity", + fallback_environment_light_intensity_); + + // Create an empty/black indirect light to ensure that the skybox is oriented + // to respect mujoco's Z-up convention. + filament::IndirectLight* empty_ibl = + model_objects_->CreateIndirectLight(-1, 100000); + scene_view_->AddToScene(empty_ibl); + + PrepareLights(); +} + +SceneBridge::~SceneBridge() { + for (auto& iter : lights_) { + scene_view_->RemoveFromScene(iter.get()); + } + lights_.clear(); + + for (auto& iter : drawables_) { + scene_view_->RemoveFromScene(iter.get()); + } + drawables_.clear(); +} + +void SceneBridge::SetEnvironmentLight(std::string_view filename, + float intensity) { + filament::IndirectLight* ibl = nullptr; + scene_view_->AddToScene(ibl); + object_mgr_->LoadFallbackIndirectLight(filename, intensity); + ibl = object_mgr_->GetFallbackIndirectLight(); + scene_view_->AddToScene(ibl); +} + +std::optional SceneBridge::ClipFromWorld(const float3& pos) const{ + const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f); + if (clip_pos.w == 0.0f) { + return std::nullopt; + } + return clip_pos.xyz / clip_pos.w; +} + +void SceneBridge::PrepareLights() { + filament::Engine* engine = object_mgr_->GetEngine(); + const mjModel* model = model_objects_->GetModel(); + filament::Skybox* skybox = model_objects_->CreateSkybox(); + if (skybox) { + scene_view_->AddToScene(skybox); + } + + float total_light_intensity = 0.0f; + + for (int i = 0; i < model->nlight; ++i) { + total_light_intensity += model->light_intensity[i]; + + if (model->light_type[i] == mjLIGHT_IMAGE) { + auto* indirect_light = model_objects_->CreateIndirectLight( + model->light_texid[i], model->light_intensity[i]); + if (indirect_light) { + scene_view_->AddToScene(indirect_light); + } + // Add an nullptr as a placeholder so that our indices still match. + lights_.emplace_back(nullptr); + } else { + Light::Params params; + params.color = ReadFloat3(model->light_diffuse); + params.type = (mjtLightType)model->light_type[i]; + params.castshadow = model->light_castshadow[i]; + params.bulbradius = model->light_bulbradius[i]; + params.range = model->light_range[i]; + params.intensity = model->light_intensity[i]; + params.shadow_map_size = default_shadow_map_size_; + params.vsm_blur_width = default_vsm_blur_width_; + if (params.type == mjLIGHT_SPOT) { + params.spot_cone_angle = model->light_cutoff[i]; + } + + auto light_obj = std::make_unique(engine, params); +#ifndef __EMSCRIPTEN__ + // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. + scene_view_->AddToScene(light_obj.get()); +#endif + lights_.emplace_back(std::move(light_obj)); + } + } + + // Add a placeholder (black) headlight as our last light. Going forward, we'll + // assume lights_.back() is always the headlight. + { + Light::Params params; + params.color = float3(0, 0, 0); + params.headlight = true; + params.type = mjLIGHT_DIRECTIONAL; + params.castshadow = 0; + params.intensity = 0; + auto light_obj = std::make_unique(engine, params); +#ifndef __EMSCRIPTEN__ + // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. + scene_view_->AddToScene(light_obj.get()); +#endif + lights_.emplace_back(std::move(light_obj)); + } + + // There are no "physical" lights in the scene which means we're likely + // dealing with a "classic renderer" scene. In this case, let's add a + // default environment light and set the light intensity ourselves. + if (total_light_intensity == 0.0f) { + auto* ibl = object_mgr_->GetFallbackIndirectLight(); + if (ibl) { + ibl->setIntensity(fallback_environment_light_intensity_); + scene_view_->AddToScene(ibl); + } + const float intensity = fallback_scene_light_intensity_ / lights_.size(); + for (auto& light : lights_) { + if (light) { + light->SetIntensity( + light->IsHeadlight() ? fallback_head_light_intensity_ : intensity); + } + } + } +} + +filament::math::mat4 CalculateClipFromWorld(const mjrRect& viewport, + const mjvGLCamera& cam) { + const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); + const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); + const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); + const float3 cam_at = cam_pos + cam_fwd; + const float aspect_ratio = (float)viewport.width / (float)viewport.height; + const float halfwidth = + cam.frustum_width + ? cam.frustum_width + : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom); + const float left = cam.frustum_center - halfwidth; + const float right = cam.frustum_center + halfwidth; + + mat4 projection; + if (cam.orthographic) { + projection = mat4::ortho(left, right, cam.frustum_bottom, cam.frustum_top, + cam.frustum_near, cam.frustum_far); + } else { + projection = mat4::frustum(left, right, cam.frustum_bottom, cam.frustum_top, + cam.frustum_near, cam.frustum_far); + projection[2][2] = -1.0f; + projection[3][2] = -2.0f * cam.frustum_near; + } + mat4 look_at = mat4::lookAt(cam_pos, cam_at, cam_up); + return projection * inverse(look_at); +} + +void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) { + filament::View* view = scene_view_->GetDefaultRenderView(); + view->setShadowingEnabled(scene->flags[mjRND_SHADOW] ? true : false); + + mjtNum hpos[3], hfwd[3]; + float headpos[3], gazedir[3]; + mjv_cameraInModel(hpos, hfwd, nullptr, scene); + mju_n2f(headpos, hpos, 3); + mju_n2f(gazedir, hfwd, 3); + + const mjvGLCamera gl_camera = + mjv_averageCamera(scene->camera, scene->camera + 1); + clip_from_world_ = CalculateClipFromWorld(viewport, gl_camera); + + // Remove all drawables from previous render and prepare new ones. + for (auto& iter : drawables_) { + scene_view_->RemoveFromScene(iter.get()); + } + drawables_.clear(); + for (int i = 0; i < scene->ngeom; ++i) { + const mjvGeom* geom = scene->geoms + i; + + if (geom->label[0] != 0) { + if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) { + DrawTextAt(geom->label, pos->x, pos->y, pos->z); + } + } + + auto drawable = + std::make_unique(object_mgr_, model_objects_.get(), *geom); + drawable->Update(model_objects_->GetModel(), scene, *geom); + scene_view_->AddToScene(drawable.get()); + drawables_.push_back(std::move(drawable)); + } + + bool headlight_enabled = false; + for (int i = 0; i < scene->nlight; ++i) { + const mjvLight& scene_light = scene->lights[i]; + if (scene_light.id < 0 && scene_light.headlight) { + // We position the headlight slightly behind the camera to avoid some + // odd clipping issues. + headlight_enabled = true; + headpos[0] -= gazedir[0] * 0.05f; + headpos[1] -= gazedir[1] * 0.05f; + headpos[2] -= gazedir[2] * 0.05f; + + // The headlight is always the "back" light. + std::unique_ptr& light = lights_.back(); + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir)); + continue; + } else if (scene_light.id < lights_.size() - 1) { + std::unique_ptr& light = lights_[scene_light.id]; + if (light) { + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(scene_light.pos), + ReadFloat3(scene_light.dir)); + } + } else { + mju_error("Unexpected light id: %d", scene_light.id); + } + } + + // Enable/disable the headlight based on whether or not it's in the scene. + if (headlight_enabled) { + lights_.back()->Enable(); + } else { + lights_.back()->Disable(); + } +} + +void SceneBridge::UploadMesh(const mjModel* model, int id) { + model_objects_->UploadMesh(model, id); +} + +void SceneBridge::UploadTexture(const mjModel* model, int id) { + model_objects_->UploadTexture(model, id); +} + +void SceneBridge::UploadHeightField(const mjModel* model, int id) { + model_objects_->UploadHeightField(model, id); +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/scene_bridge.h b/src/experimental/filament/filament/scene_bridge.h new file mode 100644 index 00000000..a1e56796 --- /dev/null +++ b/src/experimental/filament/filament/scene_bridge.h @@ -0,0 +1,85 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/model_objects.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +// Manages all mjModel data and updates a SceneView using an mjvScene. +class SceneBridge { + public: + SceneBridge(ObjectManager* object_mgr, const mjModel* model, + SceneView* scene_view); + ~SceneBridge(); + + // Updates the environment light using the KTX image at the given path. + void SetEnvironmentLight(std::string_view filename, float intensity); + + // Updates the environment light to the fallback light + void SetFallbackEnvironmentLight(float intensity); + + // Updates the Entities in the filament Scene to match the current mjvScene + // state. + void Update(const mjrRect& viewport, const mjvScene* scene); + + // Creates the filament objects from the mjModel. + void UploadMesh(const mjModel* model, int id); + void UploadTexture(const mjModel* model, int id); + void UploadHeightField(const mjModel* model, int id); + + SceneView* GetSceneView() const { return scene_view_; } + + SceneBridge(const SceneBridge&) = delete; + SceneBridge& operator=(const SceneBridge&) = delete; + + private: + void PrepareLights(); + + // Converts a point in world space to clip space, eg. in the range [-1,-1, 0] + // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera. + std::optional ClipFromWorld( + const filament::math::float3& pos) const; + + SceneView* scene_view_ = nullptr; + ObjectManager* object_mgr_ = nullptr; + std::unique_ptr model_objects_; + std::vector> lights_; + std::vector> drawables_; + filament::math::mat4 clip_from_world_; + int default_shadow_map_size_ = 2048; + float default_vsm_blur_width_ = 0.0f; + float fallback_head_light_intensity_ = 0.f; + float fallback_scene_light_intensity_ = 80'000.f; + float fallback_environment_light_intensity_ = 5'000.f; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_BRIDGE_H_ diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index 4f6f1ceb..3cd7f929 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -14,12 +14,10 @@ #include "experimental/filament/filament/scene_view.h" +#include #include #include #include -#include -#include -#include #include #include @@ -43,12 +41,9 @@ #include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/drawable.h" -#include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/material.h" #include "experimental/filament/filament/math_util.h" -#include "experimental/filament/filament/model_objects.h" -#include "experimental/filament/filament/model_util.h" -#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" #include "experimental/filament/filament/texture.h" @@ -56,21 +51,17 @@ namespace mujoco { using filament::math::float3; using filament::math::float4; -using filament::math::mat3; using filament::math::mat4; static constexpr int kNormalIndex = - static_cast(SceneView::DrawMode::kNormal); + static_cast(Material::DrawMode::kNormal); static constexpr int kDepthIndex = - static_cast(SceneView::DrawMode::kDepth); + static_cast(Material::DrawMode::kDepth); static constexpr int kSegmentIndex = - static_cast(SceneView::DrawMode::kSegmentation); + static_cast(Material::DrawMode::kSegmentation); -static filament::Viewport ReadViewport(mjrRect rect) { - return filament::Viewport(rect.left, rect.bottom, rect.width, rect.height); -} - -filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) { +static filament::ColorGrading::Builder ToBuilder( + const ColorGradingOptions& opts) { return filament::ColorGrading::Builder() .format(opts.format) .dimensions(opts.dimension) @@ -89,6 +80,27 @@ filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) { .curves(opts.shadow_gamma, opts.mid_point, opts.highlight_scale); } +static void SetupCamera(const mjvGLCamera& cam, + const filament::Viewport& viewport, + filament::Camera* camera) { + const filament::Camera::Projection type = + cam.orthographic ? filament::Camera::Projection::ORTHO + : filament::Camera::Projection::PERSPECTIVE; + const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); + const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); + const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); + const float3 cam_at = cam_pos + cam_fwd; + const float aspect_ratio = (float)viewport.width / (float)viewport.height; + const float halfwidth = + cam.frustum_width + ? cam.frustum_width + : 0.5f * aspect_ratio * (cam.frustum_top - cam.frustum_bottom); + camera->lookAt(cam_pos, cam_at, cam_up); + camera->setProjection(type, cam.frustum_center - halfwidth, + cam.frustum_center + halfwidth, cam.frustum_bottom, + cam.frustum_top, cam.frustum_near, cam.frustum_far); +} + // Sets up the `reflection_camera`'s projection matrix so that it is a // reflection of the `src_camera` across the plane defined by the // `surface_xform`. The generated projection is an oblique projection so that @@ -115,11 +127,7 @@ static void SetupReflectionCamera(const mat4& surface_xform, reflection_camera->setCustomProjection(oblique, near, far); } -SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) - : object_mgr_(object_mgr) { - filament::Engine* engine = object_mgr_->GetEngine(); - model_objects_ = std::make_unique(model, engine); - +SceneView::SceneView(filament::Engine* engine) : engine_(engine) { scene_ = engine->createScene(); camera_ = engine->createCamera(utils::EntityManager::get().create()); reflect_camera_ = engine->createCamera(utils::EntityManager::get().create()); @@ -136,53 +144,6 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) reflect_view_->setShadowingEnabled(false); reflect_view_->setPostProcessingEnabled(false); - // Configure options for the normal view. - auto& cg = color_grading_options_; - cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure); - cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast); - cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance); - cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation); - cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature); - cg.tint = ReadElement(model, "filament.out.tint", cg.tint); - - auto tone_mapping = - ReadElement(model, "filament.out.tone_mapping"); - if (tone_mapping == "aces") { - cg.tone_mapper = ToneMapperType::kACES; - } else if (tone_mapping == "aces_legacy") { - cg.tone_mapper = ToneMapperType::kACESLegacy; - } else if (tone_mapping == "filmic") { - cg.tone_mapper = ToneMapperType::kFilmic; - } else if (tone_mapping == "linear") { - cg.tone_mapper = ToneMapperType::kLinear; - } else if (tone_mapping == "pbr_neutral") { - cg.tone_mapper = ToneMapperType::kPBRNeutral; - } - SetColorGradingOptions(cg); - - auto ao = views_[kNormalIndex]->getAmbientOcclusionOptions(); - ao.enabled = ReadElement(model, "filament.ao.enabled", true); - ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false); - ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled); - ao.quality = filament::QualityLevel::ULTRA; - ao.lowPassFilter = filament::QualityLevel::ULTRA; - ao.upsampling = filament::QualityLevel::ULTRA; - ao.bilateralThreshold = 0.5f; - views_[kNormalIndex]->setAmbientOcclusionOptions(ao); - - auto msaa = views_[kNormalIndex]->getMultiSampleAntiAliasingOptions(); - msaa.enabled = ReadElement(model, "filament.msaa.enabled", true); - views_[kNormalIndex]->setMultiSampleAntiAliasingOptions(msaa); - - default_shadow_map_size_ = ReadElement( - model, "filament.shadows.map_size", default_shadow_map_size_); - default_vsm_blur_width_ = ReadElement( - model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); - - auto shadow_type = views_[kNormalIndex]->getShadowType(); - shadow_type = ReadElement(model, "filament.shadows.type", shadow_type); - views_[kNormalIndex]->setShadowType(shadow_type); - // Disable post processing for the depth and segmentation views to preserve // the values. views_[kDepthIndex]->setPostProcessingEnabled(false); @@ -192,79 +153,115 @@ SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) auto fog = views_[kNormalIndex]->getFogEntity(); auto& tm = engine->getTransformManager(); tm.create(fog); - auto rotation_axis = ReadElement( - model, "filament.fog.rotation_axis", float3{-1, 0, 0}); tm.setTransform(tm.getInstance(fog), - mat4::rotation(filament::math::f::PI / 2, rotation_axis)); - - auto fog_opts = views_[kNormalIndex]->getFogOptions(); - fog_opts.enabled = - ReadElement(model, "filament.fog.enabled", fog_opts.enabled); - fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color); - fog_opts.distance = ReadElement( - model, "filament.fog.distance", fog_opts.distance); - fog_opts.density = ReadElement( - model, "filament.fog.density", fog_opts.density); - fog_opts.cutOffDistance = ReadElement( - model, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); - fog_opts.maximumOpacity = ReadElement( - model, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); - fog_opts.height = ReadElement(model, "filament.fog.height", fog_opts.height); - fog_opts.heightFalloff = ReadElement( - model, "filament.fog.heightFalloff", fog_opts.heightFalloff); - fog_opts.inScatteringStart = ReadElement( - model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); - fog_opts.inScatteringSize = ReadElement( - model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); - views_[kNormalIndex]->setFogOptions(fog_opts); - - fallback_head_light_intensity_ = - ReadElement(model, "filament.fallback.head_light_intensity", - fallback_head_light_intensity_); - fallback_scene_light_intensity_ = - ReadElement(model, "filament.fallback.scene_light_intensity", - fallback_scene_light_intensity_); - fallback_environment_light_intensity_ = - ReadElement(model, "filament.fallback.environment_light_intensity", - fallback_environment_light_intensity_); - - // Create an empty/black indirect light to ensure that the skybox is oriented - // to respect mujoco's Z-up convention. - scene_->setIndirectLight(model_objects_->CreateIndirectLight(-1, 100000)); - - PrepareLights(); + mat4::rotation(filament::math::f::PI / 2, float3{-1, 0, 0})); } SceneView::~SceneView() { + for (auto& light : lights_) { + light->RemoveFromScene(scene_); + } + for (auto& drawable : drawables_) { + drawable->RemoveFromScene(scene_); + } lights_.clear(); drawables_.clear(); reflect_targets_.clear(); - - filament::Engine* engine = object_mgr_->GetEngine(); - engine->destroyCameraComponent(reflect_camera_->getEntity()); - engine->destroy(reflect_view_); - - engine->destroyCameraComponent(camera_->getEntity()); - engine->destroy(views_[kNormalIndex]->getColorGrading()); - for (auto& view : views_) { - engine->destroy(view); + engine_->destroyCameraComponent(reflect_camera_->getEntity()); + engine_->destroy(reflect_view_); + engine_->destroyCameraComponent(camera_->getEntity()); + if (color_grading_) { + engine_->destroy(color_grading_); + } + engine_->destroy(scene_); + for (auto& view : views_) { + engine_->destroy(view); } - engine->destroy(scene_); } -void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode, - RenderTarget* target) { - filament::View* view = PrepareRenderView(draw_mode); +void SceneView::AddToScene(Light* light) { + if (lights_.insert(light).second) { + light->AddToScene(scene_); + } +} + +void SceneView::RemoveFromScene(Light* light) { + if (lights_.erase(light)) { + light->RemoveFromScene(scene_); + } +} + +void SceneView::AddToScene(Drawable* drawable) { + if (drawables_.insert(drawable).second) { + drawable->AddToScene(scene_); + if (drawable->IsReflective()) { + AddReflectiveDrawable(drawable); + } + } +} + +void SceneView::RemoveFromScene(Drawable* drawable) { + if (drawables_.erase(drawable)) { + auto it = std::find(reflectives_.begin(), reflectives_.end(), drawable); + if (it != reflectives_.end()) { + reflectives_.erase(it); + } + drawable->RemoveFromScene(scene_); + } +} + +void SceneView::AddToScene(filament::Skybox* skybox) { + skybox_ = skybox; + scene_->setSkybox(skybox); +} + +void SceneView::RemoveFromScene(filament::Skybox* skybox) { + if (skybox_ == skybox) { + skybox_ = nullptr; + scene_->setSkybox(nullptr); + } +} + +void SceneView::AddToScene(filament::IndirectLight* indirect_light) { + indirect_light_ = indirect_light; + scene_->setIndirectLight(indirect_light); +} + +void SceneView::RemoveFromScene(filament::IndirectLight* indirect_light) { + if (indirect_light_ == indirect_light) { + indirect_light_ = nullptr; + scene_->setIndirectLight(nullptr); + } +} + +void SceneView::Render(filament::Renderer* renderer, + const RenderRequest& request) { + filament::Viewport viewport(request.viewport.left, request.viewport.bottom, + request.viewport.width, request.viewport.height); + for (auto& view : views_) { + view->setViewport(viewport); + } + reflect_view_->setViewport(viewport); + + SetupCamera(request.camera, viewport, camera_); + + for (auto& iter : drawables_) { + iter->SetDrawMode(request.draw_mode); + } + + filament::View* view = views_[static_cast(request.draw_mode)]; filament::MultiSampleAntiAliasingOptions options = view->getMultiSampleAntiAliasingOptions(); - if (target) { + filament::RenderTarget* render_target = + request.target ? request.target->GetFilamentRenderTarget() : nullptr; + if (render_target) { // We need to disable msaa in order to render to texture. view->setMultiSampleAntiAliasingOptions({.enabled = false}); } // Render reflection passes. - if (draw_mode == DrawMode::kNormal) { + if (request.draw_mode == DrawMode::kNormal) { for (size_t i = 0; i < reflectives_.size(); ++i) { Drawable* drawable = reflectives_[i]; @@ -283,242 +280,24 @@ void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode, } } - view->setRenderTarget(target ? target->GetFilamentRenderTarget() : nullptr); + view->setRenderTarget(render_target); renderer->render(view); view->setRenderTarget(nullptr); - if (target) { + if (request.target) { view->setMultiSampleAntiAliasingOptions(options); } } -filament::View* SceneView::PrepareRenderView(DrawMode mode) { - for (auto& iter : drawables_) { - iter->SetDrawMode(mode); - } - return views_[static_cast(mode)]; -} - -void SceneView::SetViewport(mjrRect viewport) { - auto filament_viewport = ReadViewport(viewport); - aspect_ratio_ = (float)viewport.width / (float)viewport.height; - for (auto& view : views_) { - view->setViewport(filament_viewport); - } - reflect_view_->setViewport(filament_viewport); -} - -void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { - filament::Engine* engine = object_mgr_->GetEngine(); - - auto tone_mapper = CreateToneMapper(opts.tone_mapper); - auto color_grading = ToBuilder(color_grading_options_) - .toneMapper(tone_mapper.get()) - .build(*engine); - views_[kNormalIndex]->setColorGrading(color_grading); - engine->destroy(color_grading_); - color_grading_ = color_grading; - color_grading_options_ = opts; -} - -void SceneView::SetEnvironmentLight(std::string_view filename, - float intensity) { - scene_->setIndirectLight(nullptr); - object_mgr_->LoadFallbackIndirectLight(filename, intensity); - scene_->setIndirectLight(object_mgr_->GetFallbackIndirectLight()); -} - -void SceneView::SetFallbackEnvironmentLight(float intensity) { - auto* ibl = object_mgr_->GetFallbackIndirectLight(); - if (ibl) { - ibl->setIntensity(intensity); - scene_->setIndirectLight(ibl); - } -} - -void SceneView::UpdateCamera(const mjvGLCamera* cameras) { - const mjvGLCamera cam = mjv_averageCamera(cameras, cameras + 1); - const filament::Camera::Projection type = - cam.orthographic ? filament::Camera::Projection::ORTHO - : filament::Camera::Projection::PERSPECTIVE; - float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); - float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); - float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); - float3 cam_at = cam_pos + cam_fwd; - camera_->lookAt(cam_pos, cam_at, cam_up); - float halfwidth = cam.frustum_width ? cam.frustum_width - : 0.5f * aspect_ratio_ * (cam.frustum_top - cam.frustum_bottom); - camera_->setProjection(type, cam.frustum_center - halfwidth, - cam.frustum_center + halfwidth, cam.frustum_bottom, - cam.frustum_top, cam.frustum_near, cam.frustum_far); - clip_from_world_ = camera_->getProjectionMatrix() * camera_->getViewMatrix(); -} - -std::optional SceneView::ClipFromWorld(const float3& pos) const{ - const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f); - if (clip_pos.w == 0.0f) { - return std::nullopt; - } - return clip_pos.xyz / clip_pos.w; -} - -void SceneView::PrepareLights() { - filament::Engine* engine = object_mgr_->GetEngine(); - const mjModel* model = model_objects_->GetModel(); - filament::Skybox* skybox = model_objects_->CreateSkybox(); - if (skybox) { - scene_->setSkybox(skybox); - } - - float total_light_intensity = 0.0f; - - for (int i = 0; i < model->nlight; ++i) { - total_light_intensity += model->light_intensity[i]; - - if (model->light_type[i] == mjLIGHT_IMAGE) { - auto* indirect_light = model_objects_->CreateIndirectLight( - model->light_texid[i], model->light_intensity[i]); - if (indirect_light) { - scene_->setIndirectLight(indirect_light); - } - // Add an nullptr as a placeholder so that our indices still match. - lights_.emplace_back(nullptr); - } else { - Light::Params params; - params.color = ReadFloat3(model->light_diffuse); - params.type = (mjtLightType)model->light_type[i]; - params.castshadow = model->light_castshadow[i]; - params.bulbradius = model->light_bulbradius[i]; - params.range = model->light_range[i]; - params.intensity = model->light_intensity[i]; - params.shadow_map_size = default_shadow_map_size_; - params.vsm_blur_width = default_vsm_blur_width_; - if (params.type == mjLIGHT_SPOT) { - params.spot_cone_angle = model->light_cutoff[i]; - } - - auto light_obj = std::make_unique(engine, params); -#ifndef __EMSCRIPTEN__ - // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. - light_obj->AddToScene(scene_); -#endif - lights_.emplace_back(std::move(light_obj)); - } - } - - // Add a placeholder (black) headlight as our last light. Going forward, we'll - // assume lights_.back() is always the headlight. - { - Light::Params params; - params.color = float3(0, 0, 0); - params.headlight = true; - params.type = mjLIGHT_DIRECTIONAL; - params.castshadow = 0; - params.intensity = 0; - auto light_obj = std::make_unique(engine, params); -#ifndef __EMSCRIPTEN__ - // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. - light_obj->AddToScene(scene_); -#endif - lights_.emplace_back(std::move(light_obj)); - } - - // There are no "physical" lights in the scene which means we're likely - // dealing with a "classic renderer" scene. In this case, let's add a - // default environment light and set the light intensity ourselves. - if (total_light_intensity == 0.0f) { - SetFallbackEnvironmentLight(fallback_environment_light_intensity_); - const float intensity = fallback_scene_light_intensity_ / lights_.size(); - for (auto& light : lights_) { - if (light) { - light->SetIntensity( - light->IsHeadlight() ? fallback_head_light_intensity_ : intensity); - } - } - } -} - -void SceneView::UpdateScene(const mjvScene* scene) { - views_[kNormalIndex]->setShadowingEnabled(scene->flags[mjRND_SHADOW]); - - mjtNum hpos[3], hfwd[3]; - float headpos[3], gazedir[3]; - mjv_cameraInModel(hpos, hfwd, nullptr, scene); - mju_n2f(headpos, hpos, 3); - mju_n2f(gazedir, hfwd, 3); - UpdateCamera(scene->camera); - - // Remove all drawables from previous render and prepare new ones. - for (auto& iter : drawables_) { - iter->RemoveFromScene(scene_); - } - drawables_.clear(); - reflectives_.clear(); - for (int i = 0; i < scene->ngeom; ++i) { - const mjvGeom* geom = scene->geoms + i; - - if (geom->label[0] != 0) { - if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) { - DrawTextAt(geom->label, pos->x, pos->y, pos->z); - } - } - - auto drawable = - std::make_unique(object_mgr_, model_objects_.get(), *geom); - drawable->AddToScene(scene_); - drawable->Update(model_objects_->GetModel(), scene, *geom); - if (drawable->IsReflective()) { - AddReflectiveDrawable(drawable.get()); - } - drawables_.push_back(std::move(drawable)); - } - - bool headlight_enabled = false; - for (int i = 0; i < scene->nlight; ++i) { - const mjvLight& scene_light = scene->lights[i]; - if (scene_light.id < 0 && scene_light.headlight) { - // We position the headlight slightly behind the camera to avoid some - // odd clipping issues. - headlight_enabled = true; - headpos[0] -= gazedir[0] * 0.05f; - headpos[1] -= gazedir[1] * 0.05f; - headpos[2] -= gazedir[2] * 0.05f; - - // The headlight is always the "back" light. - std::unique_ptr& light = lights_.back(); - light->SetColor(ReadFloat3(scene_light.diffuse)); - light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir)); - continue; - } else if (scene_light.id < lights_.size() - 1) { - std::unique_ptr& light = lights_[scene_light.id]; - if (light) { - light->SetColor(ReadFloat3(scene_light.diffuse)); - light->SetTransform(ReadFloat3(scene_light.pos), - ReadFloat3(scene_light.dir)); - } - } else { - mju_error("Unexpected light id: %d", scene_light.id); - } - } - - // Enable/disable the headlight based on whether or not it's in the scene. - if (headlight_enabled) { - lights_.back()->Enable(); - } else { - lights_.back()->Disable(); - } -} - void SceneView::AddReflectiveDrawable(Drawable* drawable) { const int index = reflectives_.size(); reflectives_.push_back(drawable); // Ensure we have the same number of render targets as we do reflective // drawables. - filament::Engine* engine = object_mgr_->GetEngine(); while (reflect_targets_.size() < reflectives_.size()) { reflect_targets_.push_back(std::make_unique( - engine, RenderTargetTextureType::kReflectionColor, + engine_, RenderTargetTextureType::kReflectionColor, RenderTargetTextureType::kDepth)); } @@ -529,20 +308,17 @@ void SceneView::AddReflectiveDrawable(Drawable* drawable) { drawable->UpdateReflectionTexture(target->GetColorTexture()); } -void SceneView::UploadMesh(const mjModel* model, int id) { - model_objects_->UploadMesh(model, id); -} - -void SceneView::UploadTexture(const mjModel* model, int id) { - model_objects_->UploadTexture(model, id); -} - -void SceneView::UploadHeightField(const mjModel* model, int id) { - model_objects_->UploadHeightField(model, id); -} - -filament::Engine* SceneView::GetEngine() const { - return object_mgr_->GetEngine(); +void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { + auto tone_mapper = CreateToneMapper(opts.tone_mapper); + auto color_grading = ToBuilder(color_grading_options_) + .toneMapper(tone_mapper.get()) + .build(*engine_); + views_[kNormalIndex]->setColorGrading(color_grading); + if (color_grading_) { + engine_->destroy(color_grading_); + } + color_grading_ = color_grading; + color_grading_options_ = opts; } filament::View* SceneView::GetDefaultRenderView() { diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index 8fb686e1..a7d7c3aa 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -17,120 +17,100 @@ #include #include -#include -#include +#include #include #include #include #include -#include #include #include -#include -#include -#include -#include #include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/drawable.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/material.h" -#include "experimental/filament/filament/model_objects.h" -#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" namespace mujoco { -// Creates and owns filament Scene and View classes given a mjvScene. +// Creates and owns the filament Scene and View (and Camera) classes. // -// The filament Scene is populated with the objects (e.g. lights, geoms, -// cameras, etc.) defined by the mjvScene. Multiple Views are created to allow -// different rendering modes (e.g. normal, depth, segmentation, etc.) +// The filament Scene is populated with the objects (e.g. lights, renderables, +// skybox, etc.). It manages multiple views to support a variety of draw modes +// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. class SceneView { public: - SceneView(ObjectManager* object_mgr, const mjModel* model); + SceneView(filament::Engine* engine); ~SceneView(); - // Updates all views to render into the given viewport. - void SetViewport(mjrRect viewport); - - // Updates the color grading options for the main render view. - void SetColorGradingOptions(const ColorGradingOptions& opts); - - // Updates the environment light using the KTX image at the given path. - void SetEnvironmentLight(std::string_view filename, float intensity); - - // Updates the environment light to the fallback light - void SetFallbackEnvironmentLight(float intensity); - - // Updates the Entities in the filament Scene to match the current mjvScene - // state. - void UpdateScene(const mjvScene* scene); + // Adds/removes entities from the scene. + void AddToScene(Light* light); + void RemoveFromScene(Light* light); + void AddToScene(Drawable* drawable); + void RemoveFromScene(Drawable* drawable); + void AddToScene(filament::Skybox* skybox); + void RemoveFromScene(filament::Skybox* skybox); + void AddToScene(filament::IndirectLight* indirect_light); + void RemoveFromScene(filament::IndirectLight* indirect_light); + // Parameters for rendering the scene. using DrawMode = Material::DrawMode; + struct RenderRequest { + // The draw mode (e.g. normal, depth, segmentation) to render. + DrawMode draw_mode = DrawMode::kNormal; + // The target viewport for the rendered image. + mjrRect viewport; + // The camera from which to render the scene. + mjvGLCamera camera; + // An optional render target into which the scene will be rendered. + RenderTarget* target = nullptr; + }; - void Render(filament::Renderer* renderer, DrawMode draw_mode, - RenderTarget* target = nullptr); + // Renders the scene. + void Render(filament::Renderer* renderer, const RenderRequest& request); - void UploadMesh(const mjModel* model, int id); - void UploadTexture(const mjModel* model, int id); - void UploadHeightField(const mjModel* model, int id); + // Returns the filament Engine managing the scene. + filament::Engine* GetEngine() const { return engine_; } - // Accessors. - filament::Engine* GetEngine() const; + // Returns the underlying filament View that is used for normal rendering. + // Callers can update rendering settings (e.g. post processing) directly. filament::View* GetDefaultRenderView(); + + // Helpers for managing the color grading options for the default render view. ColorGradingOptions GetColorGradingOptions() const; + void SetColorGradingOptions(const ColorGradingOptions& opts); SceneView(const SceneView&) = delete; SceneView& operator=(const SceneView&) = delete; private: - // Prepares and returns the filament View for the given draw mode. - filament::View* PrepareRenderView(DrawMode mode); - - void UpdateCamera(const mjvGLCamera* cameras); - - void PrepareLights(); - - // Registers the given drawable as a reflective surface. + // Marks a drawable as reflective. Reflective drawables have to be rendered + // in their own passes to create the reflective texture. void AddReflectiveDrawable(Drawable* drawable); - // Converts a point in world space to clip space, eg. in the range [-1,-1, 0] - // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera. - std::optional ClipFromWorld( - const filament::math::float3& pos) const; - - ObjectManager* object_mgr_ = nullptr; + filament::Engine* engine_ = nullptr; filament::Scene* scene_ = nullptr; filament::Camera* camera_ = nullptr; filament::ColorGrading* color_grading_ = nullptr; - std::vector> lights_; - std::vector> drawables_; - std::unique_ptr model_objects_; - std::array views_; - filament::math::mat4 clip_from_world_; ColorGradingOptions color_grading_options_; + std::array views_; DrawMode active_mode_ = DrawMode::kNumDrawModes; - float aspect_ratio_ = 1.0f; - int default_shadow_map_size_ = 2048; - float default_vsm_blur_width_ = 0.0f; - float fallback_head_light_intensity_ = 0.f; - float fallback_scene_light_intensity_ = 80'000.f; - float fallback_environment_light_intensity_ = 5'000.f; + + // Scene objects. + std::unordered_set lights_; + std::unordered_set drawables_; + filament::Skybox* skybox_ = nullptr; + filament::IndirectLight* indirect_light_ = nullptr; // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr; filament::Camera* reflect_camera_ = nullptr; - // The list of drawables that are reflective. + // The list of reflective drawables and their corresponding render targets. std::vector reflectives_; - - // Each reflective drawable has its own render target which is used to render - // the reflected image. std::vector> reflect_targets_; }; - } // namespace mujoco #endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_