diff --git a/src/experimental/filament/CMakeLists.txt b/src/experimental/filament/CMakeLists.txt index 5c9242aa..2a0c5c8f 100644 --- a/src/experimental/filament/CMakeLists.txt +++ b/src/experimental/filament/CMakeLists.txt @@ -47,6 +47,8 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME} filament/material.h filament/math_util.cc filament/math_util.h + filament/model_objects.cc + filament/model_objects.h filament/model_util.cc filament/model_util.h filament/object_manager.cc diff --git a/src/experimental/filament/filament/drawable.cc b/src/experimental/filament/filament/drawable.cc index 8dd575bb..30f1b9f0 100644 --- a/src/experimental/filament/filament/drawable.cc +++ b/src/experimental/filament/filament/drawable.cc @@ -35,6 +35,7 @@ #include "experimental/filament/filament/geom_util.h" #include "experimental/filament/filament/material.h" #include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/model_objects.h" #include "experimental/filament/filament/object_manager.h" namespace mujoco { @@ -85,8 +86,11 @@ static bool IsBehind(const mjtNum* headpos, const float* pos, const float* mat) (headpos[2] - pos[2]) * mat[8] < 0.0f); } -Drawable::Drawable(ObjectManager* object_mgr, const mjvGeom& geom) - : material_(object_mgr), renderables_(object_mgr->GetEngine()) { +Drawable::Drawable(ObjectManager* object_mgr, ModelObjects* model_objects, + const mjvGeom& geom) + : material_(object_mgr), + model_objs_(model_objects), + renderables_(object_mgr->GetEngine()) { if (geom.category == mjCAT_DECOR) { renderables_.SetCastShadows(false); renderables_.SetReceiveShadows(false); @@ -100,53 +104,53 @@ Drawable::Drawable(ObjectManager* object_mgr, const mjvGeom& geom) AddHeightField(geom.dataid); break; case mjGEOM_PLANE: - AddShape(ObjectManager::kPlane); + AddShape(ModelObjects::kPlane); break; case mjGEOM_SPHERE: - AddShape(ObjectManager::kSphere); + AddShape(ModelObjects::kSphere); break; case mjGEOM_ELLIPSOID: - AddShape(ObjectManager::kSphere); + AddShape(ModelObjects::kSphere); break; case mjGEOM_BOX: - AddShape(ObjectManager::kBox); + AddShape(ModelObjects::kBox); break; case mjGEOM_CAPSULE: - AddShape(ObjectManager::kTube); - AddShape(ObjectManager::kDome); - AddShape(ObjectManager::kDome); + AddShape(ModelObjects::kTube); + AddShape(ModelObjects::kDome); + AddShape(ModelObjects::kDome); break; case mjGEOM_CYLINDER: - AddShape(ObjectManager::kTube); - AddShape(ObjectManager::kDisk); - AddShape(ObjectManager::kDisk); + AddShape(ModelObjects::kTube); + AddShape(ModelObjects::kDisk); + AddShape(ModelObjects::kDisk); break; case mjGEOM_ARROW: - AddShape(ObjectManager::kTube); - AddShape(ObjectManager::kCone); - AddShape(ObjectManager::kDisk); + AddShape(ModelObjects::kTube); + AddShape(ModelObjects::kCone); + AddShape(ModelObjects::kDisk); break; case mjGEOM_ARROW1: - AddShape(ObjectManager::kTube); - AddShape(ObjectManager::kCone); - AddShape(ObjectManager::kDisk); - AddShape(ObjectManager::kDisk); + AddShape(ModelObjects::kTube); + AddShape(ModelObjects::kCone); + AddShape(ModelObjects::kDisk); + AddShape(ModelObjects::kDisk); break; case mjGEOM_ARROW2: - AddShape(ObjectManager::kTube); - AddShape(ObjectManager::kCone); - AddShape(ObjectManager::kCone); - AddShape(ObjectManager::kDisk); - AddShape(ObjectManager::kDisk); + AddShape(ModelObjects::kTube); + AddShape(ModelObjects::kCone); + AddShape(ModelObjects::kCone); + AddShape(ModelObjects::kDisk); + AddShape(ModelObjects::kDisk); break; case mjGEOM_LINE: - AddShape(ObjectManager::kLine); + AddShape(ModelObjects::kLine); break; case mjGEOM_LINEBOX: - AddShape(ObjectManager::kLineBox); + AddShape(ModelObjects::kLineBox); break; case mjGEOM_TRIANGLE: - AddShape(ObjectManager::kTriangle); + AddShape(ModelObjects::kTriangle); break; case mjGEOM_FLEX: case mjGEOM_SKIN: @@ -187,8 +191,7 @@ void Drawable::Update(const mjModel* model, const mjvScene* scene, } void Drawable::AddMesh(int data_id) { - ObjectManager* object_mgr = material_.GetObjectManager(); - const FilamentBuffers* buffers = object_mgr->GetMeshBuffer(data_id); + const FilamentBuffers* buffers = model_objs_->GetMeshBuffer(data_id); if (buffers == nullptr) { mju_error("Unknown mesh %d", data_id); } @@ -196,17 +199,15 @@ void Drawable::AddMesh(int data_id) { } void Drawable::AddHeightField(int hfield_id) { - ObjectManager* object_mgr = material_.GetObjectManager(); - const FilamentBuffers* buffers = object_mgr->GetHeightFieldBuffer(hfield_id); + const FilamentBuffers* buffers = model_objs_->GetHeightFieldBuffer(hfield_id); if (buffers == nullptr) { mju_error("Unknown height field %d", hfield_id); } renderables_.Append(*buffers); } -void Drawable::AddShape(ObjectManager::ShapeType shape_type) { - ObjectManager* object_mgr = material_.GetObjectManager(); - const FilamentBuffers* buffers = object_mgr->GetShapeBuffer(shape_type); +void Drawable::AddShape(ModelObjects::ShapeType shape_type) { + const FilamentBuffers* buffers = model_objs_->GetShapeBuffer(shape_type); if (buffers == nullptr) { mju_error("Unknown shape %d", shape_type); } @@ -352,8 +353,7 @@ void Drawable::SetTransform(const mjvGeom& geom) { void Drawable::UpdateMaterial(const mjvGeom& geom, bool use_segid_color, bool enable_reflection, const mjtNum* headpos) { - ObjectManager* object_mgr = material_.GetObjectManager(); - const mjModel* model = object_mgr->GetModel(); + const mjModel* model = model_objs_->GetModel(); float4 color = ReadFloat4(geom.rgba); if (geom.type == mjGEOM_PLANE) { @@ -370,15 +370,15 @@ void Drawable::UpdateMaterial(const mjvGeom& geom, bool use_segid_color, Material::Textures textures; if (geom.matid >= 0) { - textures.color = object_mgr->GetTexture(geom.matid, mjTEXROLE_RGB); - textures.normal = object_mgr->GetTexture(geom.matid, mjTEXROLE_NORMAL); - textures.emissive = object_mgr->GetTexture(geom.matid, mjTEXROLE_EMISSIVE); - textures.orm = object_mgr->GetTexture(geom.matid, mjTEXROLE_ORM); - textures.metallic = object_mgr->GetTexture(geom.matid, mjTEXROLE_METALLIC); + textures.color = model_objs_->GetTexture(geom.matid, mjTEXROLE_RGB); + textures.normal = model_objs_->GetTexture(geom.matid, mjTEXROLE_NORMAL); + textures.emissive = model_objs_->GetTexture(geom.matid, mjTEXROLE_EMISSIVE); + textures.orm = model_objs_->GetTexture(geom.matid, mjTEXROLE_ORM); + textures.metallic = model_objs_->GetTexture(geom.matid, mjTEXROLE_METALLIC); textures.roughness = - object_mgr->GetTexture(geom.matid, mjTEXROLE_ROUGHNESS); + model_objs_->GetTexture(geom.matid, mjTEXROLE_ROUGHNESS); textures.occlusion = - object_mgr->GetTexture(geom.matid, mjTEXROLE_OCCLUSION); + model_objs_->GetTexture(geom.matid, mjTEXROLE_OCCLUSION); material_.UpdateTextures(textures); } @@ -548,9 +548,9 @@ void Drawable::UpdateMaterial(const mjvGeom& geom, bool use_segid_color, } // Apply material multipliers from the model. - params.emissive *= object_mgr->GetEmissiveMultiplier(); - params.specular *= object_mgr->GetSpecularMultiplier(); - params.glossiness *= object_mgr->GetShininessMultiplier(); + params.emissive *= model_objs_->GetEmissiveMultiplier(); + params.specular *= model_objs_->GetSpecularMultiplier(); + params.glossiness *= model_objs_->GetShininessMultiplier(); material_.UpdateParams(params); } diff --git a/src/experimental/filament/filament/drawable.h b/src/experimental/filament/filament/drawable.h index dc1f95d0..19002de6 100644 --- a/src/experimental/filament/filament/drawable.h +++ b/src/experimental/filament/filament/drawable.h @@ -23,6 +23,7 @@ #include #include #include "experimental/filament/filament/material.h" +#include "experimental/filament/filament/model_objects.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/renderables.h" @@ -31,7 +32,8 @@ namespace mujoco { // Manages the filament Entities and MaterialInstances for a single mjvGeom. class Drawable { public: - Drawable(ObjectManager* object_mgr, const mjvGeom& geom); + Drawable(ObjectManager* object_mgr, ModelObjects* model_objects, + const mjvGeom& geom); ~Drawable() noexcept = default; Drawable(const Drawable&) = delete; @@ -71,7 +73,7 @@ class Drawable { private: void AddMesh(int data_id); void AddHeightField(int hfield_id); - void AddShape(ObjectManager::ShapeType shape_type); + void AddShape(ModelObjects::ShapeType shape_type); // Updates the transform of the drawable for rendering. void SetTransform(const mjvGeom& geom); @@ -81,6 +83,7 @@ class Drawable { bool enable_reflection, const mjtNum* headpos); Material material_; + ModelObjects* model_objs_ = nullptr; Renderables renderables_; bool reflective_ = false; filament::math::mat4 transform_; diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc index 0fa4b14d..d783cefa 100644 --- a/src/experimental/filament/filament/filament_context.cc +++ b/src/experimental/filament/filament/filament_context.cc @@ -41,10 +41,10 @@ #include "experimental/filament/filament/filament_platform_factory.h" #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/imgui_editor.h" -#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/model_util.h" -#include "experimental/filament/filament/scene_view.h" +#include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target_util.h" +#include "experimental/filament/filament/scene_view.h" #include "experimental/filament/render_context_filament.h" namespace mujoco { @@ -72,6 +72,8 @@ FilamentContext::FilamentContext(const mjrFilamentConfig* config) } #endif offscreen_swap_chain_ = engine_->createSwapChain(config_.width, config_.height); + + object_manager_ = std::make_unique(engine_); } FilamentContext::~FilamentContext() { @@ -86,8 +88,7 @@ FilamentContext::~FilamentContext() { } void FilamentContext::Init(const mjModel* model) { - object_manager_ = std::make_unique(model, engine_); - scene_view_ = std::make_unique(engine_, object_manager_.get()); + scene_view_ = std::make_unique(object_manager_.get(), model); gui_view_ = std::make_unique( engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi)); @@ -270,15 +271,24 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, } void FilamentContext::UploadMesh(const mjModel* model, int id) { - object_manager_->UploadMesh(model, id); + if (!scene_view_) { + mju_error("SceneView is not initialized."); + } + scene_view_->UploadMesh(model, id); } void FilamentContext::UploadTexture(const mjModel* model, int id) { - object_manager_->UploadTexture(model, id); + if (!scene_view_) { + mju_error("SceneView is not initialized."); + } + scene_view_->UploadTexture(model, id); } void FilamentContext::UploadHeightField(const mjModel* model, int id) { - object_manager_->UploadHeightField(model, id); + if (!scene_view_) { + mju_error("SceneView is not initialized."); + } + scene_view_->UploadHeightField(model, id); } uintptr_t FilamentContext::UploadGuiImage(uintptr_t tex_id, diff --git a/src/experimental/filament/filament/filament_context.h b/src/experimental/filament/filament/filament_context.h index ec0a7243..7e2e2a9a 100644 --- a/src/experimental/filament/filament/filament_context.h +++ b/src/experimental/filament/filament/filament_context.h @@ -26,8 +26,8 @@ #include #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/object_manager.h" -#include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/render_target_util.h" +#include "experimental/filament/filament/scene_view.h" #include "experimental/filament/render_context_filament.h" namespace mujoco { diff --git a/src/experimental/filament/filament/material.cc b/src/experimental/filament/filament/material.cc index 1557e9e5..08c167a2 100644 --- a/src/experimental/filament/filament/material.cc +++ b/src/experimental/filament/filament/material.cc @@ -26,7 +26,7 @@ namespace mujoco { Material::Material(ObjectManager* object_mgr) : object_mgr_(object_mgr) { instances_[kDepth] = - object_mgr->GetMaterial(ObjectManager::kUnlitDepth)->createInstance(); + object_mgr_->GetMaterial(ObjectManager::kUnlitDepth)->createInstance(); instances_[kSegmentation] = object_mgr_->GetMaterial(ObjectManager::kUnlitSegmentation) ->createInstance(); diff --git a/src/experimental/filament/filament/material.h b/src/experimental/filament/filament/material.h index 22452fb6..51ccac71 100644 --- a/src/experimental/filament/filament/material.h +++ b/src/experimental/filament/filament/material.h @@ -89,10 +89,6 @@ class Material { return instances_[mode]; } - // Returns the ObjectManager owning the Materials which are used to create - // the MaterialInstances. - ObjectManager* GetObjectManager() { return object_mgr_; } - private: // Updates the material instances based on the currently set parameters and // textures. diff --git a/src/experimental/filament/filament/model_objects.cc b/src/experimental/filament/filament/model_objects.cc new file mode 100644 index 00000000..ac3f4ca2 --- /dev/null +++ b/src/experimental/filament/filament/model_objects.cc @@ -0,0 +1,258 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/model_objects.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/builtins.h" +#include "experimental/filament/filament/model_util.h" +#include "experimental/filament/filament/texture_util.h" + + +namespace mujoco { + +ModelObjects::ModelObjects(const mjModel* model, filament::Engine* engine) + : model_(model), engine_(engine) { + const int nstack = model->vis.quality.numstacks; + const int nslice = model->vis.quality.numslices; + const int nquad = model->vis.quality.numquads; + shapes_[kLine] = CreateLine(engine_); + shapes_[kBox] = CreateBox(engine_, nquad); + shapes_[kLineBox] = CreateLineBox(engine_); + shapes_[kCone] = CreateCone(engine_, nstack, nslice); + shapes_[kDisk] = CreateDisk(engine_, nslice); + shapes_[kDome] = CreateDome(engine_, nstack / 2, nslice); + shapes_[kTube] = CreateTube(engine_, nstack, nslice); + shapes_[kPlane] = CreatePlane(engine_, nquad); + shapes_[kSphere] = CreateSphere(engine_, nstack, nslice); + shapes_[kTriangle] = CreateTriangle(engine_); + + for (int i = 0; i < model_->ntex; ++i) { + UploadTexture(model_, i); + } + for (int i = 0; i < model_->nmesh; ++i) { + UploadMesh(model_, i); + } + for (int i = 0; i < model_->nhfield; ++i) { + UploadHeightField(model_, i); + } + + specular_multiplier_ = ReadElement( + model_, "filament.phong.specular_multiplier", specular_multiplier_); + shininess_multiplier_ = ReadElement( + model_, "filament.phong.shininess_multiplier", shininess_multiplier_); + emissive_multiplier_ = ReadElement( + model_, "filament.phong.emissive_multiplier", emissive_multiplier_); +} + +ModelObjects::~ModelObjects() { + for (auto& iter : skyboxes_) { + engine_->destroy(iter); + } + for (auto& iter : indirect_lights_) { + engine_->destroy(iter); + } + for (auto& iter : meshes_) { + engine_->destroy(iter.second.vertex_buffer); + engine_->destroy(iter.second.index_buffer); + } + for (auto& iter : shapes_) { + engine_->destroy(iter.vertex_buffer); + engine_->destroy(iter.index_buffer); + } + for (auto& iter : textures_) { + engine_->destroy(iter.second); + } +} + +void ModelObjects::UploadMesh(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + } + + if (auto iter = meshes_.find(id); iter != meshes_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + if (auto iter = convex_hulls_.find(id); iter != convex_hulls_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + + FilamentBuffers& buffers = meshes_[id]; + buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kNormal); + buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kNormal); + + if (model->mesh_graphadr[id] >= 0) { + FilamentBuffers& hull_buffers = convex_hulls_[id]; + hull_buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kConvexHull); + hull_buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kConvexHull); + } +} + +void ModelObjects::UploadTexture(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->ntex) { + mju_error("Invalid texture index: %d", id); + } + + if (auto iter = textures_.find(id); iter != textures_.end()) { + engine_->destroy(iter->second); + } + + const int texture_type = model->tex_type[id]; + if (model->tex_height[id] == 1) { + const mjtByte* bytes = model->tex_data + model->tex_adr[id]; + const int num_bytes = model->tex_width[id]; + textures_[id] = + CreateKtxTexture(engine_, bytes, num_bytes, spherical_harmonics_[id]); + } else if (texture_type == mjTEXTURE_2D) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kNormal2d); + } else if (texture_type == mjTEXTURE_CUBE) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); + } else if (texture_type == mjTEXTURE_SKYBOX) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); + } else { + mju_error("Unsupported: Texture type: %d", texture_type); + } +} + +void ModelObjects::UploadHeightField(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->nhfield) { + mju_error("Invalid height field index %d", id); + } + + if (auto iter = height_fields_.find(id); iter != height_fields_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + + FilamentBuffers& buffers = height_fields_[id]; + buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kHeightField); + buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kHeightField); +} + +const FilamentBuffers* ModelObjects::GetMeshBuffer(int data_id) const { + // As defined by mjv_updateScene: + // original mesh: mesh_id * 2 + // convex hull: (mesh_id * 2) + 1 + const int mesh_id = data_id / 2; + if (data_id % 2 == 0) { + auto it = meshes_.find(mesh_id); + return it != meshes_.end() ? &it->second : nullptr; + } else { + auto it = convex_hulls_.find(mesh_id); + return it != convex_hulls_.end() ? &it->second : nullptr; + } +} + +const FilamentBuffers* ModelObjects::GetHeightFieldBuffer( + int hfield_id) const { + auto it = height_fields_.find(hfield_id); + return it != height_fields_.end() ? &it->second : nullptr; +} + +const FilamentBuffers* ModelObjects::GetShapeBuffer(ShapeType shape) const { + if (shape < 0 || shape >= kNumShapes) { + mju_error("Invalid shape type: %d", shape); + } + return &shapes_[shape]; +} + +const filament::Texture* ModelObjects::GetTexture(int tex_id) const { + auto it = textures_.find(tex_id); + return it != textures_.end() ? it->second : nullptr; +} + +const filament::Texture* ModelObjects::GetTexture(int mat_id, int role) const { + if (mat_id < 0 || mat_id >= model_->nmat || role < 0 || role >= mjNTEXROLE) { + return nullptr; + } + const int tex_id = model_->mat_texid[mat_id * mjNTEXROLE + role]; + return GetTexture(tex_id); +} + +filament::IndirectLight* ModelObjects::CreateIndirectLight(int tex_id, + float intensity) { + filament::Texture* texture = nullptr; + auto texture_iter = textures_.find(tex_id); + if (texture_iter != textures_.end()) { + texture = texture_iter->second; + } + + SphericalHarmonics* spherical_harmonics = nullptr; + auto sh_iter = spherical_harmonics_.find(tex_id); + if (sh_iter != spherical_harmonics_.end()) { + spherical_harmonics = &sh_iter->second; + } + + filament::IndirectLight::Builder builder; + builder.reflections(texture); + if (spherical_harmonics != nullptr) { + builder.irradiance(3, *spherical_harmonics); + } + builder.intensity(intensity); + // Rotate the light to match mujoco's Z-up convention. + builder.rotation(filament::math::mat3f::rotation( + filament::math::f::PI / 2, filament::math::float3{1, 0, 0})); + filament::IndirectLight* indirect_light = builder.build(*engine_); + indirect_lights_.push_back(indirect_light); + return indirect_light; +} + +filament::Skybox* ModelObjects::CreateSkybox() { + filament::Texture* skybox_texture = nullptr; + for (auto& iter : textures_) { + const int texture_type = model_->tex_type[iter.first]; + if (texture_type == mjTEXTURE_SKYBOX) { + skybox_texture = iter.second; + break; + } + } + + if (skybox_texture == nullptr) { + return nullptr; + } + + filament::Skybox::Builder builder; + builder.environment(skybox_texture); + filament::Skybox* skybox = builder.build(*engine_); + skyboxes_.push_back(skybox); + return skybox; +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/model_objects.h b/src/experimental/filament/filament/model_objects.h new file mode 100644 index 00000000..5de28f8a --- /dev/null +++ b/src/experimental/filament/filament/model_objects.h @@ -0,0 +1,100 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" + +namespace mujoco { + +// Creates and owns various filament objects based on the data in a mjrContext. +class ModelObjects { + public: + ModelObjects(const mjModel* model, filament::Engine* engine); + ~ModelObjects(); + + enum ShapeType { + kLine, + kLineBox, + kPlane, + kTriangle, + kBox, + kSphere, + kCone, + kDisk, + kDome, + kTube, + kNumShapes, + }; + + void UploadMesh(const mjModel* model, int id); + + void UploadTexture(const mjModel* model, int id); + + void UploadHeightField(const mjModel* model, int id); + + // Returns the filament engine used by the ModelObjects to create filament + // objects. + filament::Engine* GetEngine() const { return engine_; } + + // Returns the cached instance of a filament object created from the mjModel. + const FilamentBuffers* GetShapeBuffer(ShapeType shape) const; + const FilamentBuffers* GetMeshBuffer(int data_id) const; + const FilamentBuffers* GetHeightFieldBuffer(int hfield_id) const; + const filament::Texture* GetTexture(int tex_id) const; + const filament::Texture* GetTexture(int mat_id, int role) const; + + filament::Skybox* CreateSkybox(); + filament::IndirectLight* CreateIndirectLight(int tex_id, float intensity); + + float GetSpecularMultiplier() const { return specular_multiplier_; } + float GetShininessMultiplier() const { return shininess_multiplier_; } + float GetEmissiveMultiplier() const { return emissive_multiplier_; } + + const mjModel* GetModel() const { return model_; } + + ModelObjects(const ModelObjects&) = delete; + ModelObjects& operator=(const ModelObjects&) = delete; + + private: + using SphericalHarmonics = filament::math::float3[9]; + + const mjModel* model_ = nullptr; + filament::Engine* engine_ = nullptr; + std::vector skyboxes_; + std::vector indirect_lights_; + std::array shapes_; + std::unordered_map meshes_; + std::unordered_map convex_hulls_; + std::unordered_map height_fields_; + std::unordered_map textures_; + std::unordered_map spherical_harmonics_; + float specular_multiplier_ = 0.2f; + float shininess_multiplier_ = 0.1f; + float emissive_multiplier_ = 0.3f; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_OBJECTS_H_ diff --git a/src/experimental/filament/filament/object_manager.cc b/src/experimental/filament/filament/object_manager.cc index 4aedb120..4bf5a0cd 100644 --- a/src/experimental/filament/filament/object_manager.cc +++ b/src/experimental/filament/filament/object_manager.cc @@ -14,12 +14,9 @@ #include "experimental/filament/filament/object_manager.h" -#include #include #include #include -#include -#include #include #include @@ -29,11 +26,7 @@ #include #include #include -#include "experimental/filament/filament/buffer_util.h" -#include "experimental/filament/filament/builtins.h" -#include "experimental/filament/filament/model_util.h" #include "experimental/filament/filament/texture_util.h" -#include "experimental/filament/render_context_filament.h" #include "user/user_resource.h" namespace mujoco { @@ -64,23 +57,8 @@ struct Asset { } // namespace -ObjectManager::ObjectManager(const mjModel* model, filament::Engine* engine) - : model_(model), engine_(engine) { - const int nquad = model->vis.quality.numquads; - const int nstack = model->vis.quality.numstacks; - const int nslice = model->vis.quality.numslices; - - shapes_[kLine] = CreateLine(engine_); - shapes_[kBox] = CreateBox(engine_, nquad); - shapes_[kLineBox] = CreateLineBox(engine_); - shapes_[kCone] = CreateCone(engine_, nstack, nslice); - shapes_[kDisk] = CreateDisk(engine_, nslice); - shapes_[kDome] = CreateDome(engine_, nstack / 2, nslice); - shapes_[kTube] = CreateTube(engine_, nstack, nslice); - shapes_[kPlane] = CreatePlane(engine_, nquad); - shapes_[kSphere] = CreateSphere(engine_, nstack, nslice); - shapes_[kTriangle] = CreateTriangle(engine_); - +ObjectManager::ObjectManager(filament::Engine* engine) + : engine_(engine) { auto LoadMaterial = [this](std::string_view filename) { Asset asset(filename); filament::Material::Builder material_builder; @@ -107,16 +85,6 @@ ObjectManager::ObjectManager(const mjModel* model, filament::Engine* engine) materials_[kUnlitDepth] = LoadMaterial("unlit_depth.filamat"); materials_[kUnlitUi] = LoadMaterial("unlit_ui.filamat"); - for (int i = 0; i < model_->ntex; ++i) { - UploadTexture(model_, i); - } - for (int i = 0; i < model_->nmesh; ++i) { - UploadMesh(model_, i); - } - for (int i = 0; i < model_->nhfield; ++i) { - UploadHeightField(model_, i); - } - static uint8_t black_rgb[3] = {0, 0, 0}; fallback_black_ = Create2dTexture(engine_, 1, 1, 3, black_rgb, false); static uint8_t white_rgb[3] = {255, 255, 255}; @@ -135,37 +103,19 @@ ObjectManager::ObjectManager(const mjModel* model, filament::Engine* engine) fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_; fallback_textures_[mjTEXROLE_ORM] = fallback_orm_; - fallback_indirect_light_ = LoadFallbackIndirectLight("ibl.ktx", 1.0f); - - specular_multiplier_ = ReadElement( - model_, "filament.phong.specular_multiplier", specular_multiplier_); - shininess_multiplier_ = ReadElement( - model_, "filament.phong.shininess_multiplier", shininess_multiplier_); - emissive_multiplier_ = ReadElement( - model_, "filament.phong.emissive_multiplier", emissive_multiplier_); + LoadFallbackIndirectLight("ibl.ktx", 1.0f); } ObjectManager::~ObjectManager() { - for (auto& iter : skyboxes_) { - engine_->destroy(iter); + if (fallback_indirect_light_) { + engine_->destroy(fallback_indirect_light_); } - for (auto& iter : indirect_lights_) { - engine_->destroy(iter); + if (fallback_indirect_light_texture_) { + engine_->destroy(fallback_indirect_light_texture_); } for (auto& iter : materials_) { engine_->destroy(iter); } - for (auto& iter : meshes_) { - engine_->destroy(iter.second.vertex_buffer); - engine_->destroy(iter.second.index_buffer); - } - for (auto& iter : shapes_) { - engine_->destroy(iter.vertex_buffer); - engine_->destroy(iter.index_buffer); - } - for (auto& iter : textures_) { - engine_->destroy(iter.second); - } // fallback_textures_ maps to these textures. engine_->destroy(fallback_white_); engine_->destroy(fallback_black_); @@ -173,87 +123,6 @@ ObjectManager::~ObjectManager() { engine_->destroy(fallback_orm_); } -void ObjectManager::UploadMesh(const mjModel* model, int id) { - if (model != model_) { - mju_error("Model mismatch."); - } - if (id < 0 || id >= model->nmesh) { - mju_error("Invalid mesh index %d", id); - } - - if (auto iter = meshes_.find(id); iter != meshes_.end()) { - engine_->destroy(iter->second.vertex_buffer); - engine_->destroy(iter->second.index_buffer); - } - if (auto iter = convex_hulls_.find(id); iter != convex_hulls_.end()) { - engine_->destroy(iter->second.vertex_buffer); - engine_->destroy(iter->second.index_buffer); - } - - FilamentBuffers& buffers = meshes_[id]; - buffers.vertex_buffer = - CreateVertexBuffer(engine_, model, id, MeshType::kNormal); - buffers.index_buffer = - CreateIndexBuffer(engine_, model, id, MeshType::kNormal); - - if (model->mesh_graphadr[id] >= 0) { - FilamentBuffers& hull_buffers = convex_hulls_[id]; - hull_buffers.vertex_buffer = - CreateVertexBuffer(engine_, model, id, MeshType::kConvexHull); - hull_buffers.index_buffer = - CreateIndexBuffer(engine_, model, id, MeshType::kConvexHull); - } -} - -void ObjectManager::UploadTexture(const mjModel* model, int id) { - if (model != model_) { - mju_error("Model mismatch."); - } - if (id < 0 || id >= model->ntex) { - mju_error("Invalid texture index: %d", id); - } - - if (auto iter = textures_.find(id); iter != textures_.end()) { - engine_->destroy(iter->second); - } - - const int texture_type = model->tex_type[id]; - if (model->tex_height[id] == 1) { - const mjtByte* bytes = model->tex_data + model->tex_adr[id]; - const int num_bytes = model->tex_width[id]; - textures_[id] = - CreateKtxTexture(engine_, bytes, num_bytes, spherical_harmonics_[id]); - } else if (texture_type == mjTEXTURE_2D) { - textures_[id] = CreateTexture(engine_, model, id, TextureType::kNormal2d); - } else if (texture_type == mjTEXTURE_CUBE) { - textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); - } else if (texture_type == mjTEXTURE_SKYBOX) { - textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); - } else { - mju_error("Unsupported: Texture type: %d", texture_type); - } -} - -void ObjectManager::UploadHeightField(const mjModel* model, int id) { - if (model != model_) { - mju_error("Model mismatch."); - } - if (id < 0 || id >= model->nhfield) { - mju_error("Invalid height field index %d", id); - } - - if (auto iter = height_fields_.find(id); iter != height_fields_.end()) { - engine_->destroy(iter->second.vertex_buffer); - engine_->destroy(iter->second.index_buffer); - } - - FilamentBuffers& buffers = height_fields_[id]; - buffers.vertex_buffer = - CreateVertexBuffer(engine_, model, id, MeshType::kHeightField); - buffers.index_buffer = - CreateIndexBuffer(engine_, model, id, MeshType::kHeightField); -} - filament::Material* ObjectManager::GetMaterial(MaterialType type) const { if (type < 0 || type >= kNumMaterials) { mju_error("Invalid material type: %d", type); @@ -261,137 +130,50 @@ filament::Material* ObjectManager::GetMaterial(MaterialType type) const { return materials_[type]; } -const FilamentBuffers* ObjectManager::GetMeshBuffer(int data_id) const { - // As defined by mjv_updateScene: - // original mesh: mesh_id * 2 - // convex hull: (mesh_id * 2) + 1 - const int mesh_id = data_id / 2; - if (data_id % 2 == 0) { - auto it = meshes_.find(mesh_id); - return it != meshes_.end() ? &it->second : nullptr; - } else { - auto it = convex_hulls_.find(mesh_id); - return it != convex_hulls_.end() ? &it->second : nullptr; +const filament::Texture* ObjectManager::GetFallbackTexture( + mjtTextureRole role) const { + if (role < 0 || role >= mjNTEXROLE) { + mju_error("Invalid texture role: %d", role); } -} - -const FilamentBuffers* ObjectManager::GetHeightFieldBuffer( - int hfield_id) const { - auto it = height_fields_.find(hfield_id); - return it != height_fields_.end() ? &it->second : nullptr; -} - -const FilamentBuffers* ObjectManager::GetShapeBuffer(ShapeType shape) const { - if (shape < 0 || shape >= kNumShapes) { - mju_error("Invalid shape type: %d", shape); - } - return &shapes_[shape]; -} - -const filament::Texture* ObjectManager::GetTexture(int tex_id) const { - auto it = textures_.find(tex_id); - return it != textures_.end() ? it->second : nullptr; -} - -const filament::Texture* ObjectManager::GetTexture(int mat_id, int role) const { - if (mat_id < 0 || mat_id >= model_->nmat || role < 0 || role >= mjNTEXROLE) { - return nullptr; - } - const int tex_id = model_->mat_texid[mat_id * mjNTEXROLE + role]; - return GetTexture(tex_id); -} - -const filament::Texture* ObjectManager::GetTextureWithFallback(int mat_id, - int role) const { - if (auto texture = GetTexture(mat_id, role)) { - return texture; - } - return GetFallbackTexture(role); -} - -const filament::Texture* ObjectManager::GetFallbackTexture(int role) const { - auto iter = fallback_textures_.find(role); - if (iter != fallback_textures_.end()) { - return iter->second; - } - return nullptr; + return fallback_textures_[role]; } filament::IndirectLight* ObjectManager::GetFallbackIndirectLight() { return fallback_indirect_light_; } -filament::IndirectLight* ObjectManager::CreateIndirectLight(int tex_id, - float intensity) { - filament::Texture* texture = nullptr; - auto texture_iter = textures_.find(tex_id); - if (texture_iter != textures_.end()) { - texture = texture_iter->second; - } - - if (texture == nullptr) { - return nullptr; - } - - SphericalHarmonics* spherical_harmonics = nullptr; - auto sh_iter = spherical_harmonics_.find(tex_id); - if (sh_iter != spherical_harmonics_.end()) { - spherical_harmonics = &sh_iter->second; - } - - return CreateIndirectLight(texture, spherical_harmonics, intensity); -} - -filament::IndirectLight* ObjectManager::LoadFallbackIndirectLight( +void ObjectManager::LoadFallbackIndirectLight( std::string_view filename, float intensity) { + if (fallback_indirect_light_texture_ != nullptr) { + engine_->destroy(fallback_indirect_light_texture_); + fallback_indirect_light_texture_ = nullptr; + } + if (fallback_indirect_light_ != nullptr) { + engine_->destroy(fallback_indirect_light_); + fallback_indirect_light_ = nullptr; + } + Asset asset(filename); if (asset.size == 0) { - return nullptr; + return; } filament::math::float3 spherical_harmonics[9]; - filament::Texture* tex = + fallback_indirect_light_texture_ = CreateKtxTexture(engine_, reinterpret_cast(asset.payload), asset.size, spherical_harmonics); - return CreateIndirectLight(tex, &spherical_harmonics, intensity); -} - -filament::IndirectLight* ObjectManager::CreateIndirectLight( - filament::Texture* texture, SphericalHarmonics* spherical_harmonics, - float intensity) { - filament::IndirectLight::Builder builder; - builder.reflections(texture); - if (spherical_harmonics != nullptr) { - builder.irradiance(3, *spherical_harmonics); + if (fallback_indirect_light_texture_ == nullptr) { + return; } + + // Build the indirect light. + filament::IndirectLight::Builder builder; + builder.reflections(fallback_indirect_light_texture_); + builder.irradiance(3, spherical_harmonics); builder.intensity(intensity); // Rotate the light to match mujoco's Z-up convention. builder.rotation(filament::math::mat3f::rotation( filament::math::f::PI / 2, filament::math::float3{1, 0, 0})); - filament::IndirectLight* indirect_light = builder.build(*engine_); - indirect_lights_.push_back(indirect_light); - return indirect_light; + fallback_indirect_light_ = builder.build(*engine_); } - -filament::Skybox* ObjectManager::CreateSkybox() { - filament::Texture* skybox_texture = nullptr; - for (auto& iter : textures_) { - const int texture_type = model_->tex_type[iter.first]; - if (texture_type == mjTEXTURE_SKYBOX) { - skybox_texture = iter.second; - break; - } - } - - if (skybox_texture == nullptr) { - return nullptr; - } - - filament::Skybox::Builder builder; - builder.environment(skybox_texture); - filament::Skybox* skybox = builder.build(*engine_); - skyboxes_.push_back(skybox); - return skybox; -} - } // namespace mujoco diff --git a/src/experimental/filament/filament/object_manager.h b/src/experimental/filament/filament/object_manager.h index 6fe51dcc..37459b4d 100644 --- a/src/experimental/filament/filament/object_manager.h +++ b/src/experimental/filament/filament/object_manager.h @@ -16,25 +16,19 @@ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OBJECT_MANAGER_H_ #include -#include #include -#include -#include #include #include #include -#include -#include -#include "experimental/filament/filament/buffer_util.h" -#include "experimental/filament/render_context_filament.h" +#include namespace mujoco { // Creates and owns various filament objects based on the data in a mjrContext. class ObjectManager { public: - ObjectManager(const mjModel* model, filament::Engine* engine); + ObjectManager(filament::Engine* engine); ~ObjectManager(); enum MaterialType { @@ -59,85 +53,34 @@ class ObjectManager { kNumMaterials, }; - enum ShapeType { - kLine, - kLineBox, - kPlane, - kTriangle, - kBox, - kSphere, - kCone, - kDisk, - kDome, - kTube, - kNumShapes, - }; - - using SphericalHarmonics = filament::math::float3[9]; - - void UploadMesh(const mjModel* model, int id); - - void UploadTexture(const mjModel* model, int id); - - void UploadHeightField(const mjModel* model, int id); - - // Returns the filament engine used by the ObjectManager to create filament - // objects. + // Returns the filament Engine that owns the assets. filament::Engine* GetEngine() const { return engine_; } + // Returns the Material of the given type. filament::Material* GetMaterial(MaterialType type) const; - // Returns the cached instance of a filament object created from the mjModel. - const FilamentBuffers* GetMeshBuffer(int data_id) const; - const FilamentBuffers* GetShapeBuffer(ShapeType shape) const; - const FilamentBuffers* GetHeightFieldBuffer(int hfield_id) const; - const filament::Texture* GetTexture(int tex_id) const; - const filament::Texture* GetTexture(int mat_id, int role) const; - const filament::Texture* GetTextureWithFallback(int mat_id, int role) const; - const filament::Texture* GetFallbackTexture(int role) const; + // Returns the fallback Texture with the given role. + const filament::Texture* GetFallbackTexture(mjtTextureRole role) const; + + // Returns the fallback IndirectLight. filament::IndirectLight* GetFallbackIndirectLight(); - // Creates and returns a new instance of a filament object. The objects are - // owned by the ObjectManager and will be deleted in the destructor. - filament::Skybox* CreateSkybox(); - filament::IndirectLight* CreateIndirectLight( - filament::Texture* texture, SphericalHarmonics* spherical_harmonics, - float intensity); - filament::IndirectLight* CreateIndirectLight(int tex_id, float intensity); - filament::IndirectLight* LoadFallbackIndirectLight(std::string_view filename, - float intensity); - - float GetSpecularMultiplier() const { return specular_multiplier_; } - float GetShininessMultiplier() const { return shininess_multiplier_; } - float GetEmissiveMultiplier() const { return emissive_multiplier_; } - - const mjModel* GetModel() const { return model_; } + // Loads an indirect light from a file, setting it to the fallback. + void LoadFallbackIndirectLight(std::string_view filename, float intensity); ObjectManager(const ObjectManager&) = delete; ObjectManager& operator=(const ObjectManager&) = delete; private: - const mjModel* model_ = nullptr; filament::Engine* engine_ = nullptr; - - std::array shapes_; std::array materials_; - std::vector skyboxes_; - std::vector indirect_lights_; - std::unordered_map meshes_; - std::unordered_map convex_hulls_; - std::unordered_map height_fields_; - std::unordered_map textures_; - std::unordered_map spherical_harmonics_; - std::unordered_map fallback_textures_; + std::array fallback_textures_; filament::Texture* fallback_white_ = nullptr; filament::Texture* fallback_black_ = nullptr; filament::Texture* fallback_normal_ = nullptr; filament::Texture* fallback_orm_ = nullptr; + filament::Texture* fallback_indirect_light_texture_ = nullptr; filament::IndirectLight* fallback_indirect_light_ = nullptr; - float specular_multiplier_ = 0.2f; - float shininess_multiplier_ = 0.1f; - float emissive_multiplier_ = 0.3f; }; } // namespace mujoco diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index cba55395..aba4e0b8 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -46,6 +46,7 @@ #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/model_objects.h" #include "experimental/filament/filament/model_util.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target_util.h" @@ -113,37 +114,38 @@ static void SetupReflectionCamera(const mat4& surface_xform, reflection_camera->setCustomProjection(oblique, near, far); } -SceneView::SceneView(filament::Engine* engine, ObjectManager* object_mgr) - : object_mgr_(object_mgr), engine_(engine) { - scene_ = engine_->createScene(); - camera_ = engine_->createCamera(utils::EntityManager::get().create()); - reflect_camera_ = engine_->createCamera(utils::EntityManager::get().create()); +SceneView::SceneView(ObjectManager* object_mgr, const mjModel* model) + : object_mgr_(object_mgr) { + filament::Engine* engine = object_mgr_->GetEngine(); + model_objects_ = std::make_unique(model, engine); + + scene_ = engine->createScene(); + camera_ = engine->createCamera(utils::EntityManager::get().create()); + reflect_camera_ = engine->createCamera(utils::EntityManager::get().create()); for (auto& view : views_) { - view = engine_->createView(); + view = engine->createView(); view->setScene(scene_); view->setCamera(camera_); } - reflect_view_ = engine_->createView(); + reflect_view_ = engine->createView(); reflect_view_->setScene(scene_); reflect_view_->setCamera(reflect_camera_); reflect_view_->setShadowingEnabled(false); reflect_view_->setPostProcessingEnabled(false); - const mjModel* m = object_mgr_->GetModel(); - // Configure options for the normal view. auto& cg = color_grading_options_; - cg.exposure = ReadElement(m, "filament.out.exposure", cg.exposure); - cg.contrast = ReadElement(m, "filament.out.contrast", cg.contrast); - cg.vibrance = ReadElement(m, "filament.out.vibrance", cg.vibrance); - cg.saturation = ReadElement(m, "filament.out.saturation", cg.saturation); - cg.temperature = ReadElement(m, "filament.out.temperature", cg.temperature); - cg.tint = ReadElement(m, "filament.out.tint", cg.tint); + cg.exposure = ReadElement(model, "filament.out.exposure", cg.exposure); + cg.contrast = ReadElement(model, "filament.out.contrast", cg.contrast); + cg.vibrance = ReadElement(model, "filament.out.vibrance", cg.vibrance); + cg.saturation = ReadElement(model, "filament.out.saturation", cg.saturation); + cg.temperature = ReadElement(model, "filament.out.temperature", cg.temperature); + cg.tint = ReadElement(model, "filament.out.tint", cg.tint); auto tone_mapping = - ReadElement(m, "filament.out.tone_mapping"); + ReadElement(model, "filament.out.tone_mapping"); if (tone_mapping == "aces") { cg.tone_mapper = ToneMapperType::kACES; } else if (tone_mapping == "aces_legacy") { @@ -158,9 +160,9 @@ SceneView::SceneView(filament::Engine* engine, ObjectManager* object_mgr) SetColorGradingOptions(cg); auto ao = views_[kNormalIndex]->getAmbientOcclusionOptions(); - ao.enabled = ReadElement(m, "filament.ao.enabled", true); - ao.bentNormals = ReadElement(m, "filament.ao.bent_normals", false); - ao.ssct.enabled = ReadElement(m, "filament.ao.ssct", ao.ssct.enabled); + ao.enabled = ReadElement(model, "filament.ao.enabled", true); + ao.bentNormals = ReadElement(model, "filament.ao.bent_normals", false); + ao.ssct.enabled = ReadElement(model, "filament.ao.ssct", ao.ssct.enabled); ao.quality = filament::QualityLevel::ULTRA; ao.lowPassFilter = filament::QualityLevel::ULTRA; ao.upsampling = filament::QualityLevel::ULTRA; @@ -168,16 +170,16 @@ SceneView::SceneView(filament::Engine* engine, ObjectManager* object_mgr) views_[kNormalIndex]->setAmbientOcclusionOptions(ao); auto msaa = views_[kNormalIndex]->getMultiSampleAntiAliasingOptions(); - msaa.enabled = ReadElement(m, "filament.msaa.enabled", true); + msaa.enabled = ReadElement(model, "filament.msaa.enabled", true); views_[kNormalIndex]->setMultiSampleAntiAliasingOptions(msaa); default_shadow_map_size_ = ReadElement( - m, "filament.shadows.map_size", default_shadow_map_size_); + model, "filament.shadows.map_size", default_shadow_map_size_); default_vsm_blur_width_ = ReadElement( - m, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); + model, "filament.shadows.vsm_blur_width", default_vsm_blur_width_); auto shadow_type = views_[kNormalIndex]->getShadowType(); - shadow_type = ReadElement(m, "filament.shadows.type", shadow_type); + shadow_type = ReadElement(model, "filament.shadows.type", shadow_type); views_[kNormalIndex]->setShadowType(shadow_type); // Disable post processing for the depth and segmentation views to preserve @@ -190,44 +192,44 @@ SceneView::SceneView(filament::Engine* engine, ObjectManager* object_mgr) auto& tm = engine->getTransformManager(); tm.create(fog); auto rotation_axis = ReadElement( - m, "filament.fog.rotation_axis", float3{-1, 0, 0}); + model, "filament.fog.rotation_axis", float3{-1, 0, 0}); tm.setTransform(tm.getInstance(fog), mat4::rotation(filament::math::f::PI / 2, rotation_axis)); auto fog_opts = views_[kNormalIndex]->getFogOptions(); - fog_opts.enabled = ReadElement(m, "filament.fog.enabled", fog_opts.enabled); - fog_opts.color = ReadElement(m, "filament.fog.color", fog_opts.color); + fog_opts.enabled = + ReadElement(model, "filament.fog.enabled", fog_opts.enabled); + fog_opts.color = ReadElement(model, "filament.fog.color", fog_opts.color); fog_opts.distance = ReadElement( - m, "filament.fog.distance", fog_opts.distance); + model, "filament.fog.distance", fog_opts.distance); fog_opts.density = ReadElement( - m, "filament.fog.density", fog_opts.density); + model, "filament.fog.density", fog_opts.density); fog_opts.cutOffDistance = ReadElement( - m, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); + model, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); fog_opts.maximumOpacity = ReadElement( - m, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); - fog_opts.height = ReadElement(m, "filament.fog.height", fog_opts.height); + model, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); + fog_opts.height = ReadElement(model, "filament.fog.height", fog_opts.height); fog_opts.heightFalloff = ReadElement( - m, "filament.fog.heightFalloff", fog_opts.heightFalloff); + model, "filament.fog.heightFalloff", fog_opts.heightFalloff); fog_opts.inScatteringStart = ReadElement( - m, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); + model, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); fog_opts.inScatteringSize = ReadElement( - m, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); + model, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); views_[kNormalIndex]->setFogOptions(fog_opts); fallback_head_light_intensity_ = - ReadElement(m, "filament.fallback.head_light_intensity", + ReadElement(model, "filament.fallback.head_light_intensity", fallback_head_light_intensity_); fallback_scene_light_intensity_ = - ReadElement(m, "filament.fallback.scene_light_intensity", + ReadElement(model, "filament.fallback.scene_light_intensity", fallback_scene_light_intensity_); fallback_environment_light_intensity_ = - ReadElement(m, "filament.fallback.environment_light_intensity", + ReadElement(model, "filament.fallback.environment_light_intensity", fallback_environment_light_intensity_); // Create an empty/black indirect light to ensure that the skybox is oriented // to respect mujoco's Z-up convention. - scene_->setIndirectLight( - object_mgr_->CreateIndirectLight(nullptr, nullptr, 100000)); + scene_->setIndirectLight(model_objects_->CreateIndirectLight(-1, 100000)); PrepareLights(); } @@ -237,15 +239,16 @@ SceneView::~SceneView() { drawables_.clear(); reflect_targets_.clear(); - engine_->destroyCameraComponent(reflect_camera_->getEntity()); - engine_->destroy(reflect_view_); + filament::Engine* engine = object_mgr_->GetEngine(); + engine->destroyCameraComponent(reflect_camera_->getEntity()); + engine->destroy(reflect_view_); - engine_->destroyCameraComponent(camera_->getEntity()); - engine_->destroy(views_[kNormalIndex]->getColorGrading()); + engine->destroyCameraComponent(camera_->getEntity()); + engine->destroy(views_[kNormalIndex]->getColorGrading()); for (auto& view : views_) { - engine_->destroy(view); + engine->destroy(view); } - engine_->destroy(scene_); + engine->destroy(scene_); } void SceneView::Render(filament::Renderer* renderer, DrawMode draw_mode, @@ -304,22 +307,23 @@ void SceneView::SetViewport(mjrRect viewport) { } void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { + filament::Engine* engine = object_mgr_->GetEngine(); + auto tone_mapper = CreateToneMapper(opts.tone_mapper); auto color_grading = ToBuilder(color_grading_options_) .toneMapper(tone_mapper.get()) - .build(*engine_); + .build(*engine); views_[kNormalIndex]->setColorGrading(color_grading); - engine_->destroy(color_grading_); + engine->destroy(color_grading_); color_grading_ = color_grading; color_grading_options_ = opts; } void SceneView::SetEnvironmentLight(std::string_view filename, float intensity) { - auto* ibl = object_mgr_->LoadFallbackIndirectLight(filename, intensity); - if (ibl) { - scene_->setIndirectLight(ibl); - } + scene_->setIndirectLight(nullptr); + object_mgr_->LoadFallbackIndirectLight(filename, intensity); + scene_->setIndirectLight(object_mgr_->GetFallbackIndirectLight()); } void SceneView::SetFallbackEnvironmentLight(float intensity) { @@ -357,8 +361,9 @@ std::optional SceneView::ClipFromWorld(const float3& pos) const{ } void SceneView::PrepareLights() { - const mjModel* model = object_mgr_->GetModel(); - filament::Skybox* skybox = object_mgr_->CreateSkybox(); + filament::Engine* engine = object_mgr_->GetEngine(); + const mjModel* model = model_objects_->GetModel(); + filament::Skybox* skybox = model_objects_->CreateSkybox(); if (skybox) { scene_->setSkybox(skybox); } @@ -369,7 +374,7 @@ void SceneView::PrepareLights() { total_light_intensity += model->light_intensity[i]; if (model->light_type[i] == mjLIGHT_IMAGE) { - auto* indirect_light = object_mgr_->CreateIndirectLight( + auto* indirect_light = model_objects_->CreateIndirectLight( model->light_texid[i], model->light_intensity[i]); if (indirect_light) { scene_->setIndirectLight(indirect_light); @@ -390,7 +395,7 @@ void SceneView::PrepareLights() { params.spot_cone_angle = model->light_cutoff[i]; } - auto light_obj = std::make_unique(engine_, params); + auto light_obj = std::make_unique(engine, params); #ifndef __EMSCRIPTEN__ // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. light_obj->AddToScene(scene_); @@ -408,7 +413,7 @@ void SceneView::PrepareLights() { params.type = mjLIGHT_DIRECTIONAL; params.castshadow = 0; params.intensity = 0; - auto light_obj = std::make_unique(engine_, params); + auto light_obj = std::make_unique(engine, params); #ifndef __EMSCRIPTEN__ // TODO(b/458045799): Re-enable when lights work on glinux and chromebook. light_obj->AddToScene(scene_); @@ -456,9 +461,10 @@ void SceneView::UpdateScene(const mjvScene* scene) { } } - auto drawable = std::make_unique(object_mgr_, *geom); + auto drawable = + std::make_unique(object_mgr_, model_objects_.get(), *geom); drawable->AddToScene(scene_); - drawable->Update(object_mgr_->GetModel(), scene, *geom); + drawable->Update(model_objects_->GetModel(), scene, *geom); if (drawable->IsReflective()) { AddReflectiveDrawable(drawable.get()); } @@ -507,9 +513,10 @@ void SceneView::AddReflectiveDrawable(Drawable* drawable) { // Ensure we have the same number of render targets as we do reflective // drawables. + filament::Engine* engine = object_mgr_->GetEngine(); while (reflect_targets_.size() < reflectives_.size()) { reflect_targets_.push_back(std::make_unique( - engine_, kRenderTargetReflectionColor, kRenderTargetDepth)); + engine, kRenderTargetReflectionColor, kRenderTargetDepth)); } // Prepare a render target for the reflective drawable. @@ -519,7 +526,21 @@ void SceneView::AddReflectiveDrawable(Drawable* drawable) { drawable->UpdateReflectionTexture(target->GetColorTexture()); } -filament::Engine* SceneView::GetEngine() const { return engine_; } +void SceneView::UploadMesh(const mjModel* model, int id) { + model_objects_->UploadMesh(model, id); +} + +void SceneView::UploadTexture(const mjModel* model, int id) { + model_objects_->UploadTexture(model, id); +} + +void SceneView::UploadHeightField(const mjModel* model, int id) { + model_objects_->UploadHeightField(model, id); +} + +filament::Engine* SceneView::GetEngine() const { + return object_mgr_->GetEngine(); +} filament::View* SceneView::GetDefaultRenderView() { return views_[kNormalIndex]; diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index a46a54d2..8ced8be1 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -31,10 +31,12 @@ #include #include #include +#include #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/drawable.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/material.h" +#include "experimental/filament/filament/model_objects.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target_util.h" @@ -47,7 +49,7 @@ namespace mujoco { // different rendering modes (e.g. normal, depth, segmentation, etc.) class SceneView { public: - SceneView(filament::Engine* engine, ObjectManager* object_mgr); + SceneView(ObjectManager* object_mgr, const mjModel* model); ~SceneView(); // Updates all views to render into the given viewport. @@ -71,6 +73,10 @@ class SceneView { void Render(filament::Renderer* renderer, DrawMode draw_mode, filament::RenderTarget* target = nullptr); + void UploadMesh(const mjModel* model, int id); + void UploadTexture(const mjModel* model, int id); + void UploadHeightField(const mjModel* model, int id); + // Accessors. filament::Engine* GetEngine() const; filament::View* GetDefaultRenderView(); @@ -96,12 +102,12 @@ class SceneView { const filament::math::float3& pos) const; ObjectManager* object_mgr_ = nullptr; - filament::Engine* engine_ = nullptr; filament::Scene* scene_ = nullptr; filament::Camera* camera_ = nullptr; filament::ColorGrading* color_grading_ = nullptr; std::vector> lights_; std::vector> drawables_; + std::unique_ptr model_objects_; std::array views_; filament::math::mat4 clip_from_world_; ColorGradingOptions color_grading_options_;