From 36ea0c466f70d522b4a7a1910f8f5b91fdbd69cb Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Wed, 27 May 2026 06:54:23 -0700 Subject: [PATCH] Support multiscene, multiview rendering. Update Renderable so that it can switch its own draw states (material instance, shadow casting, reflection textures, etc.) per render request. This allows us to use the same Renderable with different effects from different views within the same frame. PiperOrigin-RevId: 922102817 Change-Id: I1cf02fddb7a00345a741db1bee88e716517765f4 --- .../filament/compat/scene_bridge.cc | 4 +- .../filament/compat/scene_geom_util.cc | 40 +--- .../filament/compat/scene_geom_util.h | 2 +- .../filament/filament/filament_context.cc | 10 + .../filament/filament/material.cc | 56 ++++- src/experimental/filament/filament/material.h | 6 + .../filament/filament/renderable.cc | 224 +++++++++++++----- .../filament/filament/renderable.h | 47 +++- .../filament/filament/scene_view.cc | 118 +++------ .../filament/filament/scene_view.h | 23 +- src/experimental/filament/filament_util.h | 6 +- .../filament/render_context_filament.h | 4 - 12 files changed, 321 insertions(+), 219 deletions(-) diff --git a/src/experimental/filament/compat/scene_bridge.cc b/src/experimental/filament/compat/scene_bridge.cc index 61a24d2f..b8869bbd 100644 --- a/src/experimental/filament/compat/scene_bridge.cc +++ b/src/experimental/filament/compat/scene_bridge.cc @@ -129,8 +129,8 @@ void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) { model_objects_->CreateSkinFlexMesh(scene, *geom); } - UniquePtr renderable = CreateGeomRenderable( - *geom, ctx_, model_objects_.get(), headpos, scene->flags); + UniquePtr renderable = + CreateGeomRenderable(*geom, ctx_, model_objects_.get(), scene->flags); mjrf_addRenderableToScene(scene_.get(), renderable.get()); renderables_.push_back(std::move(renderable)); diff --git a/src/experimental/filament/compat/scene_geom_util.cc b/src/experimental/filament/compat/scene_geom_util.cc index 83622ff1..17ad8e6d 100644 --- a/src/experimental/filament/compat/scene_geom_util.cc +++ b/src/experimental/filament/compat/scene_geom_util.cc @@ -40,12 +40,6 @@ static float GetPlaneTileSize(const mjModel* model, int matid, } } -static bool IsBehind(const float* headpos, const float* pos, const float* mat) { - return ((headpos[0] - pos[0]) * mat[2] + (headpos[1] - pos[1]) * mat[5] + - (headpos[2] - pos[2]) * mat[8] < - 0.0f); -} - static void PrepareGeomMeshes(mjrRenderable* renderable, const mjvGeom& geom, ModelObjects* model_objs) { const mjModel* model = model_objs->GetModel(); @@ -69,19 +63,8 @@ static void PrepareGeomMeshes(mjrRenderable* renderable, const mjvGeom& geom, break; case mjGEOM_PLANE: { mjrf_setRenderableGeomMesh(renderable, geom_type, nstack, nslice, nquad); - - float size[3]; - std::memcpy(size, &geom.size, 3 * sizeof(float)); - const bool is_infinite = !(size[0] > 0 && size[1] > 0); - if (is_infinite) { - // Infinite planes are scaled to match the tile size used by - // re-centering in engine_vis_visualize.c. - const float plane_scale = static_cast(mjMAXPLANEGRID) / 2.0f; - size[0] = plane_scale; - size[1] = plane_scale; - } // Planes only define an xy size, so set the z-dimension to 1.0f. - size[2] = 1.0f; + const float size[3] = {geom.size[0], geom.size[1], 1.0f}; mjrf_setRenderableSize(renderable, size); break; } @@ -160,7 +143,7 @@ static void PrepareGeomMeshes(mjrRenderable* renderable, const mjvGeom& geom, } static void UpdateGeomMaterial(mjrRenderable* renderable, const mjvGeom& geom, - ModelObjects* model_objs, const float headpos[3], + ModelObjects* model_objs, const mjtByte render_flags[mjNRNDFLAG]) { const mjModel* model = model_objs->GetModel(); @@ -175,21 +158,8 @@ static void UpdateGeomMaterial(mjrRenderable* renderable, const mjvGeom& geom, material.color[1] = geom.rgba[1]; material.color[2] = geom.rgba[2]; material.color[3] = geom.rgba[3]; - if (geom.type == mjGEOM_PLANE) { - if (IsBehind(headpos, geom.pos, geom.mat)) { - material.color[3] *= 0.3; - mjrf_setRenderableReceiveShadows(renderable, false); - material.reflective = false; - } else { - mjrf_setRenderableReceiveShadows(renderable, true); - material.reflective = geom.reflectance > 0 && material.color[3] == 1.0f; - } - } + mjrf_setRenderableLayerMask(renderable, geom.category); - if (geom.category == mjCAT_DECOR) { - mjrf_setRenderableCastShadows(renderable, false); - mjrf_setRenderableReceiveShadows(renderable, false); - } if (geom.matid >= 0 && geom.matid < model->nmat) { auto get_texture = [&](int role) -> const mjrTexture* { @@ -310,12 +280,12 @@ static void UpdateGeomMaterial(mjrRenderable* renderable, const mjvGeom& geom, UniquePtr CreateGeomRenderable( const mjvGeom& geom, mjrfContext* ctx, ModelObjects* model_objs, - const float headpos[3], const mjtByte render_flags[mjNRNDFLAG]) { + const mjtByte render_flags[mjNRNDFLAG]) { mjrRenderableParams params; mjr_defaultRenderableParams(¶ms); auto renderable = CreateRenderable(ctx, params); PrepareGeomMeshes(renderable.get(), geom, model_objs); - UpdateGeomMaterial(renderable.get(), geom, model_objs, headpos, render_flags); + UpdateGeomMaterial(renderable.get(), geom, model_objs, render_flags); return renderable; } } // namespace mujoco diff --git a/src/experimental/filament/compat/scene_geom_util.h b/src/experimental/filament/compat/scene_geom_util.h index 89e516a2..53911c79 100644 --- a/src/experimental/filament/compat/scene_geom_util.h +++ b/src/experimental/filament/compat/scene_geom_util.h @@ -26,7 +26,7 @@ namespace mujoco { // Creates a Renderable from the given mjvGeom. UniquePtr CreateGeomRenderable( const mjvGeom& geom, mjrfContext* ctx, ModelObjects* model_objs, - const float headpos[3], const mjtByte render_flags[mjNRNDFLAG]); + const mjtByte render_flags[mjNRNDFLAG]); } // namespace mujoco diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc index e381d402..e5682486 100644 --- a/src/experimental/filament/filament/filament_context.cc +++ b/src/experimental/filament/filament/filament_context.cc @@ -18,7 +18,9 @@ #include #include #include +#include #include +#include #include #include @@ -94,6 +96,14 @@ mjrFrameHandle FilamentContext::Render( ValidateSwapChains(requests); + std::unordered_map> scene_to_requests; + for (const mjrRenderRequest& request : requests) { + scene_to_requests[request.scene].push_back(&request); + } + for (auto& [scene, requests] : scene_to_requests) { + SceneView::downcast(scene)->PrepareToRender(requests); + } + bool render_began = false; mjrRenderTarget* current_target = nullptr; for (const mjrRenderRequest& request : requests) { diff --git a/src/experimental/filament/filament/material.cc b/src/experimental/filament/filament/material.cc index 21a024db..8c8b5460 100644 --- a/src/experimental/filament/filament/material.cc +++ b/src/experimental/filament/filament/material.cc @@ -13,6 +13,9 @@ // limitations under the License. #include "experimental/filament/filament/material.h" +#include +#include +#include #include #include @@ -29,6 +32,51 @@ namespace mujoco { +template +static void Combine(uint64_t& seed, const T* v) { + seed ^= std::hash()(*v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template +static uint64_t hash(const T& obj) { + static_assert(std::is_trivially_copyable_v, + "Only trivially copyable types are hashable."); + int num_bytes = sizeof(T); + const std::byte* ptr = reinterpret_cast(&obj); + uint64_t seed = 0; + while (num_bytes >= sizeof(uint64_t)) { + Combine(seed, reinterpret_cast(ptr)); + ptr += sizeof(uint64_t); + num_bytes -= sizeof(uint64_t); + } + while (num_bytes >= sizeof(uint32_t)) { + Combine(seed, reinterpret_cast(ptr)); + ptr += sizeof(uint32_t); + num_bytes -= sizeof(uint32_t); + } + while (num_bytes >= sizeof(uint16_t)) { + Combine(seed, reinterpret_cast(ptr)); + ptr += sizeof(uint16_t); + num_bytes -= sizeof(uint16_t); + } + while (num_bytes >= sizeof(uint8_t)) { + Combine(seed, reinterpret_cast(ptr)); + ptr += sizeof(uint8_t); + num_bytes -= sizeof(uint8_t); + } + return seed; +} + +uint64_t BuildMaterialKey(ObjectManager::MaterialType material_type, + const mjrMaterial& material) { + // Normally, hashing the struct by memory would be a problem because of + // padding and other uninitialized data. However, we do a memset(0) on the + // entire structure in mjr_defaultMaterial so this should be safe. + uint64_t key = hash(material); + Combine(key, &material_type); + return key; +} + ObjectManager::MaterialType GetMaterialType(const mjrMaterial& material, const Mesh* mesh) { if (material.decor_ux) { @@ -80,7 +128,7 @@ ObjectManager::MaterialType GetMaterialType(const mjrMaterial& material, if (color_texture == nullptr) { if (material.color[3] < 1.0f) { return ObjectManager::kPhongColorFade; - } else if (material.reflective) { + } else if (material.reflectance > 0) { return ObjectManager::kPhongColorReflect; } else { return ObjectManager::kPhongColor; @@ -88,7 +136,7 @@ ObjectManager::MaterialType GetMaterialType(const mjrMaterial& material, } else if (color_texture->GetSamplerType() == mjTEXTURE_CUBE) { if (material.color[3] < 1.0f) { return ObjectManager::kPhongCubeFade; - } else if (material.reflective) { + } else if (material.reflectance > 0) { return ObjectManager::kPhongCubeReflect; } else { return ObjectManager::kPhongCube; @@ -96,7 +144,7 @@ ObjectManager::MaterialType GetMaterialType(const mjrMaterial& material, } else if (has_texcoords) { if (material.color[3] < 1.0f) { return ObjectManager::kPhong2dUvFade; - } else if (material.reflective) { + } else if (material.reflectance > 0) { return ObjectManager::kPhong2dUvReflect; } else { return ObjectManager::kPhong2dUv; @@ -104,7 +152,7 @@ ObjectManager::MaterialType GetMaterialType(const mjrMaterial& material, } else { if (material.color[3] < 1.0f) { return ObjectManager::kPhong2dFade; - } else if (material.reflective) { + } else if (material.reflectance > 0) { return ObjectManager::kPhong2dReflect; } else { return ObjectManager::kPhong2d; diff --git a/src/experimental/filament/filament/material.h b/src/experimental/filament/filament/material.h index a131633d..9bb0db64 100644 --- a/src/experimental/filament/filament/material.h +++ b/src/experimental/filament/filament/material.h @@ -15,6 +15,7 @@ #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATERIAL_H_ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATERIAL_H_ +#include #include #include #include "experimental/filament/filament/mesh.h" @@ -23,6 +24,11 @@ namespace mujoco { +// Generates a hash for the given material (type) and material parameters. +using MaterialKey = uint64_t; +MaterialKey BuildMaterialKey(ObjectManager::MaterialType type, + const mjrMaterial& material); + // Returns a MaterialType that best matches the given material data and mesh. ObjectManager::MaterialType GetMaterialType( const mjrMaterial& material, const Mesh* mesh); diff --git a/src/experimental/filament/filament/renderable.cc b/src/experimental/filament/filament/renderable.cc index f181b82c..04665f22 100644 --- a/src/experimental/filament/filament/renderable.cc +++ b/src/experimental/filament/filament/renderable.cc @@ -15,13 +15,18 @@ #include "experimental/filament/filament/renderable.h" #include +#include +#include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -29,10 +34,12 @@ #include #include #include +#include "experimental/filament/filament_util.h" #include "experimental/filament/filament/builtins.h" #include "experimental/filament/filament/material.h" #include "experimental/filament/filament/mesh.h" #include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/render_target.h" #include "experimental/filament/render_context_filament.h" namespace mujoco { @@ -57,7 +64,6 @@ Renderable::Renderable(filament::Engine* engine, Renderable::~Renderable() noexcept { filament::Engine* engine = GetEngine(); utils::EntityManager& em = utils::EntityManager::get(); - for (Part& part : parts_) { if (assigned_scene_) { assigned_scene_->remove(part.entity); @@ -65,11 +71,8 @@ Renderable::~Renderable() noexcept { engine->destroy(part.entity); em.destroy(part.entity); } - for (int i = 0; i < mjNUM_DRAW_MODES; ++i) { - if (instances_[i] != nullptr) { - engine->destroy(instances_[i]); - instances_[i] = nullptr; - } + for (auto& instance : instances_) { + engine->destroy(instance.second); } } @@ -134,9 +137,6 @@ void Renderable::InitPartEntity(Part& part) { } else { builder.culling(false); } - if (instances_[static_cast(draw_mode_)] != nullptr) { - builder.material(0, instances_[static_cast(draw_mode_)]); - } builder.castShadows(params_.cast_shadows); builder.receiveShadows(params_.receive_shadows); builder.layerMask(0xff, params_.layer_mask); @@ -168,7 +168,14 @@ void Renderable::UpdateTransform() { } filament::TransformManager& tm = GetEngine()->getTransformManager(); - if (get_transform_fn_) { + if (geom_type_ == mjGEOM_PLANE && (trs_.size[0] <= 0 || trs_.size[1] <= 0)) { + infinite_plane_ = true; + const mat4f transform = + filament::math::mat4f(trs_.rotation, trs_.translation); + for (Part& part : parts_) { + tm.setTransform(tm.getInstance(part.entity), transform); + } + } else if (get_transform_fn_) { for (int i = 0; i < parts_.size(); ++i) { const mat4f transform = get_transform_fn_(i, trs_); tm.setTransform(tm.getInstance(parts_[i].entity), transform); @@ -220,68 +227,171 @@ void Renderable::RemoveFromScene(filament::Scene* scene) { void Renderable::UpdateMaterial(const mjrMaterial& material) { material_ = material; - - const Mesh* mesh = !parts_.empty() ? parts_[0].mesh : nullptr; - - const ObjectManager::MaterialType type = GetMaterialType(material, mesh); - AssignMaterial(mjDRAW_MODE_COLOR, type); - if (!material_.decor_ux) { - AssignMaterial(mjDRAW_MODE_WIREFRAME, type); - AssignMaterial(mjDRAW_MODE_DEPTH, ObjectManager::kUnlitDepth); - AssignMaterial(mjDRAW_MODE_SEGMENTATION, ObjectManager::kUnlitSegmentation); - } - - for (int i = 0; i < mjNUM_DRAW_MODES; ++i) { - if (instances_[i]) { - UpdateMaterialInstance(instances_[i], material_, object_mgr_); - } - } -} - -void Renderable::AssignMaterial(mjrDrawMode mode, - ObjectManager::MaterialType material_type) { - const int index = static_cast(mode); - - filament::Material* material = object_mgr_->GetMaterial(material_type); - if (instances_[index]) { - if (instances_[index]->getMaterial() == material) { - // The correct material is already assigned, do nothing. - return; - } else { - GetEngine()->destroy(instances_[index]); - instances_[index] = nullptr; - } - } - if (material) { - instances_[index] = material->createInstance(); - if (geom_type_ == mjGEOM_PLANE || geom_type_ == mjGEOM_TRIANGLE) { - instances_[index]->setCullingMode( - filament::MaterialInstance::CullingMode::NONE); - } - } } const mjrMaterial& Renderable::GetMaterial() const { return material_; } -void Renderable::SetDrawMode(mjrDrawMode mode) { - // Only SceneObjects support non-color draw modes. - if (material_.decor_ux) { - mode = mjDRAW_MODE_COLOR; +void Renderable::Prepare(std::span requests) { + // We assume BindMaterialInstance will be called with the same requests in + // the same order. As such, we'll just store the draw state in a deque rather + // than trying to perform any kind of matching with the requests. + if (!draw_queue_.empty()) { + mju_error( + "Expected all previous draw calls to be bound before preparing new " + "draw calls."); + } + curr_state_ = DrawState(); + + int num_reflections = 0; + for (const mjrRenderRequest* request : requests) { + DrawState draw_state; + mjrMaterial material = material_; + + draw_state.wireframe = (request->draw_mode == mjDRAW_MODE_WIREFRAME); + if (material.decor_ux) { + draw_state.cast_shadows = false; + draw_state.receive_shadows = false; + } + + if (geom_type_ == mjGEOM_PLANE) { + const float3 camera_pos = ReadFloat3(request->camera.pos); + const float3 position = transform_[3].xyz; + const float3 forward = transform_[2].xyz; + const bool is_behind_camera = dot(camera_pos - position, forward) < 0; + if (is_behind_camera) { + material.color[3] *= 0.3; + draw_state.receive_shadows = false; + } + } + + const bool reflective = request->draw_mode == mjDRAW_MODE_COLOR && + request->enable_reflections && + material.reflectance > 0.0; + if (reflective) { + // Allocate a render target as needed and assign it to the material's + // reflection texture. + if (reflect_targets_.size() == num_reflections) { + mjrRenderTargetConfig config; + mjr_defaultRenderTargetConfig(&config); + config.color_format = mjPIXEL_FORMAT_RGBA8; + config.depth_format = mjPIXEL_FORMAT_DEPTH32F; + reflect_targets_.push_back( + std::make_unique(GetEngine(), config)); + } + RenderTarget* target = reflect_targets_[num_reflections].get(); + target->Prepare(request->viewport.width, request->viewport.height); + material.reflection_texture = target->GetColorTexture(); + draw_state.reflection_idx = num_reflections; + ++num_reflections; + } + + draw_state.material_key = + PrepareMaterialInstance(material, request->draw_mode); + draw_queue_.push_back(draw_state); + } +} + +MaterialKey Renderable::PrepareMaterialInstance(const mjrMaterial& material, + mjrDrawMode draw_mode) { + ObjectManager::MaterialType type; + if (draw_mode == mjDRAW_MODE_COLOR || draw_mode == mjDRAW_MODE_WIREFRAME) { + const Mesh* mesh = !parts_.empty() ? parts_[0].mesh : nullptr; + type = GetMaterialType(material, mesh); + } else if (draw_mode == mjDRAW_MODE_DEPTH) { + type = ObjectManager::kUnlitDepth; + } else if (draw_mode == mjDRAW_MODE_SEGMENTATION) { + type = ObjectManager::kUnlitSegmentation; + } else { + mju_error("Invalid draw mode: %d", draw_mode); + return 0; } - SetWireframe(mode == mjDRAW_MODE_WIREFRAME); + const MaterialKey key = BuildMaterialKey(type, material); - filament::MaterialInstance* instance = instances_[static_cast(mode)]; - if (instance) { + auto it = instances_.find(key); + if (it == instances_.end()) { + filament::MaterialInstance* instance = + object_mgr_->GetMaterial(type)->createInstance(); + UpdateMaterialInstance(instance, material, object_mgr_); + if (geom_type_ == mjGEOM_PLANE || geom_type_ == mjGEOM_TRIANGLE) { + instance->setCullingMode(filament::MaterialInstance::CullingMode::NONE); + } + instances_[key] = instance; + } + return key; +} + +RenderTarget* Renderable::GetReflectionTarget() const { + if (curr_state_.reflection_idx < 0) { + return nullptr; + } else { + return reflect_targets_[curr_state_.reflection_idx].get(); + } +} + +void Renderable::BindMaterialInstance(const mjrRenderRequest& request) { + if (draw_queue_.empty()) { + mju_error("No material instances to bind."); + } + + if (geom_type_ == mjGEOM_PLANE && infinite_plane_) { + // Emulate an infinite plane by recentering a large quad in world space + // relative to the camera. We use the shared mjMAXPLANEGRID value as the + // size of the quad to ensure the texture scaling matches. + static constexpr float kInfiniteScale = 0.5f * mjMAXPLANEGRID; + const mat4f scaling = + mat4f::scaling(float3{kInfiniteScale, kInfiniteScale, 1.0f}); + + const float3 camera_pos = ReadFloat3(request.camera.pos); + const float3 plane_origin = transform_[3].xyz; + const mat3f plane_rotation = transform_.upperLeft(); + + const float3 vec = camera_pos - plane_origin; + const float3 plane_x = normalize(plane_rotation[0]); + const float3 plane_y = normalize(plane_rotation[1]); + + // Project camera position onto the plane's local XY axes. + float dx = dot(vec, plane_x); + float dy = dot(vec, plane_y); + + // Quantize based on uv_scale. + const float tile_size[] = {kInfiniteScale / material_.uv_scale[0], + kInfiniteScale / material_.uv_scale[1]}; + dx = tile_size[0] * mju_round(dx / tile_size[0]); + dy = tile_size[1] * mju_round(dy / tile_size[1]); + + // Calculate the new center quad as a displacement from the plane origin. + const float3 displacement = dx * plane_x + dy * plane_y; + const float3 center = plane_origin + displacement; + + const mat4f transform = mat4f(plane_rotation, center) * scaling; + filament::TransformManager& tm = GetEngine()->getTransformManager(); + for (Part& part : parts_) { + tm.setTransform(tm.getInstance(part.entity), transform); + } + } + + const DrawState& state = draw_queue_.front(); + SetCastShadows(state.cast_shadows); + SetReceiveShadows(state.receive_shadows); + SetWireframe(state.wireframe); + SetMaterialInstance(state.material_key); + curr_state_ = state; + draw_queue_.pop_front(); +} + +void Renderable::SetMaterialInstance(MaterialKey key) { + if (key != curr_state_.material_key) { + filament::MaterialInstance* instance = instances_[key]; filament::RenderableManager& rm = GetEngine()->getRenderableManager(); for (Part& part : parts_) { filament::RenderableManager::Instance ri = rm.getInstance(part.entity); rm.setMaterialInstanceAt(ri, 0, instance); } + curr_state_.material_key = key; } - draw_mode_ = mode; } std::uint8_t Renderable::SetLayerMask(std::uint8_t mask) { diff --git a/src/experimental/filament/filament/renderable.h b/src/experimental/filament/filament/renderable.h index ab0cd3b2..d1366cf8 100644 --- a/src/experimental/filament/filament/renderable.h +++ b/src/experimental/filament/filament/renderable.h @@ -16,7 +16,11 @@ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLE_H_ #include +#include #include +#include +#include +#include #include #include @@ -26,8 +30,10 @@ #include #include #include +#include "experimental/filament/filament/material.h" #include "experimental/filament/filament/mesh.h" #include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/render_target.h" #include "experimental/filament/render_context_filament.h" namespace mujoco { @@ -94,15 +100,23 @@ class Renderable : public mjrRenderable { // Removes this renderable from the filament Scene. void RemoveFromScene(filament::Scene* scene); - // Determines how this renderable will be drawn. See mjrDrawMode for details. - void SetDrawMode(mjrDrawMode mode); - // Updates the parameters and textures of the material for this renderable. void UpdateMaterial(const mjrMaterial& material); // Returns this renderable's current material. const mjrMaterial& GetMaterial() const; + // Prepares the material instances for the given render requests. + void Prepare(std::span requests); + + // Binds the material instance for the given render request. + void BindMaterialInstance(const mjrRenderRequest& request); + + // Returns the render target used for reflections, if any. The main render + // function will use this target to render a reflection for this renderable + // based on the current camera position. + RenderTarget* GetReflectionTarget() const; + static Renderable* downcast(mjrRenderable* renderable) { return static_cast(renderable); } @@ -132,6 +146,21 @@ class Renderable : public mjrRenderable { } }; + // For each render request in a batch, we may need to render the object + // differently (e.g. a different reflection texture for each camera). However, + // we cannot change the material instance during the actual frame rendering + // (i.e. between beginFrame/endFrame). Instead, we can switch out the material + // instance entirely for each request. We keep track of which instance, as + // well as other render state, to use with each render request. + struct DrawState { + MaterialKey material_key; + // Index to the reflection target to use, if any. + int reflection_idx = -1; + bool cast_shadows = true; + bool receive_shadows = true; + bool wireframe = false; + }; + // When composing a multi-part renderable, each Entity will have its own // transform offset based on the transform of the Renderable itself. using GetTransformFn = std::function; @@ -140,22 +169,26 @@ class Renderable : public mjrRenderable { void InitPartEntity(Part& part); void UpdateTransform(); - void AssignMaterial(mjrDrawMode mode, - ObjectManager::MaterialType material_type); + MaterialKey PrepareMaterialInstance(const mjrMaterial& material, + mjrDrawMode draw_mode); + void SetMaterialInstance(MaterialKey key); filament::Engine* GetEngine(); ObjectManager* object_mgr_; mjrRenderableParams params_; - filament::MaterialInstance* instances_[mjNUM_DRAW_MODES] = {nullptr}; mjtGeom geom_type_ = mjGEOM_NONE; mjrMaterial material_; - mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR; + std::unordered_map instances_; + std::vector> reflect_targets_; + std::deque draw_queue_; + DrawState curr_state_; filament::Scene* assigned_scene_ = nullptr; std::vector parts_; filament::math::mat4f transform_; GetTransformFn get_transform_fn_; Trs trs_; + bool infinite_plane_ = false; bool wireframe_ = false; }; diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index c794a298..f76f7e55 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -14,11 +14,9 @@ #include "experimental/filament/filament/scene_view.h" -#include -#include -#include #include #include +#include #include #include @@ -145,6 +143,7 @@ SceneView::SceneView(filament::Engine* engine, const mjrSceneParams& params) reflect_view_->setPostProcessingEnabled(false); reflect_view_->setFrontFaceWindingInverted(true); reflect_view_->setVisibleLayers(0xff, params.reflection_layer_mask); + reflect_view_->setMultiSampleAntiAliasingOptions({.enabled = false}); // Rotate the fog to align with mujoco's +Z up space. auto fog = main_view_->getFogEntity(); @@ -167,7 +166,6 @@ SceneView::~SceneView() { } lights_.clear(); renderables_.clear(); - reflect_targets_.clear(); engine_->destroyCameraComponent(reflect_camera_->getEntity()); engine_->destroy(reflect_view_); engine_->destroyCameraComponent(camera_->getEntity()); @@ -194,18 +192,11 @@ void SceneView::RemoveFromScene(Light* light) { void SceneView::AddToScene(Renderable* renderable) { if (renderables_.insert(renderable).second) { renderable->AddToScene(scene_); - if (renderable->GetMaterial().reflective) { - AddReflectiveRenderable(renderable); - } } } void SceneView::RemoveFromScene(Renderable* renderable) { if (renderables_.erase(renderable)) { - auto it = std::find(reflectives_.begin(), reflectives_.end(), renderable); - if (it != reflectives_.end()) { - reflectives_.erase(it); - } renderable->RemoveFromScene(scene_); } } @@ -224,34 +215,30 @@ void SceneView::SetSkybox(const Texture* skybox_texture) { } } +void SceneView::PrepareToRender(std::span requests) { + for (const mjrRenderRequest* request : requests) { + if (request->scene != this) { + mju_error("Invalid scene for SceneView::PrepareToRender."); + } + } + for (Renderable* renderable : renderables_) { + renderable->Prepare(requests); + } +} + void SceneView::Render(filament::Renderer* renderer, const mjrRenderRequest& request) { if (request.scene != this) { mju_error("Invalid scene for SceneView::Render."); } - if (request.enable_reflections) { - EnableReflections(); - } else { - DisableReflections(); - } - filament::Viewport viewport(request.viewport.left, request.viewport.bottom, request.viewport.width, request.viewport.height); - main_view_->setViewport(viewport); - depth_segment_view_->setViewport(viewport); - reflect_view_->setViewport(viewport); - - SetupCamera(request.camera, viewport, camera_); - - for (auto& iter : renderables_) { - iter->SetDrawMode(request.draw_mode); - } - filament::View* view = main_view_; if (request.draw_mode == mjDRAW_MODE_DEPTH || request.draw_mode == mjDRAW_MODE_SEGMENTATION) { view = depth_segment_view_; } + view->setViewport(viewport); view->setShadowingEnabled(request.enable_shadows); view->setPostProcessingEnabled(request.enable_post_processing); @@ -264,25 +251,30 @@ void SceneView::Render(filament::Renderer* renderer, const mjrRenderRequest& req view->setMultiSampleAntiAliasingOptions({.enabled = false}); } - // Render reflection passes. - if (request.draw_mode == mjDRAW_MODE_COLOR && reflections_enabled_) { - for (size_t i = 0; i < reflectives_.size(); ++i) { - Renderable* renderable = reflectives_[i]; + SetupCamera(request.camera, viewport, camera_); + + for (auto& iter : renderables_) { + iter->BindMaterialInstance(request); + + if (RenderTarget* target = iter->GetReflectionTarget()) { + viewport.left = 0; + viewport.bottom = 0; + reflect_view_->setViewport(viewport); // We assume the 0th entity is the reflective entity. - mat4 transform(renderable->GetTransform()); + mat4 transform(iter->GetTransform()); SetupReflectionCamera(transform, camera_, reflect_camera_); // Hide reflective surface from its own reflection pass. - std::uint8_t previous_layer_mask = renderable->SetLayerMask(0x00); + std::uint8_t previous_layer_mask = iter->SetLayerMask(0x00); // Render the reflection to its render target. - reflect_view_->setRenderTarget( - reflect_targets_[i]->GetFilamentRenderTarget()); + reflect_view_->setRenderTarget(target->GetFilamentRenderTarget()); renderer->render(reflect_view_); + reflect_view_->setRenderTarget(nullptr); // Unhide the reflective surface. - renderable->SetLayerMask(previous_layer_mask); + iter->SetLayerMask(previous_layer_mask); } } @@ -296,33 +288,6 @@ void SceneView::Render(filament::Renderer* renderer, const mjrRenderRequest& req } } -void SceneView::AddReflectiveRenderable(Renderable* renderable) { - const int index = reflectives_.size(); - reflectives_.push_back(renderable); - - // Ensure we have the same number of render targets as we do reflective - // renderables. - while (reflect_targets_.size() < reflectives_.size()) { - mjrRenderTargetConfig config; - mjr_defaultRenderTargetConfig(&config); - - config.color_format = mjPIXEL_FORMAT_RGBA8; - config.depth_format = mjPIXEL_FORMAT_DEPTH32F; - reflect_targets_.push_back(std::make_unique(engine_, config)); - } - - // Prepare a render target for the reflective renderable. - auto viewport = reflect_view_->getViewport(); - auto& target = reflect_targets_[index]; - target->Prepare(viewport.width, viewport.height); - - if (reflections_enabled_) { - mjrMaterial material = renderable->GetMaterial(); - material.reflection_texture = target->GetColorTexture(); - renderable->UpdateMaterial(material); - } -} - void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { auto tone_mapper = CreateToneMapper(opts.tone_mapper); auto color_grading = ToBuilder(color_grading_options_) @@ -336,33 +301,6 @@ void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { color_grading_options_ = opts; } -void SceneView::EnableReflections() { - if (reflections_enabled_) { - return; - } - - reflections_enabled_ = true; - for (int i = 0; i < reflectives_.size(); ++i) { - Renderable* renderable = reflectives_[i]; - mjrMaterial material = renderable->GetMaterial(); - material.reflection_texture = reflect_targets_[i]->GetColorTexture(); - renderable->UpdateMaterial(material); - } -} - -void SceneView::DisableReflections() { - if (!reflections_enabled_) { - return; - } - - reflections_enabled_ = false; - for (Renderable* renderable : reflectives_) { - mjrMaterial material = renderable->GetMaterial(); - material.reflection_texture = nullptr; - renderable->UpdateMaterial(material); - } -} - filament::View* SceneView::GetDefaultRenderView() { return main_view_; } diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index 9237ccdb..60c585e2 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -15,10 +15,8 @@ #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_ -#include -#include +#include #include -#include #include #include @@ -29,7 +27,6 @@ #include "experimental/filament/filament/color_grading_options.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/renderable.h" -#include "experimental/filament/filament/render_target.h" #include "experimental/filament/filament/texture.h" #include "experimental/filament/render_context_filament.h" @@ -55,6 +52,12 @@ class SceneView : public mjrScene { void RemoveFromScene(Renderable* renderable); void SetSkybox(const Texture* skybox_texture); + // Performs necessary preparations in order to render the given requests. + // Assumes that the Render() function will be called the same number of times + // and in the same order with the given requests. + void PrepareToRender(std::span requests); + + // Fulfills the given render request using the renderer. void Render(filament::Renderer* renderer, const mjrRenderRequest& request); // Returns the filament Engine managing the scene. @@ -80,13 +83,6 @@ class SceneView : public mjrScene { } private: - void EnableReflections(); - void DisableReflections(); - - // Marks a renderable as reflective. Reflective renderables have to be - // rendered in their own passes to create the reflective texture. - void AddReflectiveRenderable(Renderable* renderable); - filament::Engine* engine_ = nullptr; filament::Scene* scene_ = nullptr; filament::Camera* camera_ = nullptr; @@ -103,11 +99,6 @@ class SceneView : public mjrScene { // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr; filament::Camera* reflect_camera_ = nullptr; - - // The list of reflective renderables and their corresponding render targets. - bool reflections_enabled_ = true; - std::vector reflectives_; - std::vector> reflect_targets_; }; } // namespace mujoco diff --git a/src/experimental/filament/filament_util.h b/src/experimental/filament/filament_util.h index c8398522..7cc9ee33 100644 --- a/src/experimental/filament/filament_util.h +++ b/src/experimental/filament/filament_util.h @@ -52,9 +52,9 @@ template inline filament::math::mat3f ReadMat3(const T* arr, int index = 0) { // clang-format off const T* ptr = arr + (9 * index); - return filament::math::mat3f(ptr[0], ptr[3], ptr[6], - ptr[1], ptr[4], ptr[7], - ptr[2], ptr[5], ptr[8]); + return filament::math::mat3f(ptr[0], ptr[1], ptr[2], + ptr[3], ptr[4], ptr[5], + ptr[6], ptr[7], ptr[8]); // clang-format on } diff --git a/src/experimental/filament/render_context_filament.h b/src/experimental/filament/render_context_filament.h index fcc587c9..98204f75 100644 --- a/src/experimental/filament/render_context_filament.h +++ b/src/experimental/filament/render_context_filament.h @@ -139,8 +139,6 @@ typedef enum mjrDrawMode_ { mjDRAW_MODE_SEGMENTATION, } mjrDrawMode; -enum { mjNUM_DRAW_MODES = 4 }; // Number of modes in `mjrDrawMode`. - // Parameters describing the camera to use for rendering an image. typedef mjvGLCamera mjrCamera; @@ -614,8 +612,6 @@ struct mjrMaterial { // The emissive (glow) factor of the object. float emissive; - // Whether or not the object is a reflective surface. Only applies to planes. - mjtByte reflective; // The blend factor to use for reflective surfaces. A value of 1.0 means that // the surface is fully reflective (i.e. a mirror). float reflectance;