From 62cffb153637ee5037f127b2dde8317dc8580ab1 Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Wed, 15 Apr 2026 06:55:33 -0700 Subject: [PATCH] Update GuiView to use SceneView and Renderables. PiperOrigin-RevId: 900145167 Change-Id: I613c9fe171e97f8a4cd08c0c63e9706e09cbd202 --- src/experimental/filament/assets/unlit_ui.mat | 4 +- .../filament/filament/filament_context.cc | 18 +- .../filament/filament/gui_view.cc | 190 ++++++------------ src/experimental/filament/filament/gui_view.h | 49 ++--- .../filament/filament/material.cc | 5 + src/experimental/filament/filament/material.h | 1 + .../filament/filament/scene_view.cc | 37 ++++ .../filament/filament/scene_view.h | 17 +- 8 files changed, 147 insertions(+), 174 deletions(-) diff --git a/src/experimental/filament/assets/unlit_ui.mat b/src/experimental/filament/assets/unlit_ui.mat index 0148f28e..4659b002 100644 --- a/src/experimental/filament/assets/unlit_ui.mat +++ b/src/experimental/filament/assets/unlit_ui.mat @@ -17,7 +17,7 @@ material { parameters : [ { type : sampler2d, - name : glyph + name : BaseColor } ], requires : [ @@ -35,7 +35,7 @@ fragment { prepareMaterial(material); vec2 uv = getUV0(); uv.y = 1.0 - uv.y; - vec4 tex_color = texture(materialParams_glyph, uv); + vec4 tex_color = texture(materialParams_BaseColor, uv); material.baseColor = getColor() * tex_color; material.baseColor.rgb *= material.baseColor.a; } diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc index c01a95b2..a7824f4f 100644 --- a/src/experimental/filament/filament/filament_context.cc +++ b/src/experimental/filament/filament/filament_context.cc @@ -97,7 +97,7 @@ void FilamentContext::Init(const mjModel* model) { scene_bridge_ = std::make_unique(object_manager_.get(), model, scene_view_.get()); gui_view_ = std::make_unique( - engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi)); + scene_view_.get(), object_manager_->GetMaterial(ObjectManager::kUnlitUi)); // Set clear options. filament::Renderer::ClearOptions opts; @@ -131,7 +131,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { // Prepare the filament Renderable that contains the GUI draw commands. We // must call this function even if we do not plan on rendering the GUI to // ensure the ImGui state is updated. - gui_view_->UpdateRenderable(); + gui_view_->Update(); } last_render_mode_ = SceneView::DrawMode::kNormal; @@ -154,12 +154,9 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { request.draw_mode = last_render_mode_; request.viewport = viewport; request.camera = last_camera_; + request.enable_ux = (gui_swap_chain_target_ == kWindowSwapChain); + request.gui_scale = gui_view_ ? gui_view_->GetScale() : 1.0f; scene_view_->Render(renderer_, request); - - if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) { - gui_view_->Render(renderer_); - } - renderer_->endFrame(); } @@ -231,13 +228,10 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, request.viewport = viewport; request.target = color_target_.get(); request.camera = last_camera_; + request.enable_ux = (gui_swap_chain_target_ == kOffscreenSwapChain); + request.gui_scale = gui_view_ ? gui_view_->GetScale() : 1.0f; scene_view_->Render(renderer_, request); - // Render the GUI to the texture as well if requested. - if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) { - gui_view_->Render(renderer_, color_target_.get()); - } - const size_t num_bytes = viewport.width * viewport.height * 3; color_target_->ReadColorPixels(renderer_, rgb, num_bytes); diff --git a/src/experimental/filament/filament/gui_view.cc b/src/experimental/filament/filament/gui_view.cc index a6572366..5802b06d 100644 --- a/src/experimental/filament/filament/gui_view.cc +++ b/src/experimental/filament/filament/gui_view.cc @@ -18,70 +18,26 @@ #include #include #include -#include #include #include -#include -#include -#include -#include -#include +#include #include -#include -#include #include +#include "experimental/filament/filament/material.h" #include "experimental/filament/filament/mesh.h" -#include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/renderable.h" +#include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" namespace mujoco { -using filament::math::float4; - -static constexpr auto kTriangles = - filament::RenderableManager::PrimitiveType::TRIANGLES; - -GuiView::GuiView(filament::Engine* engine, filament::Material* ui_material) - : engine_(engine), material_(ui_material) { - auto& em = utils::EntityManager::get(); - scene_ = engine_->createScene(); - camera_ = engine_->createCamera(em.create()); - view_ = engine_->createView(); - renderable_ = em.create(); - view_->setScene(scene_); - view_->setCamera(camera_); - view_->setPostProcessingEnabled(false); +GuiView::GuiView(SceneView* scene_view, filament::Material* ui_material) + : scene_view_(scene_view), material_(ui_material) { } GuiView::~GuiView() { - if (num_elements_ > 0) { - scene_->remove(renderable_); - auto& rm = engine_->getRenderableManager(); - rm.destroy(renderable_); - } - auto& em = utils::EntityManager::get(); - em.destroy(renderable_); - meshes_.clear(); - for (auto& instance : instances_) { - engine_->destroy(instance); - } - textures_.clear(); - engine_->destroyCameraComponent(camera_->getEntity()); - engine_->destroy(view_); - engine_->destroy(scene_); -} - -void GuiView::ResetRenderable() { - auto& em = utils::EntityManager::get(); - if (!renderable_.isNull()) { - scene_->remove(renderable_); - auto& rm = engine_->getRenderableManager(); - rm.destroy(renderable_); - em.destroy(renderable_); - renderable_ = utils::Entity(); - } - meshes_.clear(); + PrepareRenderables(0); } uintptr_t GuiView::UploadImage(uintptr_t tex_id, const uint8_t* pixels, @@ -116,7 +72,7 @@ uintptr_t GuiView::UploadImage(uintptr_t tex_id, const uint8_t* pixels, config.target = mjTEXTURE_2D; config.format = bpp == 4 ? mjPIXEL_FORMAT_RGBA8 : mjPIXEL_FORMAT_RGB8; config.color_space = mjCOLORSPACE_LINEAR; - texture = std::make_unique(engine_, config); + texture = std::make_unique(scene_view_->GetEngine(), config); } // Create a copy of the image to pass it to filament as we don't know the @@ -153,7 +109,8 @@ void GuiView::CreateTexture(ImTextureData* data) { config.color_space = mjCOLORSPACE_LINEAR; const uintptr_t tex_id = textures_.size() + 1; - textures_[tex_id] = std::make_unique(engine_, config); + textures_[tex_id] = + std::make_unique(scene_view_->GetEngine(), config); data->SetTexID((ImTextureID)tex_id); UpdateTexture(data); } @@ -183,21 +140,22 @@ void GuiView::DestroyTexture(ImTextureData* data) { } } -void GuiView::UpdateRenderable() { +void GuiView::Update() { if (!ImGui::GetCurrentContext()) { + PrepareRenderables(0); return; } // Prepare the imgui draw commands. We must call this function even if we do // not plan on rendering anything to ensure imgui state is updated. ImGui::Render(); - auto& rm = engine_->getRenderableManager(); ImGuiIO& io = ImGui::GetIO(); const ImVec2& size = io.DisplaySize; const ImVec2& scale = io.DisplayFramebufferScale; ImDrawData* commands = ImGui::GetDrawData(); - if (!commands) { + if (!commands || size.x == 0 || size.y == 0) { + PrepareRenderables(0); return; } commands->ScaleClipRects(scale); @@ -243,39 +201,13 @@ void GuiView::UpdateRenderable() { } } - if (size.x == 0 || size.y == 0 || num_elements == 0) { - if (num_elements_ > 0) { - scene_->remove(renderable_); - rm.destroy(renderable_); - } - num_elements_ = 0; + PrepareRenderables(num_elements); + if (num_elements == 0) { return; } - view_->setViewport( - filament::Viewport(0.f, 0.f, size.x * scale.x, size.y * scale.y)); - camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0, size.x, - size.y, 0.0, 0.0, 1.0); - - if (num_elements != num_elements_) { - if (num_elements_ > 0) { - scene_->remove(renderable_); - rm.destroy(renderable_); - } - - num_elements_ = num_elements; - - filament::RenderableManager::Builder builder(num_elements_); - builder.boundingBox({{-100, -100, -100}, {100, 100, 100}}); - builder.culling(false); - builder.build(*engine_, renderable_); - scene_->addEntity(renderable_); - } meshes_.clear(); - - auto ri = rm.getInstance(renderable_); - - int drawable_index = 0; + int renderable_index = 0; for (int n = 0; n < commands->CmdListsCount; ++n) { const ImDrawList* cmds = commands->CmdLists[n]; @@ -297,70 +229,70 @@ void GuiView::UpdateRenderable() { data.indices = cmds->IdxBuffer.Data; data.index_type = mjINDEX_TYPE_USHORT; data.primitive_type = mjPRIM_TYPE_TRIANGLES; - meshes_.push_back(std::make_unique(engine_, data)); - const auto& mesh = meshes_.back(); + meshes_.push_back(std::make_unique(scene_view_->GetEngine(), data)); + + const Mesh* mesh = meshes_.back().get(); int index_offset = 0; for (const ImDrawCmd& command : cmds->CmdBuffer) { const int width = size.x * scale.x; const int height = size.y * scale.y; - int clip_left = command.ClipRect.x; - int clip_bottom = height - command.ClipRect.w; - int clip_width = command.ClipRect.z - command.ClipRect.x; - int clip_height = command.ClipRect.w - command.ClipRect.y; + auto& renderable = renderables_[renderable_index]; + if (renderable->GetNumMeshes() == 0) { + renderable->AppendMesh(mesh, index_offset, command.ElemCount); + } else { + renderable->UpdateMesh(0, mesh, index_offset, command.ElemCount); + } + + Material::Textures textures; + textures.color = textures_[command.GetTexID()].get(); + renderable->GetMaterial().UpdateTextures(textures); + + Material::Params properties; + properties.scissor[0] = command.ClipRect.x; + properties.scissor[1] = height - command.ClipRect.w; + properties.scissor[2] = command.ClipRect.z - command.ClipRect.x; + properties.scissor[3] = command.ClipRect.w - command.ClipRect.y; // Modal dialogs try to cover the whole window, but also a little outside // of it. This doesn't work well with filament's scissor test, so we clip // them to the window. - if (clip_left < 0 || clip_bottom < 0) { - clip_left = 0; - clip_bottom = 0; - clip_width = width; - clip_height = height; + if (properties.scissor[0] < 0 || properties.scissor[1] < 0) { + properties.scissor[0] = 0; + properties.scissor[1] = 0; + properties.scissor[2] = width; + properties.scissor[3] = height; } - - mjrRect clip_rect{clip_left, clip_bottom, clip_width, clip_height}; - rm.setMaterialInstanceAt( - ri, drawable_index, - GetMaterialInstance(drawable_index, clip_rect, command.GetTexID())); - rm.setGeometryAt( - ri, drawable_index, kTriangles, mesh->GetFilamentVertexBuffer(), - mesh->GetFilamentIndexBuffer(), index_offset, command.ElemCount); - rm.setBlendOrderAt(ri, drawable_index, drawable_index); + renderable->GetMaterial().UpdateParams(properties); index_offset += command.ElemCount; - ++drawable_index; + ++renderable_index; } } } -filament::MaterialInstance* GuiView::GetMaterialInstance(int index, - mjrRect rect, - uintptr_t texture_id) { - while (index >= instances_.size()) { - instances_.push_back(material_->createInstance()); - } +void GuiView::PrepareRenderables(int count) { + while (renderables_.size() < count) { + auto& r = renderables_.emplace_back( + std::make_unique(scene_view_->GetEngine())); + r->SetCastShadows(false); + r->SetReceiveShadows(false); + r->SetBlendOrder(static_cast(renderables_.size())); - auto iter = textures_.find(texture_id); - if (iter == textures_.end()) { - mju_error("Texture not found: %lu", texture_id); + Material& material = r->GetMaterial(); + Material::DrawMode mode = Material::DrawMode::kNormal; + material.SetMaterial(mode, material_); + r->SetMaterialInstance(material.GetMaterialInstance(mode)); + scene_view_->AddToUxScene(r.get()); + } + while (renderables_.size() > count) { + scene_view_->RemoveFromUxScene(renderables_.back().get()); + renderables_.pop_back(); } - - filament::MaterialInstance* instance = instances_[index]; - instance->setParameter("glyph", iter->second->GetFilamentTexture(), - filament::TextureSampler()); - instance->setScissor(rect.left, rect.bottom, rect.width, rect.height); - return instance; } -void GuiView::Render(filament::Renderer* renderer, RenderTarget* target) { - if (num_elements_ == 0) { - return; - } - - view_->setRenderTarget(target ? target->GetFilamentRenderTarget() : nullptr); - renderer->render(view_); - view_->setRenderTarget(nullptr); +float GuiView::GetScale() const { + return ImGui::GetIO().DisplayFramebufferScale.x; } static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) { diff --git a/src/experimental/filament/filament/gui_view.h b/src/experimental/filament/filament/gui_view.h index 0346ca57..25dd8530 100644 --- a/src/experimental/filament/filament/gui_view.h +++ b/src/experimental/filament/filament/gui_view.h @@ -21,60 +21,49 @@ #include #include -#include -#include #include -#include -#include -#include -#include -#include #include "experimental/filament/filament/mesh.h" -#include "experimental/filament/filament/render_target.h" +#include "experimental/filament/filament/renderable.h" +#include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" namespace mujoco { -// A filament::View that contains a filament::Scene used for rendering the GUI. +// Manages Renderables that will be added a SceneView's UX scene. class GuiView { public: - GuiView(filament::Engine* engine, filament::Material* ui_material); + GuiView(SceneView* scene_view, filament::Material* ui_material); ~GuiView(); - // Prepares the UX scene renderable using data from the current ImGui state. - // This function must be called once per frame to ensure ImGui state is - // correctly synced. - void UpdateRenderable(); + // Prepares the Renderables using data from the current ImGui state. This + // function must be called once per frame to ensure ImGui state is correctly + // synced. + void Update(); - void Render(filament::Renderer* renderer, RenderTarget* target = nullptr); + // Returns the current ImGui scale factor. + float GetScale() const; // Uploads texture to be used with ImGui's Image and ImageButton functions. uintptr_t UploadImage(uintptr_t tex_id, const uint8_t* pixels, int width, int height, int bpp); + GuiView(const GuiView&) = delete; + GuiView& operator=(const GuiView&) = delete; + private: + // Ensures exactly `count` Renderables exist, creating or destroying them as + // needed. + void PrepareRenderables(int count); + void CreateTexture(ImTextureData* data); void UpdateTexture(ImTextureData* data); void DestroyTexture(ImTextureData* data); - // Returns the filament::MaterialInstance configured to draw into the given - // scissor rect. - filament::MaterialInstance* GetMaterialInstance(int index, mjrRect rect, - uintptr_t texture_id); - - // Clears the filament::Scene of the UX renderable and releases all buffers. - void ResetRenderable(); - - filament::Engine* engine_ = nullptr; - filament::Scene* scene_ = nullptr; - filament::Camera* camera_ = nullptr; - filament::View* view_ = nullptr; + SceneView* scene_view_ = nullptr; filament::Material* material_ = nullptr; - utils::Entity renderable_; + std::vector> renderables_; std::vector meshes_; - std::vector instances_; std::unordered_map> textures_; - int num_elements_ = 0; }; // Draws text at the given screen coordinates in clip space (i.e. [-1,-1,-1] to diff --git a/src/experimental/filament/filament/material.cc b/src/experimental/filament/filament/material.cc index 10538d28..9dc7863c 100644 --- a/src/experimental/filament/filament/material.cc +++ b/src/experimental/filament/filament/material.cc @@ -72,6 +72,11 @@ void Material::UpdateMaterialInstances() { return; } + if (params_.scissor[2] != 0 && params_.scissor[3] != 0) { + instance->setScissor(params_.scissor[0], params_.scissor[1], + params_.scissor[2], params_.scissor[3]); + } + const filament::Material* material = instance->getMaterial(); if (material->hasParameter("BaseColorFactor")) { instance->setParameter("BaseColorFactor", filament::RgbaType::sRGB, diff --git a/src/experimental/filament/filament/material.h b/src/experimental/filament/filament/material.h index 72024aa9..eaba6103 100644 --- a/src/experimental/filament/filament/material.h +++ b/src/experimental/filament/filament/material.h @@ -55,6 +55,7 @@ class Material { filament::math::float2 tex_repeat = {1, 1}; filament::math::float3 uv_scale = {1, 1, 1}; filament::math::float3 uv_offset = {0, 0, 0}; + filament::math::float4 scissor = {0, 0, 0, 0}; float specular = -1.0f; float glossiness = -1.0f; float metallic = -1.0f; diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index 5c8f38b1..41f3d3ad 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -129,7 +129,9 @@ static void SetupReflectionCamera(const mat4& surface_xform, SceneView::SceneView(filament::Engine* engine) : engine_(engine) { scene_ = engine->createScene(); + ux_scene_ = engine->createScene(); camera_ = engine->createCamera(utils::EntityManager::get().create()); + ux_camera_ = engine->createCamera(utils::EntityManager::get().create()); reflect_camera_ = engine->createCamera(utils::EntityManager::get().create()); for (auto& view : views_) { @@ -139,6 +141,12 @@ SceneView::SceneView(filament::Engine* engine) : engine_(engine) { view->setVisibleLayers(0xff, mjCAT_ALL); } + ux_view_ = engine->createView(); + ux_view_->setScene(ux_scene_); + ux_view_->setCamera(ux_camera_); + ux_view_->setPostProcessingEnabled(false); + ux_view_->setShadowingEnabled(false); + reflect_view_ = engine->createView(); reflect_view_->setScene(scene_); reflect_view_->setCamera(reflect_camera_); @@ -166,16 +174,22 @@ SceneView::~SceneView() { for (auto& renderable : renderables_) { renderable->RemoveFromScene(scene_); } + for (auto& renderable : ux_renderables_) { + renderable->RemoveFromScene(ux_scene_); + } lights_.clear(); renderables_.clear(); reflect_targets_.clear(); engine_->destroyCameraComponent(reflect_camera_->getEntity()); engine_->destroy(reflect_view_); + engine_->destroyCameraComponent(ux_camera_->getEntity()); + engine_->destroy(ux_view_); engine_->destroyCameraComponent(camera_->getEntity()); if (color_grading_) { engine_->destroy(color_grading_); } engine_->destroy(scene_); + engine_->destroy(ux_scene_); for (auto& view : views_) { engine_->destroy(view); } @@ -212,6 +226,18 @@ void SceneView::RemoveFromScene(Renderable* renderable) { } } +void SceneView::AddToUxScene(Renderable* renderable) { + if (ux_renderables_.insert(renderable).second) { + renderable->AddToScene(ux_scene_); + } +} + +void SceneView::RemoveFromUxScene(Renderable* renderable) { + if (ux_renderables_.erase(renderable)) { + renderable->RemoveFromScene(ux_scene_); + } +} + void SceneView::AddToScene(filament::Skybox* skybox) { skybox_ = skybox; scene_->setSkybox(skybox); @@ -231,6 +257,7 @@ void SceneView::Render(filament::Renderer* renderer, for (auto& view : views_) { view->setViewport(viewport); } + ux_view_->setViewport(viewport); reflect_view_->setViewport(viewport); SetupCamera(request.camera, viewport, camera_); @@ -279,6 +306,16 @@ void SceneView::Render(filament::Renderer* renderer, renderer->render(view); view->setRenderTarget(nullptr); + if (request.enable_ux) { + ux_camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0f, + viewport.width / request.gui_scale, + viewport.height / request.gui_scale, 0.0f, 0.0f, + 1.0f); + ux_view_->setRenderTarget(render_target); + renderer->render(ux_view_); + ux_view_->setRenderTarget(nullptr); + } + if (request.target) { view->setMultiSampleAntiAliasingOptions(options); } diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index d7f02021..d6546d7c 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -38,7 +38,8 @@ namespace mujoco { // // The filament Scene is populated with the objects (e.g. lights, renderables, // skybox, etc.). It manages multiple views to support a variety of draw modes -// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. +// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. It +// also manages a separate scene and view for UX rendering. class SceneView { public: SceneView(filament::Engine* engine); @@ -52,6 +53,10 @@ class SceneView { void AddToScene(filament::Skybox* skybox); void RemoveFromScene(filament::Skybox* skybox); + // Adds/removes entities from the UX scene, which is rendered separately. + void AddToUxScene(Renderable* renderable); + void RemoveFromUxScene(Renderable* renderable); + // Parameters for rendering the scene. using DrawMode = Material::DrawMode; struct RenderRequest { @@ -63,6 +68,10 @@ class SceneView { mjvGLCamera camera; // An optional render target into which the scene will be rendered. RenderTarget* target = nullptr; + // Whether or not to render the UX as a separate pass. + bool enable_ux = false; + // The scale factor to use for UX rendering. + float gui_scale = 1.0f; }; // Renders the scene. @@ -89,6 +98,7 @@ class SceneView { filament::Engine* engine_ = nullptr; filament::Scene* scene_ = nullptr; + filament::Scene* ux_scene_ = nullptr; filament::Camera* camera_ = nullptr; filament::ColorGrading* color_grading_ = nullptr; ColorGradingOptions color_grading_options_; @@ -100,6 +110,11 @@ class SceneView { std::unordered_set renderables_; filament::Skybox* skybox_ = nullptr; + // Custom view for UX. + filament::View* ux_view_ = nullptr; + filament::Camera* ux_camera_ = nullptr; + std::unordered_set ux_renderables_; + // Custom view and camera for reflective surfaces. filament::View* reflect_view_ = nullptr; filament::Camera* reflect_camera_ = nullptr;