chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Update GuiView to use SceneView and Renderables.

Browse Source 
PiperOrigin-RevId: 900145167
Change-Id: I613c9fe171e97f8a4cd08c0c63e9706e09cbd202
This commit is contained in:
Haroon Qureshi
2026-04-15 06:55:33 -07:00
committed by Copybara-Service
parent 8f3ed662eb
commit 62cffb1536
8 changed files with 147 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/experimental/filament/assets/unlit_ui.mat
+2 -2
View File
@@ -17,7 +17,7 @@ material {
parameters : [
{
type : sampler2d,
name : glyph
name : BaseColor
}
],
requires : [
@@ -35,7 +35,7 @@ fragment {
prepareMaterial(material);
vec2 uv = getUV0();
uv.y = 1.0 - uv.y;
vec4 tex_color = texture(materialParams_glyph, uv);
vec4 tex_color = texture(materialParams_BaseColor, uv);
material.baseColor = getColor() * tex_color;
material.baseColor.rgb *= material.baseColor.a;
}
src/experimental/filament/filament/filament_context.cc
+6 -12
View File
@@ -97,7 +97,7 @@ void FilamentContext::Init(const mjModel* model) {
scene_bridge_ = std::make_unique<SceneBridge>(object_manager_.get(), model,
scene_view_.get());
gui_view_ = std::make_unique<GuiView>(
engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi));
scene_view_.get(), object_manager_->GetMaterial(ObjectManager::kUnlitUi));
// Set clear options.
filament::Renderer::ClearOptions opts;
@@ -131,7 +131,7 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) {
// Prepare the filament Renderable that contains the GUI draw commands. We
// must call this function even if we do not plan on rendering the GUI to
// ensure the ImGui state is updated.
gui_view_->UpdateRenderable();
gui_view_->Update();
}
last_render_mode_ = SceneView::DrawMode::kNormal;
@@ -154,12 +154,9 @@ void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) {
request.draw_mode = last_render_mode_;
request.viewport = viewport;
request.camera = last_camera_;
request.enable_ux = (gui_swap_chain_target_ == kWindowSwapChain);
request.gui_scale = gui_view_ ? gui_view_->GetScale() : 1.0f;
scene_view_->Render(renderer_, request);
if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) {
gui_view_->Render(renderer_);
}
renderer_->endFrame();
}
@@ -231,13 +228,10 @@ void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb,
request.viewport = viewport;
request.target = color_target_.get();
request.camera = last_camera_;
request.enable_ux = (gui_swap_chain_target_ == kOffscreenSwapChain);
request.gui_scale = gui_view_ ? gui_view_->GetScale() : 1.0f;
scene_view_->Render(renderer_, request);
// Render the GUI to the texture as well if requested.
if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) {
gui_view_->Render(renderer_, color_target_.get());
}
const size_t num_bytes = viewport.width * viewport.height * 3;
color_target_->ReadColorPixels(renderer_, rgb, num_bytes);
src/experimental/filament/filament/gui_view.cc
+61 -129
View File
@@ -18,70 +18,26 @@
#include <cstdint>
#include <cstring>
#include <memory>
#include <utility>
#include <vector>
#include <imgui.h>
#include <filament/Engine.h>
#include <filament/RenderableManager.h>
#include <filament/Renderer.h>
#include <filament/TextureSampler.h>
#include <filament/Viewport.h>
#include <filament/Material.h>
#include <math/vec4.h>
#include <utils/EntityManager.h>
#include <mujoco/mjrender.h>
#include <mujoco/mujoco.h>
#include "experimental/filament/filament/material.h"
#include "experimental/filament/filament/mesh.h"
#include "experimental/filament/filament/render_target.h"
#include "experimental/filament/filament/renderable.h"
#include "experimental/filament/filament/scene_view.h"
#include "experimental/filament/filament/texture.h"
namespace mujoco {
using filament::math::float4;
static constexpr auto kTriangles =
filament::RenderableManager::PrimitiveType::TRIANGLES;
GuiView::GuiView(filament::Engine* engine, filament::Material* ui_material)
: engine_(engine), material_(ui_material) {
auto& em = utils::EntityManager::get();
scene_ = engine_->createScene();
camera_ = engine_->createCamera(em.create());
view_ = engine_->createView();
renderable_ = em.create();
view_->setScene(scene_);
view_->setCamera(camera_);
view_->setPostProcessingEnabled(false);
GuiView::GuiView(SceneView* scene_view, filament::Material* ui_material)
: scene_view_(scene_view), material_(ui_material) {
}
GuiView::~GuiView() {
if (num_elements_ > 0) {
scene_->remove(renderable_);
auto& rm = engine_->getRenderableManager();
rm.destroy(renderable_);
}
auto& em = utils::EntityManager::get();
em.destroy(renderable_);
meshes_.clear();
for (auto& instance : instances_) {
engine_->destroy(instance);
}
textures_.clear();
engine_->destroyCameraComponent(camera_->getEntity());
engine_->destroy(view_);
engine_->destroy(scene_);
}
void GuiView::ResetRenderable() {
auto& em = utils::EntityManager::get();
if (!renderable_.isNull()) {
scene_->remove(renderable_);
auto& rm = engine_->getRenderableManager();
rm.destroy(renderable_);
em.destroy(renderable_);
renderable_ = utils::Entity();
}
meshes_.clear();
PrepareRenderables(0);
}
uintptr_t GuiView::UploadImage(uintptr_t tex_id, const uint8_t* pixels,
@@ -116,7 +72,7 @@ uintptr_t GuiView::UploadImage(uintptr_t tex_id, const uint8_t* pixels,
config.target = mjTEXTURE_2D;
config.format = bpp == 4 ? mjPIXEL_FORMAT_RGBA8 : mjPIXEL_FORMAT_RGB8;
config.color_space = mjCOLORSPACE_LINEAR;
texture = std::make_unique<Texture>(engine_, config);
texture = std::make_unique<Texture>(scene_view_->GetEngine(), config);
}
// Create a copy of the image to pass it to filament as we don't know the
@@ -153,7 +109,8 @@ void GuiView::CreateTexture(ImTextureData* data) {
config.color_space = mjCOLORSPACE_LINEAR;
const uintptr_t tex_id = textures_.size() + 1;
textures_[tex_id] = std::make_unique<Texture>(engine_, config);
textures_[tex_id] =
std::make_unique<Texture>(scene_view_->GetEngine(), config);
data->SetTexID((ImTextureID)tex_id);
UpdateTexture(data);
}
@@ -183,21 +140,22 @@ void GuiView::DestroyTexture(ImTextureData* data) {
}
}
void GuiView::UpdateRenderable() {
void GuiView::Update() {
if (!ImGui::GetCurrentContext()) {
PrepareRenderables(0);
return;
}
// Prepare the imgui draw commands. We must call this function even if we do
// not plan on rendering anything to ensure imgui state is updated.
ImGui::Render();
auto& rm = engine_->getRenderableManager();
ImGuiIO& io = ImGui::GetIO();
const ImVec2& size = io.DisplaySize;
const ImVec2& scale = io.DisplayFramebufferScale;
ImDrawData* commands = ImGui::GetDrawData();
if (!commands) {
if (!commands || size.x == 0 || size.y == 0) {
PrepareRenderables(0);
return;
}
commands->ScaleClipRects(scale);
@@ -243,39 +201,13 @@ void GuiView::UpdateRenderable() {
}
}
if (size.x == 0 || size.y == 0 || num_elements == 0) {
if (num_elements_ > 0) {
scene_->remove(renderable_);
rm.destroy(renderable_);
}
num_elements_ = 0;
PrepareRenderables(num_elements);
if (num_elements == 0) {
return;
}
view_->setViewport(
filament::Viewport(0.f, 0.f, size.x * scale.x, size.y * scale.y));
camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0, size.x,
size.y, 0.0, 0.0, 1.0);
if (num_elements != num_elements_) {
if (num_elements_ > 0) {
scene_->remove(renderable_);
rm.destroy(renderable_);
}
num_elements_ = num_elements;
filament::RenderableManager::Builder builder(num_elements_);
builder.boundingBox({{-100, -100, -100}, {100, 100, 100}});
builder.culling(false);
builder.build(*engine_, renderable_);
scene_->addEntity(renderable_);
}
meshes_.clear();
auto ri = rm.getInstance(renderable_);
int drawable_index = 0;
int renderable_index = 0;
for (int n = 0; n < commands->CmdListsCount; ++n) {
const ImDrawList* cmds = commands->CmdLists[n];
@@ -297,70 +229,70 @@ void GuiView::UpdateRenderable() {
data.indices = cmds->IdxBuffer.Data;
data.index_type = mjINDEX_TYPE_USHORT;
data.primitive_type = mjPRIM_TYPE_TRIANGLES;
meshes_.push_back(std::make_unique<Mesh>(engine_, data));
const auto& mesh = meshes_.back();
meshes_.push_back(std::make_unique<Mesh>(scene_view_->GetEngine(), data));
const Mesh* mesh = meshes_.back().get();
int index_offset = 0;
for (const ImDrawCmd& command : cmds->CmdBuffer) {
const int width = size.x * scale.x;
const int height = size.y * scale.y;
int clip_left = command.ClipRect.x;
int clip_bottom = height - command.ClipRect.w;
int clip_width = command.ClipRect.z - command.ClipRect.x;
int clip_height = command.ClipRect.w - command.ClipRect.y;
auto& renderable = renderables_[renderable_index];
if (renderable->GetNumMeshes() == 0) {
renderable->AppendMesh(mesh, index_offset, command.ElemCount);
} else {
renderable->UpdateMesh(0, mesh, index_offset, command.ElemCount);
}
Material::Textures textures;
textures.color = textures_[command.GetTexID()].get();
renderable->GetMaterial().UpdateTextures(textures);
Material::Params properties;
properties.scissor[0] = command.ClipRect.x;
properties.scissor[1] = height - command.ClipRect.w;
properties.scissor[2] = command.ClipRect.z - command.ClipRect.x;
properties.scissor[3] = command.ClipRect.w - command.ClipRect.y;
// Modal dialogs try to cover the whole window, but also a little outside
// of it. This doesn't work well with filament's scissor test, so we clip
// them to the window.
if (clip_left < 0 || clip_bottom < 0) {
clip_left = 0;
clip_bottom = 0;
clip_width = width;
clip_height = height;
if (properties.scissor[0] < 0 || properties.scissor[1] < 0) {
properties.scissor[0] = 0;
properties.scissor[1] = 0;
properties.scissor[2] = width;
properties.scissor[3] = height;
}
mjrRect clip_rect{clip_left, clip_bottom, clip_width, clip_height};
rm.setMaterialInstanceAt(
ri, drawable_index,
GetMaterialInstance(drawable_index, clip_rect, command.GetTexID()));
rm.setGeometryAt(
ri, drawable_index, kTriangles, mesh->GetFilamentVertexBuffer(),
mesh->GetFilamentIndexBuffer(), index_offset, command.ElemCount);
rm.setBlendOrderAt(ri, drawable_index, drawable_index);
renderable->GetMaterial().UpdateParams(properties);
index_offset += command.ElemCount;
++drawable_index;
++renderable_index;
}
}
}
filament::MaterialInstance* GuiView::GetMaterialInstance(int index,
mjrRect rect,
uintptr_t texture_id) {
while (index >= instances_.size()) {
instances_.push_back(material_->createInstance());
}
void GuiView::PrepareRenderables(int count) {
while (renderables_.size() < count) {
auto& r = renderables_.emplace_back(
std::make_unique<Renderable>(scene_view_->GetEngine()));
r->SetCastShadows(false);
r->SetReceiveShadows(false);
r->SetBlendOrder(static_cast<std::uint16_t>(renderables_.size()));
auto iter = textures_.find(texture_id);
if (iter == textures_.end()) {
mju_error("Texture not found: %lu", texture_id);
Material& material = r->GetMaterial();
Material::DrawMode mode = Material::DrawMode::kNormal;
material.SetMaterial(mode, material_);
r->SetMaterialInstance(material.GetMaterialInstance(mode));
scene_view_->AddToUxScene(r.get());
}
while (renderables_.size() > count) {
scene_view_->RemoveFromUxScene(renderables_.back().get());
renderables_.pop_back();
}
filament::MaterialInstance* instance = instances_[index];
instance->setParameter("glyph", iter->second->GetFilamentTexture(),
filament::TextureSampler());
instance->setScissor(rect.left, rect.bottom, rect.width, rect.height);
return instance;
}
void GuiView::Render(filament::Renderer* renderer, RenderTarget* target) {
if (num_elements_ == 0) {
return;
}
view_->setRenderTarget(target ? target->GetFilamentRenderTarget() : nullptr);
renderer->render(view_);
view_->setRenderTarget(nullptr);
float GuiView::GetScale() const {
return ImGui::GetIO().DisplayFramebufferScale.x;
}
static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) {
src/experimental/filament/filament/gui_view.h
+19 -30
View File
@@ -21,60 +21,49 @@
#include <vector>
#include <imgui.h>
#include <filament/Camera.h>
#include <filament/Engine.h>
#include <filament/Material.h>
#include <filament/MaterialInstance.h>
#include <filament/Scene.h>
#include <filament/Texture.h>
#include <filament/View.h>
#include <mujoco/mjrender.h>
#include "experimental/filament/filament/mesh.h"
#include "experimental/filament/filament/render_target.h"
#include "experimental/filament/filament/renderable.h"
#include "experimental/filament/filament/scene_view.h"
#include "experimental/filament/filament/texture.h"
namespace mujoco {
// A filament::View that contains a filament::Scene used for rendering the GUI.
// Manages Renderables that will be added a SceneView's UX scene.
class GuiView {
public:
GuiView(filament::Engine* engine, filament::Material* ui_material);
GuiView(SceneView* scene_view, filament::Material* ui_material);
~GuiView();
// Prepares the UX scene renderable using data from the current ImGui state.
// This function must be called once per frame to ensure ImGui state is
// correctly synced.
void UpdateRenderable();
// Prepares the Renderables using data from the current ImGui state. This
// function must be called once per frame to ensure ImGui state is correctly
// synced.
void Update();
void Render(filament::Renderer* renderer, RenderTarget* target = nullptr);
// Returns the current ImGui scale factor.
float GetScale() const;
// Uploads texture to be used with ImGui's Image and ImageButton functions.
uintptr_t UploadImage(uintptr_t tex_id, const uint8_t* pixels, int width,
int height, int bpp);
GuiView(const GuiView&) = delete;
GuiView& operator=(const GuiView&) = delete;
private:
// Ensures exactly `count` Renderables exist, creating or destroying them as
// needed.
void PrepareRenderables(int count);
void CreateTexture(ImTextureData* data);
void UpdateTexture(ImTextureData* data);
void DestroyTexture(ImTextureData* data);
// Returns the filament::MaterialInstance configured to draw into the given
// scissor rect.
filament::MaterialInstance* GetMaterialInstance(int index, mjrRect rect,
uintptr_t texture_id);
// Clears the filament::Scene of the UX renderable and releases all buffers.
void ResetRenderable();
filament::Engine* engine_ = nullptr;
filament::Scene* scene_ = nullptr;
filament::Camera* camera_ = nullptr;
filament::View* view_ = nullptr;
SceneView* scene_view_ = nullptr;
filament::Material* material_ = nullptr;
utils::Entity renderable_;
std::vector<std::unique_ptr<Renderable>> renderables_;
std::vector<MeshPtr> meshes_;
std::vector<filament::MaterialInstance*> instances_;
std::unordered_map<uintptr_t, std::unique_ptr<Texture>> textures_;
int num_elements_ = 0;
};
// Draws text at the given screen coordinates in clip space (i.e. [-1,-1,-1] to
src/experimental/filament/filament/material.cc
+5
View File
@@ -72,6 +72,11 @@ void Material::UpdateMaterialInstances() {
return;
}
if (params_.scissor[2] != 0 && params_.scissor[3] != 0) {
instance->setScissor(params_.scissor[0], params_.scissor[1],
params_.scissor[2], params_.scissor[3]);
}
const filament::Material* material = instance->getMaterial();
if (material->hasParameter("BaseColorFactor")) {
instance->setParameter("BaseColorFactor", filament::RgbaType::sRGB,
src/experimental/filament/filament/material.h
+1
View File
@@ -55,6 +55,7 @@ class Material {
filament::math::float2 tex_repeat = {1, 1};
filament::math::float3 uv_scale = {1, 1, 1};
filament::math::float3 uv_offset = {0, 0, 0};
filament::math::float4 scissor = {0, 0, 0, 0};
float specular = -1.0f;
float glossiness = -1.0f;
float metallic = -1.0f;
src/experimental/filament/filament/scene_view.cc
+37
View File
@@ -129,7 +129,9 @@ static void SetupReflectionCamera(const mat4& surface_xform,
SceneView::SceneView(filament::Engine* engine) : engine_(engine) {
scene_ = engine->createScene();
ux_scene_ = engine->createScene();
camera_ = engine->createCamera(utils::EntityManager::get().create());
ux_camera_ = engine->createCamera(utils::EntityManager::get().create());
reflect_camera_ = engine->createCamera(utils::EntityManager::get().create());
for (auto& view : views_) {
@@ -139,6 +141,12 @@ SceneView::SceneView(filament::Engine* engine) : engine_(engine) {
view->setVisibleLayers(0xff, mjCAT_ALL);
}
ux_view_ = engine->createView();
ux_view_->setScene(ux_scene_);
ux_view_->setCamera(ux_camera_);
ux_view_->setPostProcessingEnabled(false);
ux_view_->setShadowingEnabled(false);
reflect_view_ = engine->createView();
reflect_view_->setScene(scene_);
reflect_view_->setCamera(reflect_camera_);
@@ -166,16 +174,22 @@ SceneView::~SceneView() {
for (auto& renderable : renderables_) {
renderable->RemoveFromScene(scene_);
}
for (auto& renderable : ux_renderables_) {
renderable->RemoveFromScene(ux_scene_);
}
lights_.clear();
renderables_.clear();
reflect_targets_.clear();
engine_->destroyCameraComponent(reflect_camera_->getEntity());
engine_->destroy(reflect_view_);
engine_->destroyCameraComponent(ux_camera_->getEntity());
engine_->destroy(ux_view_);
engine_->destroyCameraComponent(camera_->getEntity());
if (color_grading_) {
engine_->destroy(color_grading_);
}
engine_->destroy(scene_);
engine_->destroy(ux_scene_);
for (auto& view : views_) {
engine_->destroy(view);
}
@@ -212,6 +226,18 @@ void SceneView::RemoveFromScene(Renderable* renderable) {
}
}
void SceneView::AddToUxScene(Renderable* renderable) {
if (ux_renderables_.insert(renderable).second) {
renderable->AddToScene(ux_scene_);
}
}
void SceneView::RemoveFromUxScene(Renderable* renderable) {
if (ux_renderables_.erase(renderable)) {
renderable->RemoveFromScene(ux_scene_);
}
}
void SceneView::AddToScene(filament::Skybox* skybox) {
skybox_ = skybox;
scene_->setSkybox(skybox);
@@ -231,6 +257,7 @@ void SceneView::Render(filament::Renderer* renderer,
for (auto& view : views_) {
view->setViewport(viewport);
}
ux_view_->setViewport(viewport);
reflect_view_->setViewport(viewport);
SetupCamera(request.camera, viewport, camera_);
@@ -279,6 +306,16 @@ void SceneView::Render(filament::Renderer* renderer,
renderer->render(view);
view->setRenderTarget(nullptr);
if (request.enable_ux) {
ux_camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0f,
viewport.width / request.gui_scale,
viewport.height / request.gui_scale, 0.0f, 0.0f,
1.0f);
ux_view_->setRenderTarget(render_target);
renderer->render(ux_view_);
ux_view_->setRenderTarget(nullptr);
}
if (request.target) {
view->setMultiSampleAntiAliasingOptions(options);
}
src/experimental/filament/filament/scene_view.h
+16 -1
View File
@@ -38,7 +38,8 @@ namespace mujoco {
//
// The filament Scene is populated with the objects (e.g. lights, renderables,
// skybox, etc.). It manages multiple views to support a variety of draw modes
// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces.
// (e.g. normal, depth, segmentation, etc.) as well as reflective surfaces. It
// also manages a separate scene and view for UX rendering.
class SceneView {
public:
SceneView(filament::Engine* engine);
@@ -52,6 +53,10 @@ class SceneView {
void AddToScene(filament::Skybox* skybox);
void RemoveFromScene(filament::Skybox* skybox);
// Adds/removes entities from the UX scene, which is rendered separately.
void AddToUxScene(Renderable* renderable);
void RemoveFromUxScene(Renderable* renderable);
// Parameters for rendering the scene.
using DrawMode = Material::DrawMode;
struct RenderRequest {
@@ -63,6 +68,10 @@ class SceneView {
mjvGLCamera camera;
// An optional render target into which the scene will be rendered.
RenderTarget* target = nullptr;
// Whether or not to render the UX as a separate pass.
bool enable_ux = false;
// The scale factor to use for UX rendering.
float gui_scale = 1.0f;
};
// Renders the scene.
@@ -89,6 +98,7 @@ class SceneView {
filament::Engine* engine_ = nullptr;
filament::Scene* scene_ = nullptr;
filament::Scene* ux_scene_ = nullptr;
filament::Camera* camera_ = nullptr;
filament::ColorGrading* color_grading_ = nullptr;
ColorGradingOptions color_grading_options_;
@@ -100,6 +110,11 @@ class SceneView {
std::unordered_set<Renderable*> renderables_;
filament::Skybox* skybox_ = nullptr;
// Custom view for UX.
filament::View* ux_view_ = nullptr;
filament::Camera* ux_camera_ = nullptr;
std::unordered_set<Renderable*> ux_renderables_;
// Custom view and camera for reflective surfaces.
filament::View* reflect_view_ = nullptr;
filament::Camera* reflect_camera_ = nullptr;