diff --git a/src/experimental/filament/CMakeLists.txt b/src/experimental/filament/CMakeLists.txt index 42261f11..c09455a9 100644 --- a/src/experimental/filament/CMakeLists.txt +++ b/src/experimental/filament/CMakeLists.txt @@ -53,8 +53,6 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME} filament/texture.h compat/imgui_bridge.cc compat/imgui_bridge.h - compat/mjr_filament_renderer.cc - compat/mjr_filament_renderer.h compat/model_objects.cc compat/model_objects.h compat/scene_bridge.cc diff --git a/src/experimental/filament/compat/mjr_filament_renderer.cc b/src/experimental/filament/compat/mjr_filament_renderer.cc deleted file mode 100644 index 8d9fcf34..00000000 --- a/src/experimental/filament/compat/mjr_filament_renderer.cc +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2025 DeepMind Technologies Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "experimental/filament/compat/mjr_filament_renderer.h" - -#include -#include -#include -#include - -#include -#include -#include -#include "experimental/filament/compat/imgui_bridge.h" -#include "experimental/filament/compat/scene_bridge.h" -#include "experimental/filament/filament/filament_context.h" -#include "experimental/filament/render_context_filament_cpp.h" -#include "experimental/filament/render_context_filament.h" - -namespace mujoco { - -MjrFilamentRenderer::MjrFilamentRenderer(const mjrFilamentConfig* config) { - filament_context_ = std::make_unique(config); -} - -void MjrFilamentRenderer::Init(const mjModel* model) { - scene_bridge_ = std::make_unique(filament_context_.get(), model); - imgui_bridge_ = std::make_unique(filament_context_.get()); - scene_bridge_->SetDrawTextFunction(DrawTextAt); -} - -void MjrFilamentRenderer::Render(const mjrRect& viewport, - const mjvScene* scene) { - scene_bridge_->Update(viewport, scene); - std::memcpy(render_flags_, scene->flags, mjNRNDFLAG * sizeof(mjtByte)); - // Update the UX renderable entity after processing the scene in case there - // are any elements in the scene which generate UX draw calls (e.g. labels). - if (mode_ != FrameBufferMode::OffScreen) { - imgui_bridge_->Update(); - } - - if (mode_ == FrameBufferMode::Window) { - mjrRenderRequest reqs[2]; - mjr_defaultRenderRequest(&reqs[0]); - reqs[0].scene = scene_bridge_->GetScene(); - reqs[0].draw_mode = mjDRAW_MODE_COLOR; - if (render_flags_[mjRND_SEGMENT]) { - reqs[0].draw_mode = mjDRAW_MODE_SEGMENTATION; - } else if (render_flags_[mjRND_DEPTH]) { - reqs[0].draw_mode = mjDRAW_MODE_DEPTH; - } else if (render_flags_[mjRND_WIREFRAME]) { - reqs[0].draw_mode = mjDRAW_MODE_WIREFRAME; - } - - reqs[0].camera = scene_bridge_->GetCamera(); - reqs[0].viewport = viewport; - reqs[0].enable_shadows = render_flags_[mjRND_SHADOW]; - reqs[0].enable_reflections = render_flags_[mjRND_REFLECTION]; - - mjr_defaultRenderRequest(&reqs[1]); - reqs[1].scene = imgui_bridge_->GetScene(); - reqs[1].draw_mode = mjDRAW_MODE_COLOR; - reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); - reqs[1].viewport = viewport; - reqs[1].enable_shadows = false; - reqs[1].enable_reflections = false; - reqs[1].enable_post_processing = false; - - filament_context_->Render(reqs); - } -} - -void MjrFilamentRenderer::SetFrameBuffer(int framebuffer) { - switch (framebuffer) { - case mjFB_WINDOW: - mode_ = FrameBufferMode::Window; - break; - case mjFB_OFFSCREEN: - mode_ = FrameBufferMode::OffScreen; - break; - case 2: // No official constant fo this. - mode_ = FrameBufferMode::OffScreenWithGui; - break; - default: - mju_error("Invalid framebuffer mode: %d", framebuffer); - } -} - -void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb, - float* depth) { - if (mode_ == FrameBufferMode::Window) { - mju_error("ReadPixels is only supported for offscreen rendering."); - } - - mjrRenderRequest reqs[2]; - mjr_defaultRenderRequest(&reqs[0]); - - reqs[0].scene = scene_bridge_->GetScene(); - reqs[0].draw_mode = mjDRAW_MODE_COLOR; - if (render_flags_[mjRND_SEGMENT]) { - reqs[0].draw_mode = mjDRAW_MODE_SEGMENTATION; - } else if (render_flags_[mjRND_DEPTH]) { - reqs[0].draw_mode = mjDRAW_MODE_DEPTH; - } else if (render_flags_[mjRND_WIREFRAME]) { - reqs[0].draw_mode = mjDRAW_MODE_WIREFRAME; - } - reqs[0].camera = scene_bridge_->GetCamera(); - reqs[0].viewport = viewport; - reqs[0].enable_shadows = render_flags_[mjRND_SHADOW]; - reqs[0].enable_reflections = render_flags_[mjRND_REFLECTION]; - - mjr_defaultRenderRequest(&reqs[1]); - reqs[1].scene = imgui_bridge_->GetScene(); - reqs[1].draw_mode = mjDRAW_MODE_COLOR; - reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); - reqs[1].viewport = viewport; - reqs[1].enable_shadows = false; - reqs[1].enable_reflections = false; - reqs[1].enable_post_processing = false; - - if (rgb) { - mjrRenderTargetConfig config; - mjr_defaultRenderTargetConfig(&config); - config.width = viewport.width; - config.height = viewport.height; - config.color_format = mjPIXEL_FORMAT_RGB8; - config.depth_format = mjPIXEL_FORMAT_DEPTH32F; - auto target = CreateRenderTarget(filament_context_.get(), config); - - reqs[0].target = target.get(); - reqs[1].target = target.get(); - - mjrReadPixelsRequest read_request; - mjr_defaultReadPixelsRequest(&read_request); - read_request.target = target.get(); - read_request.output = rgb; - read_request.num_bytes = viewport.width * viewport.height * 3; - - const size_t num_requests = - (mode_ == FrameBufferMode::OffScreenWithGui) ? 2 : 1; - const mjrFrameHandle frame = filament_context_->Render( - {&reqs[0], num_requests}, {&read_request, 1}); - filament_context_->WaitForFrame(frame); - } - - if (depth) { - mjrRenderTargetConfig config; - mjr_defaultRenderTargetConfig(&config); - config.width = viewport.width; - config.height = viewport.height; - config.color_format = mjPIXEL_FORMAT_R32F; - config.depth_format = mjPIXEL_FORMAT_DEPTH32F; - auto target = CreateRenderTarget(filament_context_.get(), config); - - reqs[0].draw_mode = mjDRAW_MODE_DEPTH; - reqs[0].target = target.get(); - - mjrReadPixelsRequest read_request; - mjr_defaultReadPixelsRequest(&read_request); - read_request.target = target.get(); - read_request.output = reinterpret_cast(depth); - read_request.num_bytes = viewport.width * viewport.height * sizeof(float); - - const mjrFrameHandle frame = filament_context_->Render( - {&reqs[0], 1}, {&read_request, 1}); - filament_context_->WaitForFrame(frame); - } -} - -void MjrFilamentRenderer::UploadMesh(const mjModel* model, int id) { - if (!scene_bridge_) { - mju_error("SceneBridge is not initialized."); - } - scene_bridge_->UploadMesh(model, id); -} - -void MjrFilamentRenderer::UploadTexture(const mjModel* model, int id) { - if (!scene_bridge_) { - mju_error("SceneBridge is not initialized."); - } - scene_bridge_->UploadTexture(model, id); -} - -void MjrFilamentRenderer::UploadHeightField(const mjModel* model, int id) { - if (!scene_bridge_) { - mju_error("SceneBridge is not initialized."); - } - scene_bridge_->UploadHeightField(model, id); -} - -uintptr_t MjrFilamentRenderer::UploadGuiImage(uintptr_t tex_id, - const uint8_t* pixels, int width, - int height, int bpp) { - return imgui_bridge_->UploadImage(tex_id, pixels, width, height, bpp); -} - -double MjrFilamentRenderer::GetFrameRate() const { - mjrFrameStats stats; - mjr_defaultFrameStats(&stats); - filament_context_->GetFrameStats(0, &stats); - return stats.frame_rate; -} - -void MjrFilamentRenderer::UpdateGui() { - mjrf_DEBUG_drawImguiEditor(scene_bridge_->GetScene()); -} - -} // namespace mujoco diff --git a/src/experimental/filament/compat/mjr_filament_renderer.h b/src/experimental/filament/compat/mjr_filament_renderer.h deleted file mode 100644 index 5f57c029..00000000 --- a/src/experimental/filament/compat/mjr_filament_renderer.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2025 DeepMind Technologies Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_ -#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_ - -#include -#include - -#include -#include -#include -#include -#include "experimental/filament/compat/imgui_bridge.h" -#include "experimental/filament/compat/scene_bridge.h" -#include "experimental/filament/filament/filament_context.h" -#include "experimental/filament/render_context_filament.h" - -namespace mujoco { - -// Subclass of the FilamentContext that implements the legacy mjr API. -class MjrFilamentRenderer { - public: - explicit MjrFilamentRenderer(const mjrFilamentConfig* config); - ~MjrFilamentRenderer() = default; - - // Initializes the renderer with the given model. - void Init(const mjModel* model); - - // Renders the given mjvScene to the viewport. - void Render(const mjrRect& viewport, const mjvScene* scene); - - // Configures the renderer to render to the window (0) or an offscreen - // texture (1 or 2). Rendering to the window always includes UX data from - // ImGui. A value of 1 indicates the UX should not be included in the - // offscreen render, whereas 2 indicates that it should. - void SetFrameBuffer(int framebuffer); - - // Renders the scene to a texture if the framebuffer is not 0. - void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth); - - // Uploads the mesh data from the model to the GPU. - void UploadMesh(const mjModel* model, int id); - - // Uploads the texture data from the model to the GPU. - void UploadTexture(const mjModel* model, int id); - - // Uploads the height field data from the model to the GPU. - void UploadHeightField(const mjModel* model, int id); - - // Uploads a texture that can be used with ImGui to the GPU. - uintptr_t UploadGuiImage(uintptr_t tex_id, const uint8_t* pixels, int width, - int height, int bpp); - - // Renders an ImGui window containing Filament-specific editor UI. - void UpdateGui(); - - double GetFrameRate() const; - - MjrFilamentRenderer(const MjrFilamentRenderer&) = delete; - MjrFilamentRenderer& operator=(const MjrFilamentRenderer&) = delete; - - private: - enum class FrameBufferMode { - Window, - OffScreen, - OffScreenWithGui, - }; - - std::unique_ptr filament_context_; - std::unique_ptr scene_bridge_; - std::unique_ptr imgui_bridge_; - FrameBufferMode mode_ = FrameBufferMode::Window; - mjtByte render_flags_[mjNRNDFLAG]; -}; - -} // namespace mujoco - -#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_ diff --git a/src/experimental/filament/mjr_compat.cc b/src/experimental/filament/mjr_compat.cc index c3cd27e1..d98d93f2 100644 --- a/src/experimental/filament/mjr_compat.cc +++ b/src/experimental/filament/mjr_compat.cc @@ -12,47 +12,213 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include + #include +#include "experimental/filament/compat/scene_bridge.h" #include "experimental/filament/render_context_filament.h" +#include "experimental/filament/render_context_filament_cpp.h" // This library implements the entirety of mujoco's mjr API. You can link this // library with your application (instead of the "classic" mujoco renderer) to -// use the same APIs but with Filament rendering instead. +// use the same APIs but with Filament rendering instead. Note that some +// functionality (e.g. ux-related functions like mjr_text, mjr_label, etc.) +// will call mju_error if called. // -// However, you should consider using filament's mjrf API directly as it will -// provide you with access to more features and optimizations. +// You should consider using filament's mjrf API directly as it will provide you +// with access to more features and optimizations. + +namespace mujoco { +namespace { + +class CompatContext { + public: + CompatContext(const mjrFilamentConfig* config, const mjModel* model); + + void Render(const mjrRect& viewport, const mjvScene* scene); + + void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth); + + void SetFrameBuffer(int framebuffer) { + framebuffer_ = (mjtFramebuffer)framebuffer; + } + + void UploadMesh(const mjModel* model, int id) { + scene_bridge_->UploadMesh(model, id); + } + + void UploadTexture(const mjModel* model, int id) { + scene_bridge_->UploadTexture(model, id); + } + + void UploadHeightField(const mjModel* model, int id) { + scene_bridge_->UploadHeightField(model, id); + } + + private: + mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR; + UniquePtr context_; + std::unique_ptr scene_bridge_; + mjtFramebuffer framebuffer_ = mjFB_WINDOW; +}; + +CompatContext::CompatContext(const mjrFilamentConfig* config, + const mjModel* model) + : context_(CreateContext(*config)) { + scene_bridge_ = std::make_unique(context_.get(), model); +} + +void CompatContext::Render(const mjrRect& viewport, const mjvScene* scene) { + scene_bridge_->Update(viewport, scene); + + if (scene->flags[mjRND_SEGMENT]) { + draw_mode_ = mjDRAW_MODE_SEGMENTATION; + } else if (scene->flags[mjRND_DEPTH]) { + draw_mode_ = mjDRAW_MODE_DEPTH; + } else if (scene->flags[mjRND_WIREFRAME]) { + draw_mode_ = mjDRAW_MODE_WIREFRAME; + } else { + draw_mode_ = mjDRAW_MODE_COLOR; + } + + if (framebuffer_ == mjFB_WINDOW) { + mjrRenderRequest req; + mjr_defaultRenderRequest(&req); + req.scene = scene_bridge_->GetScene(); + req.draw_mode = draw_mode_; + req.camera = scene_bridge_->GetCamera(); + req.viewport = viewport; + mjrf_render(context_.get(), &req, 1, nullptr, 0); + } +} + +void CompatContext::ReadPixels(mjrRect viewport, unsigned char* rgb, + float* depth) { + if (framebuffer_ == mjFB_WINDOW) { + mju_error("ReadPixels is only supported for offscreen rendering."); + } + + if (rgb) { + mjrRenderTargetConfig config; + mjr_defaultRenderTargetConfig(&config); + config.width = viewport.width; + config.height = viewport.height; + config.color_format = mjPIXEL_FORMAT_RGB8; + config.depth_format = mjPIXEL_FORMAT_DEPTH32F; + auto target = CreateRenderTarget(context_.get(), config); + + mjrRenderRequest req; + mjr_defaultRenderRequest(&req); + req.scene = scene_bridge_->GetScene(); + req.draw_mode = draw_mode_; + req.camera = scene_bridge_->GetCamera(); + req.viewport = viewport; + req.target = target.get(); + + mjrReadPixelsRequest read_req; + mjr_defaultReadPixelsRequest(&read_req); + read_req.target = target.get(); + read_req.output = rgb; + read_req.num_bytes = viewport.width * viewport.height * 3; + + const mjrFrameHandle frame = mjrf_render(context_.get(), &req, 1, &read_req, 1); + mjrf_waitForFrame(context_.get(), frame); + } + + if (depth) { + mjrRenderTargetConfig config; + mjr_defaultRenderTargetConfig(&config); + config.width = viewport.width; + config.height = viewport.height; + config.color_format = mjPIXEL_FORMAT_R32F; + config.depth_format = mjPIXEL_FORMAT_DEPTH32F; + auto target = CreateRenderTarget(context_.get(), config); + + mjrRenderRequest req; + mjr_defaultRenderRequest(&req); + req.scene = scene_bridge_->GetScene(); + req.draw_mode = mjDRAW_MODE_DEPTH; + req.camera = scene_bridge_->GetCamera(); + req.viewport = viewport; + req.target = target.get(); + + mjrReadPixelsRequest read_req; + mjr_defaultReadPixelsRequest(&read_req); + read_req.output = reinterpret_cast(depth); + read_req.num_bytes = viewport.width * viewport.height * sizeof(float); + + const mjrFrameHandle frame = mjrf_render(context_.get(), &req, 1, &read_req, 1); + mjrf_waitForFrame(context_.get(), frame); + } +} + +} // namespace +} // namespace mujoco + +static thread_local mujoco::CompatContext* g_context = nullptr; + +static mujoco::CompatContext* GetCheckedContext() { + if (g_context == nullptr) { + mju_error("Missing context; did you call mjr_makeContext?"); + } + return g_context; +} extern "C" { // mjr functions that are supported by the filament renderer. void mjr_defaultContext(mjrContext* con) { - mjrf_defaultContext(con); + memset(con, 0, sizeof(mjrContext)); } + +void mjr_makeFilamentContext(const mjModel* m, const mjrFilamentConfig* cfg, + mjrContext* con) { + if (g_context != nullptr) { + mju_error("Context already exists!"); + } + g_context = new mujoco::CompatContext(cfg, m); +} + void mjr_makeContext(const mjModel* m, mjrContext* con, int fontscale) { - mjrf_makeContext(m, con, fontscale); + mjr_freeContext(con); + + mjrFilamentConfig cfg; + memset(&cfg, 0, sizeof(mjrFilamentConfig)); + cfg.width = m->vis.global.offwidth; + cfg.height = m->vis.global.offheight; + mjr_makeFilamentContext(m, &cfg, con); } + void mjr_freeContext(mjrContext* con) { - mjrf_freeContext(con); + // mjr_freeContext may be called multiple times. + if (g_context) { + delete g_context; + g_context = nullptr; + } + mjr_defaultContext(con); } + void mjr_render(mjrRect viewport, mjvScene* scn, const mjrContext* con) { - mjrf_renderScene(viewport, scn, con); + GetCheckedContext()->Render(viewport, scn); } void mjr_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) { - mjrf_uploadMesh(m, con, meshid); + GetCheckedContext()->UploadMesh(m, meshid); } void mjr_uploadTexture(const mjModel* m, const mjrContext* con, int texid) { - mjrf_uploadTexture(m, con, texid); + GetCheckedContext()->UploadTexture(m, texid); } void mjr_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid) { - mjrf_uploadHField(m, con, hfieldid); + GetCheckedContext()->UploadHeightField(m, hfieldid); } void mjr_setBuffer(int framebuffer, mjrContext* con) { - mjrf_setBuffer(framebuffer, con); + GetCheckedContext()->SetFrameBuffer(framebuffer); } void mjr_readPixels(unsigned char* rgb, float* depth, mjrRect viewport, const mjrContext* con) { - mjrf_readPixels(rgb, depth, viewport, con); + GetCheckedContext()->ReadPixels(viewport, rgb, depth); } // mjr functions that are NOT supported by the filament renderer. diff --git a/src/experimental/filament/render_context_filament.cc b/src/experimental/filament/render_context_filament.cc index c6b66a7a..b3e3e2f7 100644 --- a/src/experimental/filament/render_context_filament.cc +++ b/src/experimental/filament/render_context_filament.cc @@ -21,10 +21,7 @@ #include #include #include -#include -#include #include -#include "experimental/filament/compat/mjr_filament_renderer.h" #include "experimental/filament/filament/filament_context.h" #include "experimental/filament/filament/light.h" #include "experimental/filament/filament/mesh.h" @@ -33,19 +30,6 @@ #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" - -#if defined(TLS_FILAMENT_CONTEXT) -static thread_local mujoco::MjrFilamentRenderer* g_filament_context = nullptr; -#else -static mujoco::MjrFilamentRenderer* g_filament_context = nullptr; -#endif - -static void CheckFilamentContext() { - if (g_filament_context == nullptr) { - mju_error("Missing context; did you call mjrf_makeFilamentContext?"); - } -} - template static void setf(float (&arr)[N], const std::array& values) { for (int i = 0; i < N; ++i) { @@ -53,7 +37,6 @@ static void setf(float (&arr)[N], const std::array& values) { } } - extern "C" { void mjrf_defaultFilamentConfig(mjrFilamentConfig* config) { @@ -340,89 +323,4 @@ void mjrf_getFrameStats(mjrfContext* ctx, mjrFrameHandle frame, mjrFrameStats* stats_out) { mujoco::FilamentContext::downcast(ctx)->GetFrameStats(frame, stats_out); } - -// Legacy API, to be deprecated. - -void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con, - const mjrFilamentConfig* config) { - // TODO: Support multiple contexts and multiple threads. For now, we'll just - // assume a single, global context. - if (g_filament_context != nullptr) { - mju_error("Context already exists!"); - } - g_filament_context = new mujoco::MjrFilamentRenderer(config); - g_filament_context->Init(m); -} - -void mjrf_defaultContext(mjrContext* con) { - memset(con, 0, sizeof(mjrContext)); -} - -void mjrf_makeContext(const mjModel* m, mjrContext* con, int fontscale) { - mjrf_freeContext(con); - mjrFilamentConfig cfg; - mjrf_defaultFilamentConfig(&cfg); - cfg.width = m->vis.global.offwidth; - cfg.height = m->vis.global.offheight; - mjrf_makeFilamentContext(m, con, &cfg); -} - -void mjrf_freeContext(mjrContext* con) { - // mjr_freeContext may be called multiple times. - if (g_filament_context) { - delete g_filament_context; - g_filament_context = nullptr; - } - mjrf_defaultContext(con); -} - -void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con) { - CheckFilamentContext(); - g_filament_context->Render(viewport, scn); -} - -void mjrf_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) { - CheckFilamentContext(); - g_filament_context->UploadMesh(m, meshid); -} - -void mjrf_uploadTexture(const mjModel* m, const mjrContext* con, int texid) { - CheckFilamentContext(); - g_filament_context->UploadTexture(m, texid); -} - -void mjrf_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid) { - CheckFilamentContext(); - g_filament_context->UploadHeightField(m, hfieldid); -} - -void mjrf_setBuffer(int framebuffer, mjrContext* con) { - CheckFilamentContext(); - g_filament_context->SetFrameBuffer(framebuffer); -} - -void mjrf_readPixels(unsigned char* rgb, float* depth, mjrRect viewport, - const mjrContext* con) { - CheckFilamentContext(); - g_filament_context->ReadPixels(viewport, rgb, depth); -} - -uintptr_t mjrf_uploadGuiImage(uintptr_t tex_id, const unsigned char* pixels, - int width, int height, int bpp, - const mjrContext* con) { - CheckFilamentContext(); - return g_filament_context->UploadGuiImage(tex_id, pixels, width, height, bpp); -} - -double mjrf_getFrameRate(const mjrContext* con) { - CheckFilamentContext(); - return g_filament_context->GetFrameRate(); -} - -void mjrf_updateGui(const mjrContext* con) { - if (g_filament_context != nullptr) { - g_filament_context->UpdateGui(); - } -} - } // extern "C" diff --git a/src/experimental/filament/render_context_filament.h b/src/experimental/filament/render_context_filament.h index 43c7fb88..e7043bec 100644 --- a/src/experimental/filament/render_context_filament.h +++ b/src/experimental/filament/render_context_filament.h @@ -747,40 +747,6 @@ void mjrf_destroyRenderTarget(mjrRenderTarget* render_target); // settings. void mjrf_DEBUG_drawImguiEditor(mjrScene* scene); -// Legacy API, to be deprecated. - -void mjrf_defaultFilamentConfig(mjrFilamentConfig* config); - -void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con, - const mjrFilamentConfig* config); - -void mjrf_defaultContext(mjrContext* con); - -void mjrf_makeContext(const mjModel* m, mjrContext* con, int fontscale); - -void mjrf_freeContext(mjrContext* con); - -void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con); - -void mjrf_uploadMesh(const mjModel* m, const mjrContext* con, int meshid); - -void mjrf_uploadTexture(const mjModel* m, const mjrContext* con, int texid); - -void mjrf_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid); - -void mjrf_setBuffer(int framebuffer, mjrContext* con); - -void mjrf_readPixels(unsigned char* rgb, float* depth, mjrRect viewport, - const mjrContext* con); - -double mjrf_getFrameRate(const mjrContext* con); - -uintptr_t mjrf_uploadGuiImage(uintptr_t tex_id, const unsigned char* pixels, - int width, int height, int bpp, - const mjrContext* con); - -void mjrf_updateGui(const mjrContext* con); - #if defined(__cplusplus) } // extern "C" #endif diff --git a/src/experimental/platform/hal/renderer.cc b/src/experimental/platform/hal/renderer.cc index c7d8c75b..65160f6e 100644 --- a/src/experimental/platform/hal/renderer.cc +++ b/src/experimental/platform/hal/renderer.cc @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include @@ -25,12 +27,22 @@ #if !defined(__EMSCRIPTEN__) && !defined(__APPLE__) #include "experimental/platform/hal/egl_utils.h" #endif +#include "experimental/filament/compat/imgui_bridge.h" +#include "experimental/filament/compat/scene_bridge.h" #include "experimental/filament/render_context_filament.h" +#include "experimental/filament/render_context_filament_cpp.h" #include "experimental/platform/hal/graphics_mode.h" #include "experimental/platform/ux/plugin.h" namespace mujoco::platform { +static std::function g_update_gui_callback = nullptr; +static void PluginUpdate(GuiPlugin* plugin) { + if (g_update_gui_callback) { + g_update_gui_callback(); + } +} + static void FlipImage(unsigned char* pixels, int width, int height, int bpp) { const int row_size = width * bpp; for (int i = 0; i < height / 2; ++i) { @@ -54,9 +66,16 @@ Renderer::Renderer(void* native_window, GraphicsMode gfx) ImGui_ImplOpenGL3_Init(); } } + + g_update_gui_callback = [this]() { + if (scene_bridge_) { + mjrf_DEBUG_drawImguiEditor(scene_bridge_->GetScene()); + } + }; } Renderer::~Renderer() { + g_update_gui_callback = nullptr; if (IsClassic(gfx_)) { if (ImGui::GetCurrentContext()) { ImGui_ImplOpenGL3_Shutdown(); @@ -72,38 +91,22 @@ void Renderer::Init(const mjModel* model) { if (IsClassic(gfx_)) { mjr_defaultContext(&render_context_); mjr_makeContext(model, &render_context_, mjFONTSCALE_150); - render_ = [&](mjrRect rect, mjvScene* scene) { - mjr_render(rect, scene, &render_context_); - }; - set_buffer_ = [&](int framebuffer) { - mjr_setBuffer(framebuffer, &render_context_); - }; - read_pixels_ = [&](unsigned char* pixels, mjrRect rect) { - mjr_readPixels(pixels, nullptr, rect, &render_context_); - mjr_setBuffer(mjFB_WINDOW, &render_context_); - }; + } else { - mjrf_defaultContext(&render_context_); - mjrFilamentConfig render_config; - mjrf_defaultFilamentConfig(&render_config); - render_config.native_window = native_window_; - render_config.width = model->vis.global.offwidth; - render_config.height = model->vis.global.offheight; - render_config.force_software_rendering = IsSoftware(gfx_); - render_config.graphics_api = IsOpenGl(gfx_) || IsWebGl(gfx_) - ? mjGRAPHICS_API_OPENGL - : mjGRAPHICS_API_VULKAN; - mjrf_makeFilamentContext(model, &render_context_, &render_config); - render_ = [&](mjrRect rect, mjvScene* scene) { - mjrf_renderScene(rect, scene, &render_context_); - }; - set_buffer_ = [&](int framebuffer) { - mjrf_setBuffer(framebuffer, &render_context_); - }; - read_pixels_ = [&](unsigned char* pixels, mjrRect rect) { - mjrf_readPixels(pixels, nullptr, rect, &render_context_); - mjrf_setBuffer(mjFB_WINDOW, &render_context_); - }; + mjrFilamentConfig cfg; + mjrf_defaultFilamentConfig(&cfg); + cfg.native_window = native_window_; + cfg.width = model->vis.global.offwidth; + cfg.height = model->vis.global.offheight; + cfg.force_software_rendering = IsSoftware(gfx_); + cfg.graphics_api = IsOpenGl(gfx_) || IsWebGl(gfx_) + ? mjGRAPHICS_API_OPENGL + : mjGRAPHICS_API_VULKAN; + filament_context_ = CreateContext(cfg); + scene_bridge_ = + std::make_unique(filament_context_.get(), model); + imgui_bridge_ = std::make_unique(filament_context_.get()); + scene_bridge_->SetDrawTextFunction(DrawTextAt); } mjv_defaultScene(&scene_); @@ -118,7 +121,9 @@ void Renderer::Deinit() { if (IsClassic(gfx_)) { mjr_freeContext(&render_context_); } else { - mjrf_freeContext(&render_context_); + scene_bridge_.reset(); + imgui_bridge_.reset(); + filament_context_.reset(); } initialized_ = false; } @@ -163,11 +168,10 @@ void Renderer::Render(const mjModel* model, mjData* data, // The filament backend supports two offscreen framebuffers. // mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the // ImGui draw data. - set_buffer_(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1); + DoSetBuffer(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1); } - const mjrRect viewport = {0, 0, width, height}; - render_(viewport, &scene_); + DoRender(width, height); // The filament backend knows how to renders the ImGui draw data. For the // classic backend, we need to render the ImGui draw data ourselves. @@ -179,10 +183,7 @@ void Renderer::Render(const mjModel* model, mjData* data, if (render_to_texture) { unsigned char* ptr = reinterpret_cast(pixels.data()); - read_pixels_(ptr, viewport); - if (IsClassic(gfx_)) { - FlipImage(ptr, width, height, 3); - } + DoReadPixels(width, height, ptr); } UpdateFps(); @@ -195,17 +196,12 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data, return; } - const mjrRect viewport = {0, 0, width, height}; mjv_updateCamera(model, data, camera, &scene_); - - set_buffer_(mjFB_OFFSCREEN); - render_(viewport, &scene_); - unsigned char* ptr = reinterpret_cast(output); - read_pixels_(ptr, viewport); - if (IsClassic(gfx_)) { - FlipImage(ptr, width, height, 3); - } + + DoSetBuffer(mjFB_OFFSCREEN); + DoRender(width, height); + DoReadPixels(width, height, ptr); } int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width, @@ -213,12 +209,126 @@ int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width, if (IsClassic(gfx_)) { return 0; } else { - return mjrf_uploadGuiImage(texture_id, - reinterpret_cast(pixels), - width, height, bpp, &render_context_); + return imgui_bridge_->UploadImage( + texture_id, reinterpret_cast(pixels), width, + height, bpp); } } +void Renderer::DoRender(int width, int height) { + const mjrRect viewport = {0, 0, width, height}; + if (IsClassic(gfx_)) { + mjr_render(viewport, &scene_, &render_context_); + } else { + scene_bridge_->Update(viewport, &scene_); + // Update the UX renderable entity after processing the scene in case there + // are any elements in the scene which generate UX draw calls (e.g. labels). + if (framebuffer_mode_ != 1) { + imgui_bridge_->Update(); + } + if (framebuffer_mode_ == 0) { + mjrDrawMode draw_mode = mjDRAW_MODE_COLOR; + if (scene_.flags[mjRND_SEGMENT]) { + draw_mode = mjDRAW_MODE_SEGMENTATION; + } else if (scene_.flags[mjRND_DEPTH]) { + draw_mode = mjDRAW_MODE_DEPTH; + } + + mjrRenderRequest reqs[2]; + + mjr_defaultRenderRequest(&reqs[0]); + reqs[0].scene = scene_bridge_->GetScene(); + reqs[0].draw_mode = draw_mode; + reqs[0].camera = scene_bridge_->GetCamera(); + reqs[0].viewport = viewport; + reqs[0].enable_shadows = scene_.flags[mjRND_SHADOW]; + reqs[0].enable_reflections = scene_.flags[mjRND_REFLECTION]; + + mjr_defaultRenderRequest(&reqs[1]); + reqs[1].scene = imgui_bridge_->GetScene(); + reqs[1].draw_mode = mjDRAW_MODE_COLOR; + reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); + reqs[1].viewport = viewport; + reqs[1].enable_shadows = false; + reqs[1].enable_reflections = false; + reqs[1].enable_post_processing = false; + + mjrf_render(filament_context_.get(), &reqs[0], 2, nullptr, 0); + } + } +} + +void Renderer::DoSetBuffer(int framebuffer) { + framebuffer_mode_ = framebuffer; + if (IsClassic(gfx_)) { + mjr_setBuffer(framebuffer, &render_context_); + } +} + +void Renderer::DoReadPixels(int width, int height, unsigned char* rgb) { + if (!rgb) { + return; + } + if (framebuffer_mode_ == 0) { + mju_warning("ReadPixels is only supported for offscreen rendering."); + return; + } + + const mjrRect viewport = {0, 0, width, height}; + if (IsClassic(gfx_)) { + mjr_readPixels(rgb, nullptr, viewport, &render_context_); + mjr_setBuffer(mjFB_WINDOW, &render_context_); + FlipImage(rgb, viewport.width, viewport.height, 3); + } else { + mjrDrawMode draw_mode = mjDRAW_MODE_COLOR; + if (scene_.flags[mjRND_SEGMENT]) { + draw_mode = mjDRAW_MODE_SEGMENTATION; + } else if (scene_.flags[mjRND_DEPTH]) { + draw_mode = mjDRAW_MODE_DEPTH; + } + + mjrRenderTargetConfig config; + mjr_defaultRenderTargetConfig(&config); + config.width = viewport.width; + config.height = viewport.height; + config.color_format = mjPIXEL_FORMAT_RGB8; + config.depth_format = mjPIXEL_FORMAT_DEPTH32F; + auto target = CreateRenderTarget(filament_context_.get(), config); + + mjrRenderRequest reqs[2]; + mjr_defaultRenderRequest(&reqs[0]); + reqs[0].scene = scene_bridge_->GetScene(); + reqs[0].draw_mode = draw_mode; + reqs[0].camera = scene_bridge_->GetCamera(); + reqs[0].target = target.get(); + reqs[0].viewport = viewport; + reqs[0].enable_shadows = scene_.flags[mjRND_SHADOW]; + reqs[0].enable_reflections = scene_.flags[mjRND_REFLECTION]; + + mjr_defaultRenderRequest(&reqs[1]); + reqs[1].scene = imgui_bridge_->GetScene(); + reqs[1].draw_mode = mjDRAW_MODE_COLOR; + reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); + reqs[1].target = target.get(); + reqs[1].viewport = viewport; + reqs[1].enable_shadows = false; + reqs[1].enable_reflections = false; + reqs[1].enable_post_processing = false; + + mjrReadPixelsRequest read_request; + mjr_defaultReadPixelsRequest(&read_request); + read_request.target = target.get(); + read_request.output = rgb; + read_request.num_bytes = viewport.width * viewport.height * 3; + + const int num_requests = (framebuffer_mode_ == 2) ? 2 : 1; + const mjrFrameHandle frame = mjrf_render( + filament_context_.get(), &reqs[0], num_requests, &read_request, 1); + mjrf_waitForFrame(filament_context_.get(), frame); + } + DoSetBuffer(mjFB_WINDOW); +} + double Renderer::GetFps() { return fps_; } void Renderer::UpdateFps() { @@ -233,7 +343,10 @@ void Renderer::UpdateFps() { frames_ = 0; } } else { - fps_ = mjrf_getFrameRate(&render_context_); + mjrFrameStats stats; + mjr_defaultFrameStats(&stats); + mjrf_getFrameStats(filament_context_.get(), 0, &stats); + fps_ = stats.frame_rate; } } @@ -243,7 +356,7 @@ mjPLUGIN_LIB_INIT(renderer) { mujoco::platform::GuiPlugin plugin; plugin.name = "Filament"; plugin.update = [](mujoco::platform::GuiPlugin* self) { - mjrf_updateGui(nullptr); + mujoco::platform::PluginUpdate(self); }; mujoco::platform::RegisterPlugin(plugin); } diff --git a/src/experimental/platform/hal/renderer.h b/src/experimental/platform/hal/renderer.h index 5efee347..5f4eb3d7 100644 --- a/src/experimental/platform/hal/renderer.h +++ b/src/experimental/platform/hal/renderer.h @@ -17,12 +17,15 @@ #include #include -#include #include #include #include #include +#include "experimental/filament/compat/imgui_bridge.h" +#include "experimental/filament/compat/scene_bridge.h" +#include "experimental/filament/render_context_filament.h" +#include "experimental/filament/render_context_filament_cpp.h" #include "experimental/platform/hal/graphics_mode.h" namespace mujoco::platform { @@ -96,13 +99,24 @@ class Renderer { void UpdateFps(); + void DoRender(int width, int height); + void DoSetBuffer(int framebuffer); + void DoReadPixels(int width, int height, unsigned char* rgb); + void* native_window_ = nullptr; GraphicsMode gfx_ = GraphicsMode::FilamentVulkan; + + // State used by the classic renderer. std::shared_ptr graphics_api_context_ = nullptr; - std::function render_; - std::function set_buffer_; - std::function read_pixels_; mjrContext render_context_; + + // State used by the filament renderer. + UniquePtr filament_context_{nullptr, nullptr}; + std::unique_ptr scene_bridge_; + std::unique_ptr imgui_bridge_; + + // Common state. + int framebuffer_mode_ = 0; mjvScene scene_; bool initialized_ = false; mjtNum last_update_time_ = -1;