// Copyright 2025 DeepMind Technologies Limited // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "experimental/filament/filament/filament_context.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "experimental/filament/filament/filament_platform_factory.h" #include "experimental/filament/filament/gui_view.h" #include "experimental/filament/filament/imgui_editor.h" #include "experimental/filament/filament/model_util.h" #include "experimental/filament/filament/object_manager.h" #include "experimental/filament/filament/render_target.h" #include "experimental/filament/filament/scene_bridge.h" #include "experimental/filament/filament/scene_view.h" #include "experimental/filament/filament/texture.h" #include "experimental/filament/render_context_filament.h" namespace mujoco { FilamentContext::FilamentContext(const mjrFilamentConfig* config) : config_(*config) { FilamentPlatformSetup setup = CreateFilamentPlatform(config_); platform_ = std::move(setup.platform); filament::Engine::Builder engine_builder; engine_builder.backend(setup.backend); engine_builder.platform(platform_.get()); engine_builder.feature("backend.disable_parallel_shader_compile", setup.disable_parallel_shader_compile); engine_ = engine_builder.build(); renderer_ = engine_->createRenderer(); #ifdef __EMSCRIPTEN__ window_swap_chain_ = engine_->createSwapChain(nullptr); #else if (config_.native_window) { window_swap_chain_ = engine_->createSwapChain(config_.native_window); } else { window_swap_chain_ = engine_->createSwapChain(config_.width, config_.height); } #endif offscreen_swap_chain_ = engine_->createSwapChain(config_.width, config_.height); object_manager_ = std::make_unique(engine_); } FilamentContext::~FilamentContext() { DestroyRenderTargets(); gui_view_.reset(); scene_bridge_.reset(); scene_view_.reset(); object_manager_.reset(); engine_->destroy(renderer_); engine_->destroy(window_swap_chain_); engine_->destroy(offscreen_swap_chain_); filament::Engine::destroy(engine_); } void FilamentContext::Init(const mjModel* model) { scene_view_ = std::make_unique(engine_); scene_bridge_ = std::make_unique(object_manager_.get(), model, scene_view_.get()); gui_view_ = std::make_unique( engine_, object_manager_->GetMaterial(ObjectManager::kUnlitUi)); // Set clear options. filament::Renderer::ClearOptions opts; opts.clear = true; opts.discard = true; opts.clearColor = ReadElement(model, "filament.clearColor", filament::math::float4(0, 0, 0, 1)); renderer_->setClearOptions(opts); } void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene) { // If we're rendering to the window, and the window size has changed, we need // to reacquire the swap chain. if (scene_swap_chain_target_ == kWindowSwapChain && (viewport.width != window_width_ || viewport.height != window_height_)) { if (window_width_ != 0 && window_height_ != 0) { if constexpr (UTILS_HAS_THREADING) { engine_->flushAndWait(); } engine_->destroy(window_swap_chain_); window_swap_chain_ = engine_->createSwapChain(config_.native_window); } window_width_ = viewport.width; window_height_ = viewport.height; } scene_bridge_->Update(viewport, scene); // Update the UX renderable entity after processing the scene in case there // are any elements in the scene which generate UX draw calls (e.g. labels). if (gui_view_ && gui_swap_chain_target_ == scene_swap_chain_target_) { // Prepare the filament Renderable that contains the GUI draw commands. We // must call this function even if we do not plan on rendering the GUI to // ensure the ImGui state is updated. gui_view_->UpdateRenderable(); } last_render_mode_ = SceneView::DrawMode::kNormal; if (scene->flags[mjRND_SEGMENT]) { last_render_mode_ = SceneView::DrawMode::kSegmentation; } else if (scene->flags[mjRND_DEPTH]) { last_render_mode_ = SceneView::DrawMode::kDepth; } last_camera_ = mjv_averageCamera(scene->camera, scene->camera + 1); // Render the frame if we're not rendering to a texture. if (scene_swap_chain_target_ == kWindowSwapChain) { if constexpr (UTILS_HAS_THREADING) { // Wait until previous frame is completed before requesting a new frame. engine_->flushAndWait(); } if (renderer_->beginFrame(window_swap_chain_)) { SceneView::RenderRequest request; request.draw_mode = last_render_mode_; request.viewport = viewport; request.camera = last_camera_; scene_view_->Render(renderer_, request); if (gui_view_ && gui_swap_chain_target_ == kWindowSwapChain) { gui_view_->Render(renderer_); } renderer_->endFrame(); } if constexpr (!UTILS_HAS_THREADING) { engine_->execute(); } } } void FilamentContext::SetFrameBuffer(int framebuffer) { switch (framebuffer) { case mjFB_WINDOW: scene_swap_chain_target_ = kWindowSwapChain; gui_swap_chain_target_ = kWindowSwapChain; break; case mjFB_OFFSCREEN: scene_swap_chain_target_ = kOffscreenSwapChain; gui_swap_chain_target_ = kWindowSwapChain; break; case 2: // No official constant fo this. scene_swap_chain_target_ = kOffscreenSwapChain; gui_swap_chain_target_ = kOffscreenSwapChain; break; default: mju_error("Invalid framebuffer mode: %d", framebuffer); } if (framebuffer == 0) { DestroyRenderTargets(); } } void FilamentContext::PrepareRenderTargets(int width, int height) { color_target_ = std::make_unique( engine_, RenderTargetTextureType::kColor, RenderTargetTextureType::kDepth); color_target_->Prepare(width, height); depth_target_ = std::make_unique( engine_, RenderTargetTextureType::kDepthColor, RenderTargetTextureType::kDepth); depth_target_->Prepare(width, height); } void FilamentContext::DestroyRenderTargets() { depth_target_.reset(); color_target_.reset(); } static void ReadColorPixels(filament::Renderer* renderer, RenderTarget* target, mjrRect viewport, unsigned char* buffer, size_t num_bytes) { filament::backend::PixelBufferDescriptor descriptor( buffer, num_bytes, filament::backend::PixelDataFormat::RGB, filament::backend::PixelDataType::UBYTE); renderer->readPixels(target->GetFilamentRenderTarget(), viewport.left, viewport.bottom, viewport.width, viewport.height, std::move(descriptor)); } static void ReadDepthPixels(filament::Renderer* renderer, RenderTarget* target, mjrRect viewport, float* buffer, size_t num_bytes) { filament::backend::PixelBufferDescriptor descriptor( buffer, num_bytes, filament::backend::PixelDataFormat::R, filament::backend::PixelDataType::FLOAT); renderer->readPixels(target->GetFilamentRenderTarget(), viewport.left, viewport.bottom, viewport.width, viewport.height, std::move(descriptor)); } void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth) { if (scene_swap_chain_target_ != kOffscreenSwapChain) { mju_error("Cannot read pixels unless framebuffer is set."); } if (color_target_ == nullptr || depth_target_ == nullptr) { if (viewport.left != 0) { mju_error("Reading subpixels not supported."); } if (viewport.bottom != 0) { mju_error("Reading subpixels not supported."); } PrepareRenderTargets(viewport.width, viewport.height); } if (rgb) { if (renderer_->beginFrame(offscreen_swap_chain_)) { SceneView::RenderRequest request; request.draw_mode = last_render_mode_; request.viewport = viewport; request.target = color_target_.get(); request.camera = last_camera_; scene_view_->Render(renderer_, request); // Render the GUI to the texture as well if requested. if (gui_view_ && gui_swap_chain_target_ == kOffscreenSwapChain) { gui_view_->Render(renderer_, color_target_.get()); } const size_t num_bytes = viewport.width * viewport.height * 3; ReadColorPixels(renderer_, color_target_.get(), viewport, rgb, num_bytes); renderer_->endFrame(); } } if (depth) { if (renderer_->beginFrame(offscreen_swap_chain_)) { SceneView::RenderRequest request; request.draw_mode = SceneView::DrawMode::kDepth; request.viewport = viewport; request.target = depth_target_.get(); request.camera = last_camera_; scene_view_->Render(renderer_, request); const size_t num_bytes = viewport.width * viewport.height * sizeof(float); ReadDepthPixels(renderer_, depth_target_.get(), viewport, depth, num_bytes); renderer_->endFrame(); } } if (rgb || depth) { if constexpr (UTILS_HAS_THREADING) { // Wait for rendering to copy back to buffer to complete. engine_->flushAndWait(); } } } void FilamentContext::UploadMesh(const mjModel* model, int id) { if (!scene_bridge_) { mju_error("SceneBridge is not initialized."); } scene_bridge_->UploadMesh(model, id); } void FilamentContext::UploadTexture(const mjModel* model, int id) { if (!scene_bridge_) { mju_error("SceneBridge is not initialized."); } scene_bridge_->UploadTexture(model, id); } void FilamentContext::UploadHeightField(const mjModel* model, int id) { if (!scene_bridge_) { mju_error("SceneBridge is not initialized."); } scene_bridge_->UploadHeightField(model, id); } uintptr_t FilamentContext::UploadGuiImage(uintptr_t tex_id, const uint8_t* pixels, int width, int height, int bpp) { if (gui_view_) { return gui_view_->UploadImage(tex_id, pixels, width, height, bpp); } return 0; } double FilamentContext::GetFrameRate() const { utils::FixedCapacityVector frame_info = renderer_->getFrameInfoHistory(1); if (frame_info.empty()) { return 0; } const int64_t ns = frame_info[0].denoisedGpuFrameDuration; return 1.0e9 / static_cast(ns); } void FilamentContext::UpdateGui() { DrawGui(scene_bridge_.get()); } } // namespace mujoco