diff --git a/src/experimental/filament/compat/imgui_bridge.cc b/src/experimental/filament/compat/imgui_bridge.cc index 461ef955..9af52b74 100644 --- a/src/experimental/filament/compat/imgui_bridge.cc +++ b/src/experimental/filament/compat/imgui_bridge.cc @@ -302,6 +302,29 @@ void ImguiBridge::PrepareRenderables(int count) { } } +mjrScene* ImguiBridge::GetScene() const { return scene_.get(); } + +mjrCamera ImguiBridge::GetCamera(int width, int height) const { + mjrCamera camera; + camera.orthographic = true; + camera.pos[0] = 0.0f; + camera.pos[1] = 0.0f; + camera.pos[2] = 1.0f; + camera.forward[0] = 0.0f; + camera.forward[1] = 0.0f; + camera.forward[2] = -1.0f; + camera.up[0] = 0.0f; + camera.up[1] = 1.0f; + camera.up[2] = 0.0f; + camera.frustum_top = 0.0f; + camera.frustum_near = 0.0f; + camera.frustum_far = 1.0f; + camera.frustum_center = width / 2.0f; + camera.frustum_width = width / 2.0f; + camera.frustum_bottom = height; + return camera; +} + static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) { const ImVec2& display_size = ImGui::GetIO().DisplaySize; const float pos_x = display_size.x * ((x + 1) * 0.5f); diff --git a/src/experimental/filament/compat/imgui_bridge.h b/src/experimental/filament/compat/imgui_bridge.h index ac479292..9625018b 100644 --- a/src/experimental/filament/compat/imgui_bridge.h +++ b/src/experimental/filament/compat/imgui_bridge.h @@ -37,7 +37,8 @@ class ImguiBridge { void Update(); // Returns the managed UX scene. - mjrScene* GetScene() const { return scene_.get(); } + mjrScene* GetScene() const; + mjrCamera GetCamera(int width, int height) const; // Uploads texture to be used with ImGui's Image and ImageButton functions. uintptr_t UploadImage(uintptr_t tex_id, const uint8_t* pixels, int width, diff --git a/src/experimental/filament/compat/mjr_filament_renderer.cc b/src/experimental/filament/compat/mjr_filament_renderer.cc index a23de31b..9331f93c 100644 --- a/src/experimental/filament/compat/mjr_filament_renderer.cc +++ b/src/experimental/filament/compat/mjr_filament_renderer.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -42,31 +41,6 @@ void MjrFilamentRenderer::Init(const mjModel* model) { scene_bridge_ = std::make_unique(filament_context_.get(), model); imgui_bridge_ = std::make_unique(filament_context_.get()); scene_bridge_->SetDrawTextFunction(DrawTextAt); - - mjr_defaultRenderRequest(&render_requests_[0]); - mjr_defaultRenderRequest(&render_requests_[1]); - - render_requests_[0].scene = scene_bridge_->GetScene(); - render_requests_[0].draw_mode = mjDRAW_MODE_COLOR; - - render_requests_[1].scene = imgui_bridge_->GetScene(); - render_requests_[1].draw_mode = mjDRAW_MODE_COLOR; - - // The UX camera is a fixed orthographic camera. We only need to change the - // width/height based on the viewport per frame. - render_requests_[1].camera.orthographic = true; - render_requests_[1].camera.pos[0] = 0.0f; - render_requests_[1].camera.pos[1] = 0.0f; - render_requests_[1].camera.pos[2] = 1.0f; - render_requests_[1].camera.forward[0] = 0.0f; - render_requests_[1].camera.forward[1] = 0.0f; - render_requests_[1].camera.forward[2] = -1.0f; - render_requests_[1].camera.up[0] = 0.0f; - render_requests_[1].camera.up[1] = 1.0f; - render_requests_[1].camera.up[2] = 0.0f; - render_requests_[1].camera.frustum_top = 0.0f; - render_requests_[1].camera.frustum_near = 0.0f; - render_requests_[1].camera.frustum_far = 1.0f; } void MjrFilamentRenderer::Render(const mjrRect& viewport, @@ -78,29 +52,22 @@ void MjrFilamentRenderer::Render(const mjrRect& viewport, imgui_bridge_->Update(); } - if (scene->flags[mjRND_SEGMENT]) { - render_requests_[0].draw_mode = mjDRAW_MODE_SEGMENTATION; - } else if (scene->flags[mjRND_DEPTH]) { - render_requests_[0].draw_mode = mjDRAW_MODE_DEPTH; - } else { - render_requests_[0].draw_mode = mjDRAW_MODE_COLOR; - } - - render_requests_[0].width = viewport.width; - render_requests_[0].height = viewport.height; - render_requests_[1].width = viewport.width; - render_requests_[1].height = viewport.height; - - render_requests_[0].camera = - mjv_averageCamera(scene->camera, scene->camera + 1); - render_requests_[1].camera.frustum_center = viewport.width / 2.0f; - render_requests_[1].camera.frustum_width = viewport.width / 2.0f; - render_requests_[1].camera.frustum_bottom = viewport.height; - if (mode_ == FrameBufferMode::Window) { - render_requests_[0].target = nullptr; - render_requests_[1].target = nullptr; - filament_context_->Render(render_requests_); + mjrRenderRequest reqs[2]; + mjr_defaultRenderRequest(&reqs[0]); + reqs[0].scene = scene_bridge_->GetScene(); + reqs[0].draw_mode = scene_bridge_->GetDrawMode(); + reqs[0].camera = scene_bridge_->GetCamera(); + reqs[0].width = viewport.width; + reqs[0].height = viewport.height; + + mjr_defaultRenderRequest(&reqs[1]); + reqs[1].scene = imgui_bridge_->GetScene(); + reqs[1].draw_mode = mjDRAW_MODE_COLOR; + reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); + reqs[1].width = viewport.width; + reqs[1].height = viewport.height; + filament_context_->Render(reqs); } } @@ -126,10 +93,21 @@ void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb, mju_error("ReadPixels is only supported for offscreen rendering."); } - render_requests_[0].width = viewport.width; - render_requests_[0].height = viewport.height; - render_requests_[1].width = viewport.width; - render_requests_[1].height = viewport.height; + mjrRenderRequest reqs[2]; + mjr_defaultRenderRequest(&reqs[0]); + + reqs[0].scene = scene_bridge_->GetScene(); + reqs[0].draw_mode = scene_bridge_->GetDrawMode(); + reqs[0].camera = scene_bridge_->GetCamera(); + reqs[0].width = viewport.width; + reqs[0].height = viewport.height; + + mjr_defaultRenderRequest(&reqs[1]); + reqs[1].scene = imgui_bridge_->GetScene(); + reqs[1].draw_mode = mjDRAW_MODE_COLOR; + reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height); + reqs[1].width = viewport.width; + reqs[1].height = viewport.height; if (rgb) { mjrRenderTargetConfig config; @@ -139,22 +117,21 @@ void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb, config.color_format = mjPIXEL_FORMAT_RGB8; config.depth_format = mjPIXEL_FORMAT_DEPTH32F; auto target = CreateRenderTarget(filament_context_.get(), config); - render_requests_[0].target = target.get(); - render_requests_[1].target = target.get(); - const size_t num_requests = - (mode_ == FrameBufferMode::OffScreenWithGui) ? 2 : 1; + reqs[0].target = target.get(); + reqs[1].target = target.get(); mjrReadPixelsRequest read_request; mjr_defaultReadPixelsRequest(&read_request); + read_request.target = target.get(); read_request.output = rgb; read_request.num_bytes = viewport.width * viewport.height * 3; - const mjrFrameHandle frame = filament_context_->Render( - {&render_requests_[0], num_requests}, {&read_request, 1}); - filament_context_->WaitForFrame(frame); - render_requests_[0].target = nullptr; - render_requests_[1].target = nullptr; + const size_t num_requests = + (mode_ == FrameBufferMode::OffScreenWithGui) ? 2 : 1; + const mjrFrameHandle frame = filament_context_->Render( + {&reqs[0], num_requests}, {&read_request, 1}); + filament_context_->WaitForFrame(frame); } if (depth) { @@ -165,23 +142,19 @@ void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb, config.color_format = mjPIXEL_FORMAT_R32F; config.depth_format = mjPIXEL_FORMAT_DEPTH32F; auto target = CreateRenderTarget(filament_context_.get(), config); - render_requests_[0].target = target.get(); - render_requests_[1].target = target.get(); - mjrDrawMode last_draw_mode = render_requests_[0].draw_mode; - render_requests_[0].draw_mode = mjDRAW_MODE_DEPTH; + reqs[0].draw_mode = mjDRAW_MODE_DEPTH; + reqs[0].target = target.get(); mjrReadPixelsRequest read_request; mjr_defaultReadPixelsRequest(&read_request); + read_request.target = target.get(); read_request.output = reinterpret_cast(depth); read_request.num_bytes = viewport.width * viewport.height * sizeof(float); - const mjrFrameHandle frame = filament_context_->Render( - {&render_requests_[0], 1}, {&read_request, 1}); - filament_context_->WaitForFrame(frame); - render_requests_[0].target = nullptr; - render_requests_[1].target = nullptr; - render_requests_[0].draw_mode = last_draw_mode; + const mjrFrameHandle frame = filament_context_->Render( + {&reqs[0], 1}, {&read_request, 1}); + filament_context_->WaitForFrame(frame); } } diff --git a/src/experimental/filament/compat/mjr_filament_renderer.h b/src/experimental/filament/compat/mjr_filament_renderer.h index ab77f2c8..f1f2f69d 100644 --- a/src/experimental/filament/compat/mjr_filament_renderer.h +++ b/src/experimental/filament/compat/mjr_filament_renderer.h @@ -80,10 +80,9 @@ class MjrFilamentRenderer { }; std::unique_ptr filament_context_; - FrameBufferMode mode_ = FrameBufferMode::Window; - mjrRenderRequest render_requests_[2]; std::unique_ptr scene_bridge_; std::unique_ptr imgui_bridge_; + FrameBufferMode mode_ = FrameBufferMode::Window; }; } // namespace mujoco diff --git a/src/experimental/filament/compat/scene_bridge.cc b/src/experimental/filament/compat/scene_bridge.cc index a7e6a5e7..ad6d1518 100644 --- a/src/experimental/filament/compat/scene_bridge.cc +++ b/src/experimental/filament/compat/scene_bridge.cc @@ -252,7 +252,7 @@ void SceneBridge::PrepareLights() { mjrf_setSceneSkybox(scene_.get(), model_objects_->GetSkyboxTexture()); } -mat4 CalculateClipFromWorld(const mjrRect& viewport, const mjvGLCamera& cam) { +mat4 CalculateClipFromWorld(const mjrRect& viewport, const mjrCamera& cam) { const float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); const float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); const float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); @@ -282,6 +282,13 @@ mat4 CalculateClipFromWorld(const mjrRect& viewport, const mjvGLCamera& cam) { void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) { mjrf_setSceneShadowsEnabled(scene_.get(), scene->flags[mjRND_SHADOW]); mjrf_setSceneReflectionsEnabled(scene_.get(), scene->flags[mjRND_REFLECTION]); + if (scene->flags[mjRND_SEGMENT]) { + draw_mode_ = mjDRAW_MODE_SEGMENTATION; + } else if (scene->flags[mjRND_DEPTH]) { + draw_mode_ = mjDRAW_MODE_DEPTH; + } else { + draw_mode_ = mjDRAW_MODE_COLOR; + } mjtNum hpos[3], hfwd[3]; float headpos[3], gazedir[3]; @@ -289,9 +296,8 @@ void SceneBridge::Update(const mjrRect& viewport, const mjvScene* scene) { mju_n2f(headpos, hpos, 3); mju_n2f(gazedir, hfwd, 3); - const mjvGLCamera gl_camera = - mjv_averageCamera(scene->camera, scene->camera + 1); - clip_from_world_ = CalculateClipFromWorld(viewport, gl_camera); + camera_ = mjv_averageCamera(scene->camera, scene->camera + 1); + clip_from_world_ = CalculateClipFromWorld(viewport, camera_); // Remove all drawables from previous render and prepare new ones. for (auto& iter : renderables_) { @@ -365,4 +371,10 @@ void SceneBridge::SetDrawTextFunction(DrawTextAtFn fn) { draw_text_callback_ = std::move(fn); } +mjrScene* SceneBridge::GetScene() const { return scene_.get(); } + +mjrCamera SceneBridge::GetCamera() const { return camera_; } + +mjrDrawMode SceneBridge::GetDrawMode() const { return draw_mode_; } + } // namespace mujoco diff --git a/src/experimental/filament/compat/scene_bridge.h b/src/experimental/filament/compat/scene_bridge.h index 3de3f2f0..0f04e402 100644 --- a/src/experimental/filament/compat/scene_bridge.h +++ b/src/experimental/filament/compat/scene_bridge.h @@ -56,7 +56,9 @@ class SceneBridge { void SetDrawTextFunction(DrawTextAtFn fn); // Returns the managed scene. - mjrScene* GetScene() const { return scene_.get(); } + mjrScene* GetScene() const; + mjrCamera GetCamera() const; + mjrDrawMode GetDrawMode() const; SceneBridge(const SceneBridge&) = delete; SceneBridge& operator=(const SceneBridge&) = delete; @@ -71,6 +73,8 @@ class SceneBridge { mjrfContext* ctx_ = nullptr; std::unique_ptr model_objects_; + mjrCamera camera_; + mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR; DrawTextAtFn draw_text_callback_; UniquePtr scene_{nullptr, nullptr}; UniquePtr fallback_ibl_{nullptr, nullptr}; diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc index 8546940b..4ddd103b 100644 --- a/src/experimental/filament/filament/scene_view.cc +++ b/src/experimental/filament/filament/scene_view.cc @@ -76,7 +76,7 @@ static filament::ColorGrading::Builder ToBuilder( .curves(opts.shadow_gamma, opts.mid_point, opts.highlight_scale); } -static void SetupCamera(const mjvGLCamera& cam, +static void SetupCamera(const mjrCamera& cam, const filament::Viewport& viewport, filament::Camera* camera) { const filament::Camera::Projection type = diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h index 0187ee0f..1522fe3c 100644 --- a/src/experimental/filament/filament/scene_view.h +++ b/src/experimental/filament/filament/scene_view.h @@ -63,7 +63,7 @@ class SceneView : public mjrScene { // The target viewport for the rendered image. mjrRect viewport; // The camera from which to render the scene. - mjvGLCamera camera; + mjrCamera camera; // An optional render target into which the scene will be rendered. RenderTarget* target = nullptr; }; diff --git a/src/experimental/filament/render_context_filament.h b/src/experimental/filament/render_context_filament.h index 04074d97..fadc10ca 100644 --- a/src/experimental/filament/render_context_filament.h +++ b/src/experimental/filament/render_context_filament.h @@ -122,6 +122,7 @@ typedef std::uint64_t mjrFrameHandle; typedef mjtTexture mjrTextureTarget; typedef mjtColorSpace mjrColorSpace; typedef mjtLightType mjrLightType; +typedef mjvGLCamera mjrCamera; // The textures that can be assigned to the drawable's material. struct mjrMaterialTextures { @@ -367,7 +368,7 @@ struct mjrRenderRequest { mjrDrawMode draw_mode; // The camera from which to render the scene. - mjvGLCamera camera; + mjrCamera camera; // The dimensions of the output image. int width;