chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make MjrFilamentRenderer an internal detail of compat library.

Browse Source 
Renames MjrFilamentRenderer to CompatContext and makes it an
internal detail of the compatibility layer.  This allows us to
remove all legacy functions from the mjrf namespace, making
them available only in the compatibility layer.

Update studio to use non-legacy mjrf APIs. This requires it to
use some of the compat helper classes. These will be refactored
in the future to make them reusable components.

PiperOrigin-RevId: 917730783
Change-Id: Ic8f68e6e75059f7d394ea3266a346cbd6c1fcaed
This commit is contained in:
Haroon Qureshi
2026-05-19 03:25:21 -07:00
committed by Copybara-Service
parent a1056676a0
commit 303c1bd0aa
8 changed files with 362 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/experimental/filament/CMakeLists.txt
-2
View File
@@ -53,8 +53,6 @@ target_sources(${MUJOCO_FILAMENT_TARGET_NAME}
filament/texture.h
compat/imgui_bridge.cc
compat/imgui_bridge.h
compat/mjr_filament_renderer.cc
compat/mjr_filament_renderer.h
compat/model_objects.cc
compat/model_objects.h
compat/scene_bridge.cc
src/experimental/filament/compat/mjr_filament_renderer.cc
-219
View File
@@ -1,219 +0,0 @@
// Copyright 2025 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "experimental/filament/compat/mjr_filament_renderer.h"
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <memory>
#include <mujoco/mjmodel.h>
#include <mujoco/mjvisualize.h>
#include <mujoco/mujoco.h>
#include "experimental/filament/compat/imgui_bridge.h"
#include "experimental/filament/compat/scene_bridge.h"
#include "experimental/filament/filament/filament_context.h"
#include "experimental/filament/render_context_filament_cpp.h"
#include "experimental/filament/render_context_filament.h"
namespace mujoco {
MjrFilamentRenderer::MjrFilamentRenderer(const mjrFilamentConfig* config) {
filament_context_ = std::make_unique<FilamentContext>(config);
}
void MjrFilamentRenderer::Init(const mjModel* model) {
scene_bridge_ = std::make_unique<SceneBridge>(filament_context_.get(), model);
imgui_bridge_ = std::make_unique<ImguiBridge>(filament_context_.get());
scene_bridge_->SetDrawTextFunction(DrawTextAt);
}
void MjrFilamentRenderer::Render(const mjrRect& viewport,
const mjvScene* scene) {
scene_bridge_->Update(viewport, scene);
std::memcpy(render_flags_, scene->flags, mjNRNDFLAG * sizeof(mjtByte));
// Update the UX renderable entity after processing the scene in case there
// are any elements in the scene which generate UX draw calls (e.g. labels).
if (mode_ != FrameBufferMode::OffScreen) {
imgui_bridge_->Update();
}
if (mode_ == FrameBufferMode::Window) {
mjrRenderRequest reqs[2];
mjr_defaultRenderRequest(&reqs[0]);
reqs[0].scene = scene_bridge_->GetScene();
reqs[0].draw_mode = mjDRAW_MODE_COLOR;
if (render_flags_[mjRND_SEGMENT]) {
reqs[0].draw_mode = mjDRAW_MODE_SEGMENTATION;
} else if (render_flags_[mjRND_DEPTH]) {
reqs[0].draw_mode = mjDRAW_MODE_DEPTH;
} else if (render_flags_[mjRND_WIREFRAME]) {
reqs[0].draw_mode = mjDRAW_MODE_WIREFRAME;
}
reqs[0].camera = scene_bridge_->GetCamera();
reqs[0].viewport = viewport;
reqs[0].enable_shadows = render_flags_[mjRND_SHADOW];
reqs[0].enable_reflections = render_flags_[mjRND_REFLECTION];
mjr_defaultRenderRequest(&reqs[1]);
reqs[1].scene = imgui_bridge_->GetScene();
reqs[1].draw_mode = mjDRAW_MODE_COLOR;
reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
reqs[1].viewport = viewport;
reqs[1].enable_shadows = false;
reqs[1].enable_reflections = false;
reqs[1].enable_post_processing = false;
filament_context_->Render(reqs);
}
}
void MjrFilamentRenderer::SetFrameBuffer(int framebuffer) {
switch (framebuffer) {
case mjFB_WINDOW:
mode_ = FrameBufferMode::Window;
break;
case mjFB_OFFSCREEN:
mode_ = FrameBufferMode::OffScreen;
break;
case 2: // No official constant fo this.
mode_ = FrameBufferMode::OffScreenWithGui;
break;
default:
mju_error("Invalid framebuffer mode: %d", framebuffer);
}
}
void MjrFilamentRenderer::ReadPixels(mjrRect viewport, unsigned char* rgb,
float* depth) {
if (mode_ == FrameBufferMode::Window) {
mju_error("ReadPixels is only supported for offscreen rendering.");
}
mjrRenderRequest reqs[2];
mjr_defaultRenderRequest(&reqs[0]);
reqs[0].scene = scene_bridge_->GetScene();
reqs[0].draw_mode = mjDRAW_MODE_COLOR;
if (render_flags_[mjRND_SEGMENT]) {
reqs[0].draw_mode = mjDRAW_MODE_SEGMENTATION;
} else if (render_flags_[mjRND_DEPTH]) {
reqs[0].draw_mode = mjDRAW_MODE_DEPTH;
} else if (render_flags_[mjRND_WIREFRAME]) {
reqs[0].draw_mode = mjDRAW_MODE_WIREFRAME;
}
reqs[0].camera = scene_bridge_->GetCamera();
reqs[0].viewport = viewport;
reqs[0].enable_shadows = render_flags_[mjRND_SHADOW];
reqs[0].enable_reflections = render_flags_[mjRND_REFLECTION];
mjr_defaultRenderRequest(&reqs[1]);
reqs[1].scene = imgui_bridge_->GetScene();
reqs[1].draw_mode = mjDRAW_MODE_COLOR;
reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
reqs[1].viewport = viewport;
reqs[1].enable_shadows = false;
reqs[1].enable_reflections = false;
reqs[1].enable_post_processing = false;
if (rgb) {
mjrRenderTargetConfig config;
mjr_defaultRenderTargetConfig(&config);
config.width = viewport.width;
config.height = viewport.height;
config.color_format = mjPIXEL_FORMAT_RGB8;
config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
auto target = CreateRenderTarget(filament_context_.get(), config);
reqs[0].target = target.get();
reqs[1].target = target.get();
mjrReadPixelsRequest read_request;
mjr_defaultReadPixelsRequest(&read_request);
read_request.target = target.get();
read_request.output = rgb;
read_request.num_bytes = viewport.width * viewport.height * 3;
const size_t num_requests =
(mode_ == FrameBufferMode::OffScreenWithGui) ? 2 : 1;
const mjrFrameHandle frame = filament_context_->Render(
{&reqs[0], num_requests}, {&read_request, 1});
filament_context_->WaitForFrame(frame);
}
if (depth) {
mjrRenderTargetConfig config;
mjr_defaultRenderTargetConfig(&config);
config.width = viewport.width;
config.height = viewport.height;
config.color_format = mjPIXEL_FORMAT_R32F;
config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
auto target = CreateRenderTarget(filament_context_.get(), config);
reqs[0].draw_mode = mjDRAW_MODE_DEPTH;
reqs[0].target = target.get();
mjrReadPixelsRequest read_request;
mjr_defaultReadPixelsRequest(&read_request);
read_request.target = target.get();
read_request.output = reinterpret_cast<uint8_t*>(depth);
read_request.num_bytes = viewport.width * viewport.height * sizeof(float);
const mjrFrameHandle frame = filament_context_->Render(
{&reqs[0], 1}, {&read_request, 1});
filament_context_->WaitForFrame(frame);
}
}
void MjrFilamentRenderer::UploadMesh(const mjModel* model, int id) {
if (!scene_bridge_) {
mju_error("SceneBridge is not initialized.");
}
scene_bridge_->UploadMesh(model, id);
}
void MjrFilamentRenderer::UploadTexture(const mjModel* model, int id) {
if (!scene_bridge_) {
mju_error("SceneBridge is not initialized.");
}
scene_bridge_->UploadTexture(model, id);
}
void MjrFilamentRenderer::UploadHeightField(const mjModel* model, int id) {
if (!scene_bridge_) {
mju_error("SceneBridge is not initialized.");
}
scene_bridge_->UploadHeightField(model, id);
}
uintptr_t MjrFilamentRenderer::UploadGuiImage(uintptr_t tex_id,
const uint8_t* pixels, int width,
int height, int bpp) {
return imgui_bridge_->UploadImage(tex_id, pixels, width, height, bpp);
}
double MjrFilamentRenderer::GetFrameRate() const {
mjrFrameStats stats;
mjr_defaultFrameStats(&stats);
filament_context_->GetFrameStats(0, &stats);
return stats.frame_rate;
}
void MjrFilamentRenderer::UpdateGui() {
mjrf_DEBUG_drawImguiEditor(scene_bridge_->GetScene());
}
} // namespace mujoco
src/experimental/filament/compat/mjr_filament_renderer.h
-90
View File
@@ -1,90 +0,0 @@
// Copyright 2025 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_
#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_
#include <cstdint>
#include <memory>
#include <mujoco/mjmodel.h>
#include <mujoco/mjrender.h>
#include <mujoco/mjtnum.h>
#include <mujoco/mjvisualize.h>
#include "experimental/filament/compat/imgui_bridge.h"
#include "experimental/filament/compat/scene_bridge.h"
#include "experimental/filament/filament/filament_context.h"
#include "experimental/filament/render_context_filament.h"
namespace mujoco {
// Subclass of the FilamentContext that implements the legacy mjr API.
class MjrFilamentRenderer {
public:
explicit MjrFilamentRenderer(const mjrFilamentConfig* config);
~MjrFilamentRenderer() = default;
// Initializes the renderer with the given model.
void Init(const mjModel* model);
// Renders the given mjvScene to the viewport.
void Render(const mjrRect& viewport, const mjvScene* scene);
// Configures the renderer to render to the window (0) or an offscreen
// texture (1 or 2). Rendering to the window always includes UX data from
// ImGui. A value of 1 indicates the UX should not be included in the
// offscreen render, whereas 2 indicates that it should.
void SetFrameBuffer(int framebuffer);
// Renders the scene to a texture if the framebuffer is not 0.
void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth);
// Uploads the mesh data from the model to the GPU.
void UploadMesh(const mjModel* model, int id);
// Uploads the texture data from the model to the GPU.
void UploadTexture(const mjModel* model, int id);
// Uploads the height field data from the model to the GPU.
void UploadHeightField(const mjModel* model, int id);
// Uploads a texture that can be used with ImGui to the GPU.
uintptr_t UploadGuiImage(uintptr_t tex_id, const uint8_t* pixels, int width,
int height, int bpp);
// Renders an ImGui window containing Filament-specific editor UI.
void UpdateGui();
double GetFrameRate() const;
MjrFilamentRenderer(const MjrFilamentRenderer&) = delete;
MjrFilamentRenderer& operator=(const MjrFilamentRenderer&) = delete;
private:
enum class FrameBufferMode {
Window,
OffScreen,
OffScreenWithGui,
};
std::unique_ptr<FilamentContext> filament_context_;
std::unique_ptr<SceneBridge> scene_bridge_;
std::unique_ptr<ImguiBridge> imgui_bridge_;
FrameBufferMode mode_ = FrameBufferMode::Window;
mjtByte render_flags_[mjNRNDFLAG];
};
} // namespace mujoco
#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_COMPAT_MJR_FILAMENT_RENDERER_H_
src/experimental/filament/mjr_compat.cc
+178 -12
View File
@@ -12,47 +12,213 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <cstring>
#include <memory>
#include <mujoco/mujoco.h>
#include "experimental/filament/compat/scene_bridge.h"
#include "experimental/filament/render_context_filament.h"
#include "experimental/filament/render_context_filament_cpp.h"
// This library implements the entirety of mujoco's mjr API. You can link this
// library with your application (instead of the "classic" mujoco renderer) to
// use the same APIs but with Filament rendering instead.
// use the same APIs but with Filament rendering instead. Note that some
// functionality (e.g. ux-related functions like mjr_text, mjr_label, etc.)
// will call mju_error if called.
//
// However, you should consider using filament's mjrf API directly as it will
// provide you with access to more features and optimizations.
// You should consider using filament's mjrf API directly as it will provide you
// with access to more features and optimizations.
namespace mujoco {
namespace {
class CompatContext {
public:
CompatContext(const mjrFilamentConfig* config, const mjModel* model);
void Render(const mjrRect& viewport, const mjvScene* scene);
void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth);
void SetFrameBuffer(int framebuffer) {
framebuffer_ = (mjtFramebuffer)framebuffer;
}
void UploadMesh(const mjModel* model, int id) {
scene_bridge_->UploadMesh(model, id);
}
void UploadTexture(const mjModel* model, int id) {
scene_bridge_->UploadTexture(model, id);
}
void UploadHeightField(const mjModel* model, int id) {
scene_bridge_->UploadHeightField(model, id);
}
private:
mjrDrawMode draw_mode_ = mjDRAW_MODE_COLOR;
UniquePtr<mjrfContext> context_;
std::unique_ptr<SceneBridge> scene_bridge_;
mjtFramebuffer framebuffer_ = mjFB_WINDOW;
};
CompatContext::CompatContext(const mjrFilamentConfig* config,
const mjModel* model)
: context_(CreateContext(*config)) {
scene_bridge_ = std::make_unique<SceneBridge>(context_.get(), model);
}
void CompatContext::Render(const mjrRect& viewport, const mjvScene* scene) {
scene_bridge_->Update(viewport, scene);
if (scene->flags[mjRND_SEGMENT]) {
draw_mode_ = mjDRAW_MODE_SEGMENTATION;
} else if (scene->flags[mjRND_DEPTH]) {
draw_mode_ = mjDRAW_MODE_DEPTH;
} else if (scene->flags[mjRND_WIREFRAME]) {
draw_mode_ = mjDRAW_MODE_WIREFRAME;
} else {
draw_mode_ = mjDRAW_MODE_COLOR;
}
if (framebuffer_ == mjFB_WINDOW) {
mjrRenderRequest req;
mjr_defaultRenderRequest(&req);
req.scene = scene_bridge_->GetScene();
req.draw_mode = draw_mode_;
req.camera = scene_bridge_->GetCamera();
req.viewport = viewport;
mjrf_render(context_.get(), &req, 1, nullptr, 0);
}
}
void CompatContext::ReadPixels(mjrRect viewport, unsigned char* rgb,
float* depth) {
if (framebuffer_ == mjFB_WINDOW) {
mju_error("ReadPixels is only supported for offscreen rendering.");
}
if (rgb) {
mjrRenderTargetConfig config;
mjr_defaultRenderTargetConfig(&config);
config.width = viewport.width;
config.height = viewport.height;
config.color_format = mjPIXEL_FORMAT_RGB8;
config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
auto target = CreateRenderTarget(context_.get(), config);
mjrRenderRequest req;
mjr_defaultRenderRequest(&req);
req.scene = scene_bridge_->GetScene();
req.draw_mode = draw_mode_;
req.camera = scene_bridge_->GetCamera();
req.viewport = viewport;
req.target = target.get();
mjrReadPixelsRequest read_req;
mjr_defaultReadPixelsRequest(&read_req);
read_req.target = target.get();
read_req.output = rgb;
read_req.num_bytes = viewport.width * viewport.height * 3;
const mjrFrameHandle frame = mjrf_render(context_.get(), &req, 1, &read_req, 1);
mjrf_waitForFrame(context_.get(), frame);
}
if (depth) {
mjrRenderTargetConfig config;
mjr_defaultRenderTargetConfig(&config);
config.width = viewport.width;
config.height = viewport.height;
config.color_format = mjPIXEL_FORMAT_R32F;
config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
auto target = CreateRenderTarget(context_.get(), config);
mjrRenderRequest req;
mjr_defaultRenderRequest(&req);
req.scene = scene_bridge_->GetScene();
req.draw_mode = mjDRAW_MODE_DEPTH;
req.camera = scene_bridge_->GetCamera();
req.viewport = viewport;
req.target = target.get();
mjrReadPixelsRequest read_req;
mjr_defaultReadPixelsRequest(&read_req);
read_req.output = reinterpret_cast<uint8_t*>(depth);
read_req.num_bytes = viewport.width * viewport.height * sizeof(float);
const mjrFrameHandle frame = mjrf_render(context_.get(), &req, 1, &read_req, 1);
mjrf_waitForFrame(context_.get(), frame);
}
}
} // namespace
} // namespace mujoco
static thread_local mujoco::CompatContext* g_context = nullptr;
static mujoco::CompatContext* GetCheckedContext() {
if (g_context == nullptr) {
mju_error("Missing context; did you call mjr_makeContext?");
}
return g_context;
}
extern "C" {
// mjr functions that are supported by the filament renderer.
void mjr_defaultContext(mjrContext* con) {
mjrf_defaultContext(con);
memset(con, 0, sizeof(mjrContext));
}
void mjr_makeFilamentContext(const mjModel* m, const mjrFilamentConfig* cfg,
mjrContext* con) {
if (g_context != nullptr) {
mju_error("Context already exists!");
}
g_context = new mujoco::CompatContext(cfg, m);
}
void mjr_makeContext(const mjModel* m, mjrContext* con, int fontscale) {
mjrf_makeContext(m, con, fontscale);
mjr_freeContext(con);
mjrFilamentConfig cfg;
memset(&cfg, 0, sizeof(mjrFilamentConfig));
cfg.width = m->vis.global.offwidth;
cfg.height = m->vis.global.offheight;
mjr_makeFilamentContext(m, &cfg, con);
}
void mjr_freeContext(mjrContext* con) {
mjrf_freeContext(con);
// mjr_freeContext may be called multiple times.
if (g_context) {
delete g_context;
g_context = nullptr;
}
mjr_defaultContext(con);
}
void mjr_render(mjrRect viewport, mjvScene* scn, const mjrContext* con) {
mjrf_renderScene(viewport, scn, con);
GetCheckedContext()->Render(viewport, scn);
}
void mjr_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) {
mjrf_uploadMesh(m, con, meshid);
GetCheckedContext()->UploadMesh(m, meshid);
}
void mjr_uploadTexture(const mjModel* m, const mjrContext* con, int texid) {
mjrf_uploadTexture(m, con, texid);
GetCheckedContext()->UploadTexture(m, texid);
}
void mjr_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid) {
mjrf_uploadHField(m, con, hfieldid);
GetCheckedContext()->UploadHeightField(m, hfieldid);
}
void mjr_setBuffer(int framebuffer, mjrContext* con) {
mjrf_setBuffer(framebuffer, con);
GetCheckedContext()->SetFrameBuffer(framebuffer);
}
void mjr_readPixels(unsigned char* rgb, float* depth, mjrRect viewport,
const mjrContext* con) {
mjrf_readPixels(rgb, depth, viewport, con);
GetCheckedContext()->ReadPixels(viewport, rgb, depth);
}
// mjr functions that are NOT supported by the filament renderer.
src/experimental/filament/render_context_filament.cc
-102
View File
@@ -21,10 +21,7 @@
#include <math/mat3.h>
#include <math/vec3.h>
#include <mujoco/mjmodel.h>
#include <mujoco/mjrender.h>
#include <mujoco/mjvisualize.h>
#include <mujoco/mujoco.h>
#include "experimental/filament/compat/mjr_filament_renderer.h"
#include "experimental/filament/filament/filament_context.h"
#include "experimental/filament/filament/light.h"
#include "experimental/filament/filament/mesh.h"
@@ -33,19 +30,6 @@
#include "experimental/filament/filament/scene_view.h"
#include "experimental/filament/filament/texture.h"
#if defined(TLS_FILAMENT_CONTEXT)
static thread_local mujoco::MjrFilamentRenderer* g_filament_context = nullptr;
#else
static mujoco::MjrFilamentRenderer* g_filament_context = nullptr;
#endif
static void CheckFilamentContext() {
if (g_filament_context == nullptr) {
mju_error("Missing context; did you call mjrf_makeFilamentContext?");
}
}
template <int N>
static void setf(float (&arr)[N], const std::array<float, N>& values) {
for (int i = 0; i < N; ++i) {
@@ -53,7 +37,6 @@ static void setf(float (&arr)[N], const std::array<float, N>& values) {
}
}
extern "C" {
void mjrf_defaultFilamentConfig(mjrFilamentConfig* config) {
@@ -340,89 +323,4 @@ void mjrf_getFrameStats(mjrfContext* ctx, mjrFrameHandle frame,
mjrFrameStats* stats_out) {
mujoco::FilamentContext::downcast(ctx)->GetFrameStats(frame, stats_out);
}
// Legacy API, to be deprecated.
void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con,
const mjrFilamentConfig* config) {
// TODO: Support multiple contexts and multiple threads. For now, we'll just
// assume a single, global context.
if (g_filament_context != nullptr) {
mju_error("Context already exists!");
}
g_filament_context = new mujoco::MjrFilamentRenderer(config);
g_filament_context->Init(m);
}
void mjrf_defaultContext(mjrContext* con) {
memset(con, 0, sizeof(mjrContext));
}
void mjrf_makeContext(const mjModel* m, mjrContext* con, int fontscale) {
mjrf_freeContext(con);
mjrFilamentConfig cfg;
mjrf_defaultFilamentConfig(&cfg);
cfg.width = m->vis.global.offwidth;
cfg.height = m->vis.global.offheight;
mjrf_makeFilamentContext(m, con, &cfg);
}
void mjrf_freeContext(mjrContext* con) {
// mjr_freeContext may be called multiple times.
if (g_filament_context) {
delete g_filament_context;
g_filament_context = nullptr;
}
mjrf_defaultContext(con);
}
void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con) {
CheckFilamentContext();
g_filament_context->Render(viewport, scn);
}
void mjrf_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) {
CheckFilamentContext();
g_filament_context->UploadMesh(m, meshid);
}
void mjrf_uploadTexture(const mjModel* m, const mjrContext* con, int texid) {
CheckFilamentContext();
g_filament_context->UploadTexture(m, texid);
}
void mjrf_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid) {
CheckFilamentContext();
g_filament_context->UploadHeightField(m, hfieldid);
}
void mjrf_setBuffer(int framebuffer, mjrContext* con) {
CheckFilamentContext();
g_filament_context->SetFrameBuffer(framebuffer);
}
void mjrf_readPixels(unsigned char* rgb, float* depth, mjrRect viewport,
const mjrContext* con) {
CheckFilamentContext();
g_filament_context->ReadPixels(viewport, rgb, depth);
}
uintptr_t mjrf_uploadGuiImage(uintptr_t tex_id, const unsigned char* pixels,
int width, int height, int bpp,
const mjrContext* con) {
CheckFilamentContext();
return g_filament_context->UploadGuiImage(tex_id, pixels, width, height, bpp);
}
double mjrf_getFrameRate(const mjrContext* con) {
CheckFilamentContext();
return g_filament_context->GetFrameRate();
}
void mjrf_updateGui(const mjrContext* con) {
if (g_filament_context != nullptr) {
g_filament_context->UpdateGui();
}
}
} // extern "C"
src/experimental/filament/render_context_filament.h
-34
View File
@@ -747,40 +747,6 @@ void mjrf_destroyRenderTarget(mjrRenderTarget* render_target);
// settings.
void mjrf_DEBUG_drawImguiEditor(mjrScene* scene);
// Legacy API, to be deprecated.
void mjrf_defaultFilamentConfig(mjrFilamentConfig* config);
void mjrf_makeFilamentContext(const mjModel* m, mjrContext* con,
const mjrFilamentConfig* config);
void mjrf_defaultContext(mjrContext* con);
void mjrf_makeContext(const mjModel* m, mjrContext* con, int fontscale);
void mjrf_freeContext(mjrContext* con);
void mjrf_renderScene(mjrRect viewport, mjvScene* scn, const mjrContext* con);
void mjrf_uploadMesh(const mjModel* m, const mjrContext* con, int meshid);
void mjrf_uploadTexture(const mjModel* m, const mjrContext* con, int texid);
void mjrf_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid);
void mjrf_setBuffer(int framebuffer, mjrContext* con);
void mjrf_readPixels(unsigned char* rgb, float* depth, mjrRect viewport,
const mjrContext* con);
double mjrf_getFrameRate(const mjrContext* con);
uintptr_t mjrf_uploadGuiImage(uintptr_t tex_id, const unsigned char* pixels,
int width, int height, int bpp,
const mjrContext* con);
void mjrf_updateGui(const mjrContext* con);
#if defined(__cplusplus)
} // extern "C"
#endif
src/experimental/platform/hal/renderer.cc
+166 -53
View File
@@ -16,6 +16,8 @@
#include <chrono>
#include <cstddef>
#include <functional>
#include <memory>
#include <span>
#include <utility>
@@ -25,12 +27,22 @@
#if !defined(__EMSCRIPTEN__) && !defined(__APPLE__)
#include "experimental/platform/hal/egl_utils.h"
#endif
#include "experimental/filament/compat/imgui_bridge.h"
#include "experimental/filament/compat/scene_bridge.h"
#include "experimental/filament/render_context_filament.h"
#include "experimental/filament/render_context_filament_cpp.h"
#include "experimental/platform/hal/graphics_mode.h"
#include "experimental/platform/ux/plugin.h"
namespace mujoco::platform {
static std::function<void()> g_update_gui_callback = nullptr;
static void PluginUpdate(GuiPlugin* plugin) {
if (g_update_gui_callback) {
g_update_gui_callback();
}
}
static void FlipImage(unsigned char* pixels, int width, int height, int bpp) {
const int row_size = width * bpp;
for (int i = 0; i < height / 2; ++i) {
@@ -54,9 +66,16 @@ Renderer::Renderer(void* native_window, GraphicsMode gfx)
ImGui_ImplOpenGL3_Init();
}
}
g_update_gui_callback = [this]() {
if (scene_bridge_) {
mjrf_DEBUG_drawImguiEditor(scene_bridge_->GetScene());
}
};
}
Renderer::~Renderer() {
g_update_gui_callback = nullptr;
if (IsClassic(gfx_)) {
if (ImGui::GetCurrentContext()) {
ImGui_ImplOpenGL3_Shutdown();
@@ -72,38 +91,22 @@ void Renderer::Init(const mjModel* model) {
if (IsClassic(gfx_)) {
mjr_defaultContext(&render_context_);
mjr_makeContext(model, &render_context_, mjFONTSCALE_150);
render_ = [&](mjrRect rect, mjvScene* scene) {
mjr_render(rect, scene, &render_context_);
};
set_buffer_ = [&](int framebuffer) {
mjr_setBuffer(framebuffer, &render_context_);
};
read_pixels_ = [&](unsigned char* pixels, mjrRect rect) {
mjr_readPixels(pixels, nullptr, rect, &render_context_);
mjr_setBuffer(mjFB_WINDOW, &render_context_);
};
} else {
mjrf_defaultContext(&render_context_);
mjrFilamentConfig render_config;
mjrf_defaultFilamentConfig(&render_config);
render_config.native_window = native_window_;
render_config.width = model->vis.global.offwidth;
render_config.height = model->vis.global.offheight;
render_config.force_software_rendering = IsSoftware(gfx_);
render_config.graphics_api = IsOpenGl(gfx_) || IsWebGl(gfx_)
? mjGRAPHICS_API_OPENGL
: mjGRAPHICS_API_VULKAN;
mjrf_makeFilamentContext(model, &render_context_, &render_config);
render_ = [&](mjrRect rect, mjvScene* scene) {
mjrf_renderScene(rect, scene, &render_context_);
};
set_buffer_ = [&](int framebuffer) {
mjrf_setBuffer(framebuffer, &render_context_);
};
read_pixels_ = [&](unsigned char* pixels, mjrRect rect) {
mjrf_readPixels(pixels, nullptr, rect, &render_context_);
mjrf_setBuffer(mjFB_WINDOW, &render_context_);
};
mjrFilamentConfig cfg;
mjrf_defaultFilamentConfig(&cfg);
cfg.native_window = native_window_;
cfg.width = model->vis.global.offwidth;
cfg.height = model->vis.global.offheight;
cfg.force_software_rendering = IsSoftware(gfx_);
cfg.graphics_api = IsOpenGl(gfx_) || IsWebGl(gfx_)
? mjGRAPHICS_API_OPENGL
: mjGRAPHICS_API_VULKAN;
filament_context_ = CreateContext(cfg);
scene_bridge_ =
std::make_unique<SceneBridge>(filament_context_.get(), model);
imgui_bridge_ = std::make_unique<ImguiBridge>(filament_context_.get());
scene_bridge_->SetDrawTextFunction(DrawTextAt);
}
mjv_defaultScene(&scene_);
@@ -118,7 +121,9 @@ void Renderer::Deinit() {
if (IsClassic(gfx_)) {
mjr_freeContext(&render_context_);
} else {
mjrf_freeContext(&render_context_);
scene_bridge_.reset();
imgui_bridge_.reset();
filament_context_.reset();
}
initialized_ = false;
}
@@ -163,11 +168,10 @@ void Renderer::Render(const mjModel* model, mjData* data,
// The filament backend supports two offscreen framebuffers.
// mjFB_OFFSCREEN renders just the mjvScene data. +1 also includes the
// ImGui draw data.
set_buffer_(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1);
DoSetBuffer(IsClassic(gfx_) ? mjFB_OFFSCREEN : mjFB_OFFSCREEN + 1);
}
const mjrRect viewport = {0, 0, width, height};
render_(viewport, &scene_);
DoRender(width, height);
// The filament backend knows how to renders the ImGui draw data. For the
// classic backend, we need to render the ImGui draw data ourselves.
@@ -179,10 +183,7 @@ void Renderer::Render(const mjModel* model, mjData* data,
if (render_to_texture) {
unsigned char* ptr = reinterpret_cast<unsigned char*>(pixels.data());
read_pixels_(ptr, viewport);
if (IsClassic(gfx_)) {
FlipImage(ptr, width, height, 3);
}
DoReadPixels(width, height, ptr);
}
UpdateFps();
@@ -195,17 +196,12 @@ void Renderer::RenderToTexture(const mjModel* model, mjData* data,
return;
}
const mjrRect viewport = {0, 0, width, height};
mjv_updateCamera(model, data, camera, &scene_);
set_buffer_(mjFB_OFFSCREEN);
render_(viewport, &scene_);
unsigned char* ptr = reinterpret_cast<unsigned char*>(output);
read_pixels_(ptr, viewport);
if (IsClassic(gfx_)) {
FlipImage(ptr, width, height, 3);
}
DoSetBuffer(mjFB_OFFSCREEN);
DoRender(width, height);
DoReadPixels(width, height, ptr);
}
int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
@@ -213,12 +209,126 @@ int Renderer::UploadImage(int texture_id, const std::byte* pixels, int width,
if (IsClassic(gfx_)) {
return 0;
} else {
return mjrf_uploadGuiImage(texture_id,
reinterpret_cast<const unsigned char*>(pixels),
width, height, bpp, &render_context_);
return imgui_bridge_->UploadImage(
texture_id, reinterpret_cast<const unsigned char*>(pixels), width,
height, bpp);
}
}
void Renderer::DoRender(int width, int height) {
const mjrRect viewport = {0, 0, width, height};
if (IsClassic(gfx_)) {
mjr_render(viewport, &scene_, &render_context_);
} else {
scene_bridge_->Update(viewport, &scene_);
// Update the UX renderable entity after processing the scene in case there
// are any elements in the scene which generate UX draw calls (e.g. labels).
if (framebuffer_mode_ != 1) {
imgui_bridge_->Update();
}
if (framebuffer_mode_ == 0) {
mjrDrawMode draw_mode = mjDRAW_MODE_COLOR;
if (scene_.flags[mjRND_SEGMENT]) {
draw_mode = mjDRAW_MODE_SEGMENTATION;
} else if (scene_.flags[mjRND_DEPTH]) {
draw_mode = mjDRAW_MODE_DEPTH;
}
mjrRenderRequest reqs[2];
mjr_defaultRenderRequest(&reqs[0]);
reqs[0].scene = scene_bridge_->GetScene();
reqs[0].draw_mode = draw_mode;
reqs[0].camera = scene_bridge_->GetCamera();
reqs[0].viewport = viewport;
reqs[0].enable_shadows = scene_.flags[mjRND_SHADOW];
reqs[0].enable_reflections = scene_.flags[mjRND_REFLECTION];
mjr_defaultRenderRequest(&reqs[1]);
reqs[1].scene = imgui_bridge_->GetScene();
reqs[1].draw_mode = mjDRAW_MODE_COLOR;
reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
reqs[1].viewport = viewport;
reqs[1].enable_shadows = false;
reqs[1].enable_reflections = false;
reqs[1].enable_post_processing = false;
mjrf_render(filament_context_.get(), &reqs[0], 2, nullptr, 0);
}
}
}
void Renderer::DoSetBuffer(int framebuffer) {
framebuffer_mode_ = framebuffer;
if (IsClassic(gfx_)) {
mjr_setBuffer(framebuffer, &render_context_);
}
}
void Renderer::DoReadPixels(int width, int height, unsigned char* rgb) {
if (!rgb) {
return;
}
if (framebuffer_mode_ == 0) {
mju_warning("ReadPixels is only supported for offscreen rendering.");
return;
}
const mjrRect viewport = {0, 0, width, height};
if (IsClassic(gfx_)) {
mjr_readPixels(rgb, nullptr, viewport, &render_context_);
mjr_setBuffer(mjFB_WINDOW, &render_context_);
FlipImage(rgb, viewport.width, viewport.height, 3);
} else {
mjrDrawMode draw_mode = mjDRAW_MODE_COLOR;
if (scene_.flags[mjRND_SEGMENT]) {
draw_mode = mjDRAW_MODE_SEGMENTATION;
} else if (scene_.flags[mjRND_DEPTH]) {
draw_mode = mjDRAW_MODE_DEPTH;
}
mjrRenderTargetConfig config;
mjr_defaultRenderTargetConfig(&config);
config.width = viewport.width;
config.height = viewport.height;
config.color_format = mjPIXEL_FORMAT_RGB8;
config.depth_format = mjPIXEL_FORMAT_DEPTH32F;
auto target = CreateRenderTarget(filament_context_.get(), config);
mjrRenderRequest reqs[2];
mjr_defaultRenderRequest(&reqs[0]);
reqs[0].scene = scene_bridge_->GetScene();
reqs[0].draw_mode = draw_mode;
reqs[0].camera = scene_bridge_->GetCamera();
reqs[0].target = target.get();
reqs[0].viewport = viewport;
reqs[0].enable_shadows = scene_.flags[mjRND_SHADOW];
reqs[0].enable_reflections = scene_.flags[mjRND_REFLECTION];
mjr_defaultRenderRequest(&reqs[1]);
reqs[1].scene = imgui_bridge_->GetScene();
reqs[1].draw_mode = mjDRAW_MODE_COLOR;
reqs[1].camera = imgui_bridge_->GetCamera(viewport.width, viewport.height);
reqs[1].target = target.get();
reqs[1].viewport = viewport;
reqs[1].enable_shadows = false;
reqs[1].enable_reflections = false;
reqs[1].enable_post_processing = false;
mjrReadPixelsRequest read_request;
mjr_defaultReadPixelsRequest(&read_request);
read_request.target = target.get();
read_request.output = rgb;
read_request.num_bytes = viewport.width * viewport.height * 3;
const int num_requests = (framebuffer_mode_ == 2) ? 2 : 1;
const mjrFrameHandle frame = mjrf_render(
filament_context_.get(), &reqs[0], num_requests, &read_request, 1);
mjrf_waitForFrame(filament_context_.get(), frame);
}
DoSetBuffer(mjFB_WINDOW);
}
double Renderer::GetFps() { return fps_; }
void Renderer::UpdateFps() {
@@ -233,7 +343,10 @@ void Renderer::UpdateFps() {
frames_ = 0;
}
} else {
fps_ = mjrf_getFrameRate(&render_context_);
mjrFrameStats stats;
mjr_defaultFrameStats(&stats);
mjrf_getFrameStats(filament_context_.get(), 0, &stats);
fps_ = stats.frame_rate;
}
}
@@ -243,7 +356,7 @@ mjPLUGIN_LIB_INIT(renderer) {
mujoco::platform::GuiPlugin plugin;
plugin.name = "Filament";
plugin.update = [](mujoco::platform::GuiPlugin* self) {
mjrf_updateGui(nullptr);
mujoco::platform::PluginUpdate(self);
};
mujoco::platform::RegisterPlugin(plugin);
}
src/experimental/platform/hal/renderer.h
+18 -4
View File
@@ -17,12 +17,15 @@
#include <chrono>
#include <cstddef>
#include <functional>
#include <memory>
#include <ratio>
#include <span>
#include <mujoco/mujoco.h>
#include "experimental/filament/compat/imgui_bridge.h"
#include "experimental/filament/compat/scene_bridge.h"
#include "experimental/filament/render_context_filament.h"
#include "experimental/filament/render_context_filament_cpp.h"
#include "experimental/platform/hal/graphics_mode.h"
namespace mujoco::platform {
@@ -96,13 +99,24 @@ class Renderer {
void UpdateFps();
void DoRender(int width, int height);
void DoSetBuffer(int framebuffer);
void DoReadPixels(int width, int height, unsigned char* rgb);
void* native_window_ = nullptr;
GraphicsMode gfx_ = GraphicsMode::FilamentVulkan;
// State used by the classic renderer.
std::shared_ptr<void> graphics_api_context_ = nullptr;
std::function<void(mjrRect, mjvScene*)> render_;
std::function<void(int)> set_buffer_;
std::function<void(unsigned char*, mjrRect)> read_pixels_;
mjrContext render_context_;
// State used by the filament renderer.
UniquePtr<mjrfContext> filament_context_{nullptr, nullptr};
std::unique_ptr<SceneBridge> scene_bridge_;
std::unique_ptr<ImguiBridge> imgui_bridge_;
// Common state.
int framebuffer_mode_ = 0;
mjvScene scene_;
bool initialized_ = false;
mjtNum last_update_time_ = -1;