Move mjrfilament.h to mujoco/include.

PiperOrigin-RevId: 934931460
Change-Id: I63bf34b3a7197b277ea7a5caf3d84d840f6210a1
This commit is contained in:
Haroon Qureshi
2026-06-19 08:00:10 -07:00
committed by Copybara-Service
parent f663139256
commit d93c102190
45 changed files with 82 additions and 79 deletions
+494
View File
@@ -0,0 +1,494 @@
// Copyright 2025 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MUJOCO_MJRFILAMENT_H_
#define MUJOCO_MJRFILAMENT_H_
#include <stdint.h>
#include <mujoco/mjmodel.h>
#include <mujoco/mjrender.h>
#include <mujoco/mjtype.h>
#if defined(__cplusplus)
extern "C" {
#endif
// IMPORTANT: This API should still be considered experimental and is likely change frequently.
// This library provides a C API for the filament rendering library (github.com/google/filament)
// that is designed to work with the MuJoCo library for visualizing simulations.
//
// The filament renderer is a real-time physically based rendering (PBR) engine developed by Google.
// It is designed to be as small as possible and as efficient as possible, while still providing
// high-quality results. It works across all major platforms (Linux, Windows, macOS, Android, iOS,
// Web) and supports OpenGL, Vulkan, and Metal.
//
// For the purposes of this API, we assume the reader has a basic understanding of rendering
// concepts (e.g. textures, vertices, cameras, framebuffers, etc.). We will also highlight some of
// the key differences between this renderer and the legacy/classic MuJoCo (mjr) renderer.
//
// ## API Overview
//
// There are seven key components: Context, Texture, Mesh, Scene, Light, Renderable, and
// RenderTarget. We'll describe these in detail further below.
//
// Each object is created using a `create` function and destroyed using a `destroy` function, e.g.
// `mjrf_createTexture` and `mjrf_destroyTexture`. The `create` functions accept a pointer to a
// configuration struct (e.g. `mjrTextureConfig`) which describes the parameters for the object to
// be created. Each of these structs has a corresponding `default` function (e.g.
// `mjrf_defaultTextureConfig`) which can be used to initialize the struct to default values. Default
// values are assumed to be 0/NULL unless otherwise specified.
//
// For now, we'll just define opaque handles for each of our components.
typedef struct mjrfContext_ {} mjrfContext;
typedef struct mjrfTexture_ {} mjrfTexture;
typedef struct mjrfMesh_ {} mjrfMesh;
typedef struct mjrfScene_ {} mjrfScene;
typedef struct mjrfLight_ {} mjrfLight;
typedef struct mjrfRenderable_ {} mjrfRenderable;
typedef struct mjrfRenderTarget_ {} mjrfRenderTarget;
// ## Rendering Context (mjrfContext)
//
// The Context is the main entry point for the library. It manages all the core filament objects
// that are responsible for the rendering of an image.
//
// All other objects (e.g. Textures, Meshes, Scenes, etc.) need a Context in order to be created.
// Otherwise, the main function to use with the Context is `mjrf_render()` which does the actual
// rendering.
//
// Filament uses a separate thread for doing the actual rendering. However, despite that, this API
// is not thread-safe; calls are expected to be made from a single thread. Also, due to the
// asynchronous nature of filament, some APIs provide handles or callbacks to signal when an
// operation is complete. (Note: for WASM builds, filament does not use a separate thread.)
//
// There are two key differences between the mjrfContext and the classic mjrContext. Firstly, the
// filament context will manage the underlying graphics context itself. This means users do not need
// to initialize EGL or similar libraries beforehand. Secondly, the filament context is independent
// of a MuJoCo model. That means you can use a single mjrfContext to render images for multiple
// models.
// Callback function type for rendering operations.
typedef void (*mjrfCallback)(void* user_data);
typedef enum mjrGraphicsApi_ { // underlying graphics API to use for rendering
mjGRAPHICS_API_DEFAULT = 0, // default (platform-dependent)
mjGRAPHICS_API_OPENGL, // desktop, mobile (GLES), web (WebGL)
mjGRAPHICS_API_VULKAN, // vulkan
} mjrGraphicsApi;
struct mjrfContextConfig_ { // parameters for creating filament context (mjrfContext)
int graphics_api; // mjrGraphicsApi; rendering graphics API
mjtBool force_software_rendering; // force backend to use software rendering
void* native_window; // platform-dependent window handle (or nullptr for windowless)
};
typedef struct mjrfContextConfig_ mjrfContextConfig;
// Initializes the mjrfContextConfig to default values.
void mjrf_defaultContextConfig(mjrfContextConfig* config);
// Creates a filament rendering context.
mjrfContext* mjrf_createContext(const mjrfContextConfig* config);
// Destroys the filament rendering context.
void mjrf_destroyContext(mjrfContext* ctx);
typedef enum mjrDrawMode_ { // how to draw objects in the scene
mjDRAW_MODE_DEFAULT, // default colors and lighting
mjDRAW_MODE_DEFAULT_NO_TEXTURES, // default, but without textures
mjDRAW_MODE_WIREFRAME, // wireframe rendering
mjDRAW_MODE_DEPTH, // grayscale depth map
mjDRAW_MODE_ISLANDS, // color objects based on island and sleep state
mjDRAW_MODE_SEGMENTATION_BY_ID, // color objects based on segmentation id
mjDRAW_MODE_SEGMENTATION_BY_COLOR, // generate visually distinct colors using segmentation id
} mjrDrawMode;
struct mjrfRenderRequest_ { // a single rendering operation
mjrfScene* scene; // scene to render
mjrCamera camera; // camera (viewpoint) from which to render scene
mjrRect viewport; // viewport (rect area) into which to render
mjrfRenderTarget* target; // target used for rendering (or nullptr for window rendering)
int draw_mode; // mjrDrawMode; method to use for drawing objects
mjtBool enable_post_processing; // enable post processing, enabled by default
mjtBool enable_reflections; // enable reflections, enabled by default
mjtBool enable_shadows; // enable shadows, enabled by default
};
typedef struct mjrfRenderRequest_ mjrfRenderRequest;
// Initializes the mjrfRenderRequest to default values.
void mjrf_defaultRenderRequest(mjrfRenderRequest* request);
struct mjrfReadPixelsRequest_ { // a single read operation
mjrfRenderTarget* target; // render target from which to read the image pixels
void* output; // buffer into which the pixels will be stored
mjtSize num_bytes; // size of output buffer
mjrfCallback read_completed; // callback when read is complete; can use to free output
void* user_data; // user data for read_completed_callback
};
typedef struct mjrfReadPixelsRequest_ mjrfReadPixelsRequest;
// Initializes the mjrfReadPixelsRequest to default values.
void mjrf_defaultReadPixelsRequest(mjrfReadPixelsRequest* request);
// Unique handle assigned to each render request; used to block until request is completed or query
// the status of the request.
typedef uint64_t mjrfFrameHandle;
// Submits the given requests for rendering. Because rendering happens asynchronously, callers have
// to submit both the render and read requests in the same call. Multiple requests and reads can be
// submitted in a single call. These requests will be processed in order, so some care must be
// taken. Firstly, requests should be grouped by target. Next, the combined area of the viewports
// for all requests for a given target must be contained within the dimensions of the target itself.
//
// Callbacks will be invoked from within this function, though there is no guarantee on when exactly
// that will be done.
mjrfFrameHandle mjrf_render(mjrfContext* ctx, const mjrfRenderRequest* req, int nreq,
const mjrfReadPixelsRequest* read_req, int nread_req);
// Waits for all rendering operations to complete for the given frame handle,
// triggering any callbacks as needed.
void mjrf_waitForFrame(mjrfContext* ctx, mjrfFrameHandle frame);
// Sets the clear color for the renderer.
void mjrf_setClearColor(mjrfContext* ctx, const float color[3]);
struct mjrfFrameStats_ { // stats for a single frame of rendering
double frame_rate; // frame rate, in frames per second
};
typedef struct mjrfFrameStats_ mjrfFrameStats;
// Initializes the mjrFrameStats to default values.
void mjrf_defaultFrameStats(mjrfFrameStats* stats);
// Returns the stats for the given frame but updating the given `stats_out`.
void mjrf_getFrameStats(mjrfContext* ctx, mjrfFrameHandle frame, mjrfFrameStats* stats_out);
// ## Textures (mjrfTexture)
//
// A texture is a 2D or 3D (cubemap) image that adds visual detail to a rendered model, such as
// color or bumpiness, without increasing geometric complexity.
//
// For textures intended to be used for image-based lights (see `mjrfLight` below), you should use
// filament's `cmgen` tool to generate a KTX image from your source image. This tool will calculate
// additional data (i.e. the spherical harmonics) and encode that information into the KTX file.
struct mjrfTextureConfig_ { // parameters for creating a texture (mjrfTexture)
int width; // texture width; or number of bytes for compressed data (e.g. KTX)
int height; // texture height; or 0 for compressed data (e.g. KTX)
int format; // mjrPixelFormat; (e.g. RGB8, RGBA8, KTX, etc.)
int color_space; // mjrColorSpace; (e.g. LINEAR, sRGB, etc.)
int sampler_type; // mjrSamplerType; texture sampler (e.g. 2D, cube, etc.)
};
typedef struct mjrfTextureConfig_ mjrfTextureConfig;
// Initializes the mjrfTextureConfig to default values.
void mjrf_defaultTextureConfig(mjrfTextureConfig* config);
// Creates a filament texture. Note that the texture will not be created on the
// GPU until `mjrf_setTextureData()` is called.
mjrfTexture* mjrf_createTexture(mjrfContext* ctx,
const mjrfTextureConfig* config);
// Destroys the texture.
void mjrf_destroyTexture(mjrfTexture* texture);
struct mjrfTextureData_ { // binary data for a texture (mjrfTexture)
const void* bytes; // pointer to image data, or nullptr for empty texture
mjtSize num_bytes; // number of bytes in the image data
mjrfCallback release; // callback when data has finished uploading
void* user_data; // user data for release callback
};
typedef struct mjrfTextureData_ mjrfTextureData;
// Initializes the mjrfTextureData to default values.
void mjrf_defaultTextureData(mjrfTextureData* data);
// Uploads the given texture data to the texture.
void mjrf_setTextureData(mjrfTexture* texture, const mjrfTextureData* data);
// Returns the width of the texture.
int mjrf_getTextureWidth(const mjrfTexture* texture);
// Returns the height of the texture.
int mjrf_getTextureHeight(const mjrfTexture* texture);
// Returns the sampler type (mjrSamplerType) used by the texture.
int mjrf_getSamplerType(const mjrfTexture* texture);
// ## Meshes (mjrfMesh)
//
// A mesh describes the surface geometry of an object to be rendered. It is defined as a collection
// of vertices (i.e. a VertexBuffer), a set of indices (i.e. an IndexBuffer) that describes the
// order in which the vertices should be processed, and a primitive type that defined how the
// vertices are to be interpreted (e.g. triangles, lines, etc.) when rendering the surface.
//
// Filament does not directly support normals. Instead, it encodes the normal, tangent, and
// bitangent into a 4-component quaternion describing the "orientation" of the vertex. Ideally, you
// should preprocess your assets to generate this data offline, but we will compute it on the fly if
// needed (at a performance cost).
//
// Vertex data may or may not be interleaved. Interleaved data assumes that the attributes are
// packed in the order specified in the attributes array, with no padding in-between. Additionally,
// the `data` pointer for each attribute is assumed to point to the first element of that type. For
// non-interleaved data, each attribute is assumed to be stored in a separate array.
//
// Additionally, the bounds of the mesh should be computed in order to allow the filament renderer
// to perform frustum-based culling. Alternatively, the bounds can be computed at runtime (though
// there is a small performance cost). If no bounds are provided (or calculated), then frustum
// culling will not be performed.
// Maximum number of vertex attributes in a mesh.
enum { mjMAX_VERTEX_ATTRIBUTES = 16 };
struct mjrfMeshData_ { // binary data for a mesh (mjrfMesh)
mjtSize num_vertices; // number of vertices; all vertex attributes share this size
int num_attributes; // number of attributes defined
mjrVertexAttribute attributes[mjMAX_VERTEX_ATTRIBUTES]; // per-vertex attribute information
mjtBool interleaved; // true if vertex attributes are interleaved
mjtSize num_indices; // number of indices
const void* indices; // indices data array
int index_type; // mjrIndexType; (e.g. UINT16 or UINT32)
int primitive_type; // mjrMeshPrimitiveType; (e.g. TRIANGLES, etc.)
mjtBool compute_bounds; // if true, compute bounds from vertex positions
float bounds_min[3]; // min/max bounds; assume unset if bounds_min == bounds_max
float bounds_max[3];
mjrfCallback release; // callback when data has finished uploading
void* user_data; // user data for release callback
};
typedef struct mjrfMeshData_ mjrfMeshData;
// Initializes the mjrfMeshData to default values.
void mjrf_defaultMeshData(mjrfMeshData* data);
// Creates a mesh with the given data.
mjrfMesh* mjrf_createMesh(mjrfContext* ctx, const mjrfMeshData* data);
// Destroys the mesh.
void mjrf_destroyMesh(mjrfMesh* mesh);
// ## Scenes (mjrfScene)
//
// A scene is a collection of entities (Lights and Renderables) that describes what is to be
// rendered.
struct mjrfSceneParams_ { // parameters for creating a scene (mjrfScene)
};
typedef struct mjrfSceneParams_ mjrfSceneParams;
// Initializes the mjrfSceneParams to default values.
void mjrf_defaultSceneParams(mjrfSceneParams* params);
// Creates a scene with the given parameters.
mjrfScene* mjrf_createScene(mjrfContext* ctx, const mjrfSceneParams* params);
// Destroys the scene.
void mjrf_destroyScene(mjrfScene* scene);
// Adds a light to the scene.
void mjrf_addLightToScene(mjrfScene* scene, mjrfLight* light);
// Removes the light from the scene.
void mjrf_removeLightFromScene(mjrfScene* scene, mjrfLight* light);
// Adds a renderable to the scene.
void mjrf_addRenderableToScene(mjrfScene* scene, mjrfRenderable* renderable);
// Removes the renderable from the scene.
void mjrf_removeRenderableFromScene(mjrfScene* scene, mjrfRenderable* renderable);
// Sets the skybox (cube texture) for the scene.
void mjrf_setSceneSkybox(mjrfScene* scene, const mjrfTexture* texture);
// Configures the scene based on the parameters in an mjModel.
void mjrf_configureSceneFromModel(mjrfScene* scene, const mjModel* model);
// ## Lights (mjrfLight)
//
// A light is a source of illumination in the scene. (Without lights, a scene will be completely
// black.) There are several different types of lights such as directional, spot, point, and image
// lights.
//
// The primary light in a scene is the image light (also sometimes known as the environment light).
// This is a light that "surrounds" the entire scene and is defined as a 3D texture. Each "pixel" of
// the cubemap is interpreted as the color of projected into the scene from a particular direction.
//
// Directional lights are the next most common type of light and is usually used to simulate the
// sun; a uniformly colored light that is emitted in a single direction.
//
// Filament only supports a single image and directional light. You can define as many point or spot
// lights as you want. Each light source (except image based lights) may or may not cast shadows.
// Each shadow-casting light incurs a performance cost.
struct mjrfLightParams_ { // parameters for creating a light (mjrfLight)
int type; // mjrLightType; type of light (e.g. spot, point, image, etc.)
const mjrfTexture* texture; // texture; only for image lights
float color[3]; // RGB color
float intensity; // light intensity, in candela
mjtBool cast_shadows; // if true, cast shadows
float range; // effective range of light, in meters
float spot_cone_angle; // spot light cone angle, in degrees
int shadow_map_size; // size of shadow map texture, 0 to use default size
float bulb_radius; // bulb radius, used for soft shadows
float vsm_blur_width; // variance shadow map blur width
};
typedef struct mjrfLightParams_ mjrfLightParams;
// Initializes the mjrfLightParams to default values.
void mjrf_defaultLightParams(mjrfLightParams* params);
// Creates a light for the filament renderer.
mjrfLight* mjrf_createLight(mjrfContext* ctx, const mjrfLightParams* params);
// Destroys the light.
void mjrf_destroyLight(mjrfLight* light);
// Enables or disables the light.
void mjrf_setLightEnabled(mjrfLight* light, mjtBool enabled);
// Sets the intensity of the light, in candela.
void mjrf_setLightIntensity(mjrfLight* light, float intensity);
// Sets the RGB color of the light.
void mjrf_setLightColor(mjrfLight* light, const float color[3]);
// Sets the position and direction of the light.
void mjrf_setLightTransform(mjrfLight* light, const float position[3], const float direction[3]);
// Returns the type of the light (mjrLightType).
int mjrf_getLightType(const mjrfLight* light);
// ## Renderables (mjrfRenderable)
//
// A renderable is a single drawable object in the scene. It is defined as a combination of a mesh
// (i.e. surface geometry) and a material (i.e. surface appearance and properties).
//
// In terms of materials, there are three lighting models currently supported:
//
// 1. Metallic-roughness (PBR): this is the preferred model for rendering models based standard
// metallic-roughness workflows.
// 2. Specular-glossiness (non-PBR): this is a legacy model designed to be compatible with classic
// mjr renderer, though it is not 100% identical.
// 3. Unlit: this model ignores lighting and used for rendering UX or decorative elements like
// contact forces and labels.
//
// Which lighting model is used is determined by the mjrfMaterial properties.
struct mjrfMaterial_ { // material properties for a renderable (mjrfMaterial)
float color[4]; // object color; defaults to white
int32_t segmentation_id; // ID for segmentation rendering; maps to RGB8 color (i.e. 24 bits)
int32_t island_id; // ID to which the renderable belongs
int sleep_state; // mjtSleepState; sleep state of the renderable
float uv_scale[3]; // scale applied to UV coordinates; defaults to (1,1,1)
float uv_offset[3]; // offset applied to UV coordinates; defaults to (0,0,0)
float scissor[4]; // if non-zero, applies scissor testing when rendering
float metallic; // metallic factory [0, 1]; disabled if < 0
float roughness; // roughness factor [0, 1]; disabled if < 0
float specular; // specular factor [0, 1]; disabled if < 0
float glossiness; // glossiness factor [0, 1]; disabled if < 0
float emissive; // emissive/glow factor [0, 1]; disabled if < 0
float reflectance; // blend factor for reflective surfaces [0, 1]; applies only to planes
mjtBool decor_ux; // for ux elements, does not apply any lighting
mjtBool selected; // for "selected" ux elements, adds additional styling
const mjrfTexture* color_texture; // color/albedo texture (RGB8)
const mjrfTexture* opacity_texture; // opacity texture (A8)
const mjrfTexture* normal_texture; // normal map texture (RGB8)
const mjrfTexture* metallic_texture; // metallic map texture (R8)
const mjrfTexture* roughness_texture; // roughness map texture (R8)
const mjrfTexture* occlusion_texture; // ambient occlusion texture (R8)
const mjrfTexture* orm_texture; // occlusion/roughness/metallic texture (RGB8)
const mjrfTexture* emissive_texture; // emissive texture (RGB8)
const mjrfTexture* reflection_texture; // reflection texture, for internal use only
};
typedef struct mjrfMaterial_ mjrfMaterial;
// Initializes the mjrfMaterial to default values.
void mjrf_defaultMaterial(mjrfMaterial* material);
struct mjrfRenderableParams_ { // parameters for creating a renderable (mjrfRenderable)
mjtBool cast_shadows; // if true, casts shadows
mjtBool receive_shadows; // if true, receives shadows
uint16_t blend_order; // controls draw order for transparent objects [0, 8]
};
typedef struct mjrfRenderableParams_ mjrfRenderableParams;
// Initializes the mjrfRenderableParams to default values.
void mjrf_defaultRenderableParams(mjrfRenderableParams* params);
// Creates a renderable with the given parameters.
mjrfRenderable* mjrf_createRenderable(mjrfContext* ctx, const mjrfRenderableParams* params);
// Destroys the renderable.
void mjrf_destroyRenderable(mjrfRenderable* renderable);
// Sets the mesh of the renderable.
void mjrf_setRenderableMesh(mjrfRenderable* renderable, const mjrfMesh* mesh, int elem_offset,
int elem_count);
// Sets the mesh of the renderable to a built-in mesh based on the geom type. Note: using the same
// parameters (nstack, nslice, nquad) will have better performance as the internal mesh data can be
// shared across renderables.
void mjrf_setRenderableGeomMesh(mjrfRenderable* renderable, int type, int nstack, int nslice,
int nquad);
// Sets the material properties and textures of the renderable.
void mjrf_setRenderableMaterial(mjrfRenderable* renderable, const mjrfMaterial* material);
// Copies the material properties of the renderable into the given mjrfMaterial.
void mjrf_getRenderableMaterial(mjrfRenderable* renderable, mjrfMaterial* material);
// Sets the transform position and rotation of the renderable.
void mjrf_setRenderableTransform(mjrfRenderable* renderable, const float position[3],
const float rotation[9]);
// Sets the size of the renderable. Note that, for most renderables, this is equivalent to setting
// the scale. However, for some geom-based renderables, the size scale is not applied uniformly
// (e.g. the spherical ends of a capsule are scaled such that they always remain spherical).
void mjrf_setRenderableSize(mjrfRenderable* renderable, const float size[3]);
// ## Render Targets (mjrfRenderTarget)
//
// A render target is a memory buffer that holds the results of a rendering operation. (This is an
// alternative to rendering directly to the screen.) See mjrf_render for more details.
struct mjrfRenderTargetConfig_ { // parameters for creating a render target (mjrfRenderTarget)
int width; // texture width
int height; // texture height
int color_format; // mjrPixelFormat; pixel format for color buffer
int depth_format; // mjrPixelFormat; pixel format for depth buffer
};
typedef struct mjrfRenderTargetConfig_ mjrfRenderTargetConfig;
// Initializes the RenderTargetConfig to default values.
void mjrf_defaultRenderTargetConfig(mjrfRenderTargetConfig* config);
// Creates a render target for the filament renderer.
mjrfRenderTarget* mjrf_createRenderTarget(mjrfContext* ctx, const mjrfRenderTargetConfig* config);
// Destroys the render target.
void mjrf_destroyRenderTarget(mjrfRenderTarget* render_target);
// Resizes the render target to the given width and height.
void mjrf_resizeRenderTarget(mjrfRenderTarget* render_target, int width, int height);
// ## Debug-only functions.
// Draws an ImGui editor for the given scene, exposing filament-specific settings.
void mjrf_DEBUG_drawImguiEditor(mjrfScene* scene);
#if defined(__cplusplus)
} // extern "C"
#endif
#endif // MUJOCO_MJRFILAMENT_H_