chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate filament header documentation to docs.

Browse Source 
PiperOrigin-RevId: 948294877
Change-Id: I6e110bbdc4d2e329a243f1b96d560698174de26b
This commit is contained in:
Haroon Qureshi
2026-07-15 06:14:09 -07:00
committed by Copybara-Service
parent b60609d2a1
commit 1e85ce176f
10 changed files with 1104 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/mujoco/mjrfilament.h
+37 -144
View File
@@ -25,33 +25,9 @@
extern "C" {
#endif
// IMPORTANT: This API should still be considered experimental and is likely change frequently.
//---------------------------------- Filament rendering --------------------------------------------
// This library provides a C API for the filament rendering library (github.com/google/filament)
// that is designed to work with the MuJoCo library for visualizing simulations.
//
// The filament renderer is a real-time physically based rendering (PBR) engine developed by Google.
// It is designed to be as small as possible and as efficient as possible, while still providing
// high-quality results. It works across all major platforms (Linux, Windows, macOS, Android, iOS,
// Web) and supports OpenGL, Vulkan, and Metal.
//
// For the purposes of this API, we assume the reader has a basic understanding of rendering
// concepts (e.g. textures, vertices, cameras, framebuffers, etc.). We will also highlight some of
// the key differences between this renderer and the legacy/classic MuJoCo (mjr) renderer.
//
// ## API Overview
//
// There are seven key components: Context, Texture, Mesh, Scene, Light, Renderable, and
// RenderTarget. We'll describe these in detail further below.
//
// Each object is created using a `create` function and destroyed using a `destroy` function, e.g.
// `mjrf_createTexture` and `mjrf_destroyTexture`. The `create` functions accept a pointer to a
// configuration struct (e.g. `mjrTextureConfig`) which describes the parameters for the object to
// be created. Each of these structs has a corresponding `default` function (e.g.
// `mjrf_defaultTextureConfig`) which can be used to initialize the struct to default values. Default
// values are assumed to be 0/NULL unless otherwise specified.
//
// For now, we'll just define opaque handles for each of our components.
// Opaque handles for each of our components.
typedef struct mjrfContext_ mjrfContext;
typedef struct mjrfTexture_ mjrfTexture;
typedef struct mjrfMesh_ mjrfMesh;
@@ -60,36 +36,18 @@ typedef struct mjrfLight_ mjrfLight;
typedef struct mjrfRenderable_ mjrfRenderable;
typedef struct mjrfRenderTarget_ mjrfRenderTarget;
// ## Rendering Context (mjrfContext)
//
// The Context is the main entry point for the library. It manages all the core filament objects
// that are responsible for the rendering of an image.
//
// All other objects (e.g. Textures, Meshes, Scenes, etc.) need a Context in order to be created.
// Otherwise, the main function to use with the Context is `mjrf_render()` which does the actual
// rendering.
//
// Filament uses a separate thread for doing the actual rendering. However, despite that, this API
// is not thread-safe; calls are expected to be made from a single thread. Also, due to the
// asynchronous nature of filament, some APIs provide handles or callbacks to signal when an
// operation is complete. (Note: for WASM builds, filament does not use a separate thread.)
//
// There are two key differences between the mjrfContext and the classic mjrContext. Firstly, the
// filament context will manage the underlying graphics context itself. This means users do not need
// to initialize EGL or similar libraries beforehand. Secondly, the filament context is independent
// of a MuJoCo model. That means you can use a single mjrfContext to render images for multiple
// models.
// Callback function type for rendering operations.
typedef void (*mjrfCallback)(void* user_data);
typedef enum mjrGraphicsApi_ { // underlying graphics API to use for rendering
// Underlying graphics API to use for rendering.
typedef enum mjrGraphicsApi_ {
mjGRAPHICS_API_DEFAULT = 0, // default (platform-dependent)
mjGRAPHICS_API_OPENGL, // desktop, mobile (GLES), web (WebGL)
mjGRAPHICS_API_VULKAN, // vulkan
} mjrGraphicsApi;
typedef struct mjrfContextConfig_ { // parameters for creating filament context (mjrfContext)
// Parameters for creating filament graphics context (mjrfContext).
typedef struct mjrfContextConfig_ {
int graphics_api; // rendering graphics API [mjrGraphicsApi]
mjtBool force_software_rendering; // force backend to use software rendering
void* native_window; // platform-dependent window handle (or nullptr for windowless)
@@ -107,7 +65,8 @@ void mjrf_destroyContext(mjrfContext* ctx);
// Gets active renderer information for the given filament context.
void mjrf_getRendererInfo(mjrfContext* ctx, mjrRendererInfo* info);
typedef enum mjrDrawMode_ { // how to draw objects in the scene
// High-level control for how to draw objects in the scene.
typedef enum mjrDrawMode_ {
mjDRAW_MODE_DEFAULT, // default colors and lighting
mjDRAW_MODE_DEFAULT_NO_TEXTURES, // default, but without textures
mjDRAW_MODE_WIREFRAME, // wireframe rendering
@@ -117,7 +76,8 @@ typedef enum mjrDrawMode_ { // how to draw objects in the scene
mjDRAW_MODE_SEGMENTATION_BY_COLOR, // generate visually distinct colors using segmentation id
} mjrDrawMode;
typedef struct mjrfRenderRequest_ { // a single rendering operation
// A single rendering operation.
typedef struct mjrfRenderRequest_ {
mjrfScene* scene; // scene to render
mjrCamera camera; // camera (viewpoint) from which to render scene
mjrRect viewport; // viewport (rect area) into which to render
@@ -131,7 +91,8 @@ typedef struct mjrfRenderRequest_ { // a single rendering operation
// Initializes the mjrfRenderRequest to default values.
void mjrf_defaultRenderRequest(mjrfRenderRequest* request);
typedef struct mjrfReadPixelsRequest_ { // a single read operation
// A single pixel read operation.
typedef struct mjrfReadPixelsRequest_ {
mjrfRenderTarget* target; // render target from which to read the image pixels
void* output; // buffer into which the pixels will be stored
mjtSize num_bytes; // size of output buffer
@@ -152,19 +113,20 @@ typedef uint64_t mjrfFrameHandle;
// taken. Firstly, requests should be grouped by target. Next, the combined area of the viewports
// for all requests for a given target must be contained within the dimensions of the target itself.
//
// Callbacks will be invoked from within this function, though there is no guarantee on when exactly
// that will be done.
// Callbacks will be invoked from within this function, though there is no guarantee on which
// invocation of this function it will be done.
mjrfFrameHandle mjrf_render(mjrfContext* ctx, const mjrfRenderRequest* req, int nreq,
const mjrfReadPixelsRequest* read_req, int nread_req);
// Waits for all rendering operations to complete for the given frame handle,
// triggering any callbacks as needed.
// Waits for all rendering operations to complete for the given frame handle, triggering any
// callbacks as needed.
void mjrf_waitForFrame(mjrfContext* ctx, mjrfFrameHandle frame);
// Sets the clear color for the renderer.
void mjrf_setClearColor(mjrfContext* ctx, const float color[3]);
typedef struct mjrfFrameStats_ { // stats for a single frame of rendering
// Information for a single frame of rendering.
typedef struct mjrfFrameStats_ {
double frame_rate; // frame rate, in frames per second
} mjrfFrameStats;
@@ -174,16 +136,8 @@ void mjrf_defaultFrameStats(mjrfFrameStats* stats);
// Returns the stats for the given frame but updating the given `stats_out`.
void mjrf_getFrameStats(mjrfContext* ctx, mjrfFrameHandle frame, mjrfFrameStats* stats_out);
// ## Textures (mjrfTexture)
//
// A texture is a 2D or 3D (cubemap) image that adds visual detail to a rendered model, such as
// color or bumpiness, without increasing geometric complexity.
//
// For textures intended to be used for image-based lights (see `mjrfLight` below), you should use
// filament's `cmgen` tool to generate a KTX image from your source image. This tool will calculate
// additional data (i.e. the spherical harmonics) and encode that information into the KTX file.
typedef struct mjrfTextureConfig_ { // parameters for creating a texture (mjrfTexture)
// Parameters for creating a texture (mjrfTexture).
typedef struct mjrfTextureConfig_ {
int width; // texture width, or number of bytes for compressed data (e.g. KTX)
int height; // texture height, or 0 for compressed data (e.g. KTX)
int format; // pixel format (e.g. RGB8, RGBA8, KTX, etc.) [mjrPixelFormat]
@@ -194,15 +148,15 @@ typedef struct mjrfTextureConfig_ { // parameters for creating a texture (mjrfT
// Initializes the mjrfTextureConfig to default values.
void mjrf_defaultTextureConfig(mjrfTextureConfig* config);
// Creates a filament texture. Note that the texture will not be created on the
// GPU until `mjrf_setTextureData()` is called.
mjrfTexture* mjrf_createTexture(mjrfContext* ctx,
const mjrfTextureConfig* config);
// Creates a filament texture. Note that the texture will not be created on the GPU until
// `mjrf_setTextureData()` is called.
mjrfTexture* mjrf_createTexture(mjrfContext* ctx, const mjrfTextureConfig* config);
// Destroys the texture.
void mjrf_destroyTexture(mjrfTexture* texture);
typedef struct mjrfTextureData_ { // binary data for a texture (mjrfTexture)
// Binary data payload for an mjrfTexture.
typedef struct mjrfTextureData_ {
const void* bytes; // pointer to image data, or nullptr for empty texture
mjtSize num_bytes; // number of bytes in the image data
mjrfCallback release; // callback when data has finished uploading
@@ -225,32 +179,11 @@ int mjrf_getTextureHeight(const mjrfTexture* texture);
// [returns: mjrSamplerType]
int mjrf_getTextureSamplerType(const mjrfTexture* texture);
// ## Meshes (mjrfMesh)
//
// A mesh describes the surface geometry of an object to be rendered. It is defined as a collection
// of vertices (i.e. a VertexBuffer), a set of indices (i.e. an IndexBuffer) that describes the
// order in which the vertices should be processed, and a primitive type that defined how the
// vertices are to be interpreted (e.g. triangles, lines, etc.) when rendering the surface.
//
// Filament does not directly support normals. Instead, it encodes the normal, tangent, and
// bitangent into a 4-component quaternion describing the "orientation" of the vertex. Ideally, you
// should preprocess your assets to generate this data offline, but we will compute it on the fly if
// needed (at a performance cost).
//
// Vertex data may or may not be interleaved. Interleaved data assumes that the attributes are
// packed in the order specified in the attributes array, with no padding in-between. Additionally,
// the `data` pointer for each attribute is assumed to point to the first element of that type. For
// non-interleaved data, each attribute is assumed to be stored in a separate array.
//
// Additionally, the bounds of the mesh should be computed in order to allow the filament renderer
// to perform frustum-based culling. Alternatively, the bounds can be computed at runtime (though
// there is a small performance cost). If no bounds are provided (or calculated), then frustum
// culling will not be performed.
// Maximum number of vertex attributes in a mesh.
enum { mjMAX_VERTEX_ATTRIBUTES = 16 };
typedef struct mjrfMeshData_ { // binary data for a mesh (mjrfMesh)
// Binary data used for creating a mesh (mjrfMesh).
typedef struct mjrfMeshData_ {
mjtSize num_vertices; // number of vertices; all vertex attributes share this size
int num_attributes; // number of attributes defined
mjrVertexAttribute attributes[mjMAX_VERTEX_ATTRIBUTES]; // per-vertex attribute information
@@ -275,12 +208,8 @@ mjrfMesh* mjrf_createMesh(mjrfContext* ctx, const mjrfMeshData* data);
// Destroys the mesh.
void mjrf_destroyMesh(mjrfMesh* mesh);
// ## Scenes (mjrfScene)
//
// A scene is a collection of entities (Lights and Renderables) that describes what is to be
// rendered.
typedef struct mjrfSceneParams_ { // parameters for creating a scene (mjrfScene)
// Parameters for creating a scene (mjrfScene).
typedef struct mjrfSceneParams_ {
} mjrfSceneParams;
// Initializes the mjrfSceneParams to default values.
@@ -310,24 +239,8 @@ void mjrf_setSceneSkybox(mjrfScene* scene, const mjrfTexture* texture);
// Configures the scene based on the parameters in an mjModel.
void mjrf_configureSceneFromModel(mjrfScene* scene, const mjModel* model);
// ## Lights (mjrfLight)
//
// A light is a source of illumination in the scene. (Without lights, a scene will be completely
// black.) There are several different types of lights such as directional, spot, point, and image
// lights.
//
// The primary light in a scene is the image light (also sometimes known as the environment light).
// This is a light that "surrounds" the entire scene and is defined as a 3D texture. Each "pixel" of
// the cubemap is interpreted as the color of projected into the scene from a particular direction.
//
// Directional lights are the next most common type of light and is usually used to simulate the
// sun; a uniformly colored light that is emitted in a single direction.
//
// Filament only supports a single image and directional light. You can define as many point or spot
// lights as you want. Each light source (except image based lights) may or may not cast shadows.
// Each shadow-casting light incurs a performance cost.
typedef struct mjrfLightParams_ { // parameters for creating a light (mjrfLight)
// Parameters for creating a light (mjrfLight).
typedef struct mjrfLightParams_ {
int type; // type of light (e.g. spot, point, image, etc.) [mjrLightType]
const mjrfTexture* texture; // texture; only for image lights
float color[3]; // RGB color
@@ -364,23 +277,8 @@ void mjrf_setLightTransform(mjrfLight* light, const float position[3], const flo
// Returns the type of the light (mjrLightType).
int mjrf_getLightType(const mjrfLight* light);
// ## Renderables (mjrfRenderable)
//
// A renderable is a single drawable object in the scene. It is defined as a combination of a mesh
// (i.e. surface geometry) and a material (i.e. surface appearance and properties).
//
// In terms of materials, there are three lighting models currently supported:
//
// 1. Metallic-roughness (PBR): this is the preferred model for rendering models based standard
// metallic-roughness workflows.
// 2. Specular-glossiness (non-PBR): this is a legacy model designed to be compatible with classic
// mjr renderer, though it is not 100% identical.
// 3. Unlit: this model ignores lighting and used for rendering UX or decorative elements like
// contact forces and labels.
//
// Which lighting model is used is determined by the mjrfMaterial properties.
typedef struct mjrfMaterial_ { // material properties for a renderable (mjrfMaterial)
// Material properties for a renderable (mjrfRenderable).
typedef struct mjrfMaterial_ {
float color[4]; // object color; defaults to white
int32_t segmentation_id; // ID for segmentation rendering; maps to RGB8 color (i.e. 24 bits)
int32_t island_id; // ID to which the renderable belongs
@@ -410,7 +308,8 @@ typedef struct mjrfMaterial_ { // material properties for a renderable (mjrfMat
// Initializes the mjrfMaterial to default values.
void mjrf_defaultMaterial(mjrfMaterial* material);
typedef struct mjrfRenderableParams_ { // parameters for creating a renderable (mjrfRenderable)
// Parameters for creating a renderable (mjrfRenderable).
typedef struct mjrfRenderableParams_ {
mjtBool cast_shadows; // if true, casts shadows
mjtBool receive_shadows; // if true, receives shadows
uint16_t blend_order; // controls draw order for transparent objects [0, 8]
@@ -451,12 +350,8 @@ void mjrf_setRenderableTransform(mjrfRenderable* renderable, const float positio
// (e.g. the spherical ends of a capsule are scaled such that they always remain spherical).
void mjrf_setRenderableSize(mjrfRenderable* renderable, const float size[3]);
// ## Render Targets (mjrfRenderTarget)
//
// A render target is a memory buffer that holds the results of a rendering operation. (This is an
// alternative to rendering directly to the screen.) See mjrf_render for more details.
typedef struct mjrfRenderTargetConfig_ { // parameters for creating a render target (mjrfRenderTarget)
// Parameters for creating a render target (mjrfRenderTarget).
typedef struct mjrfRenderTargetConfig_ {
int width; // texture width
int height; // texture height
int color_format; // pixel format for color buffer [mjrPixelFormat]
@@ -475,8 +370,6 @@ void mjrf_destroyRenderTarget(mjrfRenderTarget* render_target);
// Resizes the render target to the given width and height.
void mjrf_resizeRenderTarget(mjrfRenderTarget* render_target, int width, int height);
// ## Debug-only functions.
// Draws an ImGui editor for the given scene, exposing filament-specific settings.
void mjrf_DEBUG_drawImguiEditor(mjrfScene* scene);