From 6a7a7230931e5cfd664af7cf6cc2e8d238f46858 Mon Sep 17 00:00:00 2001 From: Taylor Howell Date: Thu, 30 Jul 2026 02:59:22 -0700 Subject: [PATCH 1/6] MuJoCo Warp documentation: Per-world assets PiperOrigin-RevId: 956399999 Change-Id: I8d951e5e52e16442325c66cc33ac693bf68939e3 --- doc/mjwarp/index.rst | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/mjwarp/index.rst b/doc/mjwarp/index.rst index 0e3d4877..e76fae6f 100644 --- a/doc/mjwarp/index.rst +++ b/doc/mjwarp/index.rst @@ -501,19 +501,22 @@ Certain fields are safe to modify directly without compilation, enabling on-devi `GitHub issue 893 `__ tracks adding on-device updates for a subset of fields. -Per-world meshes +Per-world assets ---------------- -Per-world meshes enable heterogeneous worlds where different worlds simulate different meshes. The workflow -is: +Per-world assets enable heterogeneous worlds where different worlds simulate different +`assets `__ including meshes, height fields, materials, +and textures. The general workflow is: -1. Create an :ref:`mjSpec` with **all** mesh assets and the **maximum** number of geom slots needed across variants. +1. Create an :ref:`mjSpec` with **all** assets. 2. Compile each variant by mutating the spec and calling ``spec.compile()``. 3. Compile a **base** model and create :class:`mjw.Model ` from it. 4. Override the relevant :class:`mjw.Model ` fields with per-world arrays built from the compiled variants. -**Example 1 — Geom-level** randomization (1 body, 1 geom, 2 mesh assets): +.. rubric:: Per-world meshes + +**Example 1 — Per-world meshes: Geom-level** randomization (1 body, 1 geom, 2 mesh assets): The base scene includes all mesh assets. The geom references one mesh (``mesh_a``); a second mesh (``mesh_b``) is available for per-world substitution. @@ -612,7 +615,7 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a` m.body_ipos = wp.array(body_ipos, dtype=wp.vec3) m.body_iquat = wp.array(body_iquat, dtype=wp.quat) -**Example 2 — Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets): +**Example 2 — Per-world meshes: Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets): .. admonition:: Maximum geom count :class: important @@ -784,6 +787,14 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a` - ``wp.quat`` - ``(nworld, nbody)`` +Per-world height fields, materials, and textures can be similarly formulated. + +.. admonition:: Per-world asset dependent field construction + :class: note + + MJWarp enables per-world asset functionality but does not provide utilities for construction of dependent per-world + field variants. Construction is left to the user or environment authoring frameworks. + Batch Rendering =============== From c2e95b4161276ac3b631897694cb4b99c2c03e4d Mon Sep 17 00:00:00 2001 From: Matija Kecman Date: Thu, 30 Jul 2026 03:00:11 -0700 Subject: [PATCH 2/6] Add StatePayload and serialization helpers for web viewer PiperOrigin-RevId: 956400340 Change-Id: Ibe049afe10eca85e5fb6d718a78863a641b864bb --- .../experimental/studio/web/state_payload.cc | 171 ++++++++++++++++++ .../experimental/studio/web/state_payload.h | 128 +++++++++++++ .../studio/web/state_payload_py.cc | 72 ++++++++ 3 files changed, 371 insertions(+) create mode 100644 python/mujoco/experimental/studio/web/state_payload.cc create mode 100644 python/mujoco/experimental/studio/web/state_payload.h create mode 100644 python/mujoco/experimental/studio/web/state_payload_py.cc diff --git a/python/mujoco/experimental/studio/web/state_payload.cc b/python/mujoco/experimental/studio/web/state_payload.cc new file mode 100644 index 00000000..608f881b --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload.cc @@ -0,0 +1,171 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wire-format implementation for the web viewer's state payload (see +// state_payload.h). Compiled into BOTH sides of the wire: the state_payload +// pybind module (serializer, via state_payload_py.cc) and the wasm +// web_client (parser, via web_client_session.cc). It must therefore stay +// free of python- or browser-specific dependencies. + +#include "state_payload.h" + +#include +#include +#include +#include + +#include + +namespace mujoco::studio { +namespace { + +// Appends raw bytes to the payload buffer. +void AppendBytes(std::vector& buffer, const void* data, + size_t size) { + const std::byte* bytes = static_cast(data); + buffer.insert(buffer.end(), bytes, bytes + size); +} + +// Appends a complete [u32 tag][u32 size][payload] block. +void AppendStateBlock(std::vector& buffer, uint32_t tag, + const void* data, size_t size) { + StateBlockHeader block_header{tag, static_cast(size)}; + AppendBytes(buffer, &block_header, sizeof(block_header)); + AppendBytes(buffer, data, size); +} + +// Serializes the render state (exactly kRenderStateSize bytes) into `ptr`. +void SerializeRenderStateInto(std::byte* ptr, const mjvCamera& camera, + const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, + const mjVisual& vis, const mjStatistic& stat, + const std::vector& render_flags) { + memcpy(ptr, &camera, sizeof(mjvCamera)); + ptr += sizeof(mjvCamera); + + memcpy(ptr, &perturb, sizeof(mjvPerturb)); + ptr += sizeof(mjvPerturb); + + memcpy(ptr, &vis_options, sizeof(mjvOption)); + ptr += sizeof(mjvOption); + + memcpy(ptr, &opt, sizeof(mjOption)); + ptr += sizeof(mjOption); + + memcpy(ptr, &vis, sizeof(mjVisual)); + ptr += sizeof(mjVisual); + + memcpy(ptr, &stat, sizeof(mjStatistic)); + ptr += sizeof(mjStatistic); + + // Pack render flags (mjNRNDFLAG bytes). + memset(ptr, 0, mjNRNDFLAG); + for (size_t i = 0; i < mjNRNDFLAG && i < render_flags.size(); ++i) { + ptr[i] = static_cast(render_flags[i]); + } +} + +} // namespace + +size_t MaxStatePayloadSize(size_t physics_bytes) { + return sizeof(StatePayloadHeader) + 3 * sizeof(StateBlockHeader) + + (sizeof(int32_t) + physics_bytes) + kRenderStateSize + + kMaxExtraGeoms * sizeof(mjvGeom); +} + +std::vector SerializeStatePayload( + uint32_t model_crc32, int32_t physics_spec, const void* physics, + size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis, + const mjStatistic& stat, const std::vector& render_flags, + const mjvGeom* extra_geoms, size_t extra_geom_count) { + extra_geom_count = + extra_geom_count > kMaxExtraGeoms ? kMaxExtraGeoms : extra_geom_count; + std::vector buffer; + buffer.reserve(MaxStatePayloadSize(physics_bytes)); + + StatePayloadHeader header; + header.nblocks = extra_geom_count > 0 ? 3 : 2; + header.model_crc32 = model_crc32; + AppendBytes(buffer, &header, sizeof(header)); + + // Physics state: [i32 spec][mjtNum values...]. + StateBlockHeader physics_header{ + kTagPhysicsState, static_cast(sizeof(int32_t) + physics_bytes)}; + AppendBytes(buffer, &physics_header, sizeof(physics_header)); + AppendBytes(buffer, &physics_spec, sizeof(int32_t)); + AppendBytes(buffer, physics, physics_bytes); + + // Render state, serialized into place. + StateBlockHeader render_header{kTagRenderState, + static_cast(kRenderStateSize)}; + AppendBytes(buffer, &render_header, sizeof(render_header)); + const size_t render_offset = buffer.size(); + buffer.resize(render_offset + kRenderStateSize); + SerializeRenderStateInto(buffer.data() + render_offset, camera, perturb, + vis_options, opt, vis, stat, render_flags); + + // Extra geoms (only when present). + if (extra_geom_count > 0) { + AppendStateBlock(buffer, kTagExtraGeoms, extra_geoms, + extra_geom_count * sizeof(mjvGeom)); + } + + return buffer; +} + +bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out) { + const std::byte* bytes = static_cast(data); + if (size < sizeof(StatePayloadHeader)) return false; + + StatePayloadHeader header; + memcpy(&header, bytes, sizeof(header)); + if (header.magic != kStatePayloadMagic) return false; + if (header.version != kStatePayloadVersion) return false; + out->model_crc32 = header.model_crc32; + + size_t offset = sizeof(StatePayloadHeader); + for (uint16_t i = 0; i < header.nblocks; ++i) { + if (offset + sizeof(StateBlockHeader) > size) return false; + StateBlockHeader block; + memcpy(&block, bytes + offset, sizeof(block)); + offset += sizeof(StateBlockHeader); + if (offset + block.size > size) return false; + const std::byte* payload = bytes + offset; + + switch (block.tag) { + case kTagPhysicsState: + if (block.size < sizeof(int32_t)) return false; + memcpy(&out->physics_spec, payload, sizeof(int32_t)); + out->physics = payload + sizeof(int32_t); + out->physics_bytes = block.size - sizeof(int32_t); + break; + case kTagRenderState: + if (block.size != kRenderStateSize) return false; + out->render_state = payload; + break; + case kTagExtraGeoms: + if (block.size % sizeof(mjvGeom) != 0) return false; + out->extra_geoms = payload; + out->extra_geom_count = block.size / sizeof(mjvGeom); + break; + default: + break; // Unknown tag: skip. + } + offset += block.size; + } + return true; +} + +} // namespace mujoco::studio diff --git a/python/mujoco/experimental/studio/web/state_payload.h b/python/mujoco/experimental/studio/web/state_payload.h new file mode 100644 index 00000000..0ceacf92 --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload.h @@ -0,0 +1,128 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the serialization format for the web viewer's browser +// client render payload containing the data needed so that the browser can +// render the scene using the following call: +// +// Render(model, data, perturb, camera, vis_options, width, height, extra_geoms) +// +// The arguments come from the Python process: +// +// * model : fetched once over HTTP as /model.mjb; its runtime-mutable +// parts (opt/vis/stat) are re-sent in the render state block. +// * data : streamed as the physics state vector (mjSTATE_INTEGRATION); +// the browser recomputes the rest via mj_setState/mj_forward. +// * width/height: the browser canvas size. +// * extra_geoms : optional variable-size kTagExtraGeoms block. +// * ... : the rest of the arguments are sent as a fixed-size block +// +// The payload (SerializeStatePayload) is a sequence of tagged blocks: +// +// [StatePayloadHeader][u32 tag][u32 size][payload]... +// +// The payload is serialized by Python, sent over the /state WebSocket, and +// parsed by the browser. +// +// TODO(matijak): Try shrinking the physics block: float32 (or quantized) values +// instead of doubles, and/or delta-encoding against the client's last-acked +// payload. The /state ack (web_server.py) tells the server which snapshot each +// client last applied, which is the baseline that delta compression needs. For +// 100humanoids.xml the payload is ~181 KB of doubles and dominates slow links. + +#ifndef MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ +#define MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ + +#include +#include +#include + +#include + +namespace mujoco::studio { + +// "MJWS" as little-endian bytes. This magic constant identifies the +// StateServer WebSocket payload header and helps detect malformed or +// misrouted messages. +constexpr uint32_t kStatePayloadMagic = + 'M' | ('J' << 8) | ('W' << 16) | ('S' << 24); +constexpr uint16_t kStatePayloadVersion = 1; + +struct StatePayloadHeader { + uint32_t magic = kStatePayloadMagic; + uint16_t version = kStatePayloadVersion; + uint16_t nblocks = 0; + // CRC32 of the model's MJB bytes. When this changes, the browser must + // refetch /model.mjb before applying any further state. + uint32_t model_crc32 = 0; +}; +static_assert(sizeof(StatePayloadHeader) == 12); + +// Block tags. Readers must skip unknown tags. +enum StateBlockTag : uint32_t { + kTagPhysicsState = 1, // [i32 mjtState spec signature][mjtNum values...] + kTagRenderState = 2, // fixed-size block of kRenderStateSize bytes + kTagExtraGeoms = 3, // n x mjvGeom (n = size / sizeof(mjvGeom)) +}; + +struct StateBlockHeader { + uint32_t tag = 0; + uint32_t size = 0; +}; +static_assert(sizeof(StateBlockHeader) == 8); + +// Fixed byte size of the render state block appended after physics state. +// These are plain C structs of int/float/double members whose total size is +// fixed, independent of the model and generally negligible compared to the size +// of the physics state +constexpr size_t kRenderStateSize = + sizeof(mjvCamera) + sizeof(mjvPerturb) + sizeof(mjvOption) + + sizeof(mjOption) + sizeof(mjVisual) + sizeof(mjStatistic) + mjNRNDFLAG; + +// Maximum number of extra geoms serialized per frame. Bounds the shared +// memory buffer the StateServer allocates; WebViewer truncates longer lists. +constexpr uint32_t kMaxExtraGeoms = 1024; + +// Upper bound of a serialized payload, used to size the StateServer's shared +// memory buffer. `physics_bytes` is mj_stateSize(...) * sizeof(mjtNum). +size_t MaxStatePayloadSize(size_t physics_bytes); + +// Serialize the complete state payload sent over the state WebSocket. +std::vector SerializeStatePayload( + uint32_t model_crc32, int32_t physics_spec, const void* physics, + size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis, + const mjStatistic& stat, const std::vector& render_flags, + const mjvGeom* extra_geoms, size_t extra_geom_count); + +// Parsed view into a serialized payload. Pointers alias the input buffer and +// are NOT guaranteed to be aligned; so you must memcpy the data out before use. +struct StatePayloadView { + uint32_t model_crc32 = 0; + int32_t physics_spec = 0; + const std::byte* physics = nullptr; + size_t physics_bytes = 0; + const std::byte* render_state = nullptr; // kRenderStateSize bytes when non-null + const std::byte* extra_geoms = nullptr; // extra_geom_count * sizeof(mjvGeom) + size_t extra_geom_count = 0; +}; + +// Parses a payload produced by SerializeStatePayload. Returns false if the +// buffer is malformed (bad magic/version or out-of-bounds block). Blocks +// with unknown tags are skipped. +bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out); + +} // namespace mujoco::studio + +#endif // MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ diff --git a/python/mujoco/experimental/studio/web/state_payload_py.cc b/python/mujoco/experimental/studio/web/state_payload_py.cc new file mode 100644 index 00000000..9be00030 --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload_py.cc @@ -0,0 +1,72 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Python bindings for the state payload wire format (state_payload.h). +// +// WebViewer serializes the /state WebSocket payload with this module each +// frame; the browser parses it with the same header (web_client_session). + +#include +#include +#include +#include + +#include +#include "state_payload.h" +#include "structs.h" +#include +#include + +namespace py = pybind11; + +// Serialize the complete state WebSocket payload (see state_payload.h): +// physics state, render state and extra geoms as tagged blocks. +static py::bytes SerializeStatePayload( + uint32_t model_crc32, int physics_spec, const py::bytes& physics_state, + const mujoco::python::MjvCameraWrapper& camera, + const mujoco::python::MjvPerturbWrapper& perturb, + const mujoco::python::MjvOptionWrapper& vis_options, + const mujoco::python::MjModelWrapper& model, + const std::vector& render_flags, + const std::vector& extra_geoms) { + std::vector geoms; + geoms.reserve(extra_geoms.size()); + for (const mujoco::python::MjvGeomWrapper& geom_wrapper : extra_geoms) { + if (geom_wrapper.get()) { + geoms.push_back(*geom_wrapper.get()); + } + } + + std::string physics = physics_state; + const std::vector buffer = mujoco::studio::SerializeStatePayload( + model_crc32, physics_spec, physics.data(), physics.size(), *camera.get(), + *perturb.get(), *vis_options.get(), model.get()->opt, model.get()->vis, + model.get()->stat, render_flags, geoms.data(), geoms.size()); + return py::bytes(reinterpret_cast(buffer.data()), buffer.size()); +} + +// Upper bound of a serialized payload for a model whose physics state is +// `physics_bytes` long. Used to size the StateServer's shared memory. +static size_t MaxStatePayloadSize(size_t physics_bytes) { + return mujoco::studio::MaxStatePayloadSize(physics_bytes); +} + +PYBIND11_MODULE(state_payload, m, pybind11::mod_gil_not_used()) { + py::module_::import("mujoco._structs"); + m.doc() = "MuJoCo web viewer state payload serialization"; + + m.def("serialize_state_payload", &SerializeStatePayload); + m.def("max_state_payload_size", &MaxStatePayloadSize); + m.attr("MAX_EXTRA_GEOMS") = mujoco::studio::kMaxExtraGeoms; +} From 82471cbdfb282517e6ad6b6495aaba39b0270c6e Mon Sep 17 00:00:00 2001 From: Taylor Howell Date: Thu, 30 Jul 2026 05:33:32 -0700 Subject: [PATCH 3/6] MuJoCo Warp documentation: Large scenes initialize trees asleep ![docs](https://screenshot-v2.corp.google.com/image/222i7rtheo5eg.png) PiperOrigin-RevId: 956462670 Change-Id: I0c6f211126e5d86bfacebff49591c17cb503b062 --- doc/mjwarp/index.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/mjwarp/index.rst b/doc/mjwarp/index.rst index e76fae6f..3a598a6b 100644 --- a/doc/mjwarp/index.rst +++ b/doc/mjwarp/index.rst @@ -444,6 +444,15 @@ high-performance tensor/matrix operations optimized for fixed tile sizes. to a tight upper bound of the expected active DOFs significantly reduces GPU memory usage and improves throughput. +.. TODO(taylorhowell): update example with correct island cycles initialization + +4. When setting ``nvmax < nv`` it is recommended to initialize all trees to asleep in order to avoid initial dof + overflow. + + .. code-block:: python + + d.tree_asleep.assign(np.array(np.arange(mjm.ntree, dtype=np.int32)), dtype=np.int32) + .. note:: Consider increasing the sleep tolerance setting (e.g., ``sleep_tolerance="0.01"`` in XML options or ``spec.option.sleep_tolerance = 0.01`` in Python) from its default value (0.001) to more quickly From 60889f5e85b1c5c2c3bc2c8b2b8000ccee474c48 Mon Sep 17 00:00:00 2001 From: Yuval Tassa Date: Thu, 30 Jul 2026 06:08:29 -0700 Subject: [PATCH 4/6] Add favicon to MuJoCo Live. PiperOrigin-RevId: 956476524 Change-Id: I3f33cd09764e1236999b7d269045a2f8c2a8236e --- .github/workflows/live.yml | 1 + src/experimental/studio/favicon.ico | Bin 0 -> 15086 bytes src/experimental/studio/live.html | 1 + 3 files changed, 2 insertions(+) create mode 100644 src/experimental/studio/favicon.ico diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml index 59823e1d..e8c6e4cb 100644 --- a/.github/workflows/live.yml +++ b/.github/workflows/live.yml @@ -35,6 +35,7 @@ jobs: cp src/experimental/studio/live.html dist/index.html cp src/experimental/studio/live.css dist/live.css cp src/experimental/studio/live.js dist/live.js + cp src/experimental/studio/favicon.ico dist/favicon.ico COMMIT_HASH=$(git rev-parse HEAD 2>/dev/null || true) if [ -n "$COMMIT_HASH" ]; then sed -i "s/__COMMIT_HASH_PLACEHOLDER__/$COMMIT_HASH/g" dist/index.html diff --git a/src/experimental/studio/favicon.ico b/src/experimental/studio/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..1db85336808c3b5371455f4634938f3936e8fcd7 GIT binary patch literal 15086 zcmd^mcUV=|wl7Ica*`OM^v(uBdhfmW&TjNx1yPzJ2uLrMG)>UhyC%jSJ8Ga4d>c(mH8WEj@f==sG{a6 zeRE4I5;XNql00n^8)x_r2-kfXavilH1;3Lp^-U;n)`EE6YY=1f2JrE_C85Ry>4iy$ogXfsp274}8-}U_0p*vSsro zGWvc_UJfa7(_-?RHJH!0eF)N_C*b_*OR#s#1~|OyV=#JQ0-Qav4_0>8LxRyOVEx)l zux9njvG;oxbdv%n9VXLSjj4#XgY&0OLztBzNM#b(y>$x+d_Cdpi9@iWxdM_+CxWy3 zKVW(9hhy(=*tnh)xvI0dcIr$`tQ&LY^jXLjir~i^mtpfet?=FV-@%njU&Fgq*^p*2 ziS;#jL-EeyMPu)uK7EE1x~s`6gRGRzsiDj;%fj|8>!3Q?0hZSmfP?_-{&+K68s^C4 z*r`K=xvH$WscG!}`wt(ILJv)*B{fRXQ=Fn?0N~c8vyiCs3RL*(v*!-)WS6wfV`7b8 zRTjHyDw7=arA7I9WAFD5_LI^8Gq$@>Bwt*XsW`lKJ#2oji!JrkQSuF5lAYPLhS}D; zkcrlPQC{k%D^K^FHk6bQH}*adh7|Z1vt5NENk<-E+Fc}4&d-RK*CqJK@}1R(d&^T4 zJ$aFAwvC#!$XQ#O7vvxd2@W26|NK`Mh}%?MvZyqDs3kk$NlR8je@jl{Kx0aD|Ev(( zr-d%+!}-o?l6+^?ff8qp!Lkf7DJq>Q9PR(>H*cyvdHe@4)0#}?Bu5NY#QP0aCiqDz z>n?0hJ#fJ(_a;#tlLUaDNHGTGVT# zWbxO?^}&M&_C`cTYLr#X8h!o$s3d5G<2VxM|A%t?B^4F=E}dt1C;=#Re&YV%#4~=L zp>xFlZGY1;(^CZk!T!+ja1t3EMPg%PNo-skiBE_p@$vB_F)5KG@skLj&&TVek@|!L z5*ZUs!lEKca99`#3=1W}5urq!o=(=T->|Qtxh3VlqbFY=w3{}~jF^}hlOXiP1Z*QE zEuEyKrjZoq=@GHf*<%|;W_ zthFH>%M4p}$h6adEIUo^ce;%hWZKiZXY~&1-18~cT9Ax5k`YI;l^W!ETR=&&0NOh` z$T#0#C3W)}oJKLYdwZ`onPyG`z1+Z8?{&y=&}Oq8v{?Kui*2(a)Ki4tO2PJ1@jA<1 zjcYF%?UI5z({|Eq)v#V2W#l>fJQcC^2!8rpsI?I+Xm2Gy{q`IAVEISwqwTwRcoGX+ zTM}Yz$my2rtj^{*YO)!2YAoMem36~)#Z&7Ih{wLAwx+tK+NgraUK=8?pW~*fp{+;i zDULA1Dd4631_bE832|ohS;T3j4%uEd5M*ZxUthjLK0A2mx6$^UTwIBrlM9Kq(}yA# zJ;-rVV{@Hw-EvfCGtq8=wo@SpC5$&21n6^VXmh%LuU^ms9ko>uV)zzUpMiZMworvQ z7Yhinn+h>bCJ;SU4N|f1Qqc}MURK~~G#Ng}b?v~R!}8Jg3rnYyn8ZX9WuphBUb@(> z9+T&&%cNVXlHI)v;K|@qP~uwW^42RL#D1l=^j3WdhUHQi!nN3H!ix}RPJL7p0ySR( z1!3Uv)2Hx3YaK)wO~gLY0<D_qo3X@@Y|Co@OythSiCX;!cA0xkGAnu ze;J+(4Z*!1e}K7}Nf1AE5}+?32J{;bGjP^@19t7*M|SPqJJx*L4JkNUXlz6MdIYT&Q)I_%!O4sPGS4|ncAfOX54g8f^s zLKNCM%3=!q_V@|>`p094wwQ#rQbm8(f^0|hO*bQOHJA*a?A%2@*|B4+{RN8`lluAd zNeKGVbWa^@UxO)f(PpN*>M>b1Z?R7X2l0Dq&F&y=|tFk+K`u!Cs(M?e1pl?7Q~_QDHr@AfVD`PbiI&$<0Pmctj6|6{eS!J zU1aO_Pl*q2BFywPW@h@DpiE^d{7jevM=j?23uocy-+za_d-p=BAd$Ns7#aj2Zvv#7 zy$pjA3E-RpiP7P3^vGfO8LvM-{29cVyoTdggUxeR<0QpVPZ?&*Q?z$=llPV^9czCF z`r_`-_7iWziDV|WSrKTdtPC(`ssk<14m!-4y`R7@*#BQ%xB&YP9N>K6#+8eZZ#xl+ z945h|+t)cvJ9q7Z3l}fqv%kY9t9v2I^i4L;Nt?}gnL^u#WCuM(q@$U9USlI!xajS% z_V*n)L@r+bmbmM^&dv%jRWu1hl?_RO%7z4QWlfkJv!Zbp``hC`;O>J5@DTg<-mM!@ z9%2PEJ#?VlT?-oHo#D}gyKw)(h(A919NYDo=8hrJVfq%P+9z?M{f{y{;&5 zQN?yu6-6$(@=;+Jt0-00RUndg z=Zhpg1!7ryW}IwePZJ}N%gCC=UCf+VZ{^$=H|4AVb49tIp?s#7fxN^;UACpCo>8z2 zDN6`Y3d|?T3tT75ie0r)v`}>9d~3D-G=HZd4_`kLjr+;b_7CI!4A&T9tNJq3#<)RO zQHs1hPax~a6ApLf3#Hv?i)C}ml)vMcZpu$nHi*KR27Z8IR)Dp<+;^(H+}l82?x`=Y zi?&n9rB5N%!%T_mv%JVvO;+rpFQrn182D)av;dbu8%Jm2?&&?&e&31D4xU;d!Yomp|N=!yoR*6G*!Y1oDsO&z8+kkCnA##L1dcW27}PUc(h3HqtV0 zBiRfuLrJldru?I-EGEr%qP)mWU4c3eqe#&H!$q#z(wM1J9_K{(OLciB#4jlL$*BH& z5A44qkxNNJkRK5_=#b7rv7|L8X}C2v>1lgz(h#2AhPw*|vhKo^;ZC$mOIExDaY$#! z2M$$5x(t{37|BY#45Z~=`jV=Esn|B^?XE7T?H9Rf4i&mh9xQUxkcJpeewq^>seDUQ zn1T6Y<=KB?{W(X=@IDpgAky zX;VgA|6E~2e{DkWz>I*Yh{0Gg-CbW=;;t{xTT6Isr_zRoQ8Vda!2AI`YfUKa=UjC8VON zCTFz0zbZdkxw7`#@4vrGPuhPa4BnwKavC4G9<5ViEfwb3`^SzQyV=^>I{UBM`?Grg zz(AF?8`d`;ICOCNsWYec^_@6&@aVC=y+`}{_8jRuy7x$5-`-=NA3xZ4{Mf<1V|_=C zoj86J>-!ELIox-+@5s?U{JrgyPgeCVU)F4AXZu`V|5=;=v$_}Y+vDZG|0N_ch)hMr zs9#0pt$QjeuO3%Xd1*dU2LIbazk3hAPldJzDk@F1ZOT(Ppk$+r@f4=d^Avdd|A{|; z*E5}6JrnW@3QW>6(gRXbQ{u%UVX{alNEV3%cp(s>2*d)uKqN>Kh=fUaLm*1w9-t2i z_?!p@^jatu3Gj737JQKyZ{tNWUL>ccr3!QN^J8-h3jMS4@{KKQtS9^(rtwDqvuE*p zQ<4PyjJUYCPa-2D?;>v_LEeUqiH!lQgShy3Ha;V&AUZA{qGID9G%5;w zgMuI~kq<3x9c25Con-%^!|bN5+m0@N`#rz$I{X#>x`w6~ZES5zOij%m^NdZ1wT&&L zWn@BkS0}u`XaOv4Z-JhcM(A#7fbRK?c-@58P0)pPU0C+Cj{NSNkN5GJuBLg=(NGWV zjdSswQwN<*C|I^P%tfh%*19>+GIusK&8dcFEEjaPL04B7w0CyHsx@oj_~$3-n(_0y zOFqaB3Jrf@RF6NyKO~W=AfJ{t&D?xYLraTz`}vW!mS$+LtAYH42s~dpLV}YS#Mv1` ztQ`;H97cq4VKKHw5MyV=aZ0haJg$tkH9$^G7kM&0h_up0!O~g}qDI9=f8-kZg;+a7 z5ICEGz{>$BC$+G-0eV0D5Ej4tF6=pQfE+${Y+&V@wPRf2pW!bkp6=@I<@r!sN0$VI zgplIAY)J5Sgh&f*NW?b8cE~;1BKL%R5#`AKT1L1c`ka{Km_~Rej!UE55!P|?j7t(B zheG)z$}6Qg8$zDH6Nuccp*&RpE0=vp`8D|Z+biVcnKKVMx)(UoJ{bRJnz_Ym-hKhQ zrde4N2lVy0s7MI3Fhu@h3UVtNl0?Ia$l7H z8WH3?&<^w&`d&Ka@sPWr{8=i+kL_eS>Oo0>J>&+sKvq;R^e$Nfcn*i}u3smg?%K5_ zJtO;-@%XK6ZDXu#?Pa!(jwB>B3@|nV?zfx7-vPpUTx#gAt_`kfTS&2frY((5!Ts8^w%EoAw-z>M@{=vlB34)+~{ zFD`sFxaggCeaGW>@$gt?hH)`hFK_U3cZOizoA?dN8)_lvsLkQWHaXrA{eZ& z$K!YN@OA zG;+b5eQ_L84ww3p0AmJAn`@!DsvNORf>^{g`b@eleGYxyK^u~NogfjvmE?`3n+0;t zBfNhq_bs%Cy$jJz8#nA57!3KjIS`NWi4u2Rwg7pt zTt^)y5A~qf3tco}+lPyB3|)Y;Uz~#zCr&_KOb|pEtAYS+OXpMSBLei7EN?6L{MZrL zzi$uP!vyC7RY*j;(mqHMgIS_6V0J!SyLpq&18^O=-nUyDAk_FRj{hE=lf;e& zuy6ZTxOVFnC)+lxfnemPqfAG}GUdbhI969L?&QAl&9$q5=Q?Q26NAW74Sj~L7069f zejII<;WiC|E%hN8_gNb@Z-E`VcMtqKdKHz5>S}3jQ4tg)e$M`Gh8$O(=d7jtSKhoc zGaSCXav8qAegkgYy$kC;>V;6uTZlK;z_~^fVi9LuwgA5Q?pyf&+I3DYUA_$Y@nH~X zIGOYBM0{3={;_l8TDX4eHhg#O8k|4>C6q;bVN40J+o8`RubqwjHpQRmYKr&`(Ern5 z-NsGu$&Q^*$K!9DKVMchs{#rO3%I>Y3B`|GJM|%s^LN%m-d&fii=z1k$I*vw!%g&~ z1ABKvw2cAIrRt~y#(WUlmX*DnU1=YEhdNMyTmIf6@Hcu3`#=L?alEIwo5AtJ2eB`2 zquhdHhxS9Jrzz(<*%&_>i4Bb4&u}$?5Gy0d%FKe*>o-szdJcd0!iBP$x;iK-r2fOB z{?Ek<3SG3Y)L|%hU+SjERQMROeLJ_pO|<=W^pA6A&w|j$0rwed5II#9N@9HBYvhW* z$FXr0{fv$gDqnv21&9M&AqeM48dJ%~b?E}yoIXeW@YD6HK#1ct9oGh2Z!oqnBGmrr zh(FZE80W?uShjK%Y}&eYaJ>DOF6)&wwY5SK`VXByr(?VUxj6>;F~qKqaR)ubug}bI z(_we7cn5CXy~pAI_UiXgfP2#*j9-PAOorZ`R=9cRE?m2T*wOyiFa|>PxPg7pjeBSx z9LGV}57ik-a25UjdmQ68?%jpuJMB-7{ctPvU&OWu;jyD^r4Nz z8=J`+@#D!i(TJd19$G-hwIqy)Q4({ zbHP(*B8a{0;N0m`TpiWnz~;4ZY{wStt9u;&zN1GV($Ru@wsZC@)bkGNhxl;}bk54h zF|EptTbj!=Dja@i9vfkA#ugP9!#nRU1&nb>#^YbVag&VdRz70}Xa7=99$SL`Qi8D_ zUm>FI?W+uuk4+Ew%B>=v= za*4CwJ&YZ!T)qsvEsfy-u2mEhmA>5{LzTY?G(=dyxkG#57S`QD-9Fv^F}P0EgRL7d z&WFFL4Zgq_Kw+>Gu75N>tj)z%aJ>hNv5eq%<}uNZ=1h6{Ea+Oe2tHiC{5kyFcI=R^ z#632(f3z*dZ^9sdtSt96#h8;BGt18m<4&ebHA+L21AG3&5x9fcX{>MO&Ye(RT!2`x zPtYfB-Ms@J&M5)xE2#F=g-z{MTs)A@S#;ei&dGrNpY4O&=vP#SgS)pwii;tOzK?qq z+@m<-e&w0ay<@7AkuuKJQdwPH1C7nC@XmYhKZpO*J$vMvwtbBF%Q^fMce$UbqB6)v zSrurFF)M4PCditZ9c0DK4K`1hBp zKKjs}M{PM2UXtsh~EHaqYk^4c43_AAvd;f zA?_R47qma9uf5k;!wN0bnQVuV{R*x*xOc($9PU|g9%WJ-4CD!J*7Djpb75vhB{a9R zJvaYgtWJKo?*vrheGY$sImNA*mlUj=ml&w5Pw-dHi}z8^i}hgYqnw#_ZL?YG`*a?n zK0t-W(XL#)08M;fsPNQ>YQ#>Z2K88!5dzn~y$n=Wil5Fs6hG#2VeBQD!F&!S-7Aj7 z^=Pb`Uh6Qp1}jqR^`uE2w(`b?CMZGwsBdf<8b5yb9XcfIJ9!GKE2|*N)_|=DvSjAQ zc`2GwqDNwW$&ncM3sW`;LzIn)zRU;J1?;U`H#vJ#pP};(%?H@N{2iF-Wdv0|hA0MX zm9GI?<7>#)`Wvx_H?4rXKRm>J#|VBpRxV%p8p>mRFfPR5$GD(2#^!L(f!U-hHF_@ULsU@8zOEajA3HwW58e^Fg4zK%m*{X z>C&l;$ zB&TOWS^3N-^xa?^cAhX)(UQ$qv}PqLS~C+d=1Jq8@rn+V zo*V&l{&*ie{P7VN-+h4Pp-(nroYWes{LI)&Ut_k?+lZ<3HbBu=*7zAJ>qDn9r}liz z-hKE3ho8=0ySJ=o@f@SXd0xrw718dv=3{(P6XTRxa>Oqc+h`4lyzHet-93OYQotDL znk0C#chelT|?R0P^am+?gkmb>#_V-JA&I82eQ0Thq(XxI5M1%(0_@aVK_e zvcCV6TG2H_Im?HqKz~wH`WVQm4vn$)%CAoLvDDw_m|0L&%wWEh0_RP%w9c7rOAQ9VX42XRkBnuZTV1tqGP2>xnJAWNHaY9DVOS7x0F&9@CYNB1) z=8RZHd%jry48OE1UnK3$7t0nDh-5u^0{OPL+n5J99v|Si-SFN*rXJ(z4g4TwZM287 zI@DfS5om>RWD|M04^KYJ+elvJYbdYs(Nk<&*udOF+h51MU2R&763_DTJd9%(xT@25 zvaG~amy0iBKgck@WLRXYIe^^d@PeKm*3-`)B4c8n!{7J$N!c}=PpT(%9PzUUX4v~-I`zsgOuW_y|3UO8kceJo+}PH+UiJGmX23ye~go2 z%vvC6&lO_KS|q{#kkUR_IxAms_0mQ5v#slu%~^bT8;~mpOU#Ae-ZEfPSxq*tKLdmW{Y2R8R*Ahu=wa zNNAXE`C$^olzgX5bf`1rw z7;Z=DKs`D!{=c{+MSgPkcIEP>8X1mnSx0V?yd@)EhT}k5#}Ad%#CS_8LhPk8{Vk-p zpOTe(^Q7oUQq0egFDXk_?pX1jGT#+_-c1{QURR1D84-++V{9JrOLCnxhZD`!`_qCv zrKM#vfj7;9c>4$37?1zp(IaQ>J$!(<_6H${9}l^hpVyKWMg51`9%J_WA(TOk{|7)PLMh&pVlK6$DJ@olV_|qsLJ-FBU594|S`E+eHN|`aV=3-q zB{RJZrPYDc_Tv`o-9 zHYHK9@rTCa-}dRJi@v>jm0bAx5_B~;;~ZoF=>N)|k~AsC)`v0X{rcF9Br>Cao53l zUJvJa9ccmjI_?+H=30Z8yD~U8#$771){^;5)l`ZTlc1oW7~^bdq`0hn)xx*m#%%U~ z|3hOM9bMfnTefbwfBWuTauN3u&5aFgkdqDML~5*yt}e4-b3Cb3Y4{F#mH)fAu@p@PwvLkP{c)qvvqa#kXKklatn)Z)z&vy{aO5l z)=f}XS9$OKC7I{G{Blr=IZc?442u>mf+&A)jA!UE86lqX8k}1y6N4o+{18d4IDB|+ zas*1`@LX|(WUe?|%84jkHdpveMB(x}ak!#R9HFdFj%4anBH3pG_#5yV>ad){4`z$Q z+*zTk8RKQ5p>&*PtSmq-`Q5kQfvT!XnjbQNxu{}_^RMvJdSObkO7HUJg_o{e`CZ8< z$sd>pyX}+h$SW2@xStQ>YH6x;nZ{GPPcx*#cv`#yuMc+T9k zc{6O7&2K~8cr{@%Q46KvYJ5jJkvI96_0w*l7Sxq9{L)vy-hp{tN1ptACreDu)@ z=FBmcGrJ? z>g3W37r$DI{|9K>*)Pw1dgk2O?I<6gIeT^+cGtGkXU}|m`Yis&dU_w@1Upc6ojrGU z7nZxReh)sg=hW#_drqR9I(vHe>2qgyo;dmW=Fbis=-s?^OZ}q7i=xug(^UT&(ev@s zzkBvC?onTS@tM%;@iKbvCA{|%X7XS@#)}j1DVmq@@|cYLrqBFGfBzHv|7Tx*?&(pz zM)esjY5o8B?5xD?Mu+G{?y)tyg=>#Uk6QD6#xJL literal 0 HcmV?d00001 diff --git a/src/experimental/studio/live.html b/src/experimental/studio/live.html index 08d33f37..4b4c9c2a 100644 --- a/src/experimental/studio/live.html +++ b/src/experimental/studio/live.html @@ -6,6 +6,7 @@ + MuJoCo Live From 6c86e125670f664291fb626208acfb7e01b4db3b Mon Sep 17 00:00:00 2001 From: Yuval Tassa Date: Thu, 30 Jul 2026 06:37:31 -0700 Subject: [PATCH 5/6] Make the selection outline occlusion-aware in the filament renderer. PiperOrigin-RevId: 956489408 Change-Id: I948493587eb154cac6e0fc31d5df987967405ff8 --- .../filament/assets/outline_composite.mat | 19 +++++- .../filament/assets/outline_flatten.mat | 6 +- src/render/filament/core/outliner.cc | 66 +++++++++++++------ src/render/filament/core/outliner.h | 19 +++++- src/render/filament/core/scene_view.cc | 6 +- 5 files changed, 88 insertions(+), 28 deletions(-) diff --git a/src/render/filament/assets/outline_composite.mat b/src/render/filament/assets/outline_composite.mat index 8d42edbe..6eac0a6f 100644 --- a/src/render/filament/assets/outline_composite.mat +++ b/src/render/filament/assets/outline_composite.mat @@ -16,6 +16,8 @@ // outline around an object. The outline is drawn by checking the distance to // the nearest edge point (as computed by the algorithm) and, if within the // desired width, setting the color of the pixel to the desired outline color. +// The outline is dimmed where it is occluded, i.e. where the scene contains +// geometry that is closer than the selected object at the nearest edge point. material { name : outline_composite, @@ -25,8 +27,11 @@ material { depthCulling : false, parameters : [ { type : sampler2d, name : source }, + { type : sampler2d, name : scene_depth, precision : high, filterable : false }, + { type : sampler2d, name : selection_depth, precision : high, filterable : false }, { type : float4, name : color }, - { type : float, name : width } + { type : float, name : width }, + { type : float, name : dim } ], variables : [ vertex @@ -77,6 +82,18 @@ fragment { float inner_alpha = smoothstep(0.0, 1.0, dist); float outer_alpha = smoothstep(width, width - 1.0, dist); float alpha = outer_alpha * inner_alpha; + + // Dim the outline where it is occluded. The depth buffer is reversed + // (1 = near, 0 = far), so the outline is occluded if the scene depth at + // this pixel is greater than the depth of the selected object at the + // nearest edge point. The epsilon avoids spurious dimming where the two + // depths are nearly equal, e.g. at contact points. + highp float scene_z = texture(materialParams_scene_depth, variable_vertex.xy).r; + highp float edge_z = texture(materialParams_selection_depth, edge).r; + if (scene_z > edge_z * 1.001 + 1e-6) { + alpha *= materialParams.dim; + } + postProcess.color = mix(vec4(0), materialParams.color, alpha * materialParams.color.a); } } diff --git a/src/render/filament/assets/outline_flatten.mat b/src/render/filament/assets/outline_flatten.mat index 3fa4a529..247815e9 100644 --- a/src/render/filament/assets/outline_flatten.mat +++ b/src/render/filament/assets/outline_flatten.mat @@ -16,11 +16,15 @@ // This material writes the the screen-space position of each fragment as the // output color of that fragment. This will provide us with the initial data for // the Jump Flood Algorithm. See "outline_jumpflood.mat" for more details. +// +// Depth is also written so that the render target's depth attachment holds the +// depth of the selected objects; "outline_composite.mat" samples it to dim +// occluded portions of the outline. material { name : outline_flatten, shadingModel : unlit, culling : none, - depthWrite : false + depthWrite : true } fragment { diff --git a/src/render/filament/core/outliner.cc b/src/render/filament/core/outliner.cc index 9b0bb28a..b39c8ce5 100644 --- a/src/render/filament/core/outliner.cc +++ b/src/render/filament/core/outliner.cc @@ -37,12 +37,15 @@ namespace mujoco { Outliner::Outliner(ObjectManager* object_mgr, uint8_t layer_mask, - filament::math::float4 color, float thickness) + uint8_t scene_layer_mask, filament::math::float4 color, + float thickness, float occlusion_dim) : object_mgr_(object_mgr), engine_(object_mgr->GetEngine()), layer_mask_(layer_mask), + scene_layer_mask_(scene_layer_mask), color_(color), - thickness_(thickness) {} + thickness_(thickness), + occlusion_dim_(occlusion_dim) {} Outliner::~Outliner() { Reset(); } @@ -98,9 +101,10 @@ void Outliner::Prepare(int width, int height) { } }; - // Set up two render targets. We will alternate between the two targets to - // allow for chaining passes together. - for (int i = 0; i < 2; ++i) { + // Set up the render targets: one for the scene depth, one for the flattened + // selection mask and its depth, and two ping-pong targets for chaining the + // jump flood passes together. + for (int i = 0; i < kNumTargets; ++i) { mjrfRenderTargetConfig config; mjrf_defaultRenderTargetConfig(&config); config.color_format = mjPIXEL_FORMAT_RGBA8; @@ -127,9 +131,11 @@ void Outliner::Prepare(int width, int height) { view->setMultiSampleAntiAliasingOptions({.enabled = false}); } - // In the first pass, we will render a given scene, but only render the - // objects marked as outlines. We assume that the objects have already been - // assigned the kOutlineFlatten material. + // In the first pass, we render the scene layers that can occlude the + // outline; only the resulting depth is used. In the second pass, we render + // the given scene again, but only the objects marked as outlines. We assume + // that the objects have already been assigned the kOutlineFlatten material. + views_[kPassSceneDepth]->setVisibleLayers(0xff, scene_layer_mask_); views_[kPassFlatten]->setVisibleLayers(0xff, layer_mask_); // All subsequent passes are full-screen post-processing passes. @@ -141,16 +147,19 @@ void Outliner::Prepare(int width, int height) { setup_fullscreen(kPassDrawOutline, ObjectManager::kOutlineComposite); // Chain the passes together such that the output of a pass is the input to - // the next pass. The first pass has no input (we are just rendering the - // selected objects) and the last pass has no output (we are just rendering - // the outline to the externally provided target). - bind(kPassFlatten, -1, 0); - bind(kPassJumpFlood1, 0, 1); - bind(kPassJumpFlood2, 1, 0); - bind(kPassJumpFlood3, 0, 1); - bind(kPassJumpFlood4, 1, 0); - bind(kPassJumpFlood5, 0, 1); - bind(kPassDrawOutline, 1, -1); + // the next pass. The scene passes have no input (we are just rendering the + // scene) and the last pass has no output (we are just rendering the outline + // to the externally provided target). The flatten pass has its own target + // (rather than a ping-pong target) so that its depth attachment survives the + // jump flood passes and can be sampled by the composite pass. + bind(kPassSceneDepth, -1, kTargetSceneDepth); + bind(kPassFlatten, -1, kTargetFlatten); + bind(kPassJumpFlood1, kTargetFlatten, kTargetPing); + bind(kPassJumpFlood2, kTargetPing, kTargetPong); + bind(kPassJumpFlood3, kTargetPong, kTargetPing); + bind(kPassJumpFlood4, kTargetPing, kTargetPong); + bind(kPassJumpFlood5, kTargetPong, kTargetPing); + bind(kPassDrawOutline, kTargetPing, -1); // Bind the parameters for each pass. For the jump flood passes, the step // parameter determines how far to propagate the outline in each pass. @@ -160,9 +169,22 @@ void Outliner::Prepare(int width, int height) { material_instances_[kPassJumpFlood4]->setParameter("step", 2.0f); material_instances_[kPassJumpFlood5]->setParameter("step", 1.0f); - // The final pass renders the actual outline onto a render target. + // The final pass renders the actual outline onto a render target. It samples + // the scene depth and the selection depth to dim occluded outline pixels. + const filament::TextureSampler depth_sampler( + filament::TextureSampler::MinFilter::NEAREST, + filament::TextureSampler::MagFilter::NEAREST); material_instances_[kPassDrawOutline]->setParameter("color", color_); material_instances_[kPassDrawOutline]->setParameter("width", thickness_); + material_instances_[kPassDrawOutline]->setParameter("dim", occlusion_dim_); + material_instances_[kPassDrawOutline]->setParameter( + "scene_depth", + targets_[kTargetSceneDepth]->GetDepthTexture()->GetFilamentTexture(), + depth_sampler); + material_instances_[kPassDrawOutline]->setParameter( + "selection_depth", + targets_[kTargetFlatten]->GetDepthTexture()->GetFilamentTexture(), + depth_sampler); // Commit all the material instances to the engine. for (auto& material_instance : material_instances_) { @@ -222,9 +244,13 @@ void Outliner::Render(filament::Renderer* renderer, filament::View* view, view->setViewport(viewport); } - // Re-render the view's scene to create the flattened selection mask. + // Re-render the view's scene to capture the scene depth (used for occlusion + // dimming) and to create the flattened selection mask. auto prev_clear_opts = renderer->getClearOptions(); renderer->setClearOptions({.clearColor = {0, 0, 0, 0}, .clear = true}); + views_[kPassSceneDepth]->setScene(view->getScene()); + views_[kPassSceneDepth]->setCamera(&view->getCamera()); + renderer->render(views_[kPassSceneDepth]); views_[kPassFlatten]->setScene(view->getScene()); views_[kPassFlatten]->setCamera(&view->getCamera()); renderer->render(views_[kPassFlatten]); diff --git a/src/render/filament/core/outliner.h b/src/render/filament/core/outliner.h index 1a430a76..45e27903 100644 --- a/src/render/filament/core/outliner.h +++ b/src/render/filament/core/outliner.h @@ -31,11 +31,13 @@ namespace mujoco { // Renders an outline of selected objects. // // This class uses the "jump flood" algorithm to create an outline of selected -// objects. +// objects. The outline is dimmed by `occlusion_dim` where the selected objects +// are occluded by scene geometry on the `scene_layer_mask` layers. class Outliner { public: Outliner(ObjectManager* object_mgr, uint8_t layer_mask, - filament::math::float4 color, float thickness); + uint8_t scene_layer_mask, filament::math::float4 color, + float thickness, float occlusion_dim); ~Outliner(); Outliner(const Outliner&) = delete; @@ -53,6 +55,7 @@ class Outliner { void Reset(); enum Pass { + kPassSceneDepth, kPassFlatten, kPassJumpFlood1, kPassJumpFlood2, @@ -65,17 +68,27 @@ class Outliner { kNumJumpFloodPasses = kPassJumpFlood5 - kPassJumpFlood1 + 1, }; + enum Target { + kTargetSceneDepth, // depth of the scene, used for occlusion dimming + kTargetFlatten, // selection mask (color) and selection depth (depth) + kTargetPing, // jump flood ping-pong buffer + kTargetPong, // jump flood ping-pong buffer + kNumTargets, + }; + ObjectManager* object_mgr_ = nullptr; filament::Engine* engine_ = nullptr; uint8_t layer_mask_ = 0xff; + uint8_t scene_layer_mask_ = 0xff; filament::math::float4 color_ = {1.0f, 1.0f, 1.0f, 1.0f}; float thickness_ = 2.5f; + float occlusion_dim_ = 1.0f; int width_ = 0; int height_ = 0; filament::Camera* camera_ = nullptr; - std::unique_ptr targets_[2]; + std::unique_ptr targets_[kNumTargets]; filament::View* views_[kNumPasses] = {}; filament::Scene* scenes_[kNumPasses] = {}; diff --git a/src/render/filament/core/scene_view.cc b/src/render/filament/core/scene_view.cc index 78d7a000..35748f49 100644 --- a/src/render/filament/core/scene_view.cc +++ b/src/render/filament/core/scene_view.cc @@ -321,9 +321,9 @@ void SceneView::Render(filament::Renderer* renderer, if (!selected_renderables.empty()) { if (!outliner_) { - outliner_ = - std::make_unique(object_mgr_, kLayerMask_Outline, - float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f); + outliner_ = std::make_unique( + object_mgr_, kLayerMask_Outline, kLayerMask_Object, + float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f, 0.25f); } for (Renderable* renderable : selected_renderables) { From 58a54ef47441f5c4308398e0eeccc6fc63cd16c9 Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Thu, 30 Jul 2026 07:18:40 -0700 Subject: [PATCH 6/6] Move resource provider registration into a separate library. PiperOrigin-RevId: 956506812 Change-Id: I9acde018e282171f41e0f8c756cd02b39c35dc67 --- src/experimental/platform/CMakeLists.txt | 2 + src/experimental/platform/resources.cc | 108 +++++++++++++++++++++++ src/experimental/platform/resources.h | 25 ++++++ src/experimental/studio/launcher.cc | 82 +---------------- 4 files changed, 137 insertions(+), 80 deletions(-) create mode 100644 src/experimental/platform/resources.cc create mode 100644 src/experimental/platform/resources.h diff --git a/src/experimental/platform/CMakeLists.txt b/src/experimental/platform/CMakeLists.txt index fdabc5ba..c41a0cf0 100644 --- a/src/experimental/platform/CMakeLists.txt +++ b/src/experimental/platform/CMakeLists.txt @@ -22,6 +22,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC helpers.cc helpers.h + resources.cc + resources.h sys_utils.cc sys_utils.h hal/egl_utils.cc diff --git a/src/experimental/platform/resources.cc b/src/experimental/platform/resources.cc new file mode 100644 index 00000000..b0fcb727 --- /dev/null +++ b/src/experimental/platform/resources.cc @@ -0,0 +1,108 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/platform/resources.h" + +#include // NOLINT(build/c++17) +#include +#include +#include +#include +#include + +#include +#include "experimental/platform/sys_utils.h" + +namespace mujoco::platform { + +namespace { + +std::string Resolve(std::string_view path) { + std::string_view subpath = path.substr(path.find(':') + 1); + std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve); + if (exe_dir.empty()) { + return std::string("assets/") + std::string(subpath); + } + std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources"; + if (std::filesystem::exists(resources_dir / "assets")) { + return (resources_dir / "assets" / subpath).string(); + } + return (exe_dir / "assets" / subpath).string(); +} + +class FileResource { + public: + explicit FileResource(const std::string& path) + : file_(path, std::ios::binary | std::ios::ate) { + if (!file_.is_open()) { + mju_warning("Cannot open file %s", path.c_str()); + return; + } + + size_ = file_.tellg(); + file_.seekg(0, std::ios::beg); + } + + int Read(const void** buffer) { + buffer_.resize(size_); + if (!file_.read(reinterpret_cast(buffer_.data()), size_)) { + return 0; + } + *buffer = buffer_.data(); + return size_; + } + + int Size() const { return size_; } + + FileResource(const FileResource&) = delete; + FileResource& operator=(const FileResource&) = delete; + + private: + std::ifstream file_; + std::vector buffer_; + int size_ = 0; +}; + +} // namespace + +void RegisterResourceProviders() { + mjpResourceProvider resource_provider; + mjp_defaultResourceProvider(&resource_provider); + + resource_provider.open = [](mjResource* resource) { + const std::string resolved_path = Resolve(resource->name); + FileResource* f = new FileResource(resolved_path); + if (f->Size() == 0) { + delete f; + return 0; + } + resource->data = f; + return f->Size(); + }; + resource_provider.read = [](mjResource* resource, const void** buffer) { + FileResource* f = static_cast(resource->data); + return f->Read(buffer); + }; + resource_provider.close = [](mjResource* resource) { + delete static_cast(resource->data); + resource->data = nullptr; + }; + + resource_provider.prefix = "font"; + mjp_registerResourceProvider(&resource_provider); + resource_provider.prefix = "filament"; + mjp_registerResourceProvider(&resource_provider); +} + +} // namespace mujoco::platform diff --git a/src/experimental/platform/resources.h b/src/experimental/platform/resources.h new file mode 100644 index 00000000..804a3683 --- /dev/null +++ b/src/experimental/platform/resources.h @@ -0,0 +1,25 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ +#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ + +namespace mujoco::platform { + +// Registers MuJoCo resource providers for font and filament assets. +void RegisterResourceProviders(); + +} // namespace mujoco::platform + +#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ diff --git a/src/experimental/studio/launcher.cc b/src/experimental/studio/launcher.cc index ac04c56c..4d83ba7d 100644 --- a/src/experimental/studio/launcher.cc +++ b/src/experimental/studio/launcher.cc @@ -16,99 +16,21 @@ #include #include -#include // NOLINT(build/c++17) -#include -#include #include #include -#include #include #include "experimental/platform/hal/graphics_mode.h" -#include "experimental/platform/sys_utils.h" +#include "experimental/platform/resources.h" #include "experimental/studio/app.h" namespace mujoco::studio { -namespace { - -std::string Resolve(std::string_view path) { - std::string_view subpath = path.substr(path.find(':') + 1); - std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve); - if (exe_dir.empty()) { - return std::string("assets/") + std::string(subpath); - } - std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources"; - if (std::filesystem::exists(resources_dir / "assets")) { - return (resources_dir / "assets" / subpath).string(); - } - return (exe_dir / "assets" / subpath).string(); -} - -class FileResource { - public: - explicit FileResource(const std::string& path) - : file_(path, std::ios::binary | std::ios::ate) { - if (!file_.is_open()) { - mju_warning("Cannot open file %s", path.c_str()); - return; - } - - size_ = file_.tellg(); - file_.seekg(0, std::ios::beg); - } - - int Read(const void** buffer) { - buffer_.resize(size_); - if (!file_.read(reinterpret_cast(buffer_.data()), size_)) { - return 0; - } - *buffer = buffer_.data(); - return size_; - } - - int Size() const { return size_; } - - FileResource(const FileResource&) = delete; - FileResource& operator=(const FileResource&) = delete; - - private: - std::ifstream file_; - std::vector buffer_; - int size_ = 0; -}; - -} // namespace int LaunchStudio(int argc, char** argv, LauncherConfig config) { const char* home = std::getenv("HOME"); const std::string ini_path = std::string(home ? home : ".") + "/.mujoco.ini"; - mjpResourceProvider resource_provider; - mjp_defaultResourceProvider(&resource_provider); - - resource_provider.open = [](mjResource* resource) { - const std::string resolved_path = Resolve(resource->name); - FileResource* f = new FileResource(resolved_path); - if (f->Size() == 0) { - delete f; - return 0; - } - resource->data = f; - return f->Size(); - }; - resource_provider.read = [](mjResource* resource, const void** buffer) { - FileResource* f = static_cast(resource->data); - return f->Read(buffer); - }; - resource_provider.close = [](mjResource* resource) { - delete static_cast(resource->data); - resource->data = nullptr; - }; - - resource_provider.prefix = "font"; - mjp_registerResourceProvider(&resource_provider); - resource_provider.prefix = "filament"; - mjp_registerResourceProvider(&resource_provider); + mujoco::platform::RegisterResourceProviders(); if (config.gfx_mode.empty()) { const char* display = std::getenv("DISPLAY");