Merge branch 'google-deepmind:main' into fix/actuator-velocity-index

.github/workflows/live.yml

@@ -35,6 +35,7 @@ jobs:
         cp src/experimental/studio/live.html dist/index.html
         cp src/experimental/studio/live.css dist/live.css
         cp src/experimental/studio/live.js dist/live.js
+        cp src/experimental/studio/favicon.ico dist/favicon.ico
         COMMIT_HASH=$(git rev-parse HEAD 2>/dev/null || true)
         if [ -n "$COMMIT_HASH" ]; then
           sed -i "s/__COMMIT_HASH_PLACEHOLDER__/$COMMIT_HASH/g" dist/index.html

doc/mjwarp/index.rst

@@ -444,6 +444,15 @@ high-performance tensor/matrix operations optimized for fixed tile sizes.
    to a tight upper bound of the expected active DOFs significantly reduces GPU memory usage and improves
    throughput.
 
+.. TODO(taylorhowell): update example with correct island cycles initialization
+
+4. When setting ``nvmax < nv`` it is recommended to initialize all trees to asleep in order to avoid initial dof
+   overflow.
+
+   .. code-block:: python
+
+      d.tree_asleep.assign(np.array(np.arange(mjm.ntree, dtype=np.int32)), dtype=np.int32)
+
 .. note::
    Consider increasing the sleep tolerance setting (e.g., ``sleep_tolerance="0.01"`` in XML options or
    ``spec.option.sleep_tolerance = 0.01`` in Python) from its default value (0.001) to more quickly
@@ -501,19 +510,22 @@ Certain fields are safe to modify directly without compilation, enabling on-devi
 `GitHub issue 893 <https://github.com/google-deepmind/mujoco_warp/issues/893>`__ tracks adding on-device updates for a
 subset of fields.
 
-Per-world meshes
+Per-world assets
 ----------------
 
-Per-world meshes enable heterogeneous worlds where different worlds simulate different meshes. The workflow
+Per-world assets enable heterogeneous worlds where different worlds simulate different
-is:
+`assets <https://mujoco.readthedocs.io/en/latest/XMLreference.html#asset>`__ including meshes, height fields, materials,
+and textures. The general workflow is:
 
-1. Create an :ref:`mjSpec` with **all** mesh assets and the **maximum** number of geom slots needed across variants.
+1. Create an :ref:`mjSpec` with **all** assets.
 2. Compile each variant by mutating the spec and calling ``spec.compile()``.
 3. Compile a **base** model and create :class:`mjw.Model <mujoco_warp.Model>` from it.
 4. Override the relevant :class:`mjw.Model <mujoco_warp.Model>` fields with per-world arrays built from the compiled
    variants.
 
-**Example 1 — Geom-level** randomization (1 body, 1 geom, 2 mesh assets):
+.. rubric:: Per-world meshes
+
+**Example 1 — Per-world meshes: Geom-level** randomization (1 body, 1 geom, 2 mesh assets):
 
 The base scene includes all mesh assets. The geom references one mesh (``mesh_a``); a second mesh
 (``mesh_b``) is available for per-world substitution.
@@ -612,7 +624,7 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a`
   m.body_ipos = wp.array(body_ipos, dtype=wp.vec3)
   m.body_iquat = wp.array(body_iquat, dtype=wp.quat)
 
-**Example 2 — Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets):
+**Example 2 — Per-world meshes: Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets):
 
 .. admonition:: Maximum geom count
    :class: important
@@ -784,6 +796,14 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a`
      - ``wp.quat``
      - ``(nworld, nbody)``
 
+Per-world height fields, materials, and textures can be similarly formulated.
+
+.. admonition:: Per-world asset dependent field construction
+  :class: note
+
+  MJWarp enables per-world asset functionality but does not provide utilities for construction of dependent per-world
+  field variants. Construction is left to the user or environment authoring frameworks.
+
 Batch Rendering
 ===============
 

python/mujoco/experimental/studio/web/state_payload.cc

@@ -0,0 +1,171 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Wire-format implementation for the web viewer's state payload (see
+// state_payload.h). Compiled into BOTH sides of the wire: the state_payload
+// pybind module (serializer, via state_payload_py.cc) and the wasm
+// web_client (parser, via web_client_session.cc). It must therefore stay
+// free of python- or browser-specific dependencies.
+
+#include "state_payload.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <vector>
+
+#include <mujoco/mujoco.h>
+
+namespace mujoco::studio {
+namespace {
+
+// Appends raw bytes to the payload buffer.
+void AppendBytes(std::vector<std::byte>& buffer, const void* data,
+                 size_t size) {
+  const std::byte* bytes = static_cast<const std::byte*>(data);
+  buffer.insert(buffer.end(), bytes, bytes + size);
+}
+
+// Appends a complete [u32 tag][u32 size][payload] block.
+void AppendStateBlock(std::vector<std::byte>& buffer, uint32_t tag,
+                      const void* data, size_t size) {
+  StateBlockHeader block_header{tag, static_cast<uint32_t>(size)};
+  AppendBytes(buffer, &block_header, sizeof(block_header));
+  AppendBytes(buffer, data, size);
+}
+
+// Serializes the render state (exactly kRenderStateSize bytes) into `ptr`.
+void SerializeRenderStateInto(std::byte* ptr, const mjvCamera& camera,
+                              const mjvPerturb& perturb,
+                              const mjvOption& vis_options, const mjOption& opt,
+                              const mjVisual& vis, const mjStatistic& stat,
+                              const std::vector<uint8_t>& render_flags) {
+  memcpy(ptr, &camera, sizeof(mjvCamera));
+  ptr += sizeof(mjvCamera);
+
+  memcpy(ptr, &perturb, sizeof(mjvPerturb));
+  ptr += sizeof(mjvPerturb);
+
+  memcpy(ptr, &vis_options, sizeof(mjvOption));
+  ptr += sizeof(mjvOption);
+
+  memcpy(ptr, &opt, sizeof(mjOption));
+  ptr += sizeof(mjOption);
+
+  memcpy(ptr, &vis, sizeof(mjVisual));
+  ptr += sizeof(mjVisual);
+
+  memcpy(ptr, &stat, sizeof(mjStatistic));
+  ptr += sizeof(mjStatistic);
+
+  // Pack render flags (mjNRNDFLAG bytes).
+  memset(ptr, 0, mjNRNDFLAG);
+  for (size_t i = 0; i < mjNRNDFLAG && i < render_flags.size(); ++i) {
+    ptr[i] = static_cast<std::byte>(render_flags[i]);
+  }
+}
+
+}  // namespace
+
+size_t MaxStatePayloadSize(size_t physics_bytes) {
+  return sizeof(StatePayloadHeader) + 3 * sizeof(StateBlockHeader) +
+         (sizeof(int32_t) + physics_bytes) + kRenderStateSize +
+         kMaxExtraGeoms * sizeof(mjvGeom);
+}
+
+std::vector<std::byte> SerializeStatePayload(
+    uint32_t model_crc32, int32_t physics_spec, const void* physics,
+    size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb,
+    const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis,
+    const mjStatistic& stat, const std::vector<uint8_t>& render_flags,
+    const mjvGeom* extra_geoms, size_t extra_geom_count) {
+  extra_geom_count =
+      extra_geom_count > kMaxExtraGeoms ? kMaxExtraGeoms : extra_geom_count;
+  std::vector<std::byte> buffer;
+  buffer.reserve(MaxStatePayloadSize(physics_bytes));
+
+  StatePayloadHeader header;
+  header.nblocks = extra_geom_count > 0 ? 3 : 2;
+  header.model_crc32 = model_crc32;
+  AppendBytes(buffer, &header, sizeof(header));
+
+  // Physics state: [i32 spec][mjtNum values...].
+  StateBlockHeader physics_header{
+      kTagPhysicsState, static_cast<uint32_t>(sizeof(int32_t) + physics_bytes)};
+  AppendBytes(buffer, &physics_header, sizeof(physics_header));
+  AppendBytes(buffer, &physics_spec, sizeof(int32_t));
+  AppendBytes(buffer, physics, physics_bytes);
+
+  // Render state, serialized into place.
+  StateBlockHeader render_header{kTagRenderState,
+                                 static_cast<uint32_t>(kRenderStateSize)};
+  AppendBytes(buffer, &render_header, sizeof(render_header));
+  const size_t render_offset = buffer.size();
+  buffer.resize(render_offset + kRenderStateSize);
+  SerializeRenderStateInto(buffer.data() + render_offset, camera, perturb,
+                           vis_options, opt, vis, stat, render_flags);
+
+  // Extra geoms (only when present).
+  if (extra_geom_count > 0) {
+    AppendStateBlock(buffer, kTagExtraGeoms, extra_geoms,
+                     extra_geom_count * sizeof(mjvGeom));
+  }
+
+  return buffer;
+}
+
+bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out) {
+  const std::byte* bytes = static_cast<const std::byte*>(data);
+  if (size < sizeof(StatePayloadHeader)) return false;
+
+  StatePayloadHeader header;
+  memcpy(&header, bytes, sizeof(header));
+  if (header.magic != kStatePayloadMagic) return false;
+  if (header.version != kStatePayloadVersion) return false;
+  out->model_crc32 = header.model_crc32;
+
+  size_t offset = sizeof(StatePayloadHeader);
+  for (uint16_t i = 0; i < header.nblocks; ++i) {
+    if (offset + sizeof(StateBlockHeader) > size) return false;
+    StateBlockHeader block;
+    memcpy(&block, bytes + offset, sizeof(block));
+    offset += sizeof(StateBlockHeader);
+    if (offset + block.size > size) return false;
+    const std::byte* payload = bytes + offset;
+
+    switch (block.tag) {
+      case kTagPhysicsState:
+        if (block.size < sizeof(int32_t)) return false;
+        memcpy(&out->physics_spec, payload, sizeof(int32_t));
+        out->physics = payload + sizeof(int32_t);
+        out->physics_bytes = block.size - sizeof(int32_t);
+        break;
+      case kTagRenderState:
+        if (block.size != kRenderStateSize) return false;
+        out->render_state = payload;
+        break;
+      case kTagExtraGeoms:
+        if (block.size % sizeof(mjvGeom) != 0) return false;
+        out->extra_geoms = payload;
+        out->extra_geom_count = block.size / sizeof(mjvGeom);
+        break;
+      default:
+        break;  // Unknown tag: skip.
+    }
+    offset += block.size;
+  }
+  return true;
+}
+
+}  // namespace mujoco::studio

python/mujoco/experimental/studio/web/state_payload.h

@@ -0,0 +1,128 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file defines the serialization format for the web viewer's browser
+// client render payload containing the data needed so that the browser can
+// render the scene using the following call:
+//
+// Render(model, data, perturb, camera, vis_options, width, height, extra_geoms)
+//
+// The arguments come from the Python process:
+//
+//   * model       : fetched once over HTTP as /model.mjb; its runtime-mutable
+//                   parts (opt/vis/stat) are re-sent in the render state block.
+//   * data        : streamed as the physics state vector (mjSTATE_INTEGRATION);
+//                   the browser recomputes the rest via mj_setState/mj_forward.
+//   * width/height: the browser canvas size.
+//   * extra_geoms : optional variable-size kTagExtraGeoms block.
+//   * ...         : the rest of the arguments are sent as a fixed-size block
+//
+// The payload (SerializeStatePayload) is a sequence of tagged blocks:
+//
+//   [StatePayloadHeader][u32 tag][u32 size][payload]...
+//
+// The payload is serialized by Python, sent over the /state WebSocket, and
+// parsed by the browser.
+//
+// TODO(matijak): Try shrinking the physics block: float32 (or quantized) values
+// instead of doubles, and/or delta-encoding against the client's last-acked
+// payload. The /state ack (web_server.py) tells the server which snapshot each
+// client last applied, which is the baseline that delta compression needs. For
+// 100humanoids.xml the payload is ~181 KB of doubles and dominates slow links.
+
+#ifndef MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_
+#define MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+#include <mujoco/mujoco.h>
+
+namespace mujoco::studio {
+
+// "MJWS" as little-endian bytes. This magic constant identifies the
+// StateServer WebSocket payload header and helps detect malformed or
+// misrouted messages.
+constexpr uint32_t kStatePayloadMagic =
+    'M' | ('J' << 8) | ('W' << 16) | ('S' << 24);
+constexpr uint16_t kStatePayloadVersion = 1;
+
+struct StatePayloadHeader {
+  uint32_t magic = kStatePayloadMagic;
+  uint16_t version = kStatePayloadVersion;
+  uint16_t nblocks = 0;
+  // CRC32 of the model's MJB bytes. When this changes, the browser must
+  // refetch /model.mjb before applying any further state.
+  uint32_t model_crc32 = 0;
+};
+static_assert(sizeof(StatePayloadHeader) == 12);
+
+// Block tags. Readers must skip unknown tags.
+enum StateBlockTag : uint32_t {
+  kTagPhysicsState = 1,  // [i32 mjtState spec signature][mjtNum values...]
+  kTagRenderState = 2,   // fixed-size block of kRenderStateSize bytes
+  kTagExtraGeoms = 3,    // n x mjvGeom (n = size / sizeof(mjvGeom))
+};
+
+struct StateBlockHeader {
+  uint32_t tag = 0;
+  uint32_t size = 0;
+};
+static_assert(sizeof(StateBlockHeader) == 8);
+
+// Fixed byte size of the render state block appended after physics state.
+// These are plain C structs of int/float/double members whose total size is
+// fixed, independent of the model and generally negligible compared to the size
+// of the physics state
+constexpr size_t kRenderStateSize =
+    sizeof(mjvCamera) + sizeof(mjvPerturb) + sizeof(mjvOption) +
+    sizeof(mjOption) + sizeof(mjVisual) + sizeof(mjStatistic) + mjNRNDFLAG;
+
+// Maximum number of extra geoms serialized per frame. Bounds the shared
+// memory buffer the StateServer allocates; WebViewer truncates longer lists.
+constexpr uint32_t kMaxExtraGeoms = 1024;
+
+// Upper bound of a serialized payload, used to size the StateServer's shared
+// memory buffer. `physics_bytes` is mj_stateSize(...) * sizeof(mjtNum).
+size_t MaxStatePayloadSize(size_t physics_bytes);
+
+// Serialize the complete state payload sent over the state WebSocket.
+std::vector<std::byte> SerializeStatePayload(
+    uint32_t model_crc32, int32_t physics_spec, const void* physics,
+    size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb,
+    const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis,
+    const mjStatistic& stat, const std::vector<uint8_t>& render_flags,
+    const mjvGeom* extra_geoms, size_t extra_geom_count);
+
+// Parsed view into a serialized payload. Pointers alias the input buffer and
+// are NOT guaranteed to be aligned; so you must memcpy the data out before use.
+struct StatePayloadView {
+  uint32_t model_crc32 = 0;
+  int32_t physics_spec = 0;
+  const std::byte* physics = nullptr;
+  size_t physics_bytes = 0;
+  const std::byte* render_state = nullptr;  // kRenderStateSize bytes when non-null
+  const std::byte* extra_geoms = nullptr;   // extra_geom_count * sizeof(mjvGeom)
+  size_t extra_geom_count = 0;
+};
+
+// Parses a payload produced by SerializeStatePayload. Returns false if the
+// buffer is malformed (bad magic/version or out-of-bounds block). Blocks
+// with unknown tags are skipped.
+bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out);
+
+}  // namespace mujoco::studio
+
+#endif  // MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_

python/mujoco/experimental/studio/web/state_payload_py.cc

@@ -0,0 +1,72 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Python bindings for the state payload wire format (state_payload.h).
+//
+// WebViewer serializes the /state WebSocket payload with this module each
+// frame; the browser parses it with the same header (web_client_session).
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include <mujoco/mujoco.h>
+#include "state_payload.h"
+#include "structs.h"
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+// Serialize the complete state WebSocket payload (see state_payload.h):
+// physics state, render state and extra geoms as tagged blocks.
+static py::bytes SerializeStatePayload(
+    uint32_t model_crc32, int physics_spec, const py::bytes& physics_state,
+    const mujoco::python::MjvCameraWrapper& camera,
+    const mujoco::python::MjvPerturbWrapper& perturb,
+    const mujoco::python::MjvOptionWrapper& vis_options,
+    const mujoco::python::MjModelWrapper& model,
+    const std::vector<uint8_t>& render_flags,
+    const std::vector<mujoco::python::MjvGeomWrapper>& extra_geoms) {
+  std::vector<mjvGeom> geoms;
+  geoms.reserve(extra_geoms.size());
+  for (const mujoco::python::MjvGeomWrapper& geom_wrapper : extra_geoms) {
+    if (geom_wrapper.get()) {
+      geoms.push_back(*geom_wrapper.get());
+    }
+  }
+
+  std::string physics = physics_state;
+  const std::vector<std::byte> buffer = mujoco::studio::SerializeStatePayload(
+      model_crc32, physics_spec, physics.data(), physics.size(), *camera.get(),
+      *perturb.get(), *vis_options.get(), model.get()->opt, model.get()->vis,
+      model.get()->stat, render_flags, geoms.data(), geoms.size());
+  return py::bytes(reinterpret_cast<const char*>(buffer.data()), buffer.size());
+}
+
+// Upper bound of a serialized payload for a model whose physics state is
+// `physics_bytes` long. Used to size the StateServer's shared memory.
+static size_t MaxStatePayloadSize(size_t physics_bytes) {
+  return mujoco::studio::MaxStatePayloadSize(physics_bytes);
+}
+
+PYBIND11_MODULE(state_payload, m, pybind11::mod_gil_not_used()) {
+  py::module_::import("mujoco._structs");
+  m.doc() = "MuJoCo web viewer state payload serialization";
+
+  m.def("serialize_state_payload", &SerializeStatePayload);
+  m.def("max_state_payload_size", &MaxStatePayloadSize);
+  m.attr("MAX_EXTRA_GEOMS") = mujoco::studio::kMaxExtraGeoms;
+}

src/experimental/platform/CMakeLists.txt

@@ -22,6 +22,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME}
     PUBLIC
         helpers.cc
         helpers.h
+        resources.cc
+        resources.h
         sys_utils.cc
         sys_utils.h
         hal/egl_utils.cc

src/experimental/platform/resources.cc

@@ -0,0 +1,108 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "experimental/platform/resources.h"
+
+#include <filesystem>  // NOLINT(build/c++17)
+#include <fstream>
+#include <ios>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <mujoco/mujoco.h>
+#include "experimental/platform/sys_utils.h"
+
+namespace mujoco::platform {
+
+namespace {
+
+std::string Resolve(std::string_view path) {
+  std::string_view subpath = path.substr(path.find(':') + 1);
+  std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve);
+  if (exe_dir.empty()) {
+    return std::string("assets/") + std::string(subpath);
+  }
+  std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources";
+  if (std::filesystem::exists(resources_dir / "assets")) {
+    return (resources_dir / "assets" / subpath).string();
+  }
+  return (exe_dir / "assets" / subpath).string();
+}
+
+class FileResource {
+ public:
+  explicit FileResource(const std::string& path)
+      : file_(path, std::ios::binary | std::ios::ate) {
+    if (!file_.is_open()) {
+      mju_warning("Cannot open file %s", path.c_str());
+      return;
+    }
+
+    size_ = file_.tellg();
+    file_.seekg(0, std::ios::beg);
+  }
+
+  int Read(const void** buffer) {
+    buffer_.resize(size_);
+    if (!file_.read(reinterpret_cast<char*>(buffer_.data()), size_)) {
+      return 0;
+    }
+    *buffer = buffer_.data();
+    return size_;
+  }
+
+  int Size() const { return size_; }
+
+  FileResource(const FileResource&) = delete;
+  FileResource& operator=(const FileResource&) = delete;
+
+ private:
+  std::ifstream file_;
+  std::vector<char> buffer_;
+  int size_ = 0;
+};
+
+}  // namespace
+
+void RegisterResourceProviders() {
+  mjpResourceProvider resource_provider;
+  mjp_defaultResourceProvider(&resource_provider);
+
+  resource_provider.open = [](mjResource* resource) {
+    const std::string resolved_path = Resolve(resource->name);
+    FileResource* f = new FileResource(resolved_path);
+    if (f->Size() == 0) {
+      delete f;
+      return 0;
+    }
+    resource->data = f;
+    return f->Size();
+  };
+  resource_provider.read = [](mjResource* resource, const void** buffer) {
+    FileResource* f = static_cast<FileResource*>(resource->data);
+    return f->Read(buffer);
+  };
+  resource_provider.close = [](mjResource* resource) {
+    delete static_cast<FileResource*>(resource->data);
+    resource->data = nullptr;
+  };
+
+  resource_provider.prefix = "font";
+  mjp_registerResourceProvider(&resource_provider);
+  resource_provider.prefix = "filament";
+  mjp_registerResourceProvider(&resource_provider);
+}
+
+}  // namespace mujoco::platform

src/experimental/platform/resources.h

@@ -0,0 +1,25 @@
+// Copyright 2026 DeepMind Technologies Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_
+#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_
+
+namespace mujoco::platform {
+
+// Registers MuJoCo resource providers for font and filament assets.
+void RegisterResourceProviders();
+
+}  // namespace mujoco::platform
+
+#endif  // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_

src/experimental/studio/launcher.cc

@@ -16,99 +16,21 @@
 
 #include <cstdlib>
 #include <cstring>
-#include <filesystem>  // NOLINT(build/c++17)
-#include <fstream>
-#include <ios>
 #include <string>
 #include <string_view>
-#include <vector>
 
 #include <mujoco/mujoco.h>
 #include "experimental/platform/hal/graphics_mode.h"
-#include "experimental/platform/sys_utils.h"
+#include "experimental/platform/resources.h"
 #include "experimental/studio/app.h"
 
 namespace mujoco::studio {
-namespace {
-
-std::string Resolve(std::string_view path) {
-  std::string_view subpath = path.substr(path.find(':') + 1);
-  std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve);
-  if (exe_dir.empty()) {
-    return std::string("assets/") + std::string(subpath);
-  }
-  std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources";
-  if (std::filesystem::exists(resources_dir / "assets")) {
-    return (resources_dir / "assets" / subpath).string();
-  }
-  return (exe_dir / "assets" / subpath).string();
-}
-
-class FileResource {
- public:
-  explicit FileResource(const std::string& path)
-      : file_(path, std::ios::binary | std::ios::ate) {
-    if (!file_.is_open()) {
-      mju_warning("Cannot open file %s", path.c_str());
-      return;
-    }
-
-    size_ = file_.tellg();
-    file_.seekg(0, std::ios::beg);
-  }
-
-  int Read(const void** buffer) {
-    buffer_.resize(size_);
-    if (!file_.read(reinterpret_cast<char*>(buffer_.data()), size_)) {
-      return 0;
-    }
-    *buffer = buffer_.data();
-    return size_;
-  }
-
-  int Size() const { return size_; }
-
-  FileResource(const FileResource&) = delete;
-  FileResource& operator=(const FileResource&) = delete;
-
- private:
-  std::ifstream file_;
-  std::vector<char> buffer_;
-  int size_ = 0;
-};
-
-}  // namespace
 
 int LaunchStudio(int argc, char** argv, LauncherConfig config) {
   const char* home = std::getenv("HOME");
   const std::string ini_path = std::string(home ? home : ".") + "/.mujoco.ini";
 
-  mjpResourceProvider resource_provider;
+  mujoco::platform::RegisterResourceProviders();
-  mjp_defaultResourceProvider(&resource_provider);
-
-  resource_provider.open = [](mjResource* resource) {
-    const std::string resolved_path = Resolve(resource->name);
-    FileResource* f = new FileResource(resolved_path);
-    if (f->Size() == 0) {
-      delete f;
-      return 0;
-    }
-    resource->data = f;
-    return f->Size();
-  };
-  resource_provider.read = [](mjResource* resource, const void** buffer) {
-    FileResource* f = static_cast<FileResource*>(resource->data);
-    return f->Read(buffer);
-  };
-  resource_provider.close = [](mjResource* resource) {
-    delete static_cast<FileResource*>(resource->data);
-    resource->data = nullptr;
-  };
-
-  resource_provider.prefix = "font";
-  mjp_registerResourceProvider(&resource_provider);
-  resource_provider.prefix = "filament";
-  mjp_registerResourceProvider(&resource_provider);
 
   if (config.gfx_mode.empty()) {
     const char* display = std::getenv("DISPLAY");

src/experimental/studio/live.html

@@ -6,6 +6,7 @@
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Atkinson+Hyperlegible+Next:wght@200..800&display=swap" rel="stylesheet">
     <link rel="stylesheet" href="live.css">
+    <link rel="icon" href="favicon.ico" sizes="16x16 32x32 48x48">
     <title>MuJoCo Live</title>
   </head>
   <body style="margin: 0; overflow: hidden">

src/render/filament/assets/outline_composite.mat

@@ -16,6 +16,8 @@
 // outline around an object. The outline is drawn by checking the distance to
 // the nearest edge point (as computed by the algorithm) and, if within the
 // desired width, setting the color of the pixel to the desired outline color.
+// The outline is dimmed where it is occluded, i.e. where the scene contains
+// geometry that is closer than the selected object at the nearest edge point.
 
 material {
     name : outline_composite,
@@ -25,8 +27,11 @@ material {
     depthCulling : false,
     parameters : [
         { type : sampler2d, name : source },
+        { type : sampler2d, name : scene_depth, precision : high, filterable : false },
+        { type : sampler2d, name : selection_depth, precision : high, filterable : false },
         { type : float4, name : color },
-        { type : float, name : width }
+        { type : float, name : width },
+        { type : float, name : dim }
     ],
     variables : [
         vertex
@@ -77,6 +82,18 @@ fragment {
     float inner_alpha = smoothstep(0.0, 1.0, dist);
     float outer_alpha = smoothstep(width, width - 1.0, dist);
     float alpha = outer_alpha * inner_alpha;
+
+    // Dim the outline where it is occluded. The depth buffer is reversed
+    // (1 = near, 0 = far), so the outline is occluded if the scene depth at
+    // this pixel is greater than the depth of the selected object at the
+    // nearest edge point. The epsilon avoids spurious dimming where the two
+    // depths are nearly equal, e.g. at contact points.
+    highp float scene_z = texture(materialParams_scene_depth, variable_vertex.xy).r;
+    highp float edge_z = texture(materialParams_selection_depth, edge).r;
+    if (scene_z > edge_z * 1.001 + 1e-6) {
+      alpha *= materialParams.dim;
+    }
+
     postProcess.color = mix(vec4(0), materialParams.color, alpha * materialParams.color.a);
   }
 }

src/render/filament/assets/outline_flatten.mat

@@ -16,11 +16,15 @@
 // This material writes the the screen-space position of each fragment as the
 // output color of that fragment. This will provide us with the initial data for
 // the Jump Flood Algorithm. See "outline_jumpflood.mat" for more details.
+//
+// Depth is also written so that the render target's depth attachment holds the
+// depth of the selected objects; "outline_composite.mat" samples it to dim
+// occluded portions of the outline.
 material {
     name : outline_flatten,
     shadingModel : unlit,
     culling : none,
-    depthWrite : false
+    depthWrite : true
 }
 
 fragment {

src/render/filament/core/outliner.cc

@@ -37,12 +37,15 @@
 namespace mujoco {
 
 Outliner::Outliner(ObjectManager* object_mgr, uint8_t layer_mask,
-                   filament::math::float4 color, float thickness)
+                   uint8_t scene_layer_mask, filament::math::float4 color,
+                   float thickness, float occlusion_dim)
     : object_mgr_(object_mgr),
       engine_(object_mgr->GetEngine()),
       layer_mask_(layer_mask),
+      scene_layer_mask_(scene_layer_mask),
       color_(color),
-      thickness_(thickness) {}
+      thickness_(thickness),
+      occlusion_dim_(occlusion_dim) {}
 
 Outliner::~Outliner() { Reset(); }
 
@@ -98,9 +101,10 @@ void Outliner::Prepare(int width, int height) {
     }
   };
 
-  // Set up two render targets. We will alternate between the two targets to
+  // Set up the render targets: one for the scene depth, one for the flattened
-  // allow for chaining passes together.
+  // selection mask and its depth, and two ping-pong targets for chaining the
-  for (int i = 0; i < 2; ++i) {
+  // jump flood passes together.
+  for (int i = 0; i < kNumTargets; ++i) {
     mjrfRenderTargetConfig config;
     mjrf_defaultRenderTargetConfig(&config);
     config.color_format = mjPIXEL_FORMAT_RGBA8;
@@ -127,9 +131,11 @@ void Outliner::Prepare(int width, int height) {
     view->setMultiSampleAntiAliasingOptions({.enabled = false});
   }
 
-  // In the first pass, we will render a given scene, but only render the
+  // In the first pass, we render the scene layers that can occlude the
-  // objects marked as outlines. We assume that the objects have already been
+  // outline; only the resulting depth is used. In the second pass, we render
-  // assigned the kOutlineFlatten material.
+  // the given scene again, but only the objects marked as outlines. We assume
+  // that the objects have already been assigned the kOutlineFlatten material.
+  views_[kPassSceneDepth]->setVisibleLayers(0xff, scene_layer_mask_);
   views_[kPassFlatten]->setVisibleLayers(0xff, layer_mask_);
 
   // All subsequent passes are full-screen post-processing passes.
@@ -141,16 +147,19 @@ void Outliner::Prepare(int width, int height) {
   setup_fullscreen(kPassDrawOutline, ObjectManager::kOutlineComposite);
 
   // Chain the passes together such that the output of a pass is the input to
-  // the next pass. The first pass has no input (we are just rendering the
+  // the next pass. The scene passes have no input (we are just rendering the
-  // selected objects) and the last pass has no output (we are just rendering
+  // scene) and the last pass has no output (we are just rendering the outline
-  // the outline to the externally provided target).
+  // to the externally provided target). The flatten pass has its own target
-  bind(kPassFlatten, -1, 0);
+  // (rather than a ping-pong target) so that its depth attachment survives the
-  bind(kPassJumpFlood1, 0, 1);
+  // jump flood passes and can be sampled by the composite pass.
-  bind(kPassJumpFlood2, 1, 0);
+  bind(kPassSceneDepth, -1, kTargetSceneDepth);
-  bind(kPassJumpFlood3, 0, 1);
+  bind(kPassFlatten, -1, kTargetFlatten);
-  bind(kPassJumpFlood4, 1, 0);
+  bind(kPassJumpFlood1, kTargetFlatten, kTargetPing);
-  bind(kPassJumpFlood5, 0, 1);
+  bind(kPassJumpFlood2, kTargetPing, kTargetPong);
-  bind(kPassDrawOutline, 1, -1);
+  bind(kPassJumpFlood3, kTargetPong, kTargetPing);
+  bind(kPassJumpFlood4, kTargetPing, kTargetPong);
+  bind(kPassJumpFlood5, kTargetPong, kTargetPing);
+  bind(kPassDrawOutline, kTargetPing, -1);
 
   // Bind the parameters for each pass. For the jump flood passes, the step
   // parameter determines how far to propagate the outline in each pass.
@@ -160,9 +169,22 @@ void Outliner::Prepare(int width, int height) {
   material_instances_[kPassJumpFlood4]->setParameter("step", 2.0f);
   material_instances_[kPassJumpFlood5]->setParameter("step", 1.0f);
 
-  // The final pass renders the actual outline onto a render target.
+  // The final pass renders the actual outline onto a render target. It samples
+  // the scene depth and the selection depth to dim occluded outline pixels.
+  const filament::TextureSampler depth_sampler(
+      filament::TextureSampler::MinFilter::NEAREST,
+      filament::TextureSampler::MagFilter::NEAREST);
   material_instances_[kPassDrawOutline]->setParameter("color", color_);
   material_instances_[kPassDrawOutline]->setParameter("width", thickness_);
+  material_instances_[kPassDrawOutline]->setParameter("dim", occlusion_dim_);
+  material_instances_[kPassDrawOutline]->setParameter(
+      "scene_depth",
+      targets_[kTargetSceneDepth]->GetDepthTexture()->GetFilamentTexture(),
+      depth_sampler);
+  material_instances_[kPassDrawOutline]->setParameter(
+      "selection_depth",
+      targets_[kTargetFlatten]->GetDepthTexture()->GetFilamentTexture(),
+      depth_sampler);
 
   // Commit all the material instances to the engine.
   for (auto& material_instance : material_instances_) {
@@ -222,9 +244,13 @@ void Outliner::Render(filament::Renderer* renderer, filament::View* view,
     view->setViewport(viewport);
   }
 
-  // Re-render the view's scene to create the flattened selection mask.
+  // Re-render the view's scene to capture the scene depth (used for occlusion
+  // dimming) and to create the flattened selection mask.
   auto prev_clear_opts = renderer->getClearOptions();
   renderer->setClearOptions({.clearColor = {0, 0, 0, 0}, .clear = true});
+  views_[kPassSceneDepth]->setScene(view->getScene());
+  views_[kPassSceneDepth]->setCamera(&view->getCamera());
+  renderer->render(views_[kPassSceneDepth]);
   views_[kPassFlatten]->setScene(view->getScene());
   views_[kPassFlatten]->setCamera(&view->getCamera());
   renderer->render(views_[kPassFlatten]);

src/render/filament/core/outliner.h

@@ -31,11 +31,13 @@ namespace mujoco {
 // Renders an outline of selected objects.
 //
 // This class uses the "jump flood" algorithm to create an outline of selected
-// objects.
+// objects. The outline is dimmed by `occlusion_dim` where the selected objects
+// are occluded by scene geometry on the `scene_layer_mask` layers.
 class Outliner {
  public:
   Outliner(ObjectManager* object_mgr, uint8_t layer_mask,
-           filament::math::float4 color, float thickness);
+           uint8_t scene_layer_mask, filament::math::float4 color,
+           float thickness, float occlusion_dim);
   ~Outliner();
 
   Outliner(const Outliner&) = delete;
@@ -53,6 +55,7 @@ class Outliner {
   void Reset();
 
   enum Pass {
+    kPassSceneDepth,
     kPassFlatten,
     kPassJumpFlood1,
     kPassJumpFlood2,
@@ -65,17 +68,27 @@ class Outliner {
     kNumJumpFloodPasses = kPassJumpFlood5 - kPassJumpFlood1 + 1,
   };
 
+  enum Target {
+    kTargetSceneDepth,  // depth of the scene, used for occlusion dimming
+    kTargetFlatten,     // selection mask (color) and selection depth (depth)
+    kTargetPing,        // jump flood ping-pong buffer
+    kTargetPong,        // jump flood ping-pong buffer
+    kNumTargets,
+  };
+
   ObjectManager* object_mgr_ = nullptr;
   filament::Engine* engine_ = nullptr;
   uint8_t layer_mask_ = 0xff;
+  uint8_t scene_layer_mask_ = 0xff;
   filament::math::float4 color_ = {1.0f, 1.0f, 1.0f, 1.0f};
   float thickness_ = 2.5f;
+  float occlusion_dim_ = 1.0f;
 
   int width_ = 0;
   int height_ = 0;
 
   filament::Camera* camera_ = nullptr;
-  std::unique_ptr<RenderTarget> targets_[2];
+  std::unique_ptr<RenderTarget> targets_[kNumTargets];
 
   filament::View* views_[kNumPasses] = {};
   filament::Scene* scenes_[kNumPasses] = {};

src/render/filament/core/scene_view.cc

@@ -321,9 +321,9 @@ void SceneView::Render(filament::Renderer* renderer,
 
   if (!selected_renderables.empty()) {
     if (!outliner_) {
-      outliner_ =
+      outliner_ = std::make_unique<Outliner>(
-          std::make_unique<Outliner>(object_mgr_, kLayerMask_Outline,
+          object_mgr_, kLayerMask_Outline, kLayerMask_Object,
-                                     float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f);
+          float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f, 0.25f);
     }
 
     for (Renderable* renderable : selected_renderables) {