diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml index 59823e1d..e8c6e4cb 100644 --- a/.github/workflows/live.yml +++ b/.github/workflows/live.yml @@ -35,6 +35,7 @@ jobs: cp src/experimental/studio/live.html dist/index.html cp src/experimental/studio/live.css dist/live.css cp src/experimental/studio/live.js dist/live.js + cp src/experimental/studio/favicon.ico dist/favicon.ico COMMIT_HASH=$(git rev-parse HEAD 2>/dev/null || true) if [ -n "$COMMIT_HASH" ]; then sed -i "s/__COMMIT_HASH_PLACEHOLDER__/$COMMIT_HASH/g" dist/index.html diff --git a/doc/mjwarp/index.rst b/doc/mjwarp/index.rst index 0e3d4877..3a598a6b 100644 --- a/doc/mjwarp/index.rst +++ b/doc/mjwarp/index.rst @@ -444,6 +444,15 @@ high-performance tensor/matrix operations optimized for fixed tile sizes. to a tight upper bound of the expected active DOFs significantly reduces GPU memory usage and improves throughput. +.. TODO(taylorhowell): update example with correct island cycles initialization + +4. When setting ``nvmax < nv`` it is recommended to initialize all trees to asleep in order to avoid initial dof + overflow. + + .. code-block:: python + + d.tree_asleep.assign(np.array(np.arange(mjm.ntree, dtype=np.int32)), dtype=np.int32) + .. note:: Consider increasing the sleep tolerance setting (e.g., ``sleep_tolerance="0.01"`` in XML options or ``spec.option.sleep_tolerance = 0.01`` in Python) from its default value (0.001) to more quickly @@ -501,19 +510,22 @@ Certain fields are safe to modify directly without compilation, enabling on-devi `GitHub issue 893 `__ tracks adding on-device updates for a subset of fields. -Per-world meshes +Per-world assets ---------------- -Per-world meshes enable heterogeneous worlds where different worlds simulate different meshes. The workflow -is: +Per-world assets enable heterogeneous worlds where different worlds simulate different +`assets `__ including meshes, height fields, materials, +and textures. The general workflow is: -1. Create an :ref:`mjSpec` with **all** mesh assets and the **maximum** number of geom slots needed across variants. +1. Create an :ref:`mjSpec` with **all** assets. 2. Compile each variant by mutating the spec and calling ``spec.compile()``. 3. Compile a **base** model and create :class:`mjw.Model ` from it. 4. Override the relevant :class:`mjw.Model ` fields with per-world arrays built from the compiled variants. -**Example 1 — Geom-level** randomization (1 body, 1 geom, 2 mesh assets): +.. rubric:: Per-world meshes + +**Example 1 — Per-world meshes: Geom-level** randomization (1 body, 1 geom, 2 mesh assets): The base scene includes all mesh assets. The geom references one mesh (``mesh_a``); a second mesh (``mesh_b``) is available for per-world substitution. @@ -612,7 +624,7 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a` m.body_ipos = wp.array(body_ipos, dtype=wp.vec3) m.body_iquat = wp.array(body_iquat, dtype=wp.quat) -**Example 2 — Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets): +**Example 2 — Per-world meshes: Body-level** randomization (1 body, 1 or 2 geoms, 3 mesh assets): .. admonition:: Maximum geom count :class: important @@ -784,6 +796,14 @@ The base scene includes all mesh assets. The geom references one mesh (``mesh_a` - ``wp.quat`` - ``(nworld, nbody)`` +Per-world height fields, materials, and textures can be similarly formulated. + +.. admonition:: Per-world asset dependent field construction + :class: note + + MJWarp enables per-world asset functionality but does not provide utilities for construction of dependent per-world + field variants. Construction is left to the user or environment authoring frameworks. + Batch Rendering =============== diff --git a/python/mujoco/experimental/studio/web/state_payload.cc b/python/mujoco/experimental/studio/web/state_payload.cc new file mode 100644 index 00000000..608f881b --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload.cc @@ -0,0 +1,171 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wire-format implementation for the web viewer's state payload (see +// state_payload.h). Compiled into BOTH sides of the wire: the state_payload +// pybind module (serializer, via state_payload_py.cc) and the wasm +// web_client (parser, via web_client_session.cc). It must therefore stay +// free of python- or browser-specific dependencies. + +#include "state_payload.h" + +#include +#include +#include +#include + +#include + +namespace mujoco::studio { +namespace { + +// Appends raw bytes to the payload buffer. +void AppendBytes(std::vector& buffer, const void* data, + size_t size) { + const std::byte* bytes = static_cast(data); + buffer.insert(buffer.end(), bytes, bytes + size); +} + +// Appends a complete [u32 tag][u32 size][payload] block. +void AppendStateBlock(std::vector& buffer, uint32_t tag, + const void* data, size_t size) { + StateBlockHeader block_header{tag, static_cast(size)}; + AppendBytes(buffer, &block_header, sizeof(block_header)); + AppendBytes(buffer, data, size); +} + +// Serializes the render state (exactly kRenderStateSize bytes) into `ptr`. +void SerializeRenderStateInto(std::byte* ptr, const mjvCamera& camera, + const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, + const mjVisual& vis, const mjStatistic& stat, + const std::vector& render_flags) { + memcpy(ptr, &camera, sizeof(mjvCamera)); + ptr += sizeof(mjvCamera); + + memcpy(ptr, &perturb, sizeof(mjvPerturb)); + ptr += sizeof(mjvPerturb); + + memcpy(ptr, &vis_options, sizeof(mjvOption)); + ptr += sizeof(mjvOption); + + memcpy(ptr, &opt, sizeof(mjOption)); + ptr += sizeof(mjOption); + + memcpy(ptr, &vis, sizeof(mjVisual)); + ptr += sizeof(mjVisual); + + memcpy(ptr, &stat, sizeof(mjStatistic)); + ptr += sizeof(mjStatistic); + + // Pack render flags (mjNRNDFLAG bytes). + memset(ptr, 0, mjNRNDFLAG); + for (size_t i = 0; i < mjNRNDFLAG && i < render_flags.size(); ++i) { + ptr[i] = static_cast(render_flags[i]); + } +} + +} // namespace + +size_t MaxStatePayloadSize(size_t physics_bytes) { + return sizeof(StatePayloadHeader) + 3 * sizeof(StateBlockHeader) + + (sizeof(int32_t) + physics_bytes) + kRenderStateSize + + kMaxExtraGeoms * sizeof(mjvGeom); +} + +std::vector SerializeStatePayload( + uint32_t model_crc32, int32_t physics_spec, const void* physics, + size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis, + const mjStatistic& stat, const std::vector& render_flags, + const mjvGeom* extra_geoms, size_t extra_geom_count) { + extra_geom_count = + extra_geom_count > kMaxExtraGeoms ? kMaxExtraGeoms : extra_geom_count; + std::vector buffer; + buffer.reserve(MaxStatePayloadSize(physics_bytes)); + + StatePayloadHeader header; + header.nblocks = extra_geom_count > 0 ? 3 : 2; + header.model_crc32 = model_crc32; + AppendBytes(buffer, &header, sizeof(header)); + + // Physics state: [i32 spec][mjtNum values...]. + StateBlockHeader physics_header{ + kTagPhysicsState, static_cast(sizeof(int32_t) + physics_bytes)}; + AppendBytes(buffer, &physics_header, sizeof(physics_header)); + AppendBytes(buffer, &physics_spec, sizeof(int32_t)); + AppendBytes(buffer, physics, physics_bytes); + + // Render state, serialized into place. + StateBlockHeader render_header{kTagRenderState, + static_cast(kRenderStateSize)}; + AppendBytes(buffer, &render_header, sizeof(render_header)); + const size_t render_offset = buffer.size(); + buffer.resize(render_offset + kRenderStateSize); + SerializeRenderStateInto(buffer.data() + render_offset, camera, perturb, + vis_options, opt, vis, stat, render_flags); + + // Extra geoms (only when present). + if (extra_geom_count > 0) { + AppendStateBlock(buffer, kTagExtraGeoms, extra_geoms, + extra_geom_count * sizeof(mjvGeom)); + } + + return buffer; +} + +bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out) { + const std::byte* bytes = static_cast(data); + if (size < sizeof(StatePayloadHeader)) return false; + + StatePayloadHeader header; + memcpy(&header, bytes, sizeof(header)); + if (header.magic != kStatePayloadMagic) return false; + if (header.version != kStatePayloadVersion) return false; + out->model_crc32 = header.model_crc32; + + size_t offset = sizeof(StatePayloadHeader); + for (uint16_t i = 0; i < header.nblocks; ++i) { + if (offset + sizeof(StateBlockHeader) > size) return false; + StateBlockHeader block; + memcpy(&block, bytes + offset, sizeof(block)); + offset += sizeof(StateBlockHeader); + if (offset + block.size > size) return false; + const std::byte* payload = bytes + offset; + + switch (block.tag) { + case kTagPhysicsState: + if (block.size < sizeof(int32_t)) return false; + memcpy(&out->physics_spec, payload, sizeof(int32_t)); + out->physics = payload + sizeof(int32_t); + out->physics_bytes = block.size - sizeof(int32_t); + break; + case kTagRenderState: + if (block.size != kRenderStateSize) return false; + out->render_state = payload; + break; + case kTagExtraGeoms: + if (block.size % sizeof(mjvGeom) != 0) return false; + out->extra_geoms = payload; + out->extra_geom_count = block.size / sizeof(mjvGeom); + break; + default: + break; // Unknown tag: skip. + } + offset += block.size; + } + return true; +} + +} // namespace mujoco::studio diff --git a/python/mujoco/experimental/studio/web/state_payload.h b/python/mujoco/experimental/studio/web/state_payload.h new file mode 100644 index 00000000..0ceacf92 --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload.h @@ -0,0 +1,128 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the serialization format for the web viewer's browser +// client render payload containing the data needed so that the browser can +// render the scene using the following call: +// +// Render(model, data, perturb, camera, vis_options, width, height, extra_geoms) +// +// The arguments come from the Python process: +// +// * model : fetched once over HTTP as /model.mjb; its runtime-mutable +// parts (opt/vis/stat) are re-sent in the render state block. +// * data : streamed as the physics state vector (mjSTATE_INTEGRATION); +// the browser recomputes the rest via mj_setState/mj_forward. +// * width/height: the browser canvas size. +// * extra_geoms : optional variable-size kTagExtraGeoms block. +// * ... : the rest of the arguments are sent as a fixed-size block +// +// The payload (SerializeStatePayload) is a sequence of tagged blocks: +// +// [StatePayloadHeader][u32 tag][u32 size][payload]... +// +// The payload is serialized by Python, sent over the /state WebSocket, and +// parsed by the browser. +// +// TODO(matijak): Try shrinking the physics block: float32 (or quantized) values +// instead of doubles, and/or delta-encoding against the client's last-acked +// payload. The /state ack (web_server.py) tells the server which snapshot each +// client last applied, which is the baseline that delta compression needs. For +// 100humanoids.xml the payload is ~181 KB of doubles and dominates slow links. + +#ifndef MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ +#define MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ + +#include +#include +#include + +#include + +namespace mujoco::studio { + +// "MJWS" as little-endian bytes. This magic constant identifies the +// StateServer WebSocket payload header and helps detect malformed or +// misrouted messages. +constexpr uint32_t kStatePayloadMagic = + 'M' | ('J' << 8) | ('W' << 16) | ('S' << 24); +constexpr uint16_t kStatePayloadVersion = 1; + +struct StatePayloadHeader { + uint32_t magic = kStatePayloadMagic; + uint16_t version = kStatePayloadVersion; + uint16_t nblocks = 0; + // CRC32 of the model's MJB bytes. When this changes, the browser must + // refetch /model.mjb before applying any further state. + uint32_t model_crc32 = 0; +}; +static_assert(sizeof(StatePayloadHeader) == 12); + +// Block tags. Readers must skip unknown tags. +enum StateBlockTag : uint32_t { + kTagPhysicsState = 1, // [i32 mjtState spec signature][mjtNum values...] + kTagRenderState = 2, // fixed-size block of kRenderStateSize bytes + kTagExtraGeoms = 3, // n x mjvGeom (n = size / sizeof(mjvGeom)) +}; + +struct StateBlockHeader { + uint32_t tag = 0; + uint32_t size = 0; +}; +static_assert(sizeof(StateBlockHeader) == 8); + +// Fixed byte size of the render state block appended after physics state. +// These are plain C structs of int/float/double members whose total size is +// fixed, independent of the model and generally negligible compared to the size +// of the physics state +constexpr size_t kRenderStateSize = + sizeof(mjvCamera) + sizeof(mjvPerturb) + sizeof(mjvOption) + + sizeof(mjOption) + sizeof(mjVisual) + sizeof(mjStatistic) + mjNRNDFLAG; + +// Maximum number of extra geoms serialized per frame. Bounds the shared +// memory buffer the StateServer allocates; WebViewer truncates longer lists. +constexpr uint32_t kMaxExtraGeoms = 1024; + +// Upper bound of a serialized payload, used to size the StateServer's shared +// memory buffer. `physics_bytes` is mj_stateSize(...) * sizeof(mjtNum). +size_t MaxStatePayloadSize(size_t physics_bytes); + +// Serialize the complete state payload sent over the state WebSocket. +std::vector SerializeStatePayload( + uint32_t model_crc32, int32_t physics_spec, const void* physics, + size_t physics_bytes, const mjvCamera& camera, const mjvPerturb& perturb, + const mjvOption& vis_options, const mjOption& opt, const mjVisual& vis, + const mjStatistic& stat, const std::vector& render_flags, + const mjvGeom* extra_geoms, size_t extra_geom_count); + +// Parsed view into a serialized payload. Pointers alias the input buffer and +// are NOT guaranteed to be aligned; so you must memcpy the data out before use. +struct StatePayloadView { + uint32_t model_crc32 = 0; + int32_t physics_spec = 0; + const std::byte* physics = nullptr; + size_t physics_bytes = 0; + const std::byte* render_state = nullptr; // kRenderStateSize bytes when non-null + const std::byte* extra_geoms = nullptr; // extra_geom_count * sizeof(mjvGeom) + size_t extra_geom_count = 0; +}; + +// Parses a payload produced by SerializeStatePayload. Returns false if the +// buffer is malformed (bad magic/version or out-of-bounds block). Blocks +// with unknown tags are skipped. +bool ParseStatePayload(const void* data, size_t size, StatePayloadView* out); + +} // namespace mujoco::studio + +#endif // MUJOCO_PYTHON_EXPERIMENTAL_STUDIO_WEB_STATE_PAYLOAD_H_ diff --git a/python/mujoco/experimental/studio/web/state_payload_py.cc b/python/mujoco/experimental/studio/web/state_payload_py.cc new file mode 100644 index 00000000..9be00030 --- /dev/null +++ b/python/mujoco/experimental/studio/web/state_payload_py.cc @@ -0,0 +1,72 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Python bindings for the state payload wire format (state_payload.h). +// +// WebViewer serializes the /state WebSocket payload with this module each +// frame; the browser parses it with the same header (web_client_session). + +#include +#include +#include +#include + +#include +#include "state_payload.h" +#include "structs.h" +#include +#include + +namespace py = pybind11; + +// Serialize the complete state WebSocket payload (see state_payload.h): +// physics state, render state and extra geoms as tagged blocks. +static py::bytes SerializeStatePayload( + uint32_t model_crc32, int physics_spec, const py::bytes& physics_state, + const mujoco::python::MjvCameraWrapper& camera, + const mujoco::python::MjvPerturbWrapper& perturb, + const mujoco::python::MjvOptionWrapper& vis_options, + const mujoco::python::MjModelWrapper& model, + const std::vector& render_flags, + const std::vector& extra_geoms) { + std::vector geoms; + geoms.reserve(extra_geoms.size()); + for (const mujoco::python::MjvGeomWrapper& geom_wrapper : extra_geoms) { + if (geom_wrapper.get()) { + geoms.push_back(*geom_wrapper.get()); + } + } + + std::string physics = physics_state; + const std::vector buffer = mujoco::studio::SerializeStatePayload( + model_crc32, physics_spec, physics.data(), physics.size(), *camera.get(), + *perturb.get(), *vis_options.get(), model.get()->opt, model.get()->vis, + model.get()->stat, render_flags, geoms.data(), geoms.size()); + return py::bytes(reinterpret_cast(buffer.data()), buffer.size()); +} + +// Upper bound of a serialized payload for a model whose physics state is +// `physics_bytes` long. Used to size the StateServer's shared memory. +static size_t MaxStatePayloadSize(size_t physics_bytes) { + return mujoco::studio::MaxStatePayloadSize(physics_bytes); +} + +PYBIND11_MODULE(state_payload, m, pybind11::mod_gil_not_used()) { + py::module_::import("mujoco._structs"); + m.doc() = "MuJoCo web viewer state payload serialization"; + + m.def("serialize_state_payload", &SerializeStatePayload); + m.def("max_state_payload_size", &MaxStatePayloadSize); + m.attr("MAX_EXTRA_GEOMS") = mujoco::studio::kMaxExtraGeoms; +} diff --git a/src/experimental/platform/CMakeLists.txt b/src/experimental/platform/CMakeLists.txt index fdabc5ba..c41a0cf0 100644 --- a/src/experimental/platform/CMakeLists.txt +++ b/src/experimental/platform/CMakeLists.txt @@ -22,6 +22,8 @@ target_sources(${MUJOCO_PLATFORM_TARGET_NAME} PUBLIC helpers.cc helpers.h + resources.cc + resources.h sys_utils.cc sys_utils.h hal/egl_utils.cc diff --git a/src/experimental/platform/resources.cc b/src/experimental/platform/resources.cc new file mode 100644 index 00000000..b0fcb727 --- /dev/null +++ b/src/experimental/platform/resources.cc @@ -0,0 +1,108 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/platform/resources.h" + +#include // NOLINT(build/c++17) +#include +#include +#include +#include +#include + +#include +#include "experimental/platform/sys_utils.h" + +namespace mujoco::platform { + +namespace { + +std::string Resolve(std::string_view path) { + std::string_view subpath = path.substr(path.find(':') + 1); + std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve); + if (exe_dir.empty()) { + return std::string("assets/") + std::string(subpath); + } + std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources"; + if (std::filesystem::exists(resources_dir / "assets")) { + return (resources_dir / "assets" / subpath).string(); + } + return (exe_dir / "assets" / subpath).string(); +} + +class FileResource { + public: + explicit FileResource(const std::string& path) + : file_(path, std::ios::binary | std::ios::ate) { + if (!file_.is_open()) { + mju_warning("Cannot open file %s", path.c_str()); + return; + } + + size_ = file_.tellg(); + file_.seekg(0, std::ios::beg); + } + + int Read(const void** buffer) { + buffer_.resize(size_); + if (!file_.read(reinterpret_cast(buffer_.data()), size_)) { + return 0; + } + *buffer = buffer_.data(); + return size_; + } + + int Size() const { return size_; } + + FileResource(const FileResource&) = delete; + FileResource& operator=(const FileResource&) = delete; + + private: + std::ifstream file_; + std::vector buffer_; + int size_ = 0; +}; + +} // namespace + +void RegisterResourceProviders() { + mjpResourceProvider resource_provider; + mjp_defaultResourceProvider(&resource_provider); + + resource_provider.open = [](mjResource* resource) { + const std::string resolved_path = Resolve(resource->name); + FileResource* f = new FileResource(resolved_path); + if (f->Size() == 0) { + delete f; + return 0; + } + resource->data = f; + return f->Size(); + }; + resource_provider.read = [](mjResource* resource, const void** buffer) { + FileResource* f = static_cast(resource->data); + return f->Read(buffer); + }; + resource_provider.close = [](mjResource* resource) { + delete static_cast(resource->data); + resource->data = nullptr; + }; + + resource_provider.prefix = "font"; + mjp_registerResourceProvider(&resource_provider); + resource_provider.prefix = "filament"; + mjp_registerResourceProvider(&resource_provider); +} + +} // namespace mujoco::platform diff --git a/src/experimental/platform/resources.h b/src/experimental/platform/resources.h new file mode 100644 index 00000000..804a3683 --- /dev/null +++ b/src/experimental/platform/resources.h @@ -0,0 +1,25 @@ +// Copyright 2026 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ +#define MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ + +namespace mujoco::platform { + +// Registers MuJoCo resource providers for font and filament assets. +void RegisterResourceProviders(); + +} // namespace mujoco::platform + +#endif // MUJOCO_SRC_EXPERIMENTAL_PLATFORM_RESOURCES_H_ diff --git a/src/experimental/studio/favicon.ico b/src/experimental/studio/favicon.ico new file mode 100644 index 00000000..1db85336 Binary files /dev/null and b/src/experimental/studio/favicon.ico differ diff --git a/src/experimental/studio/launcher.cc b/src/experimental/studio/launcher.cc index ac04c56c..4d83ba7d 100644 --- a/src/experimental/studio/launcher.cc +++ b/src/experimental/studio/launcher.cc @@ -16,99 +16,21 @@ #include #include -#include // NOLINT(build/c++17) -#include -#include #include #include -#include #include #include "experimental/platform/hal/graphics_mode.h" -#include "experimental/platform/sys_utils.h" +#include "experimental/platform/resources.h" #include "experimental/studio/app.h" namespace mujoco::studio { -namespace { - -std::string Resolve(std::string_view path) { - std::string_view subpath = path.substr(path.find(':') + 1); - std::filesystem::path exe_dir = mujoco::platform::GetModuleDir((void*)&Resolve); - if (exe_dir.empty()) { - return std::string("assets/") + std::string(subpath); - } - std::filesystem::path resources_dir = exe_dir.parent_path() / "Resources"; - if (std::filesystem::exists(resources_dir / "assets")) { - return (resources_dir / "assets" / subpath).string(); - } - return (exe_dir / "assets" / subpath).string(); -} - -class FileResource { - public: - explicit FileResource(const std::string& path) - : file_(path, std::ios::binary | std::ios::ate) { - if (!file_.is_open()) { - mju_warning("Cannot open file %s", path.c_str()); - return; - } - - size_ = file_.tellg(); - file_.seekg(0, std::ios::beg); - } - - int Read(const void** buffer) { - buffer_.resize(size_); - if (!file_.read(reinterpret_cast(buffer_.data()), size_)) { - return 0; - } - *buffer = buffer_.data(); - return size_; - } - - int Size() const { return size_; } - - FileResource(const FileResource&) = delete; - FileResource& operator=(const FileResource&) = delete; - - private: - std::ifstream file_; - std::vector buffer_; - int size_ = 0; -}; - -} // namespace int LaunchStudio(int argc, char** argv, LauncherConfig config) { const char* home = std::getenv("HOME"); const std::string ini_path = std::string(home ? home : ".") + "/.mujoco.ini"; - mjpResourceProvider resource_provider; - mjp_defaultResourceProvider(&resource_provider); - - resource_provider.open = [](mjResource* resource) { - const std::string resolved_path = Resolve(resource->name); - FileResource* f = new FileResource(resolved_path); - if (f->Size() == 0) { - delete f; - return 0; - } - resource->data = f; - return f->Size(); - }; - resource_provider.read = [](mjResource* resource, const void** buffer) { - FileResource* f = static_cast(resource->data); - return f->Read(buffer); - }; - resource_provider.close = [](mjResource* resource) { - delete static_cast(resource->data); - resource->data = nullptr; - }; - - resource_provider.prefix = "font"; - mjp_registerResourceProvider(&resource_provider); - resource_provider.prefix = "filament"; - mjp_registerResourceProvider(&resource_provider); + mujoco::platform::RegisterResourceProviders(); if (config.gfx_mode.empty()) { const char* display = std::getenv("DISPLAY"); diff --git a/src/experimental/studio/live.html b/src/experimental/studio/live.html index 08d33f37..4b4c9c2a 100644 --- a/src/experimental/studio/live.html +++ b/src/experimental/studio/live.html @@ -6,6 +6,7 @@ + MuJoCo Live diff --git a/src/render/filament/assets/outline_composite.mat b/src/render/filament/assets/outline_composite.mat index 8d42edbe..6eac0a6f 100644 --- a/src/render/filament/assets/outline_composite.mat +++ b/src/render/filament/assets/outline_composite.mat @@ -16,6 +16,8 @@ // outline around an object. The outline is drawn by checking the distance to // the nearest edge point (as computed by the algorithm) and, if within the // desired width, setting the color of the pixel to the desired outline color. +// The outline is dimmed where it is occluded, i.e. where the scene contains +// geometry that is closer than the selected object at the nearest edge point. material { name : outline_composite, @@ -25,8 +27,11 @@ material { depthCulling : false, parameters : [ { type : sampler2d, name : source }, + { type : sampler2d, name : scene_depth, precision : high, filterable : false }, + { type : sampler2d, name : selection_depth, precision : high, filterable : false }, { type : float4, name : color }, - { type : float, name : width } + { type : float, name : width }, + { type : float, name : dim } ], variables : [ vertex @@ -77,6 +82,18 @@ fragment { float inner_alpha = smoothstep(0.0, 1.0, dist); float outer_alpha = smoothstep(width, width - 1.0, dist); float alpha = outer_alpha * inner_alpha; + + // Dim the outline where it is occluded. The depth buffer is reversed + // (1 = near, 0 = far), so the outline is occluded if the scene depth at + // this pixel is greater than the depth of the selected object at the + // nearest edge point. The epsilon avoids spurious dimming where the two + // depths are nearly equal, e.g. at contact points. + highp float scene_z = texture(materialParams_scene_depth, variable_vertex.xy).r; + highp float edge_z = texture(materialParams_selection_depth, edge).r; + if (scene_z > edge_z * 1.001 + 1e-6) { + alpha *= materialParams.dim; + } + postProcess.color = mix(vec4(0), materialParams.color, alpha * materialParams.color.a); } } diff --git a/src/render/filament/assets/outline_flatten.mat b/src/render/filament/assets/outline_flatten.mat index 3fa4a529..247815e9 100644 --- a/src/render/filament/assets/outline_flatten.mat +++ b/src/render/filament/assets/outline_flatten.mat @@ -16,11 +16,15 @@ // This material writes the the screen-space position of each fragment as the // output color of that fragment. This will provide us with the initial data for // the Jump Flood Algorithm. See "outline_jumpflood.mat" for more details. +// +// Depth is also written so that the render target's depth attachment holds the +// depth of the selected objects; "outline_composite.mat" samples it to dim +// occluded portions of the outline. material { name : outline_flatten, shadingModel : unlit, culling : none, - depthWrite : false + depthWrite : true } fragment { diff --git a/src/render/filament/core/outliner.cc b/src/render/filament/core/outliner.cc index 9b0bb28a..b39c8ce5 100644 --- a/src/render/filament/core/outliner.cc +++ b/src/render/filament/core/outliner.cc @@ -37,12 +37,15 @@ namespace mujoco { Outliner::Outliner(ObjectManager* object_mgr, uint8_t layer_mask, - filament::math::float4 color, float thickness) + uint8_t scene_layer_mask, filament::math::float4 color, + float thickness, float occlusion_dim) : object_mgr_(object_mgr), engine_(object_mgr->GetEngine()), layer_mask_(layer_mask), + scene_layer_mask_(scene_layer_mask), color_(color), - thickness_(thickness) {} + thickness_(thickness), + occlusion_dim_(occlusion_dim) {} Outliner::~Outliner() { Reset(); } @@ -98,9 +101,10 @@ void Outliner::Prepare(int width, int height) { } }; - // Set up two render targets. We will alternate between the two targets to - // allow for chaining passes together. - for (int i = 0; i < 2; ++i) { + // Set up the render targets: one for the scene depth, one for the flattened + // selection mask and its depth, and two ping-pong targets for chaining the + // jump flood passes together. + for (int i = 0; i < kNumTargets; ++i) { mjrfRenderTargetConfig config; mjrf_defaultRenderTargetConfig(&config); config.color_format = mjPIXEL_FORMAT_RGBA8; @@ -127,9 +131,11 @@ void Outliner::Prepare(int width, int height) { view->setMultiSampleAntiAliasingOptions({.enabled = false}); } - // In the first pass, we will render a given scene, but only render the - // objects marked as outlines. We assume that the objects have already been - // assigned the kOutlineFlatten material. + // In the first pass, we render the scene layers that can occlude the + // outline; only the resulting depth is used. In the second pass, we render + // the given scene again, but only the objects marked as outlines. We assume + // that the objects have already been assigned the kOutlineFlatten material. + views_[kPassSceneDepth]->setVisibleLayers(0xff, scene_layer_mask_); views_[kPassFlatten]->setVisibleLayers(0xff, layer_mask_); // All subsequent passes are full-screen post-processing passes. @@ -141,16 +147,19 @@ void Outliner::Prepare(int width, int height) { setup_fullscreen(kPassDrawOutline, ObjectManager::kOutlineComposite); // Chain the passes together such that the output of a pass is the input to - // the next pass. The first pass has no input (we are just rendering the - // selected objects) and the last pass has no output (we are just rendering - // the outline to the externally provided target). - bind(kPassFlatten, -1, 0); - bind(kPassJumpFlood1, 0, 1); - bind(kPassJumpFlood2, 1, 0); - bind(kPassJumpFlood3, 0, 1); - bind(kPassJumpFlood4, 1, 0); - bind(kPassJumpFlood5, 0, 1); - bind(kPassDrawOutline, 1, -1); + // the next pass. The scene passes have no input (we are just rendering the + // scene) and the last pass has no output (we are just rendering the outline + // to the externally provided target). The flatten pass has its own target + // (rather than a ping-pong target) so that its depth attachment survives the + // jump flood passes and can be sampled by the composite pass. + bind(kPassSceneDepth, -1, kTargetSceneDepth); + bind(kPassFlatten, -1, kTargetFlatten); + bind(kPassJumpFlood1, kTargetFlatten, kTargetPing); + bind(kPassJumpFlood2, kTargetPing, kTargetPong); + bind(kPassJumpFlood3, kTargetPong, kTargetPing); + bind(kPassJumpFlood4, kTargetPing, kTargetPong); + bind(kPassJumpFlood5, kTargetPong, kTargetPing); + bind(kPassDrawOutline, kTargetPing, -1); // Bind the parameters for each pass. For the jump flood passes, the step // parameter determines how far to propagate the outline in each pass. @@ -160,9 +169,22 @@ void Outliner::Prepare(int width, int height) { material_instances_[kPassJumpFlood4]->setParameter("step", 2.0f); material_instances_[kPassJumpFlood5]->setParameter("step", 1.0f); - // The final pass renders the actual outline onto a render target. + // The final pass renders the actual outline onto a render target. It samples + // the scene depth and the selection depth to dim occluded outline pixels. + const filament::TextureSampler depth_sampler( + filament::TextureSampler::MinFilter::NEAREST, + filament::TextureSampler::MagFilter::NEAREST); material_instances_[kPassDrawOutline]->setParameter("color", color_); material_instances_[kPassDrawOutline]->setParameter("width", thickness_); + material_instances_[kPassDrawOutline]->setParameter("dim", occlusion_dim_); + material_instances_[kPassDrawOutline]->setParameter( + "scene_depth", + targets_[kTargetSceneDepth]->GetDepthTexture()->GetFilamentTexture(), + depth_sampler); + material_instances_[kPassDrawOutline]->setParameter( + "selection_depth", + targets_[kTargetFlatten]->GetDepthTexture()->GetFilamentTexture(), + depth_sampler); // Commit all the material instances to the engine. for (auto& material_instance : material_instances_) { @@ -222,9 +244,13 @@ void Outliner::Render(filament::Renderer* renderer, filament::View* view, view->setViewport(viewport); } - // Re-render the view's scene to create the flattened selection mask. + // Re-render the view's scene to capture the scene depth (used for occlusion + // dimming) and to create the flattened selection mask. auto prev_clear_opts = renderer->getClearOptions(); renderer->setClearOptions({.clearColor = {0, 0, 0, 0}, .clear = true}); + views_[kPassSceneDepth]->setScene(view->getScene()); + views_[kPassSceneDepth]->setCamera(&view->getCamera()); + renderer->render(views_[kPassSceneDepth]); views_[kPassFlatten]->setScene(view->getScene()); views_[kPassFlatten]->setCamera(&view->getCamera()); renderer->render(views_[kPassFlatten]); diff --git a/src/render/filament/core/outliner.h b/src/render/filament/core/outliner.h index 1a430a76..45e27903 100644 --- a/src/render/filament/core/outliner.h +++ b/src/render/filament/core/outliner.h @@ -31,11 +31,13 @@ namespace mujoco { // Renders an outline of selected objects. // // This class uses the "jump flood" algorithm to create an outline of selected -// objects. +// objects. The outline is dimmed by `occlusion_dim` where the selected objects +// are occluded by scene geometry on the `scene_layer_mask` layers. class Outliner { public: Outliner(ObjectManager* object_mgr, uint8_t layer_mask, - filament::math::float4 color, float thickness); + uint8_t scene_layer_mask, filament::math::float4 color, + float thickness, float occlusion_dim); ~Outliner(); Outliner(const Outliner&) = delete; @@ -53,6 +55,7 @@ class Outliner { void Reset(); enum Pass { + kPassSceneDepth, kPassFlatten, kPassJumpFlood1, kPassJumpFlood2, @@ -65,17 +68,27 @@ class Outliner { kNumJumpFloodPasses = kPassJumpFlood5 - kPassJumpFlood1 + 1, }; + enum Target { + kTargetSceneDepth, // depth of the scene, used for occlusion dimming + kTargetFlatten, // selection mask (color) and selection depth (depth) + kTargetPing, // jump flood ping-pong buffer + kTargetPong, // jump flood ping-pong buffer + kNumTargets, + }; + ObjectManager* object_mgr_ = nullptr; filament::Engine* engine_ = nullptr; uint8_t layer_mask_ = 0xff; + uint8_t scene_layer_mask_ = 0xff; filament::math::float4 color_ = {1.0f, 1.0f, 1.0f, 1.0f}; float thickness_ = 2.5f; + float occlusion_dim_ = 1.0f; int width_ = 0; int height_ = 0; filament::Camera* camera_ = nullptr; - std::unique_ptr targets_[2]; + std::unique_ptr targets_[kNumTargets]; filament::View* views_[kNumPasses] = {}; filament::Scene* scenes_[kNumPasses] = {}; diff --git a/src/render/filament/core/scene_view.cc b/src/render/filament/core/scene_view.cc index 78d7a000..35748f49 100644 --- a/src/render/filament/core/scene_view.cc +++ b/src/render/filament/core/scene_view.cc @@ -321,9 +321,9 @@ void SceneView::Render(filament::Renderer* renderer, if (!selected_renderables.empty()) { if (!outliner_) { - outliner_ = - std::make_unique(object_mgr_, kLayerMask_Outline, - float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f); + outliner_ = std::make_unique( + object_mgr_, kLayerMask_Outline, kLayerMask_Object, + float4{0.9f, 0.9f, 0.2f, 0.7f}, 3.5f, 0.25f); } for (Renderable* renderable : selected_renderables) {