diff --git a/src/experimental/filament/assets/ibl.ktx b/src/experimental/filament/assets/ibl.ktx new file mode 100644 index 00000000..b46b730d Binary files /dev/null and b/src/experimental/filament/assets/ibl.ktx differ diff --git a/src/experimental/filament/assets/pbr.mat b/src/experimental/filament/assets/pbr.mat new file mode 100644 index 00000000..19855986 --- /dev/null +++ b/src/experimental/filament/assets/pbr.mat @@ -0,0 +1,54 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : pbr, + shadingModel : lit, + culling: none, + flipUV: false, + parameters : [ + { type : sampler2d, name : BaseColor }, + { type : sampler2d, name : Normal }, + { type : sampler2d, name : Occlusion }, + { type : sampler2d, name : Roughness }, + { type : sampler2d, name : Metallic }, + { type : sampler2d, name : Emissive }, + { type : float4, name : BaseColorFactor }, + { type : float, name : MetallicFactor }, + { type : float, name : RoughnessFactor } + + ], + requires : [ + uv0 + ] +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = getUV0(); + + material.normal = texture(materialParams_Normal, uv).xyz * 2.0 - 1.0; + prepareMaterial(material); + + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.ambientOcclusion = texture(materialParams_Occlusion, uv).r; + material.roughness = materialParams.RoughnessFactor; + material.roughness *= texture(materialParams_Roughness, uv).r; + material.metallic = materialParams.MetallicFactor; + material.metallic *= texture(materialParams_Metallic, uv).r; + material.emissive = texture(materialParams_Emissive, uv); + material.emissive.a = 1.0 - material.emissive.a; + } +} diff --git a/src/experimental/filament/assets/pbr_packed.mat b/src/experimental/filament/assets/pbr_packed.mat new file mode 100644 index 00000000..252af662 --- /dev/null +++ b/src/experimental/filament/assets/pbr_packed.mat @@ -0,0 +1,51 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : pbr_packed, + shadingModel : lit, + culling: none, + flipUV: false, + parameters : [ + { type : sampler2d, name : BaseColor }, + { type : sampler2d, name : Normal }, + { type : sampler2d, name : ORM }, + { type : sampler2d, name : Emissive }, + { type : float4, name : BaseColorFactor }, + { type : float, name : MetallicFactor }, + { type : float, name : RoughnessFactor } + ], + requires : [ + uv0 + ] +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = getUV0(); + + material.normal = texture(materialParams_Normal, uv).xyz * 2.0 - 1.0; + prepareMaterial(material); + + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.ambientOcclusion = texture(materialParams_ORM, uv).r; + material.roughness = materialParams.RoughnessFactor; + material.roughness *= texture(materialParams_ORM, uv).g; + material.metallic = materialParams.MetallicFactor; + material.metallic *= texture(materialParams_ORM, uv).b; + material.emissive = texture(materialParams_Emissive, uv); + material.emissive.a = 1.0 - material.emissive.a; + } +} diff --git a/src/experimental/filament/assets/phong_2d.mat b/src/experimental/filament/assets/phong_2d.mat new file mode 100644 index 00000000..6218c307 --- /dev/null +++ b/src/experimental/filament/assets/phong_2d.mat @@ -0,0 +1,50 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_2d, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : float3, name : UvScale }, + { type : sampler2d, name : BaseColor } + ], + variables : [ + vertex_pos + ] +} + +vertex { + void materialVertex(inout MaterialVertexInputs material) { + material.vertex_pos = getPosition(); + } +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = variable_vertex_pos.xy * materialParams.UvScale.xy; + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_2d_fade.mat b/src/experimental/filament/assets/phong_2d_fade.mat new file mode 100644 index 00000000..ab7360a0 --- /dev/null +++ b/src/experimental/filament/assets/phong_2d_fade.mat @@ -0,0 +1,51 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_2d, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + blending: fade, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : float3, name : UvScale }, + { type : sampler2d, name : BaseColor } + ], + variables : [ + vertex_pos + ] +} + +vertex { + void materialVertex(inout MaterialVertexInputs material) { + material.vertex_pos = getPosition(); + } +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = variable_vertex_pos.xy * materialParams.UvScale.xy; + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_2d_uv.mat b/src/experimental/filament/assets/phong_2d_uv.mat new file mode 100644 index 00000000..cad22805 --- /dev/null +++ b/src/experimental/filament/assets/phong_2d_uv.mat @@ -0,0 +1,43 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_2d_uv, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : sampler2d, name : BaseColor } + ], + requires : [ + uv0 + ], +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = getUV0(); + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_2d_uv_fade.mat b/src/experimental/filament/assets/phong_2d_uv_fade.mat new file mode 100644 index 00000000..24c2834e --- /dev/null +++ b/src/experimental/filament/assets/phong_2d_uv_fade.mat @@ -0,0 +1,44 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_2d_uv, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + blending: fade, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : sampler2d, name : BaseColor } + ], + requires : [ + uv0 + ], +} + +fragment { + void material(inout MaterialInputs material) { + vec2 uv = getUV0(); + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_color.mat b/src/experimental/filament/assets/phong_color.mat new file mode 100644 index 00000000..beaec101 --- /dev/null +++ b/src/experimental/filament/assets/phong_color.mat @@ -0,0 +1,35 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_color, + shadingModel : specularGlossiness, + culling: none, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor } + ] +} + +fragment { + void material(inout MaterialInputs material) { + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_color_fade.mat b/src/experimental/filament/assets/phong_color_fade.mat new file mode 100644 index 00000000..2fd0721a --- /dev/null +++ b/src/experimental/filament/assets/phong_color_fade.mat @@ -0,0 +1,36 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_color_fade, + shadingModel : specularGlossiness, + culling: none, + blending: fade, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor } + ] +} + +fragment { + void material(inout MaterialInputs material) { + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_cube.mat b/src/experimental/filament/assets/phong_cube.mat new file mode 100644 index 00000000..0a39c3c6 --- /dev/null +++ b/src/experimental/filament/assets/phong_cube.mat @@ -0,0 +1,50 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_cube, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : float3, name : UvScale }, + { type : samplerCubemap, name : BaseColor } + ], + variables : [ + vertex_pos + ] +} + +vertex { + void materialVertex(inout MaterialVertexInputs material) { + material.vertex_pos = getPosition(); + } +} + +fragment { + void material(inout MaterialInputs material) { + vec3 uv = variable_vertex_pos.xyz * materialParams.UvScale; + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/phong_cube_fade.mat b/src/experimental/filament/assets/phong_cube_fade.mat new file mode 100644 index 00000000..3b95bec7 --- /dev/null +++ b/src/experimental/filament/assets/phong_cube_fade.mat @@ -0,0 +1,51 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : phong_cube, + shadingModel : specularGlossiness, + culling : none, + flipUV : false, + blending: fade, + parameters : [ + { type : float4, name : BaseColorFactor }, + { type : float, name : SpecularFactor }, + { type : float, name : GlossinessFactor }, + { type : float, name : EmissiveFactor }, + { type : float3, name : UvScale }, + { type : samplerCubemap, name : BaseColor } + ], + variables : [ + vertex_pos + ] +} + +vertex { + void materialVertex(inout MaterialVertexInputs material) { + material.vertex_pos = getPosition(); + } +} + +fragment { + void material(inout MaterialInputs material) { + vec3 uv = variable_vertex_pos.xyz * materialParams.UvScale; + + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + material.baseColor *= texture(materialParams_BaseColor, uv); + material.specularColor = vec3(materialParams.SpecularFactor); + material.glossiness = materialParams.GlossinessFactor; + material.emissive = vec4(materialParams.EmissiveFactor); + } +} diff --git a/src/experimental/filament/assets/unlit_depth.mat b/src/experimental/filament/assets/unlit_depth.mat new file mode 100644 index 00000000..681319d8 --- /dev/null +++ b/src/experimental/filament/assets/unlit_depth.mat @@ -0,0 +1,31 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : unlit_depth, + shadingModel : unlit, + blending : opaque, + culling: none, + depthWrite: true +} + +fragment { + void material(inout MaterialInputs material) { + prepareMaterial(material); + float3 worldPosition = getWorldPosition(); + float4 clipPosition = mulMat4x4Float3(getClipFromWorldMatrix(), worldPosition); + float3 normalizedPosition = clipPosition.xyz / clipPosition.w; + material.baseColor.rgb = vec3(normalizedPosition.z); + } +} diff --git a/src/experimental/filament/assets/unlit_segmentation.mat b/src/experimental/filament/assets/unlit_segmentation.mat new file mode 100644 index 00000000..359bb319 --- /dev/null +++ b/src/experimental/filament/assets/unlit_segmentation.mat @@ -0,0 +1,29 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : unlit_segmentation, + shadingModel : unlit, + culling: none, + parameters : [ + { type : float4, name : BaseColorFactor } + ] +} + +fragment { + void material(inout MaterialInputs material) { + prepareMaterial(material); + material.baseColor = materialParams.BaseColorFactor; + } +} diff --git a/src/experimental/filament/assets/unlit_ui.mat b/src/experimental/filament/assets/unlit_ui.mat new file mode 100644 index 00000000..2b4f402e --- /dev/null +++ b/src/experimental/filament/assets/unlit_ui.mat @@ -0,0 +1,43 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +material { + name : unlit_ui, + parameters : [ + { + type : sampler2d, + name : glyph + } + ], + requires : [ + uv0, + color + ], + shadingModel : unlit, + culling : none, + depthCulling: false, + blending : transparent, + featureLevel : 0 +} + +fragment { + void material(inout MaterialInputs material) { + prepareMaterial(material); + vec2 uv = getUV0(); + uv.y = 1.0 - uv.y; + vec4 tex_color = texture2D(materialParams_glyph, uv); + material.baseColor = getColor() * tex_color; + material.baseColor.rgb *= material.baseColor.a; + } +} diff --git a/src/experimental/filament/filament/buffer_util.cc b/src/experimental/filament/filament/buffer_util.cc new file mode 100644 index 00000000..4fc4e92f --- /dev/null +++ b/src/experimental/filament/filament/buffer_util.cc @@ -0,0 +1,34 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/buffer_util.h" + +#include + +#include "third_party/filament/filament/backend/include/backend/BufferDescriptor.h" + +namespace mujoco { + +filament::backend::BufferDescriptor CreateBufferDescriptor( + std::size_t num_bytes, const FillBufferFn& fill) { + std::byte* bytes = new std::byte[num_bytes]; + fill(bytes, num_bytes); + const auto callback = [](void* buffer, size_t size, void* user) { + auto* ptr = reinterpret_cast(user); + delete[] ptr; + }; + return filament::backend::BufferDescriptor(bytes, num_bytes, callback, bytes); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/buffer_util.h b/src/experimental/filament/filament/buffer_util.h new file mode 100644 index 00000000..8e8a4df6 --- /dev/null +++ b/src/experimental/filament/filament/buffer_util.h @@ -0,0 +1,148 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_ + +#include +#include +#include +#include +#include "third_party/filament/filament/backend/include/backend/BufferDescriptor.h" +#include +#include +#include +#include + +// Functions for creating filament vertex and index buffers. +namespace mujoco { + +// Simple tuple-type of a IndexBuffer+VertexBuffer. +struct FilamentBuffers { + filament::IndexBuffer* index_buffer = nullptr; + filament::VertexBuffer* vertex_buffer = nullptr; + filament::Box bounds = {{-1, -1, -1}, {1, 1, 1}}; +}; + +// Function that fills in the given buffer with actual data. +using FillBufferFn = std::function; + +// Creates and populates a BufferDescriptor (for vertex and index buffers). +filament::backend::BufferDescriptor CreateBufferDescriptor( + std::size_t num_bytes, const FillBufferFn& fill); + +// Creates a filament::VertexBuffer based on the VertexType. The fill function +// will be used to populate the buffer. +template +filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine, + std::size_t num_vertices, + const FillBufferFn& fill) { + int vertex_size = 0; + if constexpr (VertexType::kHasPosition) { + vertex_size += sizeof(VertexType::position); + } + if constexpr (VertexType::kHasPosition2d) { + vertex_size += sizeof(VertexType::position); + } + if constexpr (VertexType::kHasOrientation) { + vertex_size += sizeof(VertexType::orientation); + } + if constexpr (VertexType::kHasUv) { + vertex_size += sizeof(VertexType::uv); + } + if constexpr (VertexType::kHasColor) { + vertex_size += sizeof(VertexType::color); + } + + auto builder = filament::VertexBuffer::Builder(); + builder.bufferCount(1); + builder.vertexCount(num_vertices); + + int offset = 0; + if constexpr (VertexType::kHasPosition) { + builder.attribute(filament::VertexAttribute::POSITION, 0, + filament::VertexBuffer::AttributeType::FLOAT3, offset, + vertex_size); + offset += sizeof(VertexType::position); + } + if constexpr (VertexType::kHasPosition2d) { + builder.attribute(filament::VertexAttribute::POSITION, 0, + filament::VertexBuffer::AttributeType::FLOAT2, offset, + vertex_size); + offset += sizeof(VertexType::position); + } + if constexpr (VertexType::kHasOrientation) { + builder.attribute(filament::VertexAttribute::TANGENTS, 0, + filament::VertexBuffer::AttributeType::FLOAT4, offset, + vertex_size); + offset += sizeof(VertexType::orientation); + } + if constexpr (VertexType::kHasUv) { + builder.attribute(filament::VertexAttribute::UV0, 0, + filament::VertexBuffer::AttributeType::FLOAT2, offset, + vertex_size); + offset += sizeof(VertexType::uv); + } + if constexpr (VertexType::kHasColor) { + builder.attribute(filament::VertexAttribute::COLOR, 0, + filament::VertexBuffer::AttributeType::UBYTE4, offset, + vertex_size); + builder.normalized(filament::VertexAttribute::COLOR); + offset += sizeof(VertexType::color); + } + + auto vb = builder.build(*engine); + const std::size_t buffer_size = num_vertices * vertex_size; + vb->setBufferAt(*engine, 0, CreateBufferDescriptor(buffer_size, fill)); + return vb; +} + +// Creates a filament::IndexBuffer. The IndexType should be either uin16_t or +// uint32_t. The fill function will be used to populate the buffer. +template +filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine, + std::size_t num_indices, + const FillBufferFn& fill) { + static_assert(std::is_same::value || + std::is_same::value); + + constexpr auto type = std::is_same::value + ? filament::IndexBuffer::IndexType::USHORT + : filament::IndexBuffer::IndexType::UINT; + + auto builder = filament::IndexBuffer::Builder(); + builder.bufferType(type); + builder.indexCount(num_indices); + + auto ib = builder.build(*engine); + + const std::size_t buffer_size = num_indices * sizeof(IndexType); + ib->setBuffer(*engine, CreateBufferDescriptor(buffer_size, fill)); + return ib; +} + +// Fills an index buffer with a basic incrementing sequence. +template +int FillSequence(std::byte* buffer, std::size_t num_bytes) { + const T num = num_bytes / sizeof(T); + T* ptr = reinterpret_cast(buffer); + for (T i = 0; i < num; ++i) { + ptr[i] = i; + } + return num; +} + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUFFER_UTIL_H_ diff --git a/src/experimental/filament/filament/builtins.cc b/src/experimental/filament/filament/builtins.cc new file mode 100644 index 00000000..12660f5d --- /dev/null +++ b/src/experimental/filament/filament/builtins.cc @@ -0,0 +1,662 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/builtins.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/vertex_util.h" + +namespace mujoco { + +using filament::math::float2; +using filament::math::float3; +using filament::math::float4; + +static constexpr size_t kNumVerticesPerTriangle = 3; +static constexpr size_t kNumVerticesPerQuad = 4; +static constexpr size_t kNumIndicesPerTriangle = 3; +static constexpr size_t kNumIndicesPerQuad = 6; + +static int AppendQuadIndices(uint16_t* ptr, int idx, uint16_t a, uint16_t b, + uint16_t c, uint16_t d) { + ptr[idx++] = a; + ptr[idx++] = b; + ptr[idx++] = c; + ptr[idx++] = a; + ptr[idx++] = c; + ptr[idx++] = d; + return idx; +} + +std::size_t NumVerticesPerSide(int num_quads_per_axis) { + return (num_quads_per_axis + 1) * (num_quads_per_axis + 1); +} + +std::size_t NumIndicesPerSide(int num_quads_per_axis) { + return kNumIndicesPerQuad * num_quads_per_axis * num_quads_per_axis; +} + +class PlaneBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + explicit PlaneBuilder(int num_quads_per_axis) + : num_quads_per_axis_(num_quads_per_axis), + orientation_(CalculateOrientation({0, 0, 1})) {} + + std::size_t NumVertices() const { + return NumVerticesPerSide(num_quads_per_axis_); + } + + std::size_t NumIndices() const { + return NumIndicesPerSide(num_quads_per_axis_); + } + + void GenerateVertices(VertexType* ptr, size_t num) const { + const float delta = 2.0f / num_quads_per_axis_; + + int idx = 0; + for (int x = 0; x <= num_quads_per_axis_; ++x) { + for (int y = 0; y <= num_quads_per_axis_; ++y) { + const float dx = delta * static_cast(x); + const float dy = delta * static_cast(y); + ptr[idx++] = VertexType({dx - 1.0f, dy - 1.0f, 0}, orientation_); + } + } + } + + void GenerateIndices(IndexType* ptr, size_t num) const { + int idx = 0; + for (int x = 0; x < num_quads_per_axis_; ++x) { + for (int y = 0; y < num_quads_per_axis_; ++y) { + const int base_idx = x * (num_quads_per_axis_ + 1) + y; + const int i0 = base_idx + 0; + const int i1 = base_idx + 1; + const int i2 = base_idx + num_quads_per_axis_ + 2; + const int i3 = base_idx + num_quads_per_axis_ + 1; + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + } + } + + filament::Box GetBounds() const { return {{-1, -1, -0.001}, {1, 1, 0.001}}; } + + private: + int num_quads_per_axis_; + float4 orientation_; +}; + +class BoxBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + static constexpr int kNumSides = 6; + + explicit BoxBuilder(int num_quads_per_axis) + : num_quads_per_axis_(num_quads_per_axis) { + quad_size_ = 2.0f / static_cast(num_quads_per_axis_); + } + + std::size_t NumVertices() const { + return NumVerticesPerSide(num_quads_per_axis_) * kNumSides; + } + + std::size_t NumIndices() const { + return NumIndicesPerSide(num_quads_per_axis_) * kNumSides; + } + + void GenerateVertices(VertexType* ptr, size_t num) const { + int idx = 0; + idx = GenerateVerticesForSide(ptr, idx, {0, 1, 0}, [](float2 pt) { + return float3{pt.x, 1.0f, pt.y}; + }); + idx = GenerateVerticesForSide(ptr, idx, {0, -1, 0}, [](float2 pt) { + return float3{pt.x, -1.0f, pt.y}; + }); + idx = GenerateVerticesForSide(ptr, idx, {1, 0, 0}, [](float2 pt) { + return float3{1.0f, pt.x, pt.y}; + }); + idx = GenerateVerticesForSide(ptr, idx, {-1, 0, 0}, [](float2 pt) { + return float3{-1.0f, pt.x, pt.y}; + }); + idx = GenerateVerticesForSide(ptr, idx, {0, 0, 1}, [](float2 pt) { + return float3{pt.x, pt.y, 1.0f}; + }); + idx = GenerateVerticesForSide(ptr, idx, {0, 0, -1}, [](float2 pt) { + return float3{pt.x, pt.y, -1.0f}; + }); + } + + void GenerateIndices(IndexType* ptr, size_t num) const { + const int vertices_per_side = NumVerticesPerSide(num_quads_per_axis_); + + int idx = 0; + for (int i = 0; i < kNumSides; ++i) { + for (int x = 0; x < num_quads_per_axis_; ++x) { + for (int y = 0; y < num_quads_per_axis_; ++y) { + const int base_idx = + (i * vertices_per_side) + (x * (num_quads_per_axis_ + 1)) + y; + const int i0 = base_idx + 0; + const int i1 = base_idx + 1; + const int i2 = base_idx + num_quads_per_axis_ + 2; + const int i3 = base_idx + num_quads_per_axis_ + 1; + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + } + } + } + + filament::Box GetBounds() const { return {{-1, -1, -1}, {1, 1, 1}}; } + + private: + template + int GenerateVerticesForSide(VertexType* ptr, int idx, float3 normal, + const F& pt_gen) const { + float4 orientation = CalculateOrientation(normal); + for (int x = 0; x <= num_quads_per_axis_; ++x) { + for (int y = 0; y <= num_quads_per_axis_; ++y) { + const float dx = -1.0f + (quad_size_ * static_cast(x)); + const float dy = -1.0f + (quad_size_ * static_cast(y)); + const float3 position = pt_gen({dx, dy}); + ptr[idx++] = VertexType(position, orientation); + } + } + return idx; + } + + int num_quads_per_axis_; + float quad_size_; +}; + +class TubeBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + TubeBuilder(int num_stacks, int num_slices) + : num_stacks_(num_stacks), num_slices_(num_slices) {} + + std::size_t NumVertices() const { + return num_slices_ * (num_stacks_ + 1); + } + + std::size_t NumIndices() const { + return kNumIndicesPerQuad * num_slices_ * num_stacks_; + } + + void GenerateVertices(VertexType* ptr, size_t num) const { + const float delta_angle = 2.f * M_PI / (float)num_slices_; + const float delta_stack = 2.f / static_cast(num_stacks_); + + int idx = 0; + for (int i = 0; i < num_slices_; ++i) { + const float angle = static_cast(i) * delta_angle; + const float2 pt{std::cos(angle), std::sin(angle)}; + const float4 orientation = CalculateOrientation({pt.x, pt.y, 0}); + for (int j = 0; j <= num_stacks_; ++j) { + const float z = -1.0f + (static_cast(j) * delta_stack); + ptr[idx++] = VertexType({pt.x, pt.y, z}, orientation); + } + } + } + + void GenerateIndices(IndexType* ptr, size_t num) const { + const int num_vertices = NumVertices(); + const int num_vertices_in_spine = num_stacks_ + 1; + + int idx = 0; + for (int i = 0; i < num_slices_; ++i) { + for (int j = 0; j < num_stacks_; ++j) { + const int base_idx = (i * num_vertices_in_spine) + j; + const int i0 = base_idx + 0; + const int i1 = base_idx + 1; + const int i2 = (base_idx + num_stacks_ + 2) % num_vertices; + const int i3 = (base_idx + num_stacks_ + 1) % num_vertices; + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + } + } + + filament::Box GetBounds() const { return {{-1, -1, -1}, {1, 1, 1}}; } + + private: + int num_stacks_; + int num_slices_; +}; + +class ConeBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + ConeBuilder(int num_stacks, int num_slices) + : num_stacks_(num_stacks), num_slices_(num_slices) {} + + std::size_t NumVertices() const { + return (num_slices_ * kNumVerticesPerTriangle) + + ((num_stacks_ - 1) * num_slices_ * kNumVerticesPerQuad); + } + + std::size_t NumIndices() const { + return (num_slices_ * kNumIndicesPerTriangle) + + ((num_stacks_ - 1) * num_slices_ * kNumIndicesPerQuad); + } + void GenerateVertices(VertexType* ptr, std::size_t num) const { + // pole: use triangles + const float delta_angle = 2.0 * M_PI / static_cast(num_slices_); + const float delta_radius = 1.0f / static_cast(num_stacks_); + + int idx = 0; + for (int j = 0; j < num_slices_; ++j) { + const float angle1 = (j+0) * delta_angle; + const float angle2 = (j+1) * delta_angle; + + ptr[idx++] = MakeVert(angle1, delta_radius); + ptr[idx++] = MakeVert(angle2, delta_radius); + + VertexType v3; + v3.position = {0, 0, 1}; + v3.orientation = CalculateOrientation(v3.position); + ptr[idx++] = v3; + } + + // the rest: use quads + for (int i = 1; i < num_stacks_; ++i) { + const float radius1 = delta_radius * (i+0); + const float radius2 = delta_radius * (i+1); + + for (int j = 0; j < num_slices_; ++j) { + const float angle1 = (j+0) * delta_angle; + const float angle2 = (j+1) * delta_angle; + + ptr[idx++] = MakeVert(angle1, radius2); + ptr[idx++] = MakeVert(angle2, radius2); + ptr[idx++] = MakeVert(angle2, radius1); + ptr[idx++] = MakeVert(angle1, radius1); + } + } + } + + void GenerateIndices(IndexType* ptr, std::size_t num) const { + int idx = 0; + for (int j = 0; j < num_slices_ * 3; ++j) { + ptr[idx] = idx; + ++idx; + } + + int quad_idx = idx; + for (int i = 1; i < num_stacks_; ++i) { + for (int j = 0; j < num_slices_; ++j) { + const int i0 = quad_idx + 0; + const int i1 = quad_idx + 1; + const int i2 = quad_idx + 2; + const int i3 = quad_idx + 3; + quad_idx += 4; + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + } + } + + filament::Box GetBounds() const { return {{-1, -1, 0}, {1, 1, 1}}; } + + private: + static VertexType MakeVert(float theta, float radius) { + static constexpr float kNormalScale = 0.70710678118f; + const float cz = std::cos(theta); + const float sz = std::sin(theta); + const float3 pt{cz * radius, sz * radius, 1.f - radius}; + const float3 n{cz * kNormalScale, sz * kNormalScale, kNormalScale}; + return VertexType(pt, CalculateOrientation(n)); + } + + int num_stacks_; + int num_slices_; +}; + +class DiskBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + explicit DiskBuilder(int num_slices) : num_slices_(num_slices) { + orientation_ = CalculateOrientation({0, 0, 1}); + } + + std::size_t NumVertices() const { + return num_slices_ + 1; + } + + std::size_t NumIndices() const { + return num_slices_ * kNumVerticesPerTriangle; + } + + void GenerateVertices(VertexType* ptr, std::size_t num) const { + const float delta_angle = 2.0 * M_PI / static_cast(num_slices_); + + int idx = 0; + ptr[idx++] = VertexType(float3{0, 0, 0}, orientation_); + for (int i = 0; i < num_slices_; ++i) { + const float angle = static_cast(i) * delta_angle; + const float x = std::cos(angle); + const float y = std::sin(angle); + ptr[idx++] = VertexType(float3{x, y, 0}, orientation_); + } + } + + void GenerateIndices(IndexType* ptr, std::size_t num) const { + int idx = 0; + for (int i = 0; i < num_slices_; ++i) { + const int next = i < (num_slices_ - 1) ? i + 1 : 0; + ptr[idx++] = 0; + ptr[idx++] = 1 + i; + ptr[idx++] = 1 + next; + } + } + + filament::Box GetBounds() const { return {{-1, -1, -0.001}, {1, 1, 0.001}}; } + + private: + int num_slices_; + float4 orientation_; +}; + +class SphereBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + static constexpr IndexType kNorthPoleIndex = 0; + static constexpr IndexType kSouthPoleIndex = 1; + + SphereBuilder(int num_stacks, int num_slices) + : num_stacks_(num_stacks), num_slices_(num_slices) {} + + std::size_t NumVertices() const { + return (num_stacks_ * num_slices_) + 2; // +2 for poles + } + + std::size_t NumIndices() const { + const size_t num_tris_polar_cap = num_slices_; + const size_t num_quads_body = num_slices_ * (num_stacks_ - 1); + return (2 * num_tris_polar_cap * kNumIndicesPerTriangle) + + (num_quads_body * kNumIndicesPerQuad); + } + + void GenerateVertices(VertexType* ptr, size_t num) const { + const float lat_angle_delta = M_PI / static_cast(num_stacks_ + 1); + const float lon_angle_delta = 2.0 * M_PI / static_cast(num_slices_); + + // Add the north and south poles. + int idx = 0; + ptr[idx++] = MakeVert(0, 0, 1); + ptr[idx++] = MakeVert(0, 0, -1); + + // Vertices by latitude. + for (int lat = 0; lat < num_stacks_; ++lat) { + // +1 because we handle the north pole (which would be at a lat angle of + // 0-degrees) explicitly. + const float lat_angle = static_cast(lat + 1) * lat_angle_delta; + const float cos_lat_angle = std::cos(lat_angle); + const float sin_lat_angle = std::sin(lat_angle); + const float z = cos_lat_angle; + + for (int lon = 0; lon < num_slices_; ++lon) { + const float lon_angle = static_cast(lon) * lon_angle_delta; + + const float cos_lon_angle = std::cos(lon_angle); + const float sin_lon_angle = std::sin(lon_angle); + + const float x = sin_lat_angle * cos_lon_angle; + const float y = sin_lat_angle * sin_lon_angle; + ptr[idx++] = MakeVert(x, y, z); + } + } + } + + void GenerateIndices(IndexType* ptr, size_t num) const { + int idx = 0; + + // The first two vertices are the poles, so the first vertex in the first + // row starts at index 2. + IndexType row_start = kSouthPoleIndex + 1; + + // North polar cap. + for (int lon = 0; lon < num_slices_; ++lon) { + const int next = lon < (num_slices_ - 1) ? lon + 1 : 0; + ptr[idx++] = kNorthPoleIndex; + ptr[idx++] = row_start + next; + ptr[idx++] = row_start + lon; + } + + // Latitudinal triangle strips. + for (int lat = 0; lat < num_stacks_ - 1; lat++) { + const IndexType north_start = row_start; + const IndexType south_start = row_start + num_slices_; + for (int lon = 0; lon < num_slices_; ++lon) { + // The offset to the index that is adjacent to the current index. + const int adjacent = lon < (num_slices_ - 1) ? lon + 1 : 0; + + const int i0 = (north_start + lon); + const int i1 = (south_start + lon); + const int i2 = (south_start + adjacent); + const int i3 = (north_start + adjacent); + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + row_start += num_slices_; + } + + // South polar cap. + for (int lon = 0; lon < num_slices_; ++lon) { + const int adjacent = lon < (num_slices_ - 1) ? lon + 1 : 0; + ptr[idx++] = kSouthPoleIndex; + ptr[idx++] = row_start + lon; + ptr[idx++] = row_start + adjacent; + } + } + + filament::Box GetBounds() const { return {{-1, -1, -1}, {1, 1, 1}}; } + + private: + static VertexType MakeVert(float x, float y, float z) { + const float3 pt{x, y, z}; + return VertexType(pt, CalculateOrientation(pt)); + } + + int num_stacks_; + int num_slices_; +}; + +class DomeBuilder { + public: + using VertexType = VertexNoUv; + using IndexType = uint16_t; + + static constexpr IndexType kPoleIndex = 0; + + DomeBuilder(int num_stacks, int num_slices) + : num_stacks_(num_stacks), num_slices_(num_slices) {} + + std::size_t NumVertices() const { + return (num_stacks_ * num_slices_) + 1; // +1 for poles + } + + std::size_t NumIndices() const { + const size_t num_tris_polar_cap = num_slices_; + const size_t num_quads_body = num_slices_ * (num_stacks_ - 1); + return (num_tris_polar_cap * kNumIndicesPerTriangle) + + (num_quads_body * kNumIndicesPerQuad); + } + + void GenerateVertices(VertexType* ptr, size_t num) const { + const float lat_angle_delta = 0.5 * M_PI / static_cast(num_stacks_); + const float lon_angle_delta = 2.0 * M_PI / static_cast(num_slices_); + + // Add the pole. + int idx = 0; + ptr[idx++] = MakeVert(0, 0, 1); + + // Vertices by latitude. + for (int lat = 0; lat < num_stacks_; ++lat) { + // +1 because we handle the north pole (which would be at a lat angle of + // 0-degrees) explicitly. + const float lat_angle = static_cast(lat + 1) * lat_angle_delta; + const float cos_lat_angle = std::cos(lat_angle); + const float sin_lat_angle = std::sin(lat_angle); + const float z = cos_lat_angle; + + for (int lon = 0; lon < num_slices_; ++lon) { + const float lon_angle = static_cast(lon) * lon_angle_delta; + const float cos_lon_angle = std::cos(lon_angle); + const float sin_lon_angle = std::sin(lon_angle); + + const float x = sin_lat_angle * cos_lon_angle; + const float y = sin_lat_angle * sin_lon_angle; + ptr[idx++] = MakeVert(x, y, z); + } + } + } + + void GenerateIndices(IndexType* ptr, size_t num) const { + int idx = 0; + + // The first vertex is the poles, so the first vertex in the first row + // starts at index 1. + IndexType row_start = kPoleIndex + 1; + + // North polar cap. + for (int lon = 0; lon < num_slices_; ++lon) { + const int next = lon < (num_slices_ - 1) ? lon + 1 : 0; + + ptr[idx++] = kPoleIndex; + ptr[idx++] = row_start + next; + ptr[idx++] = row_start + lon; + } + + // Latitudinal quad strips. The first "stack" was handled above, so we + // only need to iterate over N-1 stacks. + for (int lat = 0; lat < num_stacks_ - 1; lat++) { + const int north_start = row_start; + const int south_start = row_start + num_slices_; + for (int lon = 0; lon < num_slices_; ++lon) { + // The offset to the index that is adjacent to the current index. + const int adjacent = lon < (num_slices_ - 1) ? lon + 1 : 0; + + const int i0 = (north_start + lon); + const int i1 = (south_start + lon); + const int i2 = (south_start + adjacent); + const int i3 = (north_start + adjacent); + idx = AppendQuadIndices(ptr, idx, i0, i1, i2, i3); + } + row_start += num_slices_; + } + } + + filament::Box GetBounds() const { return {{-1, -1, 0}, {1, 1, 1}}; } + + private: + static VertexType MakeVert(float x, float y, float z) { + const float3 pt{x, y, z}; + return VertexType(pt, CalculateOrientation(pt)); + } + + int num_stacks_; + int num_slices_; +}; + + +template +FilamentBuffers CreateFromBuilder(filament::Engine* engine, const T& builder) { + using VertexType = typename T::VertexType; + using IndexType = typename T::IndexType; + + const int num_vertices = builder.NumVertices(); + const int num_indices = builder.NumIndices(); + if (num_vertices == 0 || num_indices == 0) { + return {}; + } + + auto vertices = [&](std::byte* buffer, std::size_t len) { + auto* ptr = reinterpret_cast(buffer); + if (sizeof(*ptr) * num_vertices != len) { + mju_error("Buffer size mismatch."); + } + builder.GenerateVertices(ptr, num_vertices); + }; + + auto indices = [&](std::byte* buffer, std::size_t len) { + auto* ptr = reinterpret_cast(buffer); + if (sizeof(*ptr) * num_indices != len) { + mju_error("Buffer size mismatch."); + } + builder.GenerateIndices(ptr, num_indices); + }; + + auto vb = CreateVertexBuffer(engine, num_vertices, vertices); + auto ib = CreateIndexBuffer(engine, num_indices, indices); + return {ib, vb, builder.GetBounds()}; +} + +FilamentBuffers CreatePlane(filament::Engine* engine, const mjModel* model) { + const int num_quads = model->vis.quality.numquads; + return CreateFromBuilder(engine, PlaneBuilder(num_quads)); +} + +FilamentBuffers CreateBox(filament::Engine* engine, const mjModel* model) { + const int num_quads = model->vis.quality.numquads; + return CreateFromBuilder(engine, BoxBuilder(num_quads)); +} + +FilamentBuffers CreateSphere(filament::Engine* engine, const mjModel* model) { + const int num_stacks = model->vis.quality.numstacks; + const int num_slices = model->vis.quality.numslices; + return CreateFromBuilder(engine, SphereBuilder(num_stacks, num_slices)); +} + +FilamentBuffers CreateTube(filament::Engine* engine, const mjModel* model) { + const int num_stacks = model->vis.quality.numstacks; + const int num_slices = model->vis.quality.numslices; + return CreateFromBuilder(engine, TubeBuilder(num_stacks, num_slices)); +} + +FilamentBuffers CreateDisk(filament::Engine* engine, const mjModel* model) { + const int num_slices = model->vis.quality.numslices; + return CreateFromBuilder(engine, DiskBuilder(num_slices)); +} + +FilamentBuffers CreateDome(filament::Engine* engine, const mjModel* model) { + const int num_stacks = model->vis.quality.numstacks / 2; + const int num_slices = model->vis.quality.numslices; + return CreateFromBuilder(engine, DomeBuilder(num_stacks, num_slices)); +} + +FilamentBuffers CreateCone(filament::Engine* engine, const mjModel* model) { + const int num_stacks = model->vis.quality.numstacks; + const int num_slices = model->vis.quality.numslices; + return CreateFromBuilder(engine, ConeBuilder(num_stacks, num_slices)); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/builtins.h b/src/experimental/filament/filament/builtins.h new file mode 100644 index 00000000..e7cfd3c3 --- /dev/null +++ b/src/experimental/filament/filament/builtins.h @@ -0,0 +1,35 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUILTINS_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUILTINS_H_ + +#include +#include +#include "experimental/filament/filament/buffer_util.h" + +// Generates buffers for built-in shapes. +namespace mujoco { + +FilamentBuffers CreateBox(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreatePlane(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreateSphere(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreateTube(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreateDisk(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreateDome(filament::Engine* engine, const mjModel* model); +FilamentBuffers CreateCone(filament::Engine* engine, const mjModel* model); + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_BUILTINS_H_ diff --git a/src/experimental/filament/filament/color_grading_options.cc b/src/experimental/filament/filament/color_grading_options.cc new file mode 100644 index 00000000..7c2c2220 --- /dev/null +++ b/src/experimental/filament/filament/color_grading_options.cc @@ -0,0 +1,48 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/color_grading_options.h" + +#include + +#include +#include + +namespace mujoco { + +std::unique_ptr CreateToneMapper(ToneMapperType type) { + filament::ToneMapper* ptr = nullptr; + switch (type) { + case ToneMapperType::kPBRNeutral: + ptr = new filament::PBRNeutralToneMapper(); + break; + case ToneMapperType::kACES: + ptr = new filament::ACESToneMapper(); + break; + case ToneMapperType::kACESLegacy: + ptr = new filament::ACESLegacyToneMapper(); + break; + case ToneMapperType::kFilmic: + ptr = new filament::FilmicToneMapper(); + break; + case ToneMapperType::kLinear: + ptr = new filament::LinearToneMapper(); + break; + default: + mju_error("Unsupported tone mapper type"); + } + return std::unique_ptr(ptr); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/color_grading_options.h b/src/experimental/filament/filament/color_grading_options.h new file mode 100644 index 00000000..bf213120 --- /dev/null +++ b/src/experimental/filament/filament/color_grading_options.h @@ -0,0 +1,81 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_COLOR_GRADING_OPTIONS_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_COLOR_GRADING_OPTIONS_H_ + +#include +#include + +#include +#include +#include +#include + +namespace mujoco { + +// Filament has deprecated its ToneMapper enum. Instead, users can define their +// own ToneMapper objects directly. However, we don't want to expose that +// complexity to users of this library, so we define our own enum which can +// then be used to create predefined ToneMapper objects. +enum class ToneMapperType { + kPBRNeutral, + kACES, + kACESLegacy, + kFilmic, + kLinear, +}; + +std::unique_ptr CreateToneMapper(ToneMapperType type); + +// Structure used to initialize a filament::ColorGrading::Builder object. +// Unlike most other post processing options, the ColorGrading isn't defined +// as a struct, but rather as a write-only class with a Builder. However, we +// want to be able to change these values at runtime, so we define our own +// struct from which we can create new ColorGrading objects. +// +// The default values for this struct are chosen to match the default values +// for the Filament ColorGrading::Builder class. +struct ColorGradingOptions { + ToneMapperType tone_mapper = ToneMapperType::kPBRNeutral; + filament::ColorGrading::LutFormat format = + filament::ColorGrading::LutFormat::INTEGER; + uint8_t dimension = 32; + bool luminance_scaling = false; + bool gamut_mapping = false; + float exposure = 0.0f; + float night_adaptation = 0.0f; + float contrast = 1.0f; + float vibrance = 1.0f; + float saturation = 1.0f; + float temperature = 0.0f; + float tint = 0.0f; + filament::math::float3 out_red = {1.0f, 0.0f, 0.0f}; + filament::math::float3 out_green = {0.0f, 1.0f, 0.0f}; + filament::math::float3 out_blue = {0.0f, 0.0f, 1.0f}; + filament::math::float4 shadows = {1.0f, 1.0f, 1.0f, 1.0f}; + filament::math::float4 midtones = {1.0f, 1.0f, 1.0f, 1.0f}; + filament::math::float4 highlights = {1.0f, 1.0f, 1.0f, 1.0f}; + filament::math::float4 tonal_ranges = {0.0f, 0.333f, 0.550f, 1.0f}; + filament::math::float3 slope = {1.0f}; + filament::math::float3 offset = {0.0f}; + filament::math::float3 power = {1.0f}; + filament::math::float3 shadow_gamma = {1.0f}; + filament::math::float3 mid_point = {1.0f}; + filament::math::float3 highlight_scale = {1.0f}; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_COLOR_GRADING_OPTIONS_H_ diff --git a/src/experimental/filament/filament/drawable.cc b/src/experimental/filament/filament/drawable.cc new file mode 100644 index 00000000..cb1a95e5 --- /dev/null +++ b/src/experimental/filament/filament/drawable.cc @@ -0,0 +1,435 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/drawable.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/geom_util.h" +#include "experimental/filament/filament/material.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +using filament::math::float2; +using filament::math::float3; +using filament::math::float4; +using filament::math::mat4; + +// Planes with a 0-size dimension should be infinitely large. However, we +// don't support that directly so we use a large but finite size instead. +static constexpr float kInfinitePlaneFakeSize = 1000.0f; + +// An arbitrary scale factor for arrows. +static constexpr float kArrowScale = 1.f / 6.f; +static constexpr float kArrowHeadSize = 1.75f; + +// Some built-in geometries are actually composed of multiple simple shapes. A +// capsule, for example, is a open-ended tube with two dome ends. We use these +// constants to help identify which entity (by index) represents which part of +// the overall shape. +static constexpr int kCapsuleTopDome = 1; +static constexpr int kCapsuleBottomDome = 2; +static constexpr int kCylinderTopDisk = 1; +static constexpr int kCylinderBottomDisk = 2; +static constexpr int kArrow0Cone = 1; +static constexpr int kArrow0ConeDisk = 2; +static constexpr int kArrow0BottomDisk = 3; +static constexpr int kArrow1Cone = 1; +static constexpr int kArrow1BottomDisk = 2; +static constexpr int kArrow2TopCone = 1; +static constexpr int kArrow2BottomCone = 2; +static constexpr int kArrow2TopConeDisk = 3; +static constexpr int kArrow2BottomConeDisk = 4; + +Drawable::Drawable(ObjectManager* object_mgr, const mjvGeom& geom) + : material_(object_mgr), + renderables_(object_mgr->GetEngine()) { + if (geom.type == mjGEOM_MESH) { + AddMesh(geom.dataid); + } else if (geom.type == mjGEOM_HFIELD) { + AddHeightField(geom.dataid); + } else if (geom.type == mjGEOM_PLANE) { + AddShape(ObjectManager::kPlane); + } else if (geom.type == mjGEOM_SPHERE) { + AddShape(ObjectManager::kSphere); + } else if (geom.type == mjGEOM_ELLIPSOID) { + AddShape(ObjectManager::kSphere); + } else if (geom.type == mjGEOM_BOX) { + AddShape(ObjectManager::kBox); + } else if (geom.type == mjGEOM_CAPSULE) { + AddShape(ObjectManager::kTube); + AddShape(ObjectManager::kDome); + AddShape(ObjectManager::kDome); + } else if (geom.type == mjGEOM_CYLINDER) { + AddShape(ObjectManager::kTube); + AddShape(ObjectManager::kDisk); + AddShape(ObjectManager::kDisk); + } else if (geom.type == mjGEOM_ARROW) { + AddShape(ObjectManager::kTube); + AddShape(ObjectManager::kCone); + AddShape(ObjectManager::kDisk); + } else if (geom.type == mjGEOM_ARROW1) { + AddShape(ObjectManager::kTube); + AddShape(ObjectManager::kCone); + AddShape(ObjectManager::kDisk); + AddShape(ObjectManager::kDisk); + } else if (geom.type == mjGEOM_ARROW2) { + AddShape(ObjectManager::kTube); + AddShape(ObjectManager::kCone); + AddShape(ObjectManager::kCone); + AddShape(ObjectManager::kDisk); + AddShape(ObjectManager::kDisk); + } else if (geom.type == mjGEOM_FLEX || geom.type == mjGEOM_SKIN) { + // Flex and skin geometries are dynamically updated every frame. + } else { + mju_warning("Unsupported geom type: %d", geom.type); + } +} + +void Drawable::Update(const mjModel* model, const mjvScene* scene, + const mjvGeom& geom) { + if (geom.type == mjGEOM_FLEX || geom.type == mjGEOM_SKIN) { + // Flex geometry is updated every frame with new vertex data. + filament::Engine* engine = renderables_.GetEngine(); + FilamentBuffers buffers = CreateGeomBuffers(engine, model, scene, geom); + + if (renderables_.GetNumEntities() == 0) { + renderables_.Append(std::move(buffers)); + } else { + renderables_.Update(0, std::move(buffers)); + } + } + + SetTransform(geom); + UpdateMaterial(geom); +} + +void Drawable::AddMesh(int data_id) { + ObjectManager* object_mgr = material_.GetObjectManager(); + const FilamentBuffers* buffers = object_mgr->GetMeshBuffer(data_id); + if (buffers == nullptr) { + mju_error("Unknown mesh %d", data_id); + } + renderables_.Append(*buffers); +} + +void Drawable::AddHeightField(int hfield_id) { + ObjectManager* object_mgr = material_.GetObjectManager(); + const FilamentBuffers* buffers = object_mgr->GetHeightFieldBuffer(hfield_id); + if (buffers == nullptr) { + mju_error("Unknown height field %d", hfield_id); + } + renderables_.Append(*buffers); +} + +void Drawable::AddShape(ObjectManager::ShapeType shape_type) { + ObjectManager* object_mgr = material_.GetObjectManager(); + const FilamentBuffers* buffers = object_mgr->GetShapeBuffer(shape_type); + if (buffers == nullptr) { + mju_error("Unknown shape %d", shape_type); + } + renderables_.Append(*buffers); +} + +void Drawable::AddToScene(filament::Scene* scene) { + renderables_.AddToScene(scene); +} + +void Drawable::RemoveFromScene(filament::Scene* scene) { + renderables_.RemoveFromScene(scene); +} + +void Drawable::SetDrawMode(Material::DrawMode mode) { + renderables_.SetMaterialInstance(material_.GetMaterialInstance(mode)); +} + +void Drawable::SetTransform(const mjvGeom& geom) { + // Flex and skin geometries are in global space. + if (geom.type == mjGEOM_FLEX || geom.type == mjGEOM_SKIN) { + return; + } + + const mat4 transform(ReadMat3(geom.mat), ReadFloat3(geom.pos)); + + float3 size = ReadFloat3(geom.size); + filament::TransformManager& tm = + renderables_.GetEngine()->getTransformManager(); + for (int j = 0; j < renderables_.GetNumEntities(); ++j) { + const utils::Entity& entity = renderables_[j]; + + // Update object transform. + mat4 entity_transform = transform; + + // Some built-in drawables are composed of multiple entities. For example, + // capsules are a combination of a open tube and two dome end caps. + + if (geom.type == mjGEOM_CYLINDER) { + // Cylinders are a tube with two disks at the ends. The "bottom" disk is + // rotated so that the normals point outwards. + if (j == kCylinderTopDisk) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + } else if (j == kCylinderBottomDisk) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + } + } else if (geom.type == mjGEOM_CAPSULE) { + // Capsules are a tube with two domes at the ends. We apply an inverse + // scale to the domes to "counteract" the capsule's overall scale so that + // the domes remain spherical in shape. + const float xz_size = 0.5f * (size.x + size.y); + if (j == kCapsuleTopDome) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + entity_transform *= mat4::scaling(float3{1, 1, xz_size / size.z}); + } else if (j == kCapsuleBottomDome) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + entity_transform *= mat4::scaling(float3{1, 1, xz_size / size.z}); + } + } else if (geom.type == mjGEOM_ARROW) { + // An arrow is a tube with a cone at the end and a disk cap at the other + // end. Because the cone head's base is larger than the tube, an extra + // disk is added to the base of the cone. This disk is rotated such that + // its normal points outwards. + entity_transform *= mat4::scaling(float3{1, 1, kArrowScale}); + entity_transform *= mat4::translation(float3{0, 0, kArrowScale}); + if (j == kArrow0Cone) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } else if (j == kArrow0ConeDisk) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } else if (j == kArrow0BottomDisk) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + } + } else if (geom.type == mjGEOM_ARROW1) { + // An arrow1 is a tube with a cone at the end and a disk cap at the other + // end. + entity_transform *= mat4::scaling(float3{1, 1, kArrowScale}); + entity_transform *= mat4::translation(float3{0, 0, kArrowScale}); + if (j == kArrow1Cone) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + } else if (j == kArrow1BottomDisk) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + } + } else if (geom.type == mjGEOM_ARROW2) { + // An arrow2 is a tube with a cone at both ends. Like the standard arrow, + // an extra disk is added to the base of each cone. + entity_transform *= mat4::scaling(float3{1, 1, kArrowScale}); + entity_transform *= mat4::translation(float3{0, 0, kArrowScale}); + if (j == kArrow2TopCone) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } else if (j == kArrow2BottomCone) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } else if (j == kArrow2TopConeDisk) { + entity_transform *= mat4::translation(float3{0, 0, size.z}); + entity_transform *= mat4::rotation(M_PI, float3{1, 0, 0}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } else if (j == kArrow2BottomConeDisk) { + entity_transform *= mat4::translation(float3{0, 0, -size.z}); + entity_transform *= + mat4::scaling(float3{kArrowHeadSize, kArrowHeadSize, 1.0f}); + } + } else if (geom.type == mjGEOM_PLANE) { + // A plane with 0-size in any dimension is considered to be an infinite + // plane, but we don't really support that so just scale it so that its + // very large. + if (size.x == 0 && size.y == 0) { + size = float3(kInfinitePlaneFakeSize, kInfinitePlaneFakeSize, size.z); + } + } + + if (geom.type != mjGEOM_MESH && geom.type != mjGEOM_HFIELD) { + entity_transform *= mat4::scaling(size); + } + tm.setTransform(tm.getInstance(entity), entity_transform); + } +} + +void Drawable::UpdateMaterial(const mjvGeom& geom) { + ObjectManager* object_mgr = material_.GetObjectManager(); + const mjModel* model = object_mgr->GetModel(); + + Material::Textures textures; + if (geom.matid >= 0) { + textures.color = object_mgr->GetTexture(geom.matid, mjTEXROLE_RGB); + textures.normal = object_mgr->GetTexture(geom.matid, mjTEXROLE_NORMAL); + textures.emissive = object_mgr->GetTexture(geom.matid, mjTEXROLE_EMISSIVE); + textures.orm = object_mgr->GetTexture(geom.matid, mjTEXROLE_ORM); + textures.metallic = object_mgr->GetTexture(geom.matid, mjTEXROLE_METALLIC); + textures.roughness = object_mgr->GetTexture(geom.matid, mjTEXROLE_ROUGHNESS); + textures.occlusion = object_mgr->GetTexture(geom.matid, mjTEXROLE_OCCLUSION); + } + + if (geom.matid >= 0) { + if (textures.orm) { + material_.SetNormalMaterialType(ObjectManager::kPbrPacked); + } else if (textures.metallic) { + material_.SetNormalMaterialType(ObjectManager::kPbr); + } else if (textures.roughness) { + material_.SetNormalMaterialType(ObjectManager::kPbr); + } else if (model->mat_metallic[geom.matid] >= 0) { + material_.SetNormalMaterialType(ObjectManager::kPbr); + } else if (model->mat_roughness[geom.matid] >= 0) { + material_.SetNormalMaterialType(ObjectManager::kPbr); + } + } + + // Check to see if we're dealing with a mesh with texture coordinates. + // `data_id` is the id of the mesh in model (i.e. the geom has mesh geometry) + // and `mesh_texcoordadr` stores the address of the mesh uvs if it has them. + bool has_texcoords = false; + if ((geom.type == mjGEOM_MESH || geom.type == mjGEOM_SDF) && + geom.dataid >= 0 && model->mesh_texcoordadr[geom.dataid / 2] >= 0) { + has_texcoords = true; + } + + if (textures.color == nullptr) { + if (geom.rgba[3] < 1.0f) { + material_.SetNormalMaterialType(ObjectManager::kPhongColorFade); + } else { + material_.SetNormalMaterialType(ObjectManager::kPhongColor); + } + } else if (textures.color->getTarget() == + filament::Texture::Sampler::SAMPLER_CUBEMAP) { + if (geom.rgba[3] < 1.0f) { + material_.SetNormalMaterialType(ObjectManager::kPhongCubeFade); + } else { + material_.SetNormalMaterialType(ObjectManager::kPhongCube); + } + } else if (has_texcoords) { + if (geom.rgba[3] < 1.0f) { + material_.SetNormalMaterialType(ObjectManager::kPhong2dUvFade); + } else { + material_.SetNormalMaterialType(ObjectManager::kPhong2dUv); + } + } else { + if (geom.rgba[3] < 1.0f) { + material_.SetNormalMaterialType(ObjectManager::kPhong2dFade); + } else { + material_.SetNormalMaterialType(ObjectManager::kPhong2d); + } + } + + if (geom.matid >= 0) { + material_.UpdateTextures(textures); + } + + Material::Params params; + params.color = ReadFloat4(geom.rgba); + params.emissive = geom.emission; + params.specular = geom.specular; + params.glossiness = geom.shininess; + if (geom.matid >= 0) { + params.metallic = model->mat_metallic[geom.matid]; + params.roughness = model->mat_roughness[geom.matid]; + params.tex_uniform = model->mat_texuniform[geom.matid]; + params.tex_repeat = ReadFloat2(model->mat_texrepeat, geom.matid); + } + + if (geom.segid >= 0) { + constexpr double phi1 = 1.61803398874989484820; // Cached Phi(1). + constexpr double coef1 = 1.0 / phi1; + const double index = static_cast(geom.segid); + const double sample = std::fmod(0.5 + coef1 * index, 1.0); + uint32_t segmentation_color = 0x01000000 * sample; + const uint8_t red = (segmentation_color >> 16) & 0xff; + const uint8_t green = (segmentation_color >> 8) & 0xff; + const uint8_t blue = (segmentation_color >> 0) & 0xff; + params.segmentation_color.x = static_cast(red) / 255.0f; + params.segmentation_color.y = static_cast(green) / 255.0f; + params.segmentation_color.z = static_cast(blue) / 255.0f; + } + + // UvScale only applies to objects that don't have explicit UV coordinates + // in their vertex buffer. Instead, we set the UV coordinate to be the same + // as the vertex position. + // + // The material's `texuniform` and `texrepeat` parameters allow us to scale + // the programmatic UVs. + + if (textures.color) { + if (textures.color->getTarget() == filament::Texture::Sampler::SAMPLER_2D) { + // For 2D textures, `tex_repeat` specifies how many times the texture + // image is repeated. The `tex_uniform` flag determines if the repetition + // is applied at in object space (false) or in world space (true). + params.uv_scale.x = params.tex_repeat.x; + params.uv_scale.y = params.tex_repeat.y; + + if (geom.dataid >= 0) { + if (geom.size[0] > mjMINVAL) { + params.uv_scale.x /= geom.size[0]; + } + if (geom.size[1] > mjMINVAL) { + params.uv_scale.y /= geom.size[1]; + } + } + if (params.tex_uniform) { + if (geom.size[0] > 0) { + params.uv_scale.x *= geom.size[0]; + } + if (geom.size[1] > 0) { + params.uv_scale.y *= geom.size[1]; + } + } + if (geom.type == mjGEOM_PLANE) { + if (geom.size[0] == 0) { + params.uv_scale.x = kInfinitePlaneFakeSize; + } + if (geom.size[1] == 0) { + params.uv_scale.y = kInfinitePlaneFakeSize; + } + } + + params.uv_scale.x = 2 * params.uv_scale.x; + params.uv_scale.y = 2 * params.uv_scale.y; + } else { + // For cube maps, if `tex_uniform` is true, then scale the texture so that + // it covers a 1x1 area of world space rather than the area of the object. + if (params.tex_uniform) { + params.uv_scale.x = 1.0f / (geom.size[0] ? geom.size[0] : 1.0f); + params.uv_scale.y = 1.0f / (geom.size[1] ? geom.size[1] : 1.0f); + params.uv_scale.z = 1.0f / (geom.size[2] ? geom.size[2] : 1.0f); + } + } + } + material_.UpdateParams(params); +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/drawable.h b/src/experimental/filament/filament/drawable.h new file mode 100644 index 00000000..c44c37e4 --- /dev/null +++ b/src/experimental/filament/filament/drawable.h @@ -0,0 +1,73 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_ + +#include + +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/material.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/renderables.h" + +namespace mujoco { + +// Manages the filament Entities and MaterialInstances for a single mjvGeom. +class Drawable { + public: + Drawable(ObjectManager* object_mgr, const mjvGeom& geom); + ~Drawable() noexcept = default; + + Drawable(const Drawable&) = delete; + Drawable& operator=(const Drawable&) = delete; + + // Adds the Drawable to the given filament Scene. Note that a Drawable can + // only be assigned to a single Scene at any given time. + void AddToScene(filament::Scene* scene); + + // Removes the Drawable from the given filament Scene. + void RemoveFromScene(filament::Scene* scene); + + // Updates the drawable to reflect the current state (e.g. geometry, + // transform, material, etc.) of the geom. + void Update(const mjModel* model, const mjvScene* scene, const mjvGeom& geom); + + // Swaps the MaterialInstance that will be used to render the Drawable (e.g. + // normal, depth, segmentation, etc.). This must be called before the filament + // beginFrame/endFrame. + void SetDrawMode(Material::DrawMode mode); + + private: + void AddMesh(int data_id); + void AddHeightField(int hfield_id); + void AddShape(ObjectManager::ShapeType shape_type); + + // Updates the transform of the drawable for rendering. + void SetTransform(const mjvGeom& geom); + + // Updates the material parameters of the drawable for rendering. + void UpdateMaterial(const mjvGeom& geom); + + Material material_; + Renderables renderables_; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_DRAWABLE_H_ diff --git a/src/experimental/filament/filament/filament_context.cc b/src/experimental/filament/filament/filament_context.cc new file mode 100644 index 00000000..11883759 --- /dev/null +++ b/src/experimental/filament/filament/filament_context.cc @@ -0,0 +1,313 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/filament_context.h" + +#include +#include +#include +#include +#include + +#include "third_party/filament/filament/backend/include/backend/DriverEnums.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/gui_view.h" +#include "experimental/filament/filament/imgui_editor.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" +#include "experimental/filament/filament/texture_util.h" +#include "experimental/filament/render_context_filament.h" + +namespace mujoco { + +FilamentContext::FilamentContext(const mjrFilamentConfig* config, + const mjModel* model, mjrContext* con) + : config_(*config), context_(con), model_(model) { +#if defined( __EMSCRIPTEN__) + filament::Engine::Backend backend = filament::Engine::Backend::OPENGL; +#else + filament::Engine::Backend backend = filament::Engine::Backend::VULKAN; +#endif + + switch (config_.graphics_api) { + case mjGFX_DEFAULT: + // Use the default based on the platform above. + break; + case mjGFX_OPENGL: + backend = filament::Engine::Backend::OPENGL; + break; + case mjGFX_VULKAN: + backend = filament::Engine::Backend::VULKAN; + break; + default: + mju_error("Unsupported graphics API: %d", config_.graphics_api); + } + + engine_ = filament::Engine::create(backend); + renderer_ = engine_->createRenderer(); + #ifdef __EMSCRIPTEN__ + swap_chain_ = engine_->createSwapChain(nullptr); + #else + if (config_.native_window) { + swap_chain_ = engine_->createSwapChain(config_.native_window); + } else { + const int width = model_->vis.global.offwidth; + const int height = model_->vis.global.offheight; + swap_chain_ = engine_->createSwapChain(width, height); + } + #endif + + object_manager_ = std::make_unique(model, engine_, config); + + // Set clear options. + filament::Renderer::ClearOptions opts; + opts.clear = true; + opts.discard = true; + opts.clearColor = {0.1, 0.1, 0.1, 1}; + renderer_->setClearOptions(opts); + + // Copy parameters from model to context. + if (model_) { + context_->shadowClip = model_->stat.extent * model_->vis.map.shadowclip; + context_->shadowScale = model_->vis.map.shadowscale; + context_->offWidth = model_->vis.global.offwidth; + context_->offHeight = model_->vis.global.offheight; + context_->offSamples = model_->vis.quality.offsamples; + context_->fogStart = + (float)(model_->stat.extent * model_->vis.map.fogstart); + context_->fogEnd = (float)(model_->stat.extent * model_->vis.map.fogend); + context_->fogRGBA[0] = model_->vis.rgba.fog[0]; + context_->fogRGBA[1] = model_->vis.rgba.fog[1]; + context_->fogRGBA[2] = model_->vis.rgba.fog[2]; + context_->fogRGBA[3] = model_->vis.rgba.fog[3]; + context_->lineWidth = model_->vis.global.linewidth; + context_->shadowSize = model_->vis.quality.shadowsize; + context_->readPixelFormat = 0x1907; // 0x1907 = GL_RGB; + context_->ntexture = model_->ntex; + for (int i = 0; i < model_->ntex; ++i) { + context_->textureType[i] = model_->tex_type[i]; + } + } + + scene_view_ = std::make_unique(engine_, object_manager_.get()); + if (config_.enable_gui) { + gui_view_ = std::make_unique(engine_, object_manager_.get()); + } +} + +FilamentContext::~FilamentContext() { + DestroyRenderTargets(); + gui_view_.reset(); + scene_view_.reset(); + object_manager_.reset(); + engine_->destroy(renderer_); + engine_->destroy(swap_chain_); + filament::Engine::destroy(engine_); +} + +void FilamentContext::Render(const mjrRect& viewport, const mjvScene* scene, + const mjrContext* con) { + if (con != context_) { + mju_error("Unexpected context."); + } + + scene_view_->SetViewport(viewport); + scene_view_->UpdateScene(con, scene); + + // Draw the GUI. We do this after processing the scene in case there are any + // label elements in the scene. + if (gui_view_) { + DrawGui(scene_view_.get()); + + // Prepare the filament Renderable that contains the GUI draw commands. We + // must call this function even if we do not plan on rendering the GUI to + // ensure the ImGui state is updated. + render_gui_ = gui_view_->PrepareRenderable(); + } + + // Render the frame if we're not rendering to a texture.s + if (!render_to_texture_) { + SceneView::DrawMode mode = scene->flags[mjRND_SEGMENT] + ? SceneView::DrawMode::kSegmentation + : SceneView::DrawMode::kNormal; + + filament::View* view = scene_view_->PrepareRenderView(mode); + if (renderer_->beginFrame(swap_chain_)) { + renderer_->render(view); + if (render_gui_) { + renderer_->render(gui_view_->PrepareRenderView()); + render_gui_ = false; + } + renderer_->endFrame(); + } + #ifdef __EMSCRIPTEN__ + engine_->execute(); + #endif + } +} + +void FilamentContext::SetFrameBuffer(int framebuffer) { + render_to_texture_ = (framebuffer != 0); + if (!render_to_texture_) { + DestroyRenderTargets(); + } +} + +void FilamentContext::PrepareRenderTargets(int width, int height) { + for (int i = 0; i < kNumRenderTargetTextureTypes; ++i) { + target_textures_[i] = CreateRenderTargetTexture( + engine_, width, height, static_cast(i)); + } + + // Render target for color pass. + filament::RenderTarget::Builder color_target_builder; + color_target_builder.texture(filament::RenderTarget::AttachmentPoint::COLOR0, + target_textures_[kRenderTargetColor]); + color_target_builder.texture(filament::RenderTarget::AttachmentPoint::DEPTH, + target_textures_[kRenderTargetDepth]); + color_target_ = color_target_builder.build(*engine_); + + // Render target for depth pass. + filament::RenderTarget::Builder depth_target_builder; + depth_target_builder.texture(filament::RenderTarget::AttachmentPoint::COLOR0, + target_textures_[kRenderTargetDepthColor]); + depth_target_builder.texture(filament::RenderTarget::AttachmentPoint::DEPTH, + target_textures_[kRenderTargetDepth]); + depth_target_ = depth_target_builder.build(*engine_); +} + +void FilamentContext::DestroyRenderTargets() { + if (depth_target_) { + engine_->destroy(depth_target_); + depth_target_ = nullptr; + } + if (color_target_) { + engine_->destroy(color_target_); + color_target_ = nullptr; + } + for (int i = 0; i < kNumRenderTargetTextureTypes; ++i) { + if (target_textures_[i]) { + engine_->destroy(target_textures_[i]); + target_textures_[i] = nullptr; + } + } +} + +static void ReadColorPixels(filament::Renderer* renderer, + filament::RenderTarget* target, mjrRect viewport, + unsigned char* buffer, size_t num_bytes) { + filament::backend::PixelBufferDescriptor descriptor( + buffer, num_bytes, filament::backend::PixelDataFormat::RGB, + filament::backend::PixelDataType::UBYTE); + renderer->readPixels(target, viewport.left, viewport.bottom, viewport.width, + viewport.height, std::move(descriptor)); +} + +static void ReadDepthPixels(filament::Renderer* renderer, + filament::RenderTarget* target, mjrRect viewport, + float* buffer, size_t num_bytes) { + filament::backend::PixelBufferDescriptor descriptor( + buffer, num_bytes, filament::backend::PixelDataFormat::R, + filament::backend::PixelDataType::FLOAT); + renderer->readPixels(target, viewport.left, viewport.bottom, viewport.width, + viewport.height, std::move(descriptor)); +} + +void FilamentContext::ReadPixels(mjrRect viewport, unsigned char* rgb, + float* depth) { + if (!render_to_texture_) { + mju_error("Cannot read pixels unless framebuffer is set."); + } + if (color_target_ == nullptr || depth_target_ == nullptr) { + if (viewport.left != 0) { + mju_error("Reading subpixels not supported."); + } + if (viewport.bottom != 0) { + mju_error("Reading subpixels not supported."); + } + PrepareRenderTargets(viewport.width, viewport.height); + } + + if (rgb) { + filament::View* view = + scene_view_->PrepareRenderView(SceneView::DrawMode::kNormal); + if (renderer_->beginFrame(swap_chain_)) { + // We need to disable msaa in order to render to texture. + auto options = view->getMultiSampleAntiAliasingOptions(); + view->setMultiSampleAntiAliasingOptions({ + .enabled = false, + }); + view->setRenderTarget(color_target_); + renderer_->render(view); + view->setRenderTarget(nullptr); + view->setMultiSampleAntiAliasingOptions(options); + renderer_->endFrame(); + } + + engine_->flushAndWait(); + const size_t num_bytes = viewport.width * viewport.height * 3; + ReadColorPixels(renderer_, color_target_, viewport, rgb, num_bytes); + } + + if (depth) { + filament::View* view = + scene_view_->PrepareRenderView(SceneView::DrawMode::kDepth); + if (renderer_->beginFrame(swap_chain_)) { + view->setRenderTarget(depth_target_); + renderer_->render(view); + view->setRenderTarget(nullptr); + renderer_->endFrame(); + } + + engine_->flushAndWait(); + const size_t num_bytes = viewport.width * viewport.height * sizeof(float); + ReadDepthPixels(renderer_, depth_target_, viewport, depth, num_bytes); + } + + if (rgb || depth) { + engine_->flushAndWait(); + } +} + +void FilamentContext::UploadMesh(const mjModel* model, int id) { + object_manager_->UploadMesh(model, id); +} + +void FilamentContext::UploadTexture(const mjModel* model, int id) { + object_manager_->UploadTexture(model, id); +} + +void FilamentContext::UploadHeightField(const mjModel* model, int id) { + object_manager_->UploadHeightField(model, id); +} + +void FilamentContext::UploadFont(const uint8_t* pixels, int width, int height, + int id) { + object_manager_->UploadFont(pixels, width, height, id); +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/filament_context.h b/src/experimental/filament/filament/filament_context.h new file mode 100644 index 00000000..f957eaf9 --- /dev/null +++ b/src/experimental/filament/filament/filament_context.h @@ -0,0 +1,86 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_FILAMENT_CONTEXT_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_FILAMENT_CONTEXT_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/gui_view.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/scene_view.h" +#include "experimental/filament/filament/texture_util.h" +#include "experimental/filament/render_context_filament.h" + +namespace mujoco { + +// Manages the filament renderer that is exposed via the mjr functions. +class FilamentContext { + public: + FilamentContext(const mjrFilamentConfig* config, const mjModel* model, + mjrContext* con); + ~FilamentContext(); + + void Render(const mjrRect& viewport, const mjvScene* scene, + const mjrContext* con); + + void SetFrameBuffer(int framebuffer); + + void ReadPixels(mjrRect viewport, unsigned char* rgb, float* depth); + + void UploadMesh(const mjModel* model, int id); + + void UploadTexture(const mjModel* model, int id); + + void UploadHeightField(const mjModel* model, int id); + + void UploadFont(const uint8_t* pixels, int width, int height, int id); + + FilamentContext(const FilamentContext&) = delete; + FilamentContext& operator=(const FilamentContext&) = delete; + + private: + void PrepareRenderTargets(int width, int height); + void DestroyRenderTargets(); + + SceneView* GetSceneView(const mjvScene* scene); + + mjrFilamentConfig config_; + mjrContext* context_ = nullptr; + const mjModel* model_ = nullptr; + filament::Engine* engine_ = nullptr; + filament::SwapChain* swap_chain_ = nullptr; + filament::Renderer* renderer_ = nullptr; + filament::RenderTarget* color_target_ = nullptr; + filament::RenderTarget* depth_target_ = nullptr; + filament::Texture* target_textures_[kNumRenderTargetTextureTypes] = { + nullptr, nullptr, nullptr}; + + bool render_to_texture_ = false; + bool render_gui_ = false; + + std::unique_ptr object_manager_; + std::unique_ptr scene_view_; + std::unique_ptr gui_view_; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_FILAMENT_CONTEXT_H_ diff --git a/src/experimental/filament/filament/geom_util.cc b/src/experimental/filament/filament/geom_util.cc new file mode 100644 index 00000000..fed02258 --- /dev/null +++ b/src/experimental/filament/filament/geom_util.cc @@ -0,0 +1,171 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/geom_util.h" + +#include +#include +#include +#include + +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/vertex_util.h" + +namespace mujoco { + +static std::span GetPositions(const mjModel* model, + const mjvScene* scene, + const mjvGeom& geom) { + if (geom.type == mjGEOM_FLEX) { + const int num = 9 * scene->flexfaceused[geom.objid]; + const int addr = scene->flexfaceadr[geom.objid]; + const float* ptr = scene->flexface + (9 * addr); + return {ptr, static_cast(num)}; + } else { + const int num = 3 * scene->skinvertnum[geom.objid]; + const int addr = scene->skinvertadr[geom.objid]; + const float* ptr = scene->skinvert + (3 * addr); + return {ptr, static_cast(num)}; + } +} + +static std::span GetNormals(const mjModel* model, + const mjvScene* scene, + const mjvGeom& geom) { + if (geom.type == mjGEOM_FLEX) { + const int num = 9 * scene->flexfaceused[geom.objid]; + const int addr = scene->flexfaceadr[geom.objid]; + const float* ptr = scene->flexnormal + (9 * addr); + return {ptr, static_cast(num)}; + } else { + const int num = 3 * scene->skinvertnum[geom.objid]; + const int addr = scene->skinvertadr[geom.objid]; + const float* ptr = scene->skinnormal + (3 * addr); + return {ptr, static_cast(num)}; + } +} + +static std::span GetUvs(const mjModel* model, + const mjvScene* scene, + const mjvGeom& geom) { + if (geom.type == mjGEOM_FLEX) { + if (geom.texcoord && geom.matid >= 0) { + const int num = 6 * scene->flexfaceused[geom.objid]; + const int addr = scene->flexfaceadr[geom.objid]; + const float* ptr = scene->flextexcoord + (6 * addr); + return {ptr, static_cast(num)}; + } else { + const float* ptr = nullptr; + return {ptr, 0}; + } + } else { + if (model->skin_texcoordadr[geom.objid] >= 0) { + const int num = 3 * scene->skinvertnum[geom.objid]; + const int addr = model->skin_texcoordadr[geom.objid]; + const float* ptr = model->skin_texcoord + (2 * addr); + return {ptr, static_cast(num)}; + } else { + const float* ptr = nullptr; + return {ptr, 0}; + } + } +} + +static std::span GetIndices(const mjModel* model, + const mjvScene* scene, + const mjvGeom& geom) { + if (geom.type == mjGEOM_FLEX) { + const int* ptr = nullptr; + return {ptr, 0}; + } else { + const int num = 3 * model->skin_facenum[geom.objid]; + const int* ptr = model->skin_face + 3 * model->skin_faceadr[geom.objid]; + return {ptr, static_cast(num)}; + } +} + +template +static void FillVertices(std::byte* buffer, std::size_t len, + std::span positions, + std::span normals, + std::span uvs) { + const int num_vertices = len / sizeof(T); + T* ptr = reinterpret_cast(buffer); + for (int i = 0; i < num_vertices; ++i) { + ptr->position = ReadFloat3(positions.data(), i); + ptr->orientation = CalculateOrientation(ReadFloat3(normals.data(), i)); + if constexpr (T::kHasUv) { + ptr->uv.x = uvs[i * 2]; + ptr->uv.y = uvs[i * 2 + 1]; + } + ++ptr; + } +} + +static filament::VertexBuffer* BuildVertexBuffer( + filament::Engine* engine, std::span positions, + std::span normals, std::span uvs) { + const int num_vertices = positions.size() / 3; + if (uvs.data() != nullptr) { + using VertexType = VertexWithUv; + auto fill = [&](std::byte* buffer, std::size_t len) { + FillVertices(buffer, len, positions, normals, uvs); + }; + return CreateVertexBuffer(engine, num_vertices, fill); + } else { + using VertexType = VertexNoUv; + auto fill = [&](std::byte* buffer, std::size_t len) { + FillVertices(buffer, len, positions, normals, uvs); + }; + return CreateVertexBuffer(engine, num_vertices, fill); + } +} + +static filament::IndexBuffer* BuildIndexBuffer(filament::Engine* engine, + std::span indices, + int num_indices) { + if (indices.data() == nullptr) { + auto fill_indices = FillSequence; + return CreateIndexBuffer(engine, num_indices, fill_indices); + } else { + auto fill_indices = [&](std::byte* buffer, std::size_t len) { + std::memcpy(buffer, indices.data(), len); + }; + return CreateIndexBuffer(engine, indices.size(), fill_indices); + } +} + +FilamentBuffers CreateGeomBuffers(filament::Engine* engine, + const mjModel* model, const mjvScene* scene, + const mjvGeom& geom) { + auto positions = GetPositions(model, scene, geom); + auto normals = GetNormals(model, scene, geom); + auto uvs = GetUvs(model, scene, geom); + auto indices = GetIndices(model, scene, geom); + + int num_indices = indices.size(); + if (num_indices == 0 && geom.type == mjGEOM_FLEX) { + num_indices = 3 * scene->flexfaceused[geom.objid]; + } + + FilamentBuffers buffers; + buffers.vertex_buffer = BuildVertexBuffer(engine, positions, normals, uvs); + buffers.index_buffer = BuildIndexBuffer(engine, indices, num_indices); + return buffers; +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/geom_util.h b/src/experimental/filament/filament/geom_util.h new file mode 100644 index 00000000..b6b1e6a1 --- /dev/null +++ b/src/experimental/filament/filament/geom_util.h @@ -0,0 +1,31 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_ + +#include +#include +#include "experimental/filament/filament/buffer_util.h" + +namespace mujoco { + +// Populates the FilamentBuffers for a flex geometry. +FilamentBuffers CreateGeomBuffers(filament::Engine* engine, + const mjModel* model, const mjvScene* scene, + const mjvGeom& geom); + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GEOM_UTIL_H_ diff --git a/src/experimental/filament/filament/gui_view.cc b/src/experimental/filament/filament/gui_view.cc new file mode 100644 index 00000000..49923ecd --- /dev/null +++ b/src/experimental/filament/filament/gui_view.cc @@ -0,0 +1,244 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/gui_view.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/object_manager.h" +#include "experimental/filament/filament/vertex_util.h" + +namespace mujoco { + +using filament::math::float4; + +static constexpr auto kTriangles = + filament::RenderableManager::PrimitiveType::TRIANGLES; + +GuiView::GuiView(filament::Engine* engine, ObjectManager* object_mgr) + : object_mgr_(object_mgr), engine_(engine) { + auto& em = utils::EntityManager::get(); + scene_ = engine_->createScene(); + camera_ = engine_->createCamera(em.create()); + view_ = engine_->createView(); + view_->setScene(scene_); + view_->setCamera(camera_); + view_->setPostProcessingEnabled(false); + + material_ = object_mgr_->GetMaterial(ObjectManager::kUnlitUi); + + // Upload the ImGui font as a texture that is sampled by the UX material. + int font_width = 0; + int font_height = 0; + int font_bpp = 0; + unsigned char* pixels = nullptr; + ImGuiIO& io = ImGui::GetIO(); + io.Fonts->GetTexDataAsRGBA32(&pixels, &font_width, &font_height, &font_bpp); + object_mgr_->UploadFont(pixels, font_width, font_height, 0); +} + +GuiView::~GuiView() { + ResetRenderable(); + for (auto& instance : instances_) { + engine_->destroy(instance.second); + } + engine_->destroyCameraComponent(camera_->getEntity()); + engine_->destroy(view_); + engine_->destroy(scene_); +} + +void GuiView::ResetRenderable() { + auto& em = utils::EntityManager::get(); + if (!renderable_.isNull()) { + scene_->remove(renderable_); + auto& rm = engine_->getRenderableManager(); + rm.destroy(renderable_); + em.destroy(renderable_); + renderable_ = utils::Entity(); + } + + for (auto& buffer : buffers_) { + engine_->destroy(buffer.vertex_buffer); + engine_->destroy(buffer.index_buffer); + } + buffers_.clear(); +} + +bool GuiView::PrepareRenderable() { + ResetRenderable(); + + // Prepare the imgui draw commands. We must call this function even if we do + // not plan on rendering anything to ensure imgui state is updated. + ImGui::Render(); + + ImGuiIO& io = ImGui::GetIO(); + const ImVec2& size = io.DisplaySize; + const ImVec2& scale = io.DisplayFramebufferScale; + ImDrawData* commands = ImGui::GetDrawData(); + commands->ScaleClipRects(scale); + + int num_elements = 0; + for (int n = 0; n < commands->CmdListsCount; ++n) { + const ImDrawList* cmds = commands->CmdLists[n]; + if (sizeof(GuiVertex) != sizeof(cmds->VtxBuffer.Data[0])) { + mju_error("Invalid vertex buffer size."); + } + if (sizeof(uint16_t) != sizeof(cmds->IdxBuffer.Data[0])) { + mju_error("Invalid index buffer size."); + } + num_elements += cmds->CmdBuffer.size(); + } + + if (size.x == 0 || size.y == 0 || num_elements == 0) { + return false; + } + + view_->setViewport( + filament::Viewport(0.f, 0.f, size.x * scale.x, size.y * scale.y)); + camera_->setProjection(filament::Camera::Projection::ORTHO, 0.0, size.x, + size.y, 0.0, 0.0, 1.0); + + filament::RenderableManager::Builder builder(num_elements); + builder.boundingBox({{-100, -100, -100}, {100, 100, 100}}); + builder.culling(false); + + int drawable_index = 0; + for (int n = 0; n < commands->CmdListsCount; ++n) { + const ImDrawList* cmds = commands->CmdLists[n]; + auto vfill = [&](std::byte* dst, std::size_t size) { + if (size != cmds->VtxBuffer.size_in_bytes()) { + mju_error("Invalid vertex buffer size."); + } + std::memcpy(dst, cmds->VtxBuffer.Data, size); + }; + auto ifill = [&](std::byte* dst, std::size_t size) { + if (size != cmds->IdxBuffer.size_in_bytes()) { + mju_error("Invalid index buffer size."); + } + std::memcpy(dst, cmds->IdxBuffer.Data, size); + }; + const mujoco::FilamentBuffers& buffer = buffers_.emplace_back( + CreateIndexBuffer(engine_, cmds->IdxBuffer.Size, ifill), + CreateVertexBuffer(engine_, cmds->VtxBuffer.Size, vfill)); + + int index_offset = 0; + for (const ImDrawCmd& command : cmds->CmdBuffer) { + const int width = size.x * scale.x; + const int height = size.y * scale.y; + + int clip_left = command.ClipRect.x; + int clip_bottom = height - command.ClipRect.w; + int clip_width = command.ClipRect.z - command.ClipRect.x; + int clip_height = command.ClipRect.w - command.ClipRect.y; + // Modal dialogs try to cover the whole window, but also a little outside + // of it. This doesn't work well with filament's scissor test, so we clip + // them to the window. + if (clip_left < 0 || clip_bottom < 0) { + clip_left = 0; + clip_bottom = 0; + clip_width = width; + clip_height = height; + } + + mjrRect clip_rect(clip_left, clip_bottom, clip_width, clip_height); + builder.material(drawable_index, GetMaterialInstance(clip_rect)); + builder.geometry(drawable_index, kTriangles, buffer.vertex_buffer, + buffer.index_buffer, index_offset, + command.ElemCount); + builder.blendOrder(drawable_index, drawable_index); + + index_offset += command.ElemCount; + ++drawable_index; + } + } + + auto& em = utils::EntityManager::get(); + renderable_ = em.create(); + builder.build(*engine_, renderable_); + scene_->addEntity(renderable_); + return true; +} + +// Maps a rect into a 64-bit key to assist with lookup. +static uint64_t MakeRectKey(mjrRect rect) { + return static_cast(rect.left & 0xff) << 24 | + static_cast(rect.bottom & 0xff) << 16 | + static_cast(rect.width & 0xff) << 8 | + static_cast(rect.height & 0xff); +} + +filament::MaterialInstance* GuiView::GetMaterialInstance(mjrRect rect) { + const uint64_t key = MakeRectKey(rect); + auto iter = instances_.find(key); + if (iter != instances_.end()) { + return iter->second; + } + + auto instance = material_->createInstance(); + instance->setScissor(rect.left, rect.bottom, rect.width, rect.height); + filament::TextureSampler sampler; + instance->setParameter("glyph", object_mgr_->GetFont(0), sampler); + instances_[key] = instance; + return instance; +} + +filament::View* GuiView::PrepareRenderView() { return view_; } + +static ImVec2 ClipSpaceToWindowCoordinates(float x, float y) { + const ImVec2& display_size = ImGui::GetIO().DisplaySize; + const float pos_x = display_size.x * ((x + 1) * 0.5f); + const float pos_y = display_size.y * (1.0f - ((y + 1) * 0.5f)); + return ImVec2(pos_x, pos_y); +} + +void DrawTextAt(const char* text, float x, float y, float z) { + if (x < -1 || y < -1 || x > 1 || y > 1 || z < -1 || z > 1) { + return; + } + + const ImVec2 center_pos = ClipSpaceToWindowCoordinates(x, y); + const ImVec2 size = ImGui::CalcTextSize(text); + + const ImVec2 pos = ImVec2(center_pos.x - size.x / 2, center_pos.y); + const ImVec2 shadow_pos = ImVec2(pos.x + 2, pos.y + 2); + const int flags = ImGuiWindowFlags_NoBringToFrontOnFocus | + ImGuiWindowFlags_NoFocusOnAppearing | + ImGuiWindowFlags_NoBackground | + ImGuiWindowFlags_NoDecoration | + ImGuiWindowFlags_NoInputs | + ImGuiWindowFlags_NoNav; + + ImGui::Begin("labels", nullptr, flags); + ImGui::BeginChild("labels", ImGui::GetIO().DisplaySize, 0, flags); + ImDrawList* draw_list = ImGui::GetWindowDrawList(); + draw_list->AddText(shadow_pos, IM_COL32_BLACK, text); + draw_list->AddText(pos, IM_COL32_WHITE, text); + ImGui::EndChild(); + ImGui::End(); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/gui_view.h b/src/experimental/filament/filament/gui_view.h new file mode 100644 index 00000000..a4156053 --- /dev/null +++ b/src/experimental/filament/filament/gui_view.h @@ -0,0 +1,74 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +// A filament::View that contains a filament::Scene used for rendering the GUI. +class GuiView { + public: + GuiView(filament::Engine* engine, ObjectManager* object_mgr); + ~GuiView(); + + // Prepares the UX scene renderable using data from the current ImGui state. + // This function must be called once per frame to ensure ImGui state is + // correctly synced. + bool PrepareRenderable(); + + // Returns the filament::View used to render the UX scene. + filament::View* PrepareRenderView(); + + private: + // Returns the filament::MaterialInstance configured to draw into the given + // scissor rect. + filament::MaterialInstance* GetMaterialInstance(mjrRect rect); + + // Clears the filament::Scene of the UX renderable and releases all buffers. + void ResetRenderable(); + + ObjectManager* object_mgr_ = nullptr; + filament::Engine* engine_ = nullptr; + filament::Scene* scene_ = nullptr; + filament::Camera* camera_ = nullptr; + filament::View* view_ = nullptr; + filament::Material* material_ = nullptr; + utils::Entity renderable_; + std::vector buffers_; + std::unordered_map instances_; +}; + +// Draws text at the given screen coordinates in clip space (i.e. [-1,-1,-1] to +// [1,1,1]). +void DrawTextAt(const char* text, float x, float y, float z); + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_GUI_VIEW_H_ diff --git a/src/experimental/filament/filament/imgui_editor.cc b/src/experimental/filament/filament/imgui_editor.cc new file mode 100644 index 00000000..71b623bf --- /dev/null +++ b/src/experimental/filament/filament/imgui_editor.cc @@ -0,0 +1,691 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/imgui_editor.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +using filament::math::float3; +using filament::math::float4; +using filament::math::mat4; + +using NameList = std::span; + +NameList EnumNames(ToneMapperType v) { + static const char* names[] = {"PBR Neutral", "ACES", "ACES Legacy", "Filmic", + "Linear"}; + return names; +} + +NameList EnumNames(filament::QualityLevel v) { + static const char* names[] = {"LOW", "MEDIUM", "HIGH", "ULTRA"}; + return names; +} + +NameList EnumNames(filament::ShadowType v) { + static const char* names[] = {"PCF", "VSM", "DPCF", "PCSS"}; + return names; +} + +NameList EnumNames(filament::TemporalAntiAliasingOptions::BoxType v) { + static const char* names[] = {"AABB", "VARIANCE", "AABB_VARIANCE"}; + return names; +} + +NameList EnumNames(filament::TemporalAntiAliasingOptions::BoxClipping v) { + static const char* names[] = {"ACCURATE", "CLAMP", "NONE"}; + return names; +} + +NameList EnumNames(filament::TemporalAntiAliasingOptions::JitterPattern v) { + static const char* names[] = {"RGSS_X4", "UNIFORM_HELIX_X4", "HALTON_23_X8", + "HALTON_23_X16", "HALTON_23_X32"}; + return names; +} + +NameList EnumNames(filament::ColorGrading::LutFormat v) { + static const char* names[] = {"Integer", "Float"}; + return names; +} + +NameList EnumNames(filament::BloomOptions::BlendMode v) { + static const char* names[] = {"Add", "Interpolate"}; + return names; +} + +NameList EnumNames(filament::DepthOfFieldOptions::Filter v) { + static const char* names[] = {"NONE", "UNUSED", "MEDIAN"}; + return names; +} + +template +struct UiOpts { + std::optional min; + std::optional max; + std::optional step; + std::optional fstep; +}; + +template +bool Ui(std::string_view label, T* value, UiOpts opts = {}) { + bool changed = false; + + if constexpr (std::is_enum_v) { + int idx = static_cast(*value); + const NameList names = EnumNames(*value); + changed = ImGui::Combo(label.data(), &idx, names.data(), names.size()); + *value = static_cast(idx); + } else if constexpr (std::is_same_v) { + changed = ImGui::Checkbox(label.data(), value); + } else if constexpr (std::is_same_v) { + changed = ImGui::InputScalar(label.data(), ImGuiDataType_U8, value); + } else if constexpr (std::is_same_v) { + changed = ImGui::InputScalar(label.data(), ImGuiDataType_U16, value); + } else if constexpr (std::is_same_v) { + changed = ImGui::InputScalar(label.data(), ImGuiDataType_U32, value); + } else if constexpr (std::is_same_v) { + if (opts.min.has_value() && opts.max.has_value()) { + changed = ImGui::SliderInt(label.data(), value, *opts.min, *opts.max); + } else { + int step = opts.step.value_or(1); + int fast_step = opts.fstep.value_or(100); + changed = ImGui::InputInt(label.data(), value, step, fast_step); + } + } else if constexpr (std::is_same_v) { + if (opts.min.has_value() && opts.max.has_value()) { + changed = ImGui::SliderFloat(label.data(), value, *opts.min, *opts.max); + } else { + float step = opts.step.value_or(0.f); + float fast_step = opts.fstep.value_or(0.f); + changed = ImGui::InputFloat(label.data(), value, step, fast_step); + } + } else if constexpr (std::is_same_v) { + changed = ImGui::InputFloat3(label.data(), &value->x); + } else if constexpr (std::is_same_v) { + changed = ImGui::InputFloat4(label.data(), &value->x); + } else { + static_assert(false, "Unsupported type"); + } + return changed; +} + +void DrawAmbientOcclusionGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getAmbientOcclusionOptions(); + + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Radius (m)", &opts.radius); + changed |= Ui("Power Contrrast", &opts.power); + changed |= Ui("Bias (m)", &opts.bias); + changed |= Ui("Resolution Scale", &opts.resolution); + changed |= Ui("Intensity", &opts.intensity); + changed |= Ui("Bend Normals", &opts.bentNormals); + changed |= Ui("Bilateral Threshold", &opts.bilateralThreshold); + changed |= Ui("Min. Horizon Angle (Rad)", &opts.minHorizonAngleRad); + changed |= Ui("Quality", &opts.quality); + changed |= Ui("Low Pass Filter", &opts.lowPassFilter); + changed |= Ui("Upsampling", &opts.upsampling); + changed |= Ui("SSCT Enabled", &opts.ssct.enabled); + changed |= Ui("Cone Angle (Rad)", &opts.ssct.lightConeRad); + changed |= Ui("Shadow Distance (m)", &opts.ssct.shadowDistance); + changed |= Ui("Max Contact Distance (m)", &opts.ssct.contactDistanceMax); + changed |= Ui("Intensity", &opts.ssct.intensity); + changed |= Ui("Deeth Bias", &opts.ssct.depthBias); + changed |= Ui("Depth Slope Bias", &opts.ssct.depthSlopeBias); + changed |= Ui("Sample Count", &opts.ssct.sampleCount); + changed |= Ui("Raw Count", &opts.ssct.rayCount); + changed |= Ui("Light Direction", &opts.ssct.lightDirection); + + if (changed) { + opts.resolution = (opts.resolution < 0.75f) ? 0.f : 1.0f; + opts.power = std::max(opts.power, 0.0f); + opts.radius = std::clamp(opts.radius, 0.0f, 10.0f); + opts.bias = std::clamp(opts.bias, 0.0f, 0.001f); + opts.ssct.lightConeRad = std::clamp(opts.ssct.lightConeRad, 0.0f, + filament::math::f::PI / 2.0f); + + view->setAmbientOcclusionOptions(opts); + } +} + +void DrawScreenSpaceGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto reflection = view->getScreenSpaceReflectionsOptions(); + bool refraction_enabled = view->isScreenSpaceRefractionEnabled(); + auto guard_band = view->getGuardBandOptions(); + + bool changed = false; + changed |= Ui("Reflection Enabled", &reflection.enabled); + changed |= Ui("Refl. Thickness", &reflection.thickness); + changed |= Ui("Refl. Bias", &reflection.bias); + changed |= Ui("Refl. Max Distance", &reflection.maxDistance); + changed |= Ui("Refl. Stride", &reflection.stride); + if (changed) { + view->setScreenSpaceReflectionsOptions(reflection); + } + if (Ui("Refraction", &refraction_enabled)) { + view->setScreenSpaceRefractionEnabled(refraction_enabled); + } + if (Ui("Guard Band Enabled", &guard_band.enabled)) { + view->setGuardBandOptions(guard_band); + } +} + +void DrawShadowingGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto vsm_opts = view->getVsmShadowOptions(); + auto soft_opts = view->getSoftShadowOptions(); + + bool enabled = view->isShadowingEnabled(); + if (Ui("Shadowing", &enabled)) { + view->setShadowingEnabled(enabled); + } + auto shadow_type = view->getShadowType(); + if (Ui("Shadow Type", &shadow_type)) { + view->setShadowType(shadow_type); + } + + bool changed = false; + changed |= Ui("VSM Anisotropy", &vsm_opts.anisotropy); + changed |= Ui("VSM Mipmapping", &vsm_opts.mipmapping); + changed |= Ui("VSM MSAA Samples", &vsm_opts.msaaSamples); + changed |= Ui("VSM High Precision", &vsm_opts.highPrecision); + changed |= Ui("VSM Min Variance Scale", &vsm_opts.minVarianceScale); + changed |= Ui("VSM Light Bleed Reduction", &vsm_opts.lightBleedReduction); + if (changed) { + vsm_opts.minVarianceScale = std::max(vsm_opts.minVarianceScale, 0.0f); + vsm_opts.lightBleedReduction = + std::clamp(vsm_opts.lightBleedReduction, 0.0f, 1.0f); + + view->setVsmShadowOptions(vsm_opts); + } + + changed = false; + changed |= Ui("Soft Penumbra Scale", &soft_opts.penumbraScale); + changed |= Ui("Soft Penumbra Ratio Scale", &soft_opts.penumbraRatioScale); + if (changed) { + soft_opts.penumbraScale = std::max(soft_opts.penumbraScale, 0.0f); + soft_opts.penumbraRatioScale = std::max(soft_opts.penumbraRatioScale, 1.0f); + + view->setSoftShadowOptions(soft_opts); + } +} + +void DrawPostProcessingGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + bool enabled = view->isPostProcessingEnabled(); + if (Ui("Enabled", &enabled)) { + view->setPostProcessingEnabled(enabled); + } +} + +void DrawFxaaGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + bool enabled = view->getAntiAliasing() == filament::AntiAliasing::FXAA; + if (Ui("Enabled", &enabled)) { + view->setAntiAliasing(enabled ? filament::AntiAliasing::FXAA + : filament::AntiAliasing::NONE); + } +} + +void DrawMsaaGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getMultiSampleAntiAliasingOptions(); + + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Samples", &opts.sampleCount); + changed |= Ui("Custom Resolve", &opts.customResolve); + if (changed) { + view->setMultiSampleAntiAliasingOptions(opts); + } +} + +void DrawTaaGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getTemporalAntiAliasingOptions(); + + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Filter Width", &opts.filterWidth); + changed |= Ui("Feedback", &opts.feedback, {.min = 0.f, .max = 1.f}); + changed |= Ui("Lod Bias", &opts.lodBias); + changed |= Ui("Sharpness", &opts.sharpness); + changed |= Ui("Upscaling", &opts.upscaling); + changed |= Ui("Filter History", &opts.filterHistory); + changed |= Ui("Filter Input", &opts.filterInput); + changed |= Ui("Use YCoCg", &opts.useYCoCg); + changed |= Ui("Box Type", &opts.boxType); + changed |= Ui("Box Clipping", &opts.boxClipping); + changed |= Ui("Jitter Pattern", &opts.jitterPattern); + changed |= Ui("Variance Gamma", &opts.varianceGamma); + changed |= Ui("Prevent Flickering", &opts.preventFlickering); + + if (changed) { + opts.feedback = std::clamp(opts.feedback, 0.f, 1.f); + opts.varianceGamma = std::clamp(opts.varianceGamma, 0.75f, 1.25f); + view->setTemporalAntiAliasingOptions(opts); + } +} + +void DrawBloomGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getBloomOptions(); + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Strength", &opts.strength); + changed |= Ui("Quality", &opts.quality); + changed |= Ui("Resolution", &opts.resolution); + changed |= Ui("Levels", &opts.levels); + changed |= Ui("Blend Mode", &opts.blendMode); + changed |= Ui("Threshold", &opts.threshold); + changed |= Ui("Highlight", &opts.highlight); + changed |= Ui("Lens Flare", &opts.lensFlare); + changed |= Ui("Starburst", &opts.starburst); + changed |= Ui("Ch. Aberration", &opts.chromaticAberration); + changed |= Ui("Ghost Count", &opts.ghostCount); + changed |= Ui("Ghost Spacing", &opts.ghostSpacing); + changed |= Ui("Ghost Threshold", &opts.ghostThreshold); + changed |= Ui("Halo Radius", &opts.haloRadius); + changed |= Ui("Halo Thickness", &opts.haloThickness); + changed |= Ui("Halo Threshold", &opts.haloThreshold); + + if (changed) { + opts.strength = std::clamp(opts.strength, 0.0f, 1.0f); + opts.levels = std::clamp(opts.levels, 1, 11); + opts.ghostSpacing = std::clamp(opts.ghostSpacing, 0.0f, 1.0f); + opts.highlight = std::max(opts.highlight, 10.0f); + opts.haloThickness = std::clamp(opts.haloThickness, 0.0f, 1.0f); + opts.haloRadius = std::clamp(opts.haloRadius, 0.0f, 0.5f); + view->setBloomOptions(opts); + } +} + +void DrawColorGradingGui(SceneView* scene_view) { + auto opts = scene_view->GetColorGradingOptions(); + + bool changed = false; + changed |= Ui("Tone Mapper", &opts.tone_mapper); + changed |= Ui("Lut Format", &opts.format); + changed |= Ui("Lut Dimension", &opts.dimension); + changed |= Ui("Luminance Scaling", &opts.luminance_scaling); + changed |= Ui("Gamut Mapping", &opts.gamut_mapping); + changed |= Ui("Exposure", &opts.exposure, {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Night Adaptation", &opts.night_adaptation, + {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Contrast", &opts.contrast, {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Vibrance", &opts.vibrance, {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Saturation", &opts.saturation, + {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Temperature", &opts.temperature, + {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Tint", &opts.tint, {.step = 0.01f, .fstep = 0.1f}); + changed |= Ui("Out Red", &opts.out_red); + changed |= Ui("Out Green", &opts.out_green); + changed |= Ui("Out Blue", &opts.out_blue); + changed |= Ui("Shadows", &opts.shadows); + changed |= Ui("Midtones", &opts.midtones); + changed |= Ui("Highlights", &opts.highlights); + changed |= Ui("Tonal Ranges", &opts.tonal_ranges); + changed |= Ui("Slope", &opts.slope); + changed |= Ui("Offset", &opts.offset); + changed |= Ui("Power", &opts.power); + changed |= Ui("Shadow Gamma", &opts.shadow_gamma); + changed |= Ui("Mid Point", &opts.mid_point); + changed |= Ui("Highlight Scale", &opts.highlight_scale); + + opts.contrast = std::clamp(opts.contrast, 0.0f, 2.0f); + opts.vibrance = std::clamp(opts.vibrance, 0.0f, 2.0f); + opts.saturation = std::clamp(opts.saturation, 0.0f, 2.0f); + opts.temperature = std::clamp(opts.temperature, -1.0f, 1.0f); + opts.tint = std::clamp(opts.tint, -1.0f, 1.0f); + opts.slope.x = std::max(0.000001f, opts.slope.x); + opts.slope.y = std::max(0.000001f, opts.slope.y); + opts.slope.z = std::max(0.000001f, opts.slope.z); + opts.power.x = std::max(0.000001f, opts.power.x); + opts.power.y = std::max(0.000001f, opts.power.y); + opts.power.z = std::max(0.000001f, opts.power.z); + opts.shadow_gamma.x = std::max(0.000001f, opts.shadow_gamma.x); + opts.shadow_gamma.y = std::max(0.000001f, opts.shadow_gamma.y); + opts.shadow_gamma.z = std::max(0.000001f, opts.shadow_gamma.z); + opts.mid_point.x = std::max(0.000001f, opts.mid_point.x); + opts.mid_point.y = std::max(0.000001f, opts.mid_point.y); + opts.mid_point.z = std::max(0.000001f, opts.mid_point.z); + if (opts.highlight_scale.x == 0.0f) opts.highlight_scale.x = 0.000001f; + if (opts.highlight_scale.y == 0.0f) opts.highlight_scale.y = 0.000001f; + if (opts.highlight_scale.z == 0.0f) opts.highlight_scale.z = 0.000001f; + + if (changed) { + scene_view->SetColorGradingOptions(opts); + } +} + +void DrawDepthOfFieldGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getDepthOfFieldOptions(); + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Filter", &opts.filter); + changed |= Ui("Max. Aperture", &opts.maxApertureDiameter); + changed |= Ui("Resolution", &opts.nativeResolution); + changed |= Ui("COC Scale", &opts.cocScale); + changed |= Ui("COC Aspect Ratio", &opts.cocAspectRatio); + changed |= Ui("Max COC (Foreground)", &opts.maxForegroundCOC); + changed |= Ui("Max COC (Background)", &opts.maxBackgroundCOC); + changed |= Ui("Forground Ring Count", &opts.foregroundRingCount); + changed |= Ui("Background Ring Count", &opts.backgroundRingCount); + changed |= Ui("Fast Gather Ring Count", &opts.fastGatherRingCount); + if (changed) { + view->setDepthOfFieldOptions(opts); + } +} + +void DrawDitheringGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + bool enabled = view->getDithering() != filament::Dithering::NONE; + if (Ui("Enabled", &enabled)) { + view->setDithering(enabled ? filament::Dithering::TEMPORAL + : filament::Dithering::NONE); + } +} + +void DrawFogGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getFogOptions(); + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Color", &opts.color); + changed |= Ui("Use IBL Color", &opts.fogColorFromIbl); + changed |= Ui("Density", &opts.density); + changed |= Ui("Distance", &opts.distance); + changed |= Ui("Cutoff", &opts.cutOffDistance); + changed |= Ui("Max. Opacity", &opts.maximumOpacity); + changed |= Ui("Height", &opts.height); + changed |= Ui("Height Falloff", &opts.heightFalloff); + changed |= Ui("InScattering Start", &opts.inScatteringStart); + changed |= Ui("InScattering Size", &opts.inScatteringSize); + if (changed) { + opts.maximumOpacity = std::clamp(opts.maximumOpacity, 0.0f, 1.0f); + view->setFogOptions(opts); + } +} + +void DrawVignetteGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto opts = view->getVignetteOptions(); + bool changed = false; + changed |= Ui("Enabled", &opts.enabled); + changed |= Ui("Color", &opts.color); + changed |= Ui("Midpoint", &opts.midPoint); + changed |= Ui("Roundness", &opts.roundness); + changed |= Ui("Feature", &opts.feather); + + if (changed) { + opts.midPoint = std::clamp(opts.midPoint, 0.0f, 1.0f); + opts.roundness = std::clamp(opts.roundness, 0.0f, 1.0f); + opts.feather = std::clamp(opts.feather, 0.0f, 1.0f); + view->setVignetteOptions(opts); + } +} + +void DrawVisibleLayersGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + uint32_t layers = view->getVisibleLayers(); + bool changed = false; + + ImGui::Columns(2); + changed |= ImGui::CheckboxFlags("Layer 0", &layers, 1 << 0); + changed |= ImGui::CheckboxFlags("Layer 1", &layers, 1 << 1); + changed |= ImGui::CheckboxFlags("Layer 2", &layers, 1 << 2); + changed |= ImGui::CheckboxFlags("Layer 3", &layers, 1 << 3); + ImGui::NextColumn(); + changed |= ImGui::CheckboxFlags("Layer 4", &layers, 1 << 4); + changed |= ImGui::CheckboxFlags("Layer 5", &layers, 1 << 5); + changed |= ImGui::CheckboxFlags("Layer 6", &layers, 1 << 6); + changed |= ImGui::CheckboxFlags("Layer 7", &layers, 1 << 7); + ImGui::Columns(1); + if (changed) { + view->setVisibleLayers(0xff, static_cast(layers)); + } +} + +void DrawCameraGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + filament::Camera& camera = view->getCamera(); + auto position = float3(camera.getPosition()); + Ui("Position", &position); + auto direction = camera.getForwardVector(); + Ui("Direction", &direction); +} + +void DrawIndirectLightGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + auto ibl = view->getScene()->getIndirectLight(); + + float intensity = 10000.0f; + if (ibl) { + intensity = ibl->getIntensity(); + if (Ui("Intensity", &intensity, {.step = 1000.0f, .fstep = 10000.0f})) { + ibl->setIntensity(intensity); + } + } + + static char filename[256]; + ImGui::InputText("Filename", filename, sizeof(filename)); + if (ImGui::Button("Load")) { + scene_view->SetEnvironmentLight(filename, intensity); + } +} + +void DrawLightGui(filament::LightManager& lm, + filament::LightManager::Instance li) { + auto color = lm.getColor(li); + if (ImGui::ColorEdit3("Color", &color.x)) { + lm.setColor(li, color); + } + auto position = lm.getPosition(li); + if (Ui("Position", &position)) { + lm.setPosition(li, position); + } + auto direction = lm.getDirection(li); + if (Ui("Direction", &direction)) { + lm.setDirection(li, direction); + } + float intensity = lm.getIntensity(li); + if (Ui("Intensity", &intensity, {.step = 1000.0f, .fstep = 10000.0f})) { + lm.setIntensityCandela(li, intensity); + } + float falloff = lm.getFalloff(li); + if (Ui("Falloff Radius", &falloff)) { + lm.setFalloff(li, falloff); + } + if (lm.isSpotLight(li)) { + float spotLightInnerCone = lm.getSpotLightInnerCone(li); + float spotLightOuterCone = lm.getSpotLightOuterCone(li); + if (Ui("Spot Light Inner Cone", &spotLightInnerCone)) { + lm.setSpotLightCone(li, spotLightInnerCone, spotLightOuterCone); + } + if (Ui("Spot Light Outer Cone", &spotLightOuterCone)) { + lm.setSpotLightCone(li, spotLightInnerCone, spotLightOuterCone); + } + } + ImGui::Columns(2); + for (int i = 0; i < 8; ++i) { + std::string name = "Channel " + std::to_string(i); + bool enabled = lm.getLightChannel(li, i); + if (i == 4) { + ImGui::NextColumn(); + } + if (Ui(name, &enabled)) { + lm.setLightChannel(li, i, enabled); + } + } + ImGui::Columns(1); + if (ImGui::TreeNodeEx("Shadow Options")) { + bool enabled = lm.isShadowCaster(li); + if (Ui("Enabled", &enabled)) { + lm.setShadowCaster(li, enabled); + } + + filament::LightManager::ShadowOptions opts = lm.getShadowOptions(li); + bool changed = false; + changed |= Ui("Stable", &opts.stable); + changed |= Ui("LiSPSM", &opts.lispsm); + changed |= Ui("SS Contact Shadows", &opts.screenSpaceContactShadows); + changed |= Ui("Map Size", &opts.mapSize); + changed |= Ui("Bulb Radius", &opts.shadowBulbRadius); + changed |= Ui("Step Count", &opts.stepCount); + changed |= Ui("# Cascades", &opts.shadowCascades); + changed |= Ui("Constant Bias", &opts.constantBias); + changed |= Ui("Normal Bias", &opts.normalBias); + if (changed) { + lm.setShadowOptions(li, opts); + } + ImGui::TreePop(); + } +} + +void DrawSceneViewGui(SceneView* scene_view) { + filament::View* view = scene_view->GetDefaultRenderView(); + filament::Engine* engine = scene_view->GetEngine(); + filament::LightManager& lm = engine->getLightManager(); + + if (ImGui::TreeNodeEx("Ambient Occlusion")) { + DrawAmbientOcclusionGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Screen Space")) { + DrawScreenSpaceGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Shadowing")) { + DrawShadowingGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Post Processing")) { + DrawPostProcessingGui(scene_view); + if (ImGui::TreeNodeEx("Anti Aliasing (FXAA)")) { + DrawFxaaGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Anti Aliasing (MSAA)")) { + DrawMsaaGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Anti Aliasing (Temporal)")) { + DrawTaaGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Bloom")) { + DrawBloomGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Color Grading")) { + DrawColorGradingGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Depth of Field")) { + DrawDepthOfFieldGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Dithering")) { + DrawDitheringGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Fog")) { + DrawFogGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Vignette")) { + DrawVignetteGui(scene_view); + ImGui::TreePop(); + } + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Visibility Layers")) { + DrawVisibleLayersGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Camera")) { + DrawCameraGui(scene_view); + ImGui::TreePop(); + } + if (ImGui::TreeNodeEx("Lights")) { + if (ImGui::TreeNodeEx("Indirect (Image-based) Light")) { + DrawIndirectLightGui(scene_view); + ImGui::TreePop(); + } + view->getScene()->forEach([&](utils::Entity entity) { + auto li = lm.getInstance(entity); + if (!li.isValid()) { + return; + } + + const char* type = lm.isDirectional(li) ? " (D)" + : lm.isPointLight(li) ? " (P)" + : " (S)"; + const std::string name = "Light " + std::to_string(entity.getId()) + type; + if (ImGui::TreeNodeEx(name.c_str())) { + DrawLightGui(lm, li); + ImGui::TreePop(); + } + }); + ImGui::TreePop(); + } +} + +void DrawGui(SceneView* scene_view) { + static bool display = false; + if (ImGui::BeginMainMenuBar()) { + if (ImGui::BeginMenu("View")) { + ImGui::Separator(); + if (ImGui::MenuItem("Filament", "", display)) { + display = !display; + } + ImGui::EndMenu(); + } + ImGui::EndMainMenuBar(); + } + + if (!display) { + return; + } + + ImGui::SetNextWindowSize(ImVec2(400, 300), ImGuiCond_Appearing); + ImGui::Begin("Filament"); + DrawSceneViewGui(scene_view); + ImGui::End(); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/imgui_editor.h b/src/experimental/filament/filament/imgui_editor.h new file mode 100644 index 00000000..b8f45357 --- /dev/null +++ b/src/experimental/filament/filament/imgui_editor.h @@ -0,0 +1,27 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ + +#include "experimental/filament/filament/scene_view.h" + +namespace mujoco { + +// Generates a ImGui Window for the given scene views. +void DrawGui(SceneView* scene_views); + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_IMGUI_EDITOR_H_ diff --git a/src/experimental/filament/filament/light.cc b/src/experimental/filament/filament/light.cc new file mode 100644 index 00000000..f6cbfd62 --- /dev/null +++ b/src/experimental/filament/filament/light.cc @@ -0,0 +1,130 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/light.h" + +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +Light::Light(ObjectManager* object_mgr, const Params& params) + : engine_(object_mgr->GetEngine()), params_(params) { + filament::LightManager::Type type; + switch (params.type) { + case mjLIGHT_SPOT: + type = filament::LightManager::Type::FOCUSED_SPOT; + break; + case mjLIGHT_DIRECTIONAL: + // We break with the spec here slightly and use a spot light for the head + // light instead of a directional params. This is because filament only + // supports a single directional light, and we'd rather allow a scene + // light to be that directional params. It's also a bit odd for a + // directional light to move with the camera. + type = params.headlight ? filament::LightManager::Type::FOCUSED_SPOT + : filament::LightManager::Type::DIRECTIONAL; + break; + case mjLIGHT_POINT: + type = filament::LightManager::Type::POINT; + break; + default: + mju_error("Unsupported light type: %d", params.type); + return; + } + + filament::LightManager::Builder builder(type); + builder.color(params.color); + builder.intensityCandela(params.intensity); + builder.castShadows(params.castshadow); + if (type == filament::LightManager::Type::FOCUSED_SPOT) { + if (params.headlight) { + builder.spotLightCone(0, M_PI / 2.0f); + } else { + builder.spotLightCone(0, params.spot_cone_angle * M_PI / 180.0f); + } + } + if (type != filament::LightManager::Type::DIRECTIONAL) { + builder.falloff(params.range); + } + filament::LightManager::ShadowOptions opts; + opts.mapSize = 4096; + opts.shadowCascades = + type == filament::LightManager::Type::DIRECTIONAL ? 4 : 1; + opts.shadowBulbRadius = params.bulbradius; + builder.shadowOptions(opts); + + entity_ = utils::EntityManager::get().create(); + if (entity_.isNull()) { + mju_error("Failed to create light entity."); + } + builder.build(*engine_, entity_); +} + +Light::~Light() noexcept { + utils::EntityManager& em = utils::EntityManager::get(); + if (!entity_.isNull()) { + engine_->destroy(entity_); + em.destroy(entity_); + } +} + +void Light::AddToScene(filament::Scene* scene) { scene->addEntity(entity_); } + +void Light::RemoveFromScene(filament::Scene* scene) { scene->remove(entity_); } + +void Light::SetTransform(filament::math::float3 position, + filament::math::float3 direction) { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setPosition(li, position); + lm.setDirection(li, direction); +} + +void Light::SetColor(const filament::math::float3& color) { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setColor(li, color); +} + +void Light::SetIntensity(float intensity) { + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setIntensityCandela(li, intensity); +} + +void Light::Enable() { + if (!enabled_) { + enabled_ = true; + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setLightChannel(li, 0, enabled_); + } +} + +void Light::Disable() { + if (enabled_) { + enabled_ = false; + filament::LightManager& lm = engine_->getLightManager(); + const filament::LightManager::Instance li = lm.getInstance(entity_); + lm.setLightChannel(li, 0, enabled_); + } +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/light.h b/src/experimental/filament/filament/light.h new file mode 100644 index 00000000..b1f0930e --- /dev/null +++ b/src/experimental/filament/filament/light.h @@ -0,0 +1,88 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_LIGHT_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_LIGHT_H_ + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +// Manages the filament Entities for a single mjvLight. +class Light { + public: + // Configuration parameters for a light. + struct Params { + // The type of light (e.g. spot, point, directional, etc.) + mjtLightType type; + // The color of the light. + filament::math::float3 color = {0, 0, 0}; + // The intensity of the light, in candela. + float intensity = 0.0f; + // Whether or not the light casts shadows. + bool castshadow = true; + // The range/distance in which the light is effective, in meters. + float range = 10.0f; + // The angle of the spot light cone, in degrees. + float spot_cone_angle = 180.f; + // The radius of the bulb used for soft shadows. + float bulbradius = 0.0f; + // Whether or not the light is a headlight. + bool headlight = false; + }; + + Light(ObjectManager* object_mgr, const Params& params); + ~Light() noexcept; + + Light(const Light&) = delete; + Light& operator=(const Light&) = delete; + + // Adds the filament light Entities to the given filament Scene. + void AddToScene(filament::Scene* scene); + + // Removes the filament light Entities from the given filament Scene. + void RemoveFromScene(filament::Scene* scene); + + // Updates the light's position/rotation. + void SetTransform(filament::math::float3 position, + filament::math::float3 direction); + + // Sets the color of the light. + void SetColor(const filament::math::float3& color); + + // Sets the intensity of the light in candela. + void SetIntensity(float intensity); + + // Enables/disables the light in the scene. + void Enable(); + void Disable(); + + // Returns true if the light is a headlight. + bool IsHeadlight() const { return params_.headlight; } + + private: + filament::Engine* engine_ = nullptr; + utils::Entity entity_; + bool enabled_ = true; + Params params_; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_LIGHT_H_ diff --git a/src/experimental/filament/filament/material.cc b/src/experimental/filament/filament/material.cc new file mode 100644 index 00000000..edc43477 --- /dev/null +++ b/src/experimental/filament/filament/material.cc @@ -0,0 +1,186 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/material.h" + +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +// Various tweakable parameters for mapping mujoco values onto filament. +static constexpr float kSpecularMultiplier = 0.6f; +static constexpr float kShininessMultiplier = 0.1f; +static constexpr float kEmissiveMultiplier = 0.3f; + +Material::Material(ObjectManager* object_mgr) : object_mgr_(object_mgr) { + instances_[kDepth] = + object_mgr->GetMaterial(ObjectManager::kUnlitDepth)->createInstance(); + instances_[kSegmentation] = + object_mgr_->GetMaterial(ObjectManager::kUnlitSegmentation) + ->createInstance(); +} + +Material::~Material() noexcept { + filament::Engine* engine = object_mgr_->GetEngine(); + for (int i = 0; i < kNumDrawModes; ++i) { + if (instances_[i]) { + engine->destroy(instances_[i]); + } + } +} + +void Material::SetNormalMaterialType( + ObjectManager::MaterialType material_type) { + filament::Material* material = object_mgr_->GetMaterial(material_type); + + if (instances_[kNormal]) { + const filament::Material* current_material = + instances_[kNormal]->getMaterial(); + if (current_material == material) { + return; + } + + object_mgr_->GetEngine()->destroy(instances_[kNormal]); + instances_[kNormal] = nullptr; + } + if (material) { + instances_[kNormal] = material->createInstance(); + UpdateMaterialInstances(); + } +} + +void Material::UpdateParams(const Params& params) { + params_ = params; + UpdateMaterialInstances(); +} + +void Material::UpdateTextures(const Textures& textures) { + textures_ = textures; + UpdateMaterialInstances(); +} + +void Material::UpdateMaterialInstances() { + filament::MaterialInstance* instance = instances_[DrawMode::kNormal]; + if (instance == nullptr) { + return; + } + + const filament::Material* material = instance->getMaterial(); + if (material->hasParameter("BaseColorFactor")) { + instance->setParameter("BaseColorFactor", filament::RgbaType::sRGB, + params_.color); + } + if (material->hasParameter("EmissiveFactor")) { + instance->setParameter("EmissiveFactor", + params_.emissive * kEmissiveMultiplier); + } + if (material->hasParameter("SpecularFactor")) { + instance->setParameter("SpecularFactor", + params_.specular * kSpecularMultiplier); + } + if (material->hasParameter("GlossinessFactor")) { + instance->setParameter("GlossinessFactor", + params_.glossiness * kShininessMultiplier); + } + if (material->hasParameter("MetallicFactor")) { + instance->setParameter("MetallicFactor", + params_.metallic >= 0 ? params_.metallic : 1.0f); + } + if (material->hasParameter("RoughnessFactor")) { + instance->setParameter("RoughnessFactor", + params_.roughness >= 0 ? params_.roughness : 1.0f); + } + if (material->hasParameter("UvScale")) { + instance->setParameter("UvScale", params_.uv_scale); + } + + if (instances_[DrawMode::kSegmentation]) { + instances_[DrawMode::kSegmentation]->setParameter( + "BaseColorFactor", params_.segmentation_color); + } + + // All textures use the same default sampler. + filament::TextureSampler sampler; + sampler.setWrapModeR(filament::TextureSampler::WrapMode::REPEAT); + sampler.setWrapModeS(filament::TextureSampler::WrapMode::REPEAT); + sampler.setWrapModeT(filament::TextureSampler::WrapMode::REPEAT); + sampler.setMagFilter(filament::TextureSampler::MagFilter::LINEAR); + sampler.setMinFilter( + filament::TextureSampler::MinFilter::LINEAR_MIPMAP_LINEAR); + + if (material->hasParameter("BaseColor")) { + if (textures_.color) { + instance->setParameter("BaseColor", textures_.color, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_RGB); + instance->setParameter("BaseColor", fallback, sampler); + } + } + if (material->hasParameter("Normal")) { + if (textures_.normal) { + instance->setParameter("Normal", textures_.normal, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_NORMAL); + instance->setParameter("Normal", fallback, sampler); + } + } + if (material->hasParameter("Metallic")) { + if (textures_.metallic) { + instance->setParameter("Metallic", textures_.metallic, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_METALLIC); + instance->setParameter("Metallic", fallback, sampler); + } + } + if (material->hasParameter("Roughness")) { + if (textures_.roughness) { + instance->setParameter("Roughness", textures_.roughness, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_ROUGHNESS); + instance->setParameter("Roughness", fallback, sampler); + } + } + if (material->hasParameter("Occlusion")) { + if (textures_.occlusion) { + instance->setParameter("Occlusion", textures_.occlusion, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_OCCLUSION); + instance->setParameter("Occlusion", fallback, sampler); + } + } + if (material->hasParameter("ORM")) { + if (textures_.orm) { + instance->setParameter("ORM", textures_.orm, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_ORM); + instance->setParameter("ORM", fallback, sampler); + } + } + if (material->hasParameter("Emissive")) { + if (textures_.emissive) { + instance->setParameter("Emissive", textures_.emissive, sampler); + } else { + auto* fallback = object_mgr_->GetFallbackTexture(mjTEXROLE_EMISSIVE); + instance->setParameter("Emissive", fallback, sampler); + } + } +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/material.h b/src/experimental/filament/filament/material.h new file mode 100644 index 00000000..c21f9fb1 --- /dev/null +++ b/src/experimental/filament/filament/material.h @@ -0,0 +1,102 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATERIAL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATERIAL_H_ + +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +class Material { + public: + // The different methods for rendering objects. Each mode uses a different + // material, but all materials "share" the same textures and parameters + // (unless specifically noted otherwise). + enum DrawMode { + kNormal, + kDepth, + kSegmentation, + kNumDrawModes, + }; + + // The textures that can be assigned to the drawable's material. + struct Textures { + const filament::Texture* color = nullptr; + const filament::Texture* normal = nullptr; + const filament::Texture* metallic = nullptr; + const filament::Texture* roughness = nullptr; + const filament::Texture* occlusion = nullptr; + const filament::Texture* orm = nullptr; + const filament::Texture* emissive = nullptr; + }; + + // The parameters that can be applied to the drawable's material. + struct Params { + filament::math::float4 color = {1, 1, 1, 1}; + filament::math::float4 segmentation_color = {1, 1, 1, 1}; + filament::math::float2 tex_repeat = {1, 1}; + filament::math::float3 uv_scale = {1, 1, 1}; + float specular = -1.0f; + float glossiness = -1.0f; + float metallic = -1.0f; + float roughness = -1.0f; + float emissive = -1.0f; + bool tex_uniform = false; + }; + + Material(ObjectManager* object_mgr); + ~Material() noexcept; + + Material(const Material&) = delete; + Material& operator=(const Material&) = delete; + + // Assigns a material to the draw mode. + void SetNormalMaterialType(ObjectManager::MaterialType material_type); + + // Updates the material parameters of the drawable for rendering. + void UpdateParams(const Params& params); + + // Updates the material textures of the drawable for rendering. + void UpdateTextures(const Textures& textures); + + // Returns the material instance assigned to the draw mode. + filament::MaterialInstance* GetMaterialInstance(DrawMode mode) { + return instances_[mode]; + } + + // Returns the ObjectManager owning the Materials which are used to create + // the MaterialInstances. + ObjectManager* GetObjectManager() { return object_mgr_; } + + private: + // Updates the material instances based on the currently set parameters and + // textures. + void UpdateMaterialInstances(); + + ObjectManager* object_mgr_ = nullptr; + filament::MaterialInstance* instances_[kNumDrawModes] = {nullptr}; + Params params_; + Textures textures_; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATERIAL_H_ diff --git a/src/experimental/filament/filament/math_util.h b/src/experimental/filament/filament/math_util.h new file mode 100644 index 00000000..d76cc406 --- /dev/null +++ b/src/experimental/filament/filament/math_util.h @@ -0,0 +1,59 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_ + +#include +#include +#include +#include + +namespace mujoco { + +// Reads a float2 from an array buffer in the model/scene. +template +inline filament::math::float2 ReadFloat2(const T* arr, int index = 0) { + const T* ptr = arr + (2 * index); + return filament::math::float2(ptr[0], ptr[1]); +} + +// Reads a float3 from an array buffer in the model/scene. +template +inline filament::math::float3 ReadFloat3(const T* arr, int index = 0) { + const T* ptr = arr + (3 * index); + return filament::math::float3(ptr[0], ptr[1], ptr[2]); +} + +// Reads a float4 from an array buffer in the model/scene. +template +inline filament::math::float4 ReadFloat4(const T* arr, int index = 0) { + const T* ptr = arr + (4 * index); + return filament::math::float4(ptr[0], ptr[1], ptr[2], ptr[3]); +} + +// Reads a mat3 from an array buffer in the model/scene. +template +inline filament::math::mat3 ReadMat3(const T* arr, int index = 0) { + // clang-format off + const T* ptr = arr + (9 * index); + return filament::math::mat3(ptr[0], ptr[3], ptr[6], + ptr[1], ptr[4], ptr[7], + ptr[2], ptr[5], ptr[8]); + // clang-format on +} + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MATH_UTIL_H_ diff --git a/src/experimental/filament/filament/model_util.cc b/src/experimental/filament/filament/model_util.cc new file mode 100644 index 00000000..57c08ae7 --- /dev/null +++ b/src/experimental/filament/filament/model_util.cc @@ -0,0 +1,477 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/model_util.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/texture_util.h" +#include "experimental/filament/filament/vertex_util.h" + +namespace mujoco { + +using filament::math::float2; +using filament::math::float3; +using filament::math::float4; + +static bool UseFaceNormal(const float3& face_normal, + const float3& mesh_normal) { + // clang-format off + return face_normal[0] * mesh_normal[0] + + face_normal[1] * mesh_normal[1] + + face_normal[2] * mesh_normal[2] < 0.8f; + // clang-format on +} + +template +static void FillConvexHullBuffer(T* ptr, std::size_t num, const mjModel* model, + int meshid) { + const int numvert = model->mesh_graph[model->mesh_graphadr[meshid]]; + const int numface = model->mesh_graph[model->mesh_graphadr[meshid]+1]; + + const int vertadr = model->mesh_vertadr[meshid]; + const float* vertices = model->mesh_vert + (3 * vertadr); + const int texcoordadr = model->mesh_texcoordadr[meshid]; + const float* texcoords = model->mesh_texcoord + (2 * texcoordadr); + + if (num != numface * 3) { + mju_error("Invalid vertex count."); + return; + } + + for (int face = 0; face < numface; ++face) { + int j = model->mesh_graphadr[meshid] + 2 + 3*numvert + 3*numface + 3*face; + + const float3 p1 = ReadFloat3(vertices, model->mesh_graph[j + 0]); + const float3 p2 = ReadFloat3(vertices, model->mesh_graph[j + 1]); + const float3 p3 = ReadFloat3(vertices, model->mesh_graph[j + 2]); + const float4 orientation = CalculateOrientation(p1, p2, p3); + + ptr->position = p1; + ptr->orientation = orientation; + if constexpr (T::kHasUv) { + ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 0]); + } + ++ptr; + + ptr->position = p2; + ptr->orientation = orientation; + if constexpr (T::kHasUv) { + ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 1]); + } + ++ptr; + + ptr->position = p3; + ptr->orientation = orientation; + if constexpr (T::kHasUv) { + ptr->uv = ReadFloat2(texcoords, model->mesh_graph[j + 2]); + } + ++ptr; + } +} + +template +static void FillMeshBuffer(T* ptr, std::size_t num, const mjModel* model, + int meshid) { + const int faceadr = model->mesh_faceadr[meshid]; + const int facenum = model->mesh_facenum[meshid]; + if (num != facenum * 3) { + mju_error("Invalid vertex count."); + return; + } + + const int vertadr = model->mesh_vertadr[meshid]; + const float* vertices = model->mesh_vert + (3 * vertadr); + const int normaladr = model->mesh_normaladr[meshid]; + const float* normals = model->mesh_normal + 3 * normaladr; + const int texcoordadr = model->mesh_texcoordadr[meshid]; + const float* texcoords = model->mesh_texcoord + (2 * texcoordadr); + + for (int i = 0; i < facenum; ++i) { + const int face = 3 * (faceadr + i); + + const float3 p1 = ReadFloat3(vertices, model->mesh_face[face + 0]); + const float3 p2 = ReadFloat3(vertices, model->mesh_face[face + 1]); + const float3 p3 = ReadFloat3(vertices, model->mesh_face[face + 2]); + const float3 face_normal = CalculateNormal(p1, p2, p3); + + const float3 n1 = ReadFloat3(normals, model->mesh_facenormal[face + 0]); + const float3 n2 = ReadFloat3(normals, model->mesh_facenormal[face + 1]); + const float3 n3 = ReadFloat3(normals, model->mesh_facenormal[face + 2]); + + ptr->position = p1; + if constexpr (T::kHasUv) { + ptr->orientation = CalculateOrientation(n1); + ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 0]); + } else if (UseFaceNormal(face_normal, n1)) { + ptr->orientation = CalculateOrientation(face_normal); + } else { + ptr->orientation = CalculateOrientation(n1); + } + ++ptr; + + ptr->position = p2; + if constexpr (T::kHasUv) { + ptr->orientation = CalculateOrientation(n2); + ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 1]); + } else if (UseFaceNormal(face_normal, n2)) { + ptr->orientation = CalculateOrientation(face_normal); + } else { + ptr->orientation = CalculateOrientation(n2); + } + ++ptr; + + ptr->position = p3; + if constexpr (T::kHasUv) { + ptr->orientation = CalculateOrientation(n3); + ptr->uv = ReadFloat2(texcoords, model->mesh_facetexcoord[face + 2]); + } else if (UseFaceNormal(face_normal, n3)) { + ptr->orientation = CalculateOrientation(face_normal); + } else { + ptr->orientation = CalculateOrientation(n3); + } + ++ptr; + } +} + +static void FillHeightFieldBuffer(VertexNoUv* ptr, std::size_t num, + const mjModel* model, int hfieldid) { + int count = 0; + auto append_tri = [&](float3 a, float3 b, float3 c) { + float4 orientation = CalculateOrientation(a, b, c); + ptr[count].position = a; + ptr[count].orientation = orientation; + ++count; + ptr[count].position = b; + ptr[count].orientation = orientation; + ++count; + ptr[count].position = c; + ptr[count].orientation = orientation; + ++count; + }; + auto append_quad = [&](float3 a, float3 b, float3 c, float3 d) { + append_tri(a, b, d); + append_tri(d, b, c); + }; + + const float* data = model->hfield_data + model->hfield_adr[hfieldid]; + const int nrow = model->hfield_nrow[hfieldid]; + const int ncol = model->hfield_ncol[hfieldid]; + const float height = 0.5f * (nrow - 1); + const float width = 0.5f * (ncol - 1); + float sz[4]; + for (int i = 0; i < 4; ++i) { + sz[i] = static_cast(model->hfield_size[4 * hfieldid + i]); + } + + auto get_pos = [=](int r, int c) { + const float x = sz[0] * (c / width - 1.0f); + const float y = sz[1] * (r / height - 1.0f); + const float z = sz[2] * data[(r * ncol) + c]; + return float3{x, y, z}; + }; + + // For each quad defined by 4 points in the height field, we will create 4 + // triangles by introducing a vertex in the middle of the quad. + // a---b + // |\ /| + // | m | + // |/ \| + // d---c + for (int row = 0; row < nrow - 1; ++row) { + for (int col = 0; col < ncol - 1; ++col) { + const float3 a = get_pos(row, col); + const float3 b = get_pos(row, col + 1); + const float3 c = get_pos(row + 1, col + 1); + const float3 d = get_pos(row + 1, col); + + const float mid_x = (a.x + b.x) * 0.5f; + const float mid_y = (a.y + d.y) * 0.5f; + + // To determine the height of the middle vertex, we look at the heights + // of the opposing corners (i.e. {a, c} and {b, d}). Our goal is to avoid + // creating any odd bumps or valleys in the height field if possible. + // + // If one of the two opposing corners are of the same height, then we + // set the middle vertex such that we're effectively rendering two + // triangles, preventing an odd bump. Otherwise, we use the higher + // midpoint between two opposing corners to prevent valleys. + // 0---0 0---0 6---4 + // |\ | | /| |\ /| + // | 0 | | 0 | | 7 | + // | \| |/ | |/ \| + // 2---0 0---2 0---8 + float mid_z = 0; + if (a.z == c.z && b.z != d.z) { + mid_z = a.z; + } else if (a.z != c.z && b.z == d.z) { + mid_z = b.z; + } else { + const float mid_z_ac = (a.z + c.z) * 0.5f; + const float mid_z_bd = (b.z + d.z) * 0.5f; + mid_z = std::max(mid_z_ac, mid_z_bd); + } + + const float3 mid = {mid_x, mid_y, mid_z}; + append_tri(a, b, mid); + append_tri(b, c, mid); + append_tri(c, d, mid); + append_tri(d, a, mid); + } + } + // Build the left edge. + for (int row = 0; row < nrow - 1; ++row) { + const float3 a = get_pos(row, 0); + const float3 b = get_pos(row + 1, 0); + const float3 c = {b.x, b.y, -sz[3]}; + const float3 d = {a.x, a.y, -sz[3]}; + append_quad(a, b, c, d); + } + // Build the right edge. + for (int row = 0; row < nrow - 1; ++row) { + const float3 a = get_pos(row + 1, ncol-1); + const float3 b = get_pos(row, ncol-1); + const float3 c = {b.x, b.y, -sz[3]}; + const float3 d = {a.x, a.y, -sz[3]}; + append_quad(a, b, c, d); + } + // Build the front edge. + for (int col = 0; col < ncol - 1; ++col) { + const float3 a = get_pos(0, col); + const float3 b = get_pos(0, col + 1); + const float3 c = {b.x, b.y, -sz[3]}; + const float3 d = {a.x, a.y, -sz[3]}; + append_quad(a, b, c, d); + } + // Build the back edge. + for (int col = 0; col < ncol - 1; ++col) { + const float3 a = get_pos(nrow-1, col + 1); + const float3 b = get_pos(nrow-1, col); + const float3 c = {b.x, b.y, -sz[3]}; + const float3 d = {a.x, a.y, -sz[3]}; + append_quad(a, b, c, d); + } + // Build the base. We use the visualization quality as the size rather than + // the height field dimensions. + const float base_width = (0.5f * model->vis.quality.numquads); + const float base_height = (0.5f * model->vis.quality.numquads); + for (int row = 0; row < model->vis.quality.numquads; ++row) { + for (int col = 0; col < model->vis.quality.numquads; ++col) { + const float x0 = sz[0] * ((col + 0) / base_width - 1.0f); + const float x1 = sz[0] * ((col + 1) / base_width - 1.0f); + const float y0 = sz[1] * ((row + 0) / base_height - 1.0f); + const float y1 = sz[1] * ((row + 1) / base_height - 1.0f); + append_quad({x0, y0, -sz[3]}, + {x0, y1, -sz[3]}, + {x1, y1, -sz[3]}, + {x1, y0, -sz[3]}); + } + } + if (count != num) { + mju_error("Vertex count mismatch."); + } +} + +static int CalculateHeightFieldVertexCount(const mjModel* model, int hfieldid) { + const int nrow = model->hfield_nrow[hfieldid]; + const int ncol = model->hfield_ncol[hfieldid]; + + // For details, see the logic in FillHeightFieldBuffer for how many vertices + // we need. But, in general... + + // We use 4 triangles (i.e. 12 vertices) per quad. + const int surface_count = 12 * (nrow-1) * (ncol-1); + // We use 1 quad (i.e. 6 vertices) per edge element. We double this because + // we have two edges per dimension (e.g. left/right and front/back). + const int edge_count = (12 * (nrow - 1)) + (12 * (ncol - 1)); + // We use 1 quad (i.e. 6 vertices) per base element. We use the visualization + // quality as the size rather than the height field dimensions. + const int base_count = + 6 * model->vis.quality.numquads * model->vis.quality.numquads; + + const int total_count = surface_count + edge_count + base_count; + return total_count; +} + +template +static filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine, + const mjModel* model, int id, + int vertex_count, + FillFn fill_fn) { + return CreateVertexBuffer( + engine, vertex_count, [&](std::byte* buffer, std::size_t num_bytes) { + auto* ptr = reinterpret_cast(buffer); + fill_fn(ptr, num_bytes / sizeof(T), model, id); + }); +} + +filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine, + const mjModel* model, int id, + MeshType mesh_type) { + if (id < 0) { + mju_error("Invalid mesh index %d", id); + return nullptr; + } + + int vertex_count = 0; + switch (mesh_type) { + case MeshType::kNormal: + if (id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + return nullptr; + } + vertex_count = 3 * model->mesh_facenum[id]; + break; + case MeshType::kConvexHull: + if (id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + return nullptr; + } + vertex_count = 3 * model->mesh_graph[model->mesh_graphadr[id] + 1]; + break; + case MeshType::kHeightField: + if (id >= model->nhfield) { + mju_error("Invalid height field index %d", id); + return nullptr; + } + vertex_count = CalculateHeightFieldVertexCount(model, id); + break; + } + + if (vertex_count == 0) { + mju_error("Vertex count is zero."); + return nullptr; + } + + const bool has_texcoords = mesh_type == MeshType::kHeightField + ? false + : model->mesh_texcoordadr[id] >= 0; + if (has_texcoords) { + using VertexType = VertexWithUv; + switch (mesh_type) { + case MeshType::kNormal: + return CreateVertexBuffer(engine, model, id, vertex_count, + FillMeshBuffer); + break; + case MeshType::kConvexHull: + return CreateVertexBuffer(engine, model, id, vertex_count, + FillConvexHullBuffer); + break; + case MeshType::kHeightField: + mju_error("Height fields do not support UV coordinates."); + return nullptr; + } + } else { + using VertexType = VertexNoUv; + switch (mesh_type) { + case MeshType::kNormal: + return CreateVertexBuffer(engine, model, id, vertex_count, + FillMeshBuffer); + break; + case MeshType::kConvexHull: + return CreateVertexBuffer(engine, model, id, vertex_count, + FillConvexHullBuffer); + break; + case MeshType::kHeightField: + return CreateVertexBuffer(engine, model, id, vertex_count, + FillHeightFieldBuffer); + break; + } + } +} + +filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine, + const mjModel* model, int id, + MeshType mesh_type) { + if (id < 0) { + mju_error("Invalid index %d", id); + return nullptr; + } + + int index_count = 0; + switch (mesh_type) { + case MeshType::kNormal: + if (id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + return nullptr; + } + index_count = 3 * model->mesh_facenum[id]; + break; + case MeshType::kConvexHull: + if (id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + return nullptr; + } + index_count = 3 * model->mesh_graph[model->mesh_graphadr[id] + 1]; + break; + case MeshType::kHeightField: + if (id >= model->nhfield) { + mju_error("Invalid height field index %d", id); + return nullptr; + } + index_count = CalculateHeightFieldVertexCount(model, id); + break; + } + + if (index_count == 0) { + mju_error("Index count is zero."); + return nullptr; + } + + if (index_count >= std::numeric_limits::max()) { + return CreateIndexBuffer(engine, index_count, + FillSequence); + } else { + return CreateIndexBuffer(engine, index_count, + FillSequence); + } +} + +filament::Texture* CreateTexture(filament::Engine* engine, const mjModel* model, + int id, TextureType texture_type) { + if (id < 0 || id >= model->ntex) { + mju_error("Invalid texture index %d", id); + } + + const int width = model->tex_width[id]; + const int height = model->tex_height[id]; + const bool is_srgb = model->tex_colorspace[id] == mjCOLORSPACE_SRGB; + const int num_channels = model->tex_nchannel[id]; + const mjtByte* data = model->tex_data + model->tex_adr[id]; + filament::Texture* texture = + texture_type == TextureType::kNormal2d + ? Create2dTexture(engine, width, height, num_channels, data, is_srgb) + : CreateCubeTexture(engine, width, height, num_channels, data, + is_srgb); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + texture->generateMipmaps(*engine); + #endif + return texture; +} +} // namespace mujoco diff --git a/src/experimental/filament/filament/model_util.h b/src/experimental/filament/filament/model_util.h new file mode 100644 index 00000000..b1dcbd90 --- /dev/null +++ b/src/experimental/filament/filament/model_util.h @@ -0,0 +1,103 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mujoco { + +// The types of meshes stored in the mjModel. +enum class MeshType { + kNormal, + kConvexHull, + kHeightField, +}; + +// The types of textures stored in the mjModel. +enum class TextureType { + kNormal2d, + kCube, +}; + +// Generates a filament VertexBuffer for a given mesh in the mjModel. +filament::VertexBuffer* CreateVertexBuffer(filament::Engine* engine, + const mjModel* model, int id, + MeshType mesh_type); + +// Generates a filament IndexBuffer for a given mesh in the mjModel. +filament::IndexBuffer* CreateIndexBuffer(filament::Engine* engine, + const mjModel* model, int id, + MeshType mesh_type); + +// Generates a filament Texture for a given 2D texture in the mjModel. +filament::Texture* CreateTexture(filament::Engine* engine, const mjModel* model, + int id, TextureType texture_type); + +// Reads a value with the given name from the mjModel's data sections. The +// default_value is returned if the named element is not found. +template +T ReadElement(const mjModel* model, const char* name, T default_value = T()) { + constexpr bool is_string = + std::is_same_v || std::is_same_v; + + const int type = is_string ? mjOBJ_TEXT : mjOBJ_NUMERIC; + const int id = mj_name2id(model, type, name); + if (id < 0) { + return default_value; + } + + if constexpr (std::is_same_v) { + const char* ptr = model->text_data + model->text_adr[id]; + return ptr; + } else if constexpr (std::is_same_v) { + const char* ptr = model->text_data + model->text_adr[id]; + // Do not include the null terminator in the string view. + return std::string_view(ptr, model->text_size[id] - 1); + } else if constexpr (std::is_arithmetic_v) { + const mjtNum* ptr = model->numeric_data + model->numeric_adr[id]; + return static_cast(*ptr); + } else if constexpr (std::is_same_v) { + const mjtNum* ptr = model->numeric_data + model->numeric_adr[id]; + if (model->numeric_size[id] != 2) mju_error("Invalid numeric size."); + return T{ptr[0], ptr[1]}; + } else if constexpr (std::is_same_v) { + const mjtNum* ptr = model->numeric_data + model->numeric_adr[id]; + if (model->numeric_size[id] != 3) mju_error("Invalid numeric size."); + return T{ptr[0], ptr[1], ptr[2]}; + } else if constexpr (std::is_same_v) { + const mjtNum* ptr = model->numeric_data + model->numeric_adr[id]; + if (model->numeric_size[id] != 4) mju_error("Invalid numeric size."); + return T{ptr[0], ptr[1], ptr[2], ptr[3]}; + } else if constexpr (std::is_same_v) { + const mjtNum* ptr = model->numeric_data + model->numeric_adr[id]; + return static_cast(*ptr != 0); + } + return default_value; +} + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_MODEL_UTIL_H_ diff --git a/src/experimental/filament/filament/object_manager.cc b/src/experimental/filament/filament/object_manager.cc new file mode 100644 index 00000000..baeb1f2c --- /dev/null +++ b/src/experimental/filament/filament/object_manager.cc @@ -0,0 +1,400 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/object_manager.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/filament/builtins.h" +#include "experimental/filament/filament/model_util.h" +#include "experimental/filament/filament/texture_util.h" +#include "experimental/filament/render_context_filament.h" + +namespace mujoco { +namespace { + +// Loads binary data from a file using mjrFilamentConfig callbacks. +struct Asset { + Asset(const char* filename, const mjrFilamentConfig* config) { + const int error = config->load_asset(filename, config->load_asset_user_data, + &payload, &size); + if (error) { + mju_error("Failed to load file: %s (error: %d)", filename, error); + } + } + + ~Asset() { + if (payload) { + free(payload); + payload = nullptr; + } + } + + uint64_t size = 0; + unsigned char* payload = nullptr; + + Asset(const Asset&) = delete; + Asset& operator=(const Asset&) = delete; +}; + +} // namespace + +ObjectManager::ObjectManager(const mjModel* model, filament::Engine* engine, + const mjrFilamentConfig* config) + : model_(model), engine_(engine), config_(config) { + shapes_[kBox] = CreateBox(engine_, model_); + shapes_[kCone] = CreateCone(engine_, model_); + shapes_[kDisk] = CreateDisk(engine_, model_); + shapes_[kDome] = CreateDome(engine_, model_); + shapes_[kTube] = CreateTube(engine_, model_); + shapes_[kPlane] = CreatePlane(engine_, model_); + shapes_[kSphere] = CreateSphere(engine_, model_); + + auto LoadMaterial = [this](const char* filename) { + Asset asset(filename, config_); + filament::Material::Builder material_builder; + material_builder.package(asset.payload, asset.size); + return material_builder.build(*this->engine_); + }; + + materials_[kPbr] = LoadMaterial("pbr.filamat"); + materials_[kPbrPacked] = LoadMaterial("pbr_packed.filamat"); + materials_[kPhong2d] = LoadMaterial("phong_2d.filamat"); + materials_[kPhong2dFade] = LoadMaterial("phong_2d_fade.filamat"); + materials_[kPhong2dUv] = LoadMaterial("phong_2d_uv.filamat"); + materials_[kPhong2dUvFade] = LoadMaterial("phong_2d_uv_fade.filamat"); + materials_[kPhongColor] = LoadMaterial("phong_color.filamat"); + materials_[kPhongColorFade] = LoadMaterial("phong_color_fade.filamat"); + materials_[kPhongCube] = LoadMaterial("phong_cube.filamat"); + materials_[kPhongCubeFade] = LoadMaterial("phong_cube_fade.filamat"); + materials_[kUnlitSegmentation] = LoadMaterial("unlit_segmentation.filamat"); + materials_[kUnlitDepth] = LoadMaterial("unlit_depth.filamat"); + materials_[kUnlitUi] = LoadMaterial("unlit_ui.filamat"); + + for (int i = 0; i < model_->ntex; ++i) { + UploadTexture(model_, i); + } + for (int i = 0; i < model_->nmesh; ++i) { + UploadMesh(model_, i); + } + for (int i = 0; i < model_->nhfield; ++i) { + UploadHeightField(model_, i); + } + + static uint8_t black_rgb[3] = {0, 0, 0}; + fallback_black_ = Create2dTexture(engine_, 1, 1, 3, black_rgb, false); + static uint8_t white_rgb[3] = {255, 255, 255}; + fallback_white_ = Create2dTexture(engine_, 1, 1, 3, white_rgb, false); + static uint8_t normal_data[3] = {128, 128, 255}; + fallback_normal_ = Create2dTexture(engine_, 1, 1, 3, normal_data, false); + static uint8_t orm_data[3] = {0, 255, 0}; + fallback_orm_ = Create2dTexture(engine_, 1, 1, 3, orm_data, false); + + fallback_textures_[mjTEXROLE_USER] = fallback_black_; + fallback_textures_[mjTEXROLE_RGB] = fallback_black_; + fallback_textures_[mjTEXROLE_OCCLUSION] = fallback_black_; + fallback_textures_[mjTEXROLE_ROUGHNESS] = fallback_white_; + fallback_textures_[mjTEXROLE_METALLIC] = fallback_black_; + fallback_textures_[mjTEXROLE_NORMAL] = fallback_normal_; + fallback_textures_[mjTEXROLE_EMISSIVE] = fallback_black_; + fallback_textures_[mjTEXROLE_ORM] = fallback_orm_; + + fallback_indirect_light_ = LoadFallbackIndirectLight("ibl.ktx", 1.0f); +} + +ObjectManager::~ObjectManager() { + for (auto& iter : skyboxes_) { + engine_->destroy(iter); + } + for (auto& iter : indirect_lights_) { + engine_->destroy(iter); + } + for (auto& iter : materials_) { + engine_->destroy(iter); + } + for (auto& iter : meshes_) { + engine_->destroy(iter.second.vertex_buffer); + engine_->destroy(iter.second.index_buffer); + } + for (auto& iter : shapes_) { + engine_->destroy(iter.vertex_buffer); + engine_->destroy(iter.index_buffer); + } + for (auto& iter : textures_) { + engine_->destroy(iter.second); + } + // fallback_textures_ maps to these textures. + engine_->destroy(fallback_white_); + engine_->destroy(fallback_black_); + engine_->destroy(fallback_normal_); + engine_->destroy(fallback_orm_); + for (auto& iter : fonts_) { + engine_->destroy(iter.second); + } +} + +void ObjectManager::UploadMesh(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->nmesh) { + mju_error("Invalid mesh index %d", id); + } + + if (auto iter = meshes_.find(id); iter != meshes_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + if (auto iter = convex_hulls_.find(id); iter != convex_hulls_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + + FilamentBuffers& buffers = meshes_[id]; + buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kNormal); + buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kNormal); + + if (model->mesh_graphadr[id] >= 0) { + FilamentBuffers& hull_buffers = convex_hulls_[id]; + hull_buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kConvexHull); + hull_buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kConvexHull); + } +} + +void ObjectManager::UploadTexture(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->ntex) { + mju_error("Invalid texture index: %d", id); + } + + if (auto iter = textures_.find(id); iter != textures_.end()) { + engine_->destroy(iter->second); + } + + const int texture_type = model->tex_type[id]; + if (model->tex_height[id] == 1) { + const mjtByte* bytes = model->tex_data + model->tex_adr[id]; + const int num_bytes = model->tex_width[id]; + textures_[id] = + CreateKtxTexture(engine_, bytes, num_bytes, spherical_harmonics_[id]); + } else if (texture_type == mjTEXTURE_2D) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kNormal2d); + } else if (texture_type == mjTEXTURE_CUBE) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); + } else if (texture_type == mjTEXTURE_SKYBOX) { + textures_[id] = CreateTexture(engine_, model, id, TextureType::kCube); + } else { + mju_error("Unsupported: Texture type: %d", texture_type); + } +} + +void ObjectManager::UploadHeightField(const mjModel* model, int id) { + if (model != model_) { + mju_error("Model mismatch."); + } + if (id < 0 || id >= model->nhfield) { + mju_error("Invalid height field index %d", id); + } + + if (auto iter = height_fields_.find(id); iter != height_fields_.end()) { + engine_->destroy(iter->second.vertex_buffer); + engine_->destroy(iter->second.index_buffer); + } + + FilamentBuffers& buffers = height_fields_[id]; + buffers.vertex_buffer = + CreateVertexBuffer(engine_, model, id, MeshType::kHeightField); + buffers.index_buffer = + CreateIndexBuffer(engine_, model, id, MeshType::kHeightField); +} + +void ObjectManager::UploadFont(const uint8_t* pixels, int width, int height, + int id) { + if (auto iter = fonts_.find(id); iter != fonts_.end()) { + engine_->destroy(iter->second); + } + fonts_[id] = Create2dTexture(engine_, width, height, 4, pixels, false); +} + +filament::Material* ObjectManager::GetMaterial(MaterialType type) const { + if (type < 0 || type >= kNumMaterials) { + mju_error("Invalid material type: %d", type); + } + return materials_[type]; +} + +const FilamentBuffers* ObjectManager::GetMeshBuffer(int data_id) const { + // As defined by mjv_updateScene: + // original mesh: mesh_id * 2 + // convex hull: (mesh_id * 2) + 1 + const int mesh_id = data_id / 2; + if (data_id % 2 == 0) { + auto it = meshes_.find(mesh_id); + return it != meshes_.end() ? &it->second : nullptr; + } else { + auto it = convex_hulls_.find(mesh_id); + return it != convex_hulls_.end() ? &it->second : nullptr; + } +} + +const FilamentBuffers* ObjectManager::GetHeightFieldBuffer( + int hfield_id) const { + auto it = height_fields_.find(hfield_id); + return it != height_fields_.end() ? &it->second : nullptr; +} + +const FilamentBuffers* ObjectManager::GetShapeBuffer(ShapeType shape) const { + if (shape < 0 || shape >= kNumShapes) { + mju_error("Invalid shape type: %d", shape); + } + return &shapes_[shape]; +} + +const filament::Texture* ObjectManager::GetFont(int font_id) const { + auto it = fonts_.find(font_id); + return it != fonts_.end() ? it->second : nullptr; +} + +const filament::Texture* ObjectManager::GetTexture(int tex_id) const { + auto it = textures_.find(tex_id); + return it != textures_.end() ? it->second : nullptr; +} + +const filament::Texture* ObjectManager::GetTexture(int mat_id, int role) const { + if (mat_id < 0 || mat_id >= model_->nmat || role < 0 || role >= mjNTEXROLE) { + return nullptr; + } + const int tex_id = model_->mat_texid[mat_id * mjNTEXROLE + role]; + return GetTexture(tex_id); +} + +const filament::Texture* ObjectManager::GetTextureWithFallback(int mat_id, + int role) const { + if (auto texture = GetTexture(mat_id, role)) { + return texture; + } + return GetFallbackTexture(role); +} + +const filament::Texture* ObjectManager::GetFallbackTexture(int role) const { + auto iter = fallback_textures_.find(role); + if (iter != fallback_textures_.end()) { + return iter->second; + } + return nullptr; +} + +filament::IndirectLight* ObjectManager::GetFallbackIndirectLight() { + return fallback_indirect_light_; +} + +filament::IndirectLight* ObjectManager::CreateIndirectLight(int tex_id, + float intensity) { + filament::Texture* texture = nullptr; + auto texture_iter = textures_.find(tex_id); + if (texture_iter != textures_.end()) { + texture = texture_iter->second; + } + + if (texture == nullptr) { + return nullptr; + } + + SphericalHarmonics* spherical_harmonics = nullptr; + auto sh_iter = spherical_harmonics_.find(tex_id); + if (sh_iter != spherical_harmonics_.end()) { + spherical_harmonics = &sh_iter->second; + } + + return CreateIndirectLight(texture, spherical_harmonics, intensity); +} + +filament::IndirectLight* ObjectManager::LoadFallbackIndirectLight( + std::string_view filename, float intensity) { + Asset asset(std::string(filename).c_str(), config_); + if (asset.size == 0) { + return nullptr; + } + + filament::math::float3 spherical_harmonics[9]; + filament::Texture* tex = + CreateKtxTexture(engine_, asset.payload, asset.size, spherical_harmonics); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + tex->generateMipmaps(*engine_); + #endif + + return CreateIndirectLight(tex, &spherical_harmonics, intensity); +} + +filament::IndirectLight* ObjectManager::CreateIndirectLight( + filament::Texture* texture, SphericalHarmonics* spherical_harmonics, + float intensity) { + filament::IndirectLight::Builder builder; + builder.reflections(texture); + if (spherical_harmonics != nullptr) { + builder.irradiance(3, *spherical_harmonics); + } + builder.intensity(intensity); + // Rotate the light to match mujoco's Z-up convention. + builder.rotation(filament::math::mat3f::rotation( + filament::math::f::PI / 2, filament::math::float3{1, 0, 0})); + filament::IndirectLight* indirect_light = builder.build(*engine_); + indirect_lights_.push_back(indirect_light); + return indirect_light; +} + +filament::Skybox* ObjectManager::CreateSkybox() { + filament::Texture* skybox_texture = nullptr; + for (auto& iter : textures_) { + const int texture_type = model_->tex_type[iter.first]; + if (texture_type == mjTEXTURE_SKYBOX) { + skybox_texture = iter.second; + break; + } + } + + if (skybox_texture == nullptr) { + return nullptr; + } + + filament::Skybox::Builder builder; + builder.environment(skybox_texture); + filament::Skybox* skybox = builder.build(*engine_); + skyboxes_.push_back(skybox); + return skybox; +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/object_manager.h b/src/experimental/filament/filament/object_manager.h new file mode 100644 index 00000000..48c1bcfe --- /dev/null +++ b/src/experimental/filament/filament/object_manager.h @@ -0,0 +1,136 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OBJECT_MANAGER_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OBJECT_MANAGER_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" +#include "experimental/filament/render_context_filament.h" + +namespace mujoco { + +// Creates and owns various filament objects based on the data in a mjrContext. +class ObjectManager { + public: + ObjectManager(const mjModel* model, filament::Engine* engine, + const mjrFilamentConfig* config); + ~ObjectManager(); + + enum MaterialType { + kPbr, + kPbrPacked, + kPhong2d, + kPhong2dFade, + kPhong2dUv, + kPhong2dUvFade, + kPhongColor, + kPhongColorFade, + kPhongCube, + kPhongCubeFade, + kUnlitSegmentation, + kUnlitDepth, + kUnlitUi, + kNumMaterials, + }; + + enum ShapeType { + kPlane, + kBox, + kSphere, + kCone, + kDisk, + kDome, + kTube, + kNumShapes, + }; + + using SphericalHarmonics = filament::math::float3[9]; + + void UploadMesh(const mjModel* model, int id); + + void UploadTexture(const mjModel* model, int id); + + void UploadHeightField(const mjModel* model, int id); + + void UploadFont(const uint8_t* pixels, int width, int height, int id); + + // Returns the filament engine used by the ObjectManager to create filament + // objects. + filament::Engine* GetEngine() const { return engine_; } + + filament::Material* GetMaterial(MaterialType type) const; + + // Returns the cached instance of a filament object created from the mjModel. + const FilamentBuffers* GetMeshBuffer(int data_id) const; + const FilamentBuffers* GetShapeBuffer(ShapeType shape) const; + const FilamentBuffers* GetHeightFieldBuffer(int hfield_id) const; + const filament::Texture* GetFont(int font_id) const; + const filament::Texture* GetTexture(int tex_id) const; + const filament::Texture* GetTexture(int mat_id, int role) const; + const filament::Texture* GetTextureWithFallback(int mat_id, int role) const; + const filament::Texture* GetFallbackTexture(int role) const; + filament::IndirectLight* GetFallbackIndirectLight(); + + // Creates and returns a new instance of a filament object. The objects are + // owned by the ObjectManager and will be deleted in the destructor. + filament::Skybox* CreateSkybox(); + filament::IndirectLight* CreateIndirectLight( + filament::Texture* texture, SphericalHarmonics* spherical_harmonics, + float intensity); + filament::IndirectLight* CreateIndirectLight(int tex_id, float intensity); + filament::IndirectLight* LoadFallbackIndirectLight(std::string_view filename, + float intensity); + + const mjModel* GetModel() const { return model_; } + + ObjectManager(const ObjectManager&) = delete; + ObjectManager& operator=(const ObjectManager&) = delete; + + private: + const mjModel* model_ = nullptr; + filament::Engine* engine_ = nullptr; + const mjrFilamentConfig* config_; + + std::array shapes_; + std::array materials_; + std::vector skyboxes_; + std::vector indirect_lights_; + std::unordered_map meshes_; + std::unordered_map convex_hulls_; + std::unordered_map height_fields_; + std::unordered_map fonts_; + std::unordered_map textures_; + std::unordered_map spherical_harmonics_; + std::unordered_map fallback_textures_; + filament::Texture* fallback_white_ = nullptr; + filament::Texture* fallback_black_ = nullptr; + filament::Texture* fallback_normal_ = nullptr; + filament::Texture* fallback_orm_ = nullptr; + filament::IndirectLight* fallback_indirect_light_ = nullptr; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_OBJECT_MANAGER_H_ diff --git a/src/experimental/filament/filament/renderables.cc b/src/experimental/filament/filament/renderables.cc new file mode 100644 index 00000000..7216a188 --- /dev/null +++ b/src/experimental/filament/filament/renderables.cc @@ -0,0 +1,201 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/renderables.h" + +#include + +#include +#include +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" + +namespace mujoco { + +Renderables::Renderables(filament::Engine* engine) : engine_(engine) {} + +Renderables::~Renderables() { + while (!entities_.empty()) { + RemoveLast(); + } +} + +void Renderables::RemoveLast() { + if (entities_.empty()) { + return; + } + + utils::EntityManager& em = utils::EntityManager::get(); + utils::Entity entity = entities_.back(); + + if (assigned_scene_) { + assigned_scene_->remove(entity); + } + + engine_->destroy(entity); + em.destroy(entity); + entities_.pop_back(); + + UpdateBuffers(owned_buffers_.size() - 1, std::nullopt); + owned_buffers_.pop_back(); +} + +void Renderables::Update(int index, const FilamentBuffers& buffers) { + if (index < 0 || index >= entities_.size()) { + mju_error("Invalid index %d for renderable.", index); + } + utils::Entity& entity = entities_[index]; + UpdateEntity(entity, buffers); + UpdateBuffers(index, std::nullopt); +} + +void Renderables::Update(int index, FilamentBuffers&& buffers) { + if (index < 0 || index >= entities_.size()) { + mju_error("Invalid index %d for renderable.", index); + } + utils::Entity& entity = entities_[index]; + UpdateEntity(entity, buffers); + UpdateBuffers(index, buffers); +} + +void Renderables::Append(const FilamentBuffers& buffers) { + utils::Entity entity = CreateEntity(buffers); + entities_.push_back(entity); + owned_buffers_.push_back(std::nullopt); +} + +void Renderables::Append(FilamentBuffers&& buffers) { + utils::Entity entity = CreateEntity(buffers); + entities_.push_back(entity); + owned_buffers_.push_back(buffers); +} + +utils::Entity Renderables::CreateEntity(const FilamentBuffers& buffers) { + if (buffers.vertex_buffer == nullptr) { + mju_error("Invalid (null) vertex buffer."); + } + if (buffers.index_buffer == nullptr) { + mju_error("Invalid (null) index buffer."); + } + utils::Entity entity = utils::EntityManager::get().create(); + if (entity.isNull()) { + mju_error("Failed to create entity."); + } + + filament::RenderableManager::Builder builder(1); + builder.geometry(0, filament::RenderableManager::PrimitiveType::TRIANGLES, + buffers.vertex_buffer, buffers.index_buffer); + if (material_instance_) { + builder.material(0, material_instance_); + } + builder.boundingBox(buffers.bounds) + .culling(false) + .castShadows(true) + .receiveShadows(true) + .layerMask(1, visible_ ? 1 : 0) + .screenSpaceContactShadows(true); + + builder.build(*engine_, entity); + if (assigned_scene_) { + assigned_scene_->addEntity(entity); + } + return entity; +} + +void Renderables::UpdateEntity(utils::Entity entity, + const FilamentBuffers& buffers) { + if (buffers.vertex_buffer == nullptr) { + mju_error("Invalid (null) vertex buffer."); + } + if (buffers.index_buffer == nullptr) { + mju_error("Invalid (null) index buffer."); + } + filament::RenderableManager& rm = engine_->getRenderableManager(); + rm.setGeometryAt(rm.getInstance(entity), 0, + filament::RenderableManager::PrimitiveType::TRIANGLES, + buffers.vertex_buffer, buffers.index_buffer, 0, + buffers.index_buffer->getIndexCount()); +} + +void Renderables::UpdateBuffers(int index, + std::optional buffers) { + if (index < 0 || index >= owned_buffers_.size()) { + mju_error("Invalid index %d for renderable.", index); + } + if (owned_buffers_[index].has_value()) { + engine_->destroy(owned_buffers_[index]->vertex_buffer); + engine_->destroy(owned_buffers_[index]->index_buffer); + } + owned_buffers_[index] = buffers; +} + +void Renderables::AddToScene(filament::Scene* scene) { + if (assigned_scene_) { + if (assigned_scene_ != scene) { + mju_error("Cannot add renderable to multiple scenes."); + } + // Entities are already added to the scene. + return; + } + for (utils::Entity& entity : entities_) { + scene->addEntity(entity); + } + assigned_scene_ = scene; +} + +void Renderables::RemoveFromScene(filament::Scene* scene) { + if (assigned_scene_ != scene) { + mju_error("Attempting to remove renderable from wrong scene."); + } + for (utils::Entity& entity : entities_) { + scene->remove(entity); + } + assigned_scene_ = nullptr; +} + +void Renderables::SetMaterialInstance( + filament::MaterialInstance* instance) { + if (instance != material_instance_) { + filament::RenderableManager& rm = engine_->getRenderableManager(); + for (utils::Entity& entity : entities_) { + filament::RenderableManager::Instance ri = rm.getInstance(entity); + rm.setMaterialInstanceAt(ri, 0, instance); + } + material_instance_ = instance; + } +} + +void Renderables::Hide() { + if (visible_) { + filament::RenderableManager& rm = engine_->getRenderableManager(); + for (utils::Entity& entity : entities_) { + rm.setLayerMask(rm.getInstance(entity), 1, 0); + } + visible_ = false; + } +} + +void Renderables::Show() { + if (!visible_) { + filament::RenderableManager& rm = engine_->getRenderableManager(); + for (utils::Entity& entity : entities_) { + rm.setLayerMask(rm.getInstance(entity), 1, 1); + } + visible_ = true; + } +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/renderables.h b/src/experimental/filament/filament/renderables.h new file mode 100644 index 00000000..66372f91 --- /dev/null +++ b/src/experimental/filament/filament/renderables.h @@ -0,0 +1,91 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_ + +#include +#include +#include + +#include +#include +#include +#include "experimental/filament/filament/buffer_util.h" + +namespace mujoco { + +// Manages a collection of related filament Renderable Entities. +class Renderables { + public: + Renderables(filament::Engine* engine); + ~Renderables() noexcept; + + Renderables(const Renderables&) = delete; + Renderables& operator=(const Renderables&) = delete; + + // Appends a new renderable entity built from the given buffers. + void Append(const FilamentBuffers& buffers); + void Append(FilamentBuffers&& buffers); + + // Updates the entity at the index with new buffers. + void Update(int index, const FilamentBuffers& buffers); + void Update(int index, FilamentBuffers&& buffers); + + // Removes the last entity. + void RemoveLast(); + + // Returns the entity at the given index. + utils::Entity operator[](int index) { return entities_[index]; } + + // Returns the owned buffers at the given index. + int GetNumEntities() const { return entities_.size(); } + + // Hides all managed entities. + void Hide(); + + // Shows all managed entities. + void Show(); + + // Returns true if the entities are visible. + bool IsVisible() const { return visible_; } + + // Adds all managed entities to the given filament Scene. + void AddToScene(filament::Scene* scene); + + // Removes all managed entities from the given filament Scene. + void RemoveFromScene(filament::Scene* scene); + + // Sets the material instance for all managed entities. + void SetMaterialInstance(filament::MaterialInstance* material_instance); + + // Returns the filament Engine managing the entities in this collection. + filament::Engine* GetEngine() { return engine_; } + + private: + utils::Entity CreateEntity(const FilamentBuffers& buffers); + void UpdateEntity(utils::Entity entity, const FilamentBuffers& buffers); + void UpdateBuffers(int index, std::optional buffers); + + filament::Engine* engine_ = nullptr; + filament::Scene* assigned_scene_ = nullptr; + filament::MaterialInstance* material_instance_ = nullptr; + std::vector entities_; + std::vector> owned_buffers_; + bool visible_ = true; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_RENDERABLES_H_ diff --git a/src/experimental/filament/filament/scene_view.cc b/src/experimental/filament/filament/scene_view.cc new file mode 100644 index 00000000..72a6138d --- /dev/null +++ b/src/experimental/filament/filament/scene_view.cc @@ -0,0 +1,399 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/scene_view.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/gui_view.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/model_util.h" +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +using filament::math::float3; +using filament::math::float4; +using filament::math::mat4; + +static constexpr int kNormalIndex = + static_cast(SceneView::DrawMode::kNormal); +static constexpr int kDepthIndex = + static_cast(SceneView::DrawMode::kDepth); +static constexpr int kSegmentIndex = + static_cast(SceneView::DrawMode::kSegmentation); + +static filament::Viewport ReadViewport(mjrRect rect) { + return filament::Viewport(rect.left, rect.bottom, rect.width, rect.height); +} + +filament::ColorGrading::Builder ToBuilder(const ColorGradingOptions& opts) { + return filament::ColorGrading::Builder() + .format(opts.format) + .dimensions(opts.dimension) + .luminanceScaling(opts.luminance_scaling) + .gamutMapping(opts.gamut_mapping) + .exposure(opts.exposure) + .nightAdaptation(opts.night_adaptation) + .contrast(opts.contrast) + .vibrance(opts.vibrance) + .saturation(opts.saturation) + .whiteBalance(opts.temperature, opts.tint) + .channelMixer(opts.out_red, opts.out_green, opts.out_blue) + .shadowsMidtonesHighlights(opts.shadows, opts.midtones, opts.highlights, + opts.tonal_ranges) + .slopeOffsetPower(opts.slope, opts.offset, opts.power) + .curves(opts.shadow_gamma, opts.mid_point, opts.highlight_scale); +} + +SceneView::SceneView(filament::Engine* engine, ObjectManager* object_mgr) + : object_mgr_(object_mgr), engine_(engine) { + scene_ = engine_->createScene(); + camera_ = engine_->createCamera(utils::EntityManager::get().create()); + + for (auto& view : views_) { + view = engine_->createView(); + view->setScene(scene_); + view->setCamera(camera_); + } + + const mjModel* m = object_mgr_->GetModel(); + + // Configure options for the normal view. + auto& cg = color_grading_options_; + cg.exposure = ReadElement(m, "filament.out.exposure", cg.exposure); + cg.contrast = ReadElement(m, "filament.out.contrast", cg.contrast); + cg.vibrance = ReadElement(m, "filament.out.vibrance", cg.vibrance); + cg.saturation = ReadElement(m, "filament.out.saturation", cg.saturation); + cg.temperature = ReadElement(m, "filament.out.temperature", cg.temperature); + cg.tint = ReadElement(m, "filament.out.tint", cg.tint); + + auto tone_mapping = + ReadElement(m, "filament.out.tone_mapping"); + if (tone_mapping == "aces") { + cg.tone_mapper = ToneMapperType::kACES; + } else if (tone_mapping == "aces_legacy") { + cg.tone_mapper = ToneMapperType::kACESLegacy; + } else if (tone_mapping == "filmic") { + cg.tone_mapper = ToneMapperType::kFilmic; + } else if (tone_mapping == "linear") { + cg.tone_mapper = ToneMapperType::kLinear; + } else if (tone_mapping == "pbr_neutral") { + cg.tone_mapper = ToneMapperType::kPBRNeutral; + } + SetColorGradingOptions(cg); + + auto ao = views_[kNormalIndex]->getAmbientOcclusionOptions(); + ao.enabled = ReadElement(m, "filament.ao.enabled", true); + ao.bentNormals = ReadElement(m, "filament.ao.bent_normals", true); + ao.ssct.enabled = ReadElement(m, "filament.ao.ssct", ao.ssct.enabled); + ao.quality = filament::QualityLevel::ULTRA; + ao.lowPassFilter = filament::QualityLevel::ULTRA; + ao.upsampling = filament::QualityLevel::ULTRA; + ao.bilateralThreshold = 0.5f; + views_[kNormalIndex]->setAmbientOcclusionOptions(ao); + + auto msaa = views_[kNormalIndex]->getMultiSampleAntiAliasingOptions(); + msaa.enabled = ReadElement(m, "filament.msaa.enabled", true); + views_[kNormalIndex]->setMultiSampleAntiAliasingOptions(msaa); + + // Disable post processing for the depth and segmentation views to preserve + // the values. + views_[kDepthIndex]->setPostProcessingEnabled(false); + views_[kSegmentIndex]->setPostProcessingEnabled(false); + + // Rotate the fog to align with mujoco's +Z up space. + auto fog = views_[kNormalIndex]->getFogEntity(); + auto& tm = engine->getTransformManager(); + tm.create(fog); + auto rotation_axis = ReadElement( + m, "filament.fog.rotation_axis", float3{-1, 0, 0}); + tm.setTransform(tm.getInstance(fog), + mat4::rotation(filament::math::f::PI / 2, rotation_axis)); + + auto fog_opts = views_[kNormalIndex]->getFogOptions(); + fog_opts.enabled = ReadElement(m, "filament.fog.enabled", fog_opts.enabled); + fog_opts.color = ReadElement(m, "filament.fog.color", fog_opts.color); + fog_opts.distance = ReadElement( + m, "filament.fog.distance", fog_opts.distance); + fog_opts.density = ReadElement( + m, "filament.fog.density", fog_opts.density); + fog_opts.cutOffDistance = ReadElement( + m, "filament.fog.cutOffDistance", fog_opts.cutOffDistance); + fog_opts.maximumOpacity = ReadElement( + m, "filament.fog.maximumOpacity", fog_opts.maximumOpacity); + fog_opts.height = ReadElement(m, "filament.fog.height", fog_opts.height); + fog_opts.heightFalloff = ReadElement( + m, "filament.fog.heightFalloff", fog_opts.heightFalloff); + fog_opts.inScatteringStart = ReadElement( + m, "filament.fog.inScatteringStart", fog_opts.inScatteringStart); + fog_opts.inScatteringSize = ReadElement( + m, "filament.fog.inScatteringSize", fog_opts.inScatteringSize); + views_[kNormalIndex]->setFogOptions(fog_opts); + + // Create an empty/black indirect light to ensure that the skybox is oriented + // to respect mujoco's Z-up convention. + scene_->setIndirectLight( + object_mgr_->CreateIndirectLight(nullptr, nullptr, 100000)); + + PrepareLights(); +} + +SceneView::~SceneView() { + lights_.clear(); + drawables_.clear(); + engine_->destroyCameraComponent(camera_->getEntity()); + engine_->destroy(views_[kNormalIndex]->getColorGrading()); + for (auto& view : views_) { + engine_->destroy(view); + } + engine_->destroy(scene_); +} + +filament::View* SceneView::PrepareRenderView(DrawMode mode) { + for (auto& iter : drawables_) { + iter->SetDrawMode(mode); + } + return views_[static_cast(mode)]; +} + +void SceneView::SetViewport(mjrRect viewport) { + aspect_ratio_ = (float)viewport.width / (float)viewport.height; + for (auto& view : views_) { + view->setViewport(ReadViewport(viewport)); + } +} + +void SceneView::SetColorGradingOptions(const ColorGradingOptions& opts) { + auto tone_mapper = CreateToneMapper(opts.tone_mapper); + auto color_grading = ToBuilder(color_grading_options_) + .toneMapper(tone_mapper.get()) + .build(*engine_); + views_[kNormalIndex]->setColorGrading(color_grading); + engine_->destroy(color_grading_); + color_grading_ = color_grading; + color_grading_options_ = opts; +} + +void SceneView::SetEnvironmentLight(std::string_view filename, + float intensity) { + auto* ibl = object_mgr_->LoadFallbackIndirectLight(filename, intensity); + if (ibl) { + scene_->setIndirectLight(ibl); + } +} + +void SceneView::SetFallbackEnvironmentLight(float intensity) { + auto* ibl = object_mgr_->GetFallbackIndirectLight(); + if (ibl) { + ibl->setIntensity(intensity); + scene_->setIndirectLight(ibl); + } +} + +void SceneView::UpdateCamera(const mjvGLCamera* cameras) { + const mjvGLCamera cam = mjv_averageCamera(cameras, cameras + 1); + const filament::Camera::Projection type = + cam.orthographic ? filament::Camera::Projection::ORTHO + : filament::Camera::Projection::PERSPECTIVE; + float3 cam_pos(cam.pos[0], cam.pos[1], cam.pos[2]); + float3 cam_fwd(cam.forward[0], cam.forward[1], cam.forward[2]); + float3 cam_up(cam.up[0], cam.up[1], cam.up[2]); + float3 cam_at = cam_pos + cam_fwd; + camera_->lookAt(cam_pos, cam_at, cam_up); + float halfwidth = cam.frustum_width ? cam.frustum_width + : 0.5f * aspect_ratio_ * (cam.frustum_top - cam.frustum_bottom); + camera_->setProjection(type, cam.frustum_center - halfwidth, + cam.frustum_center + halfwidth, cam.frustum_bottom, + cam.frustum_top, cam.frustum_near, cam.frustum_far); + clip_from_world_ = camera_->getProjectionMatrix() * camera_->getViewMatrix(); +} + +std::optional SceneView::ClipFromWorld(const float3& pos) const{ + const float4 clip_pos = clip_from_world_ * float4(pos, 1.0f); + if (clip_pos.w == 0.0f) { + return std::nullopt; + } + return clip_pos.xyz / clip_pos.w; +} + +void SceneView::PrepareLights() { + const mjModel* model = object_mgr_->GetModel(); + filament::Skybox* skybox = object_mgr_->CreateSkybox(); + if (skybox) { + scene_->setSkybox(skybox); + } + + float total_light_intensity = 0.0f; + + for (int i = 0; i < model->nlight; ++i) { + total_light_intensity += model->light_intensity[i]; + + if (model->light_type[i] == mjLIGHT_IMAGE) { + auto* indirect_light = object_mgr_->CreateIndirectLight( + model->light_texid[i], model->light_intensity[i]); + if (indirect_light) { + scene_->setIndirectLight(indirect_light); + } + // Add an nullptr as a placeholder so that our indices still match. + lights_.emplace_back(nullptr); + } else { + Light::Params params; + params.color = ReadFloat3(model->light_diffuse); + params.type = (mjtLightType)model->light_type[i]; + params.castshadow = model->light_castshadow[i]; + params.bulbradius = model->light_bulbradius[i]; + params.range = model->light_range[i]; + params.intensity = model->light_intensity[i]; + if (params.type == mjLIGHT_SPOT) { + params.spot_cone_angle = model->light_cutoff[i]; + } + auto light_obj = std::make_unique(object_mgr_, params); + light_obj->AddToScene(scene_); + lights_.emplace_back(std::move(light_obj)); + } + } + + // Add a placeholder (black) headlight as our last light. Going forward, we'll + // assume lights_.back() is always the headlight. + { + Light::Params params; + params.color = float3(0, 0, 0); + params.headlight = true; + params.type = mjLIGHT_DIRECTIONAL; + params.castshadow = 0; + params.intensity = 0; + auto light_obj = std::make_unique(object_mgr_, params); + light_obj->AddToScene(scene_); + lights_.emplace_back(std::move(light_obj)); + } + + // There are no "physical" lights in the scene which means we're likely + // dealing with a "classic renderer" scene. In this case, let's add a + // default environment light and set the light intensity ourselves. + if (total_light_intensity == 0.0f) { + constexpr float kHeadlightIntensityCandela = 10'000.f; + constexpr float kTotalSceneLightIntensityCandela = 100'000.f; + constexpr float kFallbackEnvironmentLightIntensityCandela = 10'000.f; + + SetFallbackEnvironmentLight(kFallbackEnvironmentLightIntensityCandela); + const float intensity = kTotalSceneLightIntensityCandela / lights_.size(); + for (auto& light : lights_) { + light->SetIntensity(light->IsHeadlight() ? kHeadlightIntensityCandela + : intensity); + } + } +} + +void SceneView::UpdateScene(const mjrContext* context, const mjvScene* scene) { + mjtNum hpos[3], hfwd[3]; + float headpos[3], gazedir[3]; + mjv_cameraInModel(hpos, hfwd, nullptr, scene); + mju_n2f(headpos, hpos, 3); + mju_n2f(gazedir, hfwd, 3); + UpdateCamera(scene->camera); + + // Remove all drawables from previous render and prepare new ones. + for (auto& iter : drawables_) { + iter->RemoveFromScene(scene_); + } + drawables_.clear(); + for (int i = 0; i < scene->ngeom; ++i) { + const mjvGeom* geom = scene->geoms + i; + const mjtGeom geom_type = static_cast(geom->type); + + if (geom_type == mjGEOM_LABEL) { + if (geom->label[0] == 0) { + continue; + } + if (auto pos = ClipFromWorld(ReadFloat3(geom->pos))) { + DrawTextAt(geom->label, pos->x, pos->y, pos->z); + } + } else { + auto drawable = std::make_unique(object_mgr_, *geom); + drawable->AddToScene(scene_); + drawable->Update(object_mgr_->GetModel(), scene, *geom); + drawables_.push_back(std::move(drawable)); + } + } + + bool headlight_enabled = false; + for (int i = 0; i < scene->nlight; ++i) { + const mjvLight& scene_light = scene->lights[i]; + if (scene_light.id < 0 && scene_light.headlight) { + // We position the headlight slightly behind the camera to avoid some + // odd clipping issues. + headlight_enabled = true; + headpos[0] -= gazedir[0] * 0.05f; + headpos[1] -= gazedir[1] * 0.05f; + headpos[2] -= gazedir[2] * 0.05f; + + // The headlight is always the "back" light. + std::unique_ptr& light = lights_.back(); + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(headpos), ReadFloat3(gazedir)); + continue; + } else if (scene_light.id < lights_.size() - 1) { + std::unique_ptr& light = lights_[scene_light.id]; + light->SetColor(ReadFloat3(scene_light.diffuse)); + light->SetTransform(ReadFloat3(scene_light.pos), + ReadFloat3(scene_light.dir)); + } else { + mju_error("Unexpected light id: %d", scene_light.id); + } + } + + // Enable/disable the headlight based on whether or not it's in the scene. + if (headlight_enabled) { + lights_.back()->Enable(); + } else { + lights_.back()->Disable(); + } +} + +filament::Engine* SceneView::GetEngine() const { return engine_; } + +filament::View* SceneView::GetDefaultRenderView() { + return views_[kNormalIndex]; +} + +ColorGradingOptions SceneView::GetColorGradingOptions() const { + return color_grading_options_; +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/scene_view.h b/src/experimental/filament/filament/scene_view.h new file mode 100644 index 00000000..223e4940 --- /dev/null +++ b/src/experimental/filament/filament/scene_view.h @@ -0,0 +1,107 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "experimental/filament/filament/color_grading_options.h" +#include "experimental/filament/filament/drawable.h" +#include "experimental/filament/filament/light.h" +#include "experimental/filament/filament/material.h" +#include "experimental/filament/filament/object_manager.h" + +namespace mujoco { + +// Creates and owns filament Scene and View classes given a mjvScene. +// +// The filament Scene is populated with the objects (e.g. lights, geoms, +// cameras, etc.) defined by the mjvScene. Multiple Views are created to allow +// different rendering modes (e.g. normal, depth, segmentation, etc.) +class SceneView { + public: + SceneView(filament::Engine* engine, ObjectManager* object_mgr); + ~SceneView(); + + // Updates all views to render into the given viewport. + void SetViewport(mjrRect viewport); + + // Updates the color grading options for the main render view. + void SetColorGradingOptions(const ColorGradingOptions& opts); + + // Updates the environment light using the KTX image at the given path. + void SetEnvironmentLight(std::string_view filename, float intensity); + + // Updates the environment light to the fallback light + void SetFallbackEnvironmentLight(float intensity); + + // Updates the Entities in the filament Scene to match the current mjvScene + // state. + void UpdateScene(const mjrContext* context, const mjvScene* scene); + + using DrawMode = Material::DrawMode; + + // Prepares and returns the filament View for the given draw mode. + filament::View* PrepareRenderView(DrawMode mode); + + // Accessors. + filament::Engine* GetEngine() const; + filament::View* GetDefaultRenderView(); + ColorGradingOptions GetColorGradingOptions() const; + + SceneView(const SceneView&) = delete; + SceneView& operator=(const SceneView&) = delete; + + private: + void UpdateCamera(const mjvGLCamera* cameras); + + void PrepareLights(); + + // Converts a point in world space to clip space, eg. in the range [-1,-1, 0] + // to [1, 1, 1]. Returns std::nullopt if the point is behind the camera. + std::optional ClipFromWorld( + const filament::math::float3& pos) const; + + ObjectManager* object_mgr_ = nullptr; + filament::Engine* engine_ = nullptr; + filament::Scene* scene_ = nullptr; + filament::Camera* camera_ = nullptr; + filament::ColorGrading* color_grading_ = nullptr; + std::vector> lights_; + std::vector> drawables_; + std::array views_; + filament::math::mat4 clip_from_world_; + ColorGradingOptions color_grading_options_; + DrawMode active_mode_ = DrawMode::kNumDrawModes; + float aspect_ratio_ = 1.0f; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_SCENE_VIEW_H_ diff --git a/src/experimental/filament/filament/texture_util.cc b/src/experimental/filament/filament/texture_util.cc new file mode 100644 index 00000000..a629f3bf --- /dev/null +++ b/src/experimental/filament/filament/texture_util.cc @@ -0,0 +1,218 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/texture_util.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +namespace mujoco { + +static filament::Texture::Format GetTextureFormat(int num_channels) { + switch (num_channels) { + case 1: + return filament::Texture::Format::R; + case 3: + return filament::Texture::Format::RGB; + case 4: + return filament::Texture::Format::RGBA; + default: + mju_error("Unsupported number of channels: %d", num_channels); + return filament::Texture::Format::UNUSED; + } +} + +static filament::Texture::InternalFormat GetTextureInternalFormat( + int num_channels, bool is_srgb) { + if (is_srgb) { + switch (num_channels) { + case 3: + return filament::Texture::InternalFormat::SRGB8; + case 4: + return filament::Texture::InternalFormat::SRGB8_A8; + default: + mju_error("Unsupported number of channels: %d", num_channels); + return filament::Texture::InternalFormat::UNUSED; + } + } else { + switch (num_channels) { + case 1: + return filament::Texture::InternalFormat::R8; + case 3: + return filament::Texture::InternalFormat::RGB8; + case 4: + return filament::Texture::InternalFormat::RGBA8; + default: + mju_error("Unsupported number of channels: %d", num_channels); + return filament::Texture::InternalFormat::UNUSED; + } + } +} + +filament::Texture* Create2dTexture(filament::Engine* engine, int width, + int height, int num_channels, + const uint8_t* data, bool is_srgb) { + if (num_channels != 1 && num_channels != 3 && num_channels != 4) { + mju_error("Unsupported number of channels: %d", num_channels); + return nullptr; + } + + filament::Texture::Builder builder; + builder.width(width); + builder.height(height); + builder.format(GetTextureInternalFormat(num_channels, is_srgb)); + builder.sampler(filament::Texture::Sampler::SAMPLER_2D); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + if (!is_srgb) { + builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE | + filament::Texture::Usage::SAMPLEABLE | + filament::Texture::Usage::UPLOADABLE); + } + #endif + filament::Texture* texture = builder.build(*engine); + + if (data) { + const size_t num_bytes = width * height * sizeof(uint8_t) * num_channels; + const filament::Texture::Format format = GetTextureFormat(num_channels); + texture->setImage( + *engine, 0, + filament::Texture::PixelBufferDescriptor( + data, num_bytes, format, filament::Texture::Type::UBYTE)); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + texture->generateMipmaps(*engine); + #endif + } + return texture; +} + +filament::Texture* CreateCubeTexture(filament::Engine* engine, int width, + int height, int num_channels, + const uint8_t* data, bool is_srgb) { + if (num_channels != 3) { + mju_error("Only support RGB cubemaps."); + return nullptr; + } + + const int kNumFacesPerCube = 6; + + int face_height = height; + if (width != height) { + if (width * kNumFacesPerCube != height) { + mju_error("Cube maps must contain 6 square images."); + } + face_height = height / kNumFacesPerCube; + } + if (width != face_height) { + mju_error("Cube map faces must be square."); + } + + filament::Texture::Builder builder; + builder.width(width); + builder.height(face_height); + builder.format(GetTextureInternalFormat(num_channels, is_srgb)); + builder.sampler(filament::Texture::Sampler::SAMPLER_CUBEMAP); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + if (!is_srgb) { + builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE | + filament::Texture::Usage::SAMPLEABLE | + filament::Texture::Usage::UPLOADABLE); + } + #endif + filament::Texture* texture = builder.build(*engine); + + const int face_size = width * face_height * num_channels; + const int num_bytes = face_size * kNumFacesPerCube; + + uint8_t* buffer = new uint8_t[num_bytes]; + auto callback = +[](void* buffer, size_t size, void* user) { + delete [] reinterpret_cast(buffer); + }; + + filament::Texture::FaceOffsets offsets(face_size); + if (width == height) { + // Copy the image to all the faces. + for (int i = 0; i < kNumFacesPerCube; ++i) { + std::memcpy(buffer + (i * face_size), data, face_size); + } + } else { + // Use the cubemap as is. + std::memcpy(buffer, data, num_bytes); + } + + if (data) { + filament::Texture::PixelBufferDescriptor desc( + buffer, num_bytes, filament::Texture::Format::RGB, + filament::Texture::Type::UBYTE, callback); + texture->setImage(*engine, 0, std::move(desc), offsets); + // TODO: Revisit this to make it this work in WebGL + #ifndef __EMSCRIPTEN__ + texture->generateMipmaps(*engine); + #endif + } + return texture; +} + +filament::Texture* CreateKtxTexture( + filament::Engine* engine, const uint8_t* data, int size, + filament::math::float3* spherical_harmonics_out) { + image::Ktx1Bundle* bundle = new image::Ktx1Bundle(data, size); + if (spherical_harmonics_out) { + bundle->getSphericalHarmonics(spherical_harmonics_out); + } + const bool is_srgb = false; + return ktxreader::Ktx1Reader::createTexture(engine, bundle, is_srgb); +} + +filament::Texture* CreateRenderTargetTexture(filament::Engine* engine, + int width, int height, + RenderTargetTextureType type) { + filament::Texture::Builder builder; + builder.width(width); + builder.height(height); + switch (type) { + case kRenderTargetColor: + builder.usage(filament::Texture::Usage::COLOR_ATTACHMENT | + filament::Texture::Usage::BLIT_SRC); + builder.format(filament::Texture::InternalFormat::RGB8); + break; + case kRenderTargetDepth: + builder.usage(filament::Texture::Usage::DEPTH_ATTACHMENT | + filament::Texture::Usage::SAMPLEABLE); + builder.format(filament::Texture::InternalFormat::DEPTH32F); + break; + case kRenderTargetDepthColor: + builder.usage(filament::Texture::Usage::COLOR_ATTACHMENT | + filament::Texture::Usage::BLIT_SRC); + builder.format(filament::Texture::InternalFormat::R32F); + break; + default: + mju_error("Unknown type: %d", static_cast(type)); + } + return builder.build(*engine); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/texture_util.h b/src/experimental/filament/filament/texture_util.h new file mode 100644 index 00000000..447954ba --- /dev/null +++ b/src/experimental/filament/filament/texture_util.h @@ -0,0 +1,56 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_ + +#include + +#include +#include +#include + +// Functions for creating filament textures. +namespace mujoco { + +// Creates a filament Texture for the given 2D texture. +filament::Texture* Create2dTexture(filament::Engine* engine, int width, + int height, int num_channels, + const uint8_t* data, bool is_srgb); + +// Creates a filament Texture for the given cube texture. +filament::Texture* CreateCubeTexture(filament::Engine* engine, int width, + int height, int num_channels, + const uint8_t* data, bool is_srgb); + +// Creates a filament Texture for the given KTX payload. +filament::Texture* CreateKtxTexture( + filament::Engine* engine, const uint8_t* data, int size, + filament::math::float3* spherical_harmonics_out); + +enum RenderTargetTextureType { + kRenderTargetColor, + kRenderTargetDepth, + kRenderTargetDepthColor, + kNumRenderTargetTextureTypes, +}; + +// Creates a filament RenderTargetTexture for the given target type. +filament::Texture* CreateRenderTargetTexture(filament::Engine* engine, + int width, int height, + RenderTargetTextureType type); + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_ diff --git a/src/experimental/filament/filament/vertex_util.cc b/src/experimental/filament/filament/vertex_util.cc new file mode 100644 index 00000000..3304921d --- /dev/null +++ b/src/experimental/filament/filament/vertex_util.cc @@ -0,0 +1,65 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/filament/vertex_util.h" + +#include + +#include +#include +#include +#include +#include + +namespace mujoco { + +using filament::math::float3; +using filament::math::float4; +using filament::math::mat3f; +using filament::math::quatf; + +float4 CalculateOrientation(const float3& normal) { + float3 tangent; + float3 bitangent; + if (normal.y < -1.0f + std::numeric_limits::epsilon()) { + // Handle the singularity. + tangent = float3{-1.0f, 0.0f, 0.0f}; + bitangent = float3{0.0f, 0.0f, -1.0f}; + } else { + const float a = 1.0f / (1.0f + normal.y); + const float b = -normal.z * normal.x * a; + tangent = float3(b, -normal.z, 1.0f - normal.z * normal.z * a); + bitangent = float3(1.0f - normal.x * normal.x * a, -normal.x, b); + } + quatf orientation = mat3f::packTangentFrame({tangent, bitangent, normal}); + return float4(orientation.xyz, orientation.w); +} + +float3 CalculateNormal( + const filament::math::float3& p1, + const filament::math::float3& p2, + const filament::math::float3& p3) { + const float3 v12 = p2 - p1; + const float3 v13 = p3 - p1; + return normalize(cross(v12, v13)); +} + +float4 CalculateOrientation( + const filament::math::float3& p1, + const filament::math::float3& p2, + const filament::math::float3& p3) { + return CalculateOrientation(CalculateNormal(p1, p2, p3)); +} + +} // namespace mujoco diff --git a/src/experimental/filament/filament/vertex_util.h b/src/experimental/filament/filament/vertex_util.h new file mode 100644 index 00000000..f31c5e62 --- /dev/null +++ b/src/experimental/filament/filament/vertex_util.h @@ -0,0 +1,95 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_ + +#include +#include +#include + +namespace mujoco { + +// Calculates the normal of a triangle given its three vertices. +filament::math::float3 CalculateNormal( + const filament::math::float3& p1, + const filament::math::float3& p2, + const filament::math::float3& p3); + +// Calculates the orientation of a vertex given just its normal. +filament::math::float4 CalculateOrientation( + const filament::math::float3& normal); + +// Calculates the orientation of a triangle given its three vertices. +filament::math::float4 CalculateOrientation( + const filament::math::float3& p1, + const filament::math::float3& p2, + const filament::math::float3& p3); + +// A standard vertex with no UV coordinates. +struct VertexNoUv { + VertexNoUv() = default; + VertexNoUv(filament::math::float3 position, + filament::math::float4 orientation) + : position(position), orientation(orientation) {} + + filament::math::float3 position; + filament::math::float4 orientation; + + static constexpr bool kHasPosition = true; + static constexpr bool kHasPosition2d = false; + static constexpr bool kHasOrientation = true; + static constexpr bool kHasUv = false; + static constexpr bool kHasColor = false; +}; + +// A standard vertex with UV coordinates. +struct VertexWithUv { + VertexWithUv() = default; + VertexWithUv(filament::math::float3 position, + filament::math::float4 orientation, filament::math::float2 uv) + : position(position), orientation(orientation), uv(uv) {} + + filament::math::float3 position; + filament::math::float4 orientation; + filament::math::float2 uv; + + static constexpr bool kHasPosition = true; + static constexpr bool kHasPosition2d = false; + static constexpr bool kHasOrientation = true; + static constexpr bool kHasUv = true; + static constexpr bool kHasColor = false; +}; + +// A vertex for rendering GUI elements. +struct GuiVertex { + GuiVertex() = default; + GuiVertex(filament::math::float2 position, filament::math::float2 uv, + filament::math::ubyte4 color) + : position(position), uv(uv), color(color) {} + + filament::math::float2 position; + filament::math::float2 uv; + filament::math::ubyte4 color; + + static constexpr bool kHasPosition = false; + static constexpr bool kHasPosition2d = true; + static constexpr bool kHasOrientation = false; + static constexpr bool kHasUv = true; + static constexpr bool kHasColor = true; +}; + +} // namespace mujoco + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_VERTEX_UTIL_H_ diff --git a/src/experimental/filament/render_context_filament.cc b/src/experimental/filament/render_context_filament.cc new file mode 100644 index 00000000..6b9c591c --- /dev/null +++ b/src/experimental/filament/render_context_filament.cc @@ -0,0 +1,149 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "experimental/filament/render_context_filament.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "experimental/filament/filament/filament_context.h" + +#if defined(TLS_FILAMENT_CONTEXT) +static thread_local mujoco::FilamentContext* g_filament_context = nullptr; +#else +static mujoco::FilamentContext* g_filament_context = nullptr; +#endif + +// Default asset loader to use when calling mjr_makeContext. This is only +// intended for basic backwards compatibility with the existing mjr_makeContext +// API. If this doesn't work as expected, you should be calling +// mjr_makeFilamentContext instead and provide your own asset loading callbacks. +// Returns 0 on success and non-zero to indicate an error. +static int DefaultLoadAsset(const char* asset_filename, void* user_data, + unsigned char** contents, uint64_t* out_size) { + std::filesystem::path full_path; + full_path.append("filament/assets/data"); + full_path.append(asset_filename); + + std::ifstream file(full_path, std::ios::binary); + if (!file) { + mju_error("File does not exist: %s", full_path.c_str()); + return 1; + } + + file.seekg(0, std::ios::end); + *out_size = static_cast(file.tellg()); + if (*out_size == 0) { + mju_error("File is empty: %s", full_path.c_str()); + return 1; + } + file.seekg(0, std::ios::beg); + + *contents = (unsigned char*)malloc(*out_size); + file.read((char*)*contents, *out_size); + file.close(); + return 0; +} + +static void CheckFilamentContext() { + if (g_filament_context == nullptr) { + mju_error("Missing context; did you call mjr_makeFilamentContext?"); + } +} + + +extern "C" { + +void mjr_defaultFilamentConfig(mjrFilamentConfig* config) { + memset(config, 0, sizeof(mjrFilamentConfig)); +} + +void mjr_makeFilamentContext(const mjModel* m, mjrContext* con, + const mjrFilamentConfig* config) { + // TODO: Support multiple contexts and multiple threads. For now, we'll just + // assume a single, global context. + if (g_filament_context != nullptr) { + mju_error("Context already exists!"); + } + g_filament_context = new mujoco::FilamentContext(config, m, con); +} + +void mjr_defaultContext(mjrContext* con) { memset(con, 0, sizeof(mjrContext)); } + +void mjr_makeContext(const mjModel* m, mjrContext* con, int fontscale) { + mjr_freeContext(con); + mjrFilamentConfig cfg; + mjr_defaultFilamentConfig(&cfg); + cfg.load_asset = DefaultLoadAsset; + mjr_makeFilamentContext(m, con, &cfg); +} + +void mjr_freeContext(mjrContext* con) { + // mjr_freeContext may be called multiple times. + if (g_filament_context) { + delete g_filament_context; + g_filament_context = nullptr; + } + mjr_defaultContext(con); +} + +void mjr_render(mjrRect viewport, mjvScene* scn, const mjrContext* con) { + CheckFilamentContext(); + g_filament_context->Render(viewport, scn, con); +} + +void mjr_uploadMesh(const mjModel* m, const mjrContext* con, int meshid) { + CheckFilamentContext(); + g_filament_context->UploadMesh(m, meshid); +} + +void mjr_uploadTexture(const mjModel* m, const mjrContext* con, int texid) { + CheckFilamentContext(); + g_filament_context->UploadTexture(m, texid); +} + +void mjr_uploadHField(const mjModel* m, const mjrContext* con, int hfieldid) { + CheckFilamentContext(); + g_filament_context->UploadHeightField(m, hfieldid); +} + +void mjr_setBuffer(int framebuffer, mjrContext* con) { + CheckFilamentContext(); + g_filament_context->SetFrameBuffer(framebuffer); +} + +void mjr_readPixels(unsigned char* rgb, float* depth, mjrRect viewport, + const mjrContext* con) { + CheckFilamentContext(); + g_filament_context->ReadPixels(viewport, rgb, depth); +} + +void mjr_uploadFont(unsigned char* pixels, int width, int height, int bpp, + int id, const mjrContext* con) { + CheckFilamentContext(); + if (bpp != 4) { + mju_error("Only 4bpp fonts are supported, got %d", bpp); + } + g_filament_context->UploadFont(pixels, width, height, id); +} + +} // extern "C" diff --git a/src/experimental/filament/render_context_filament.h b/src/experimental/filament/render_context_filament.h new file mode 100644 index 00000000..fb1e68a7 --- /dev/null +++ b/src/experimental/filament/render_context_filament.h @@ -0,0 +1,85 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_H_ +#define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_H_ + +#include + +#include +#include +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef enum mjtGraphicsApi_ { // backend graphics API to use + mjGFX_DEFAULT = 0, // default based on platform + mjGFX_OPENGL, // OpenGL (desktop) + mjGFX_VULKAN // Vulkan +} mjtGraphicsApi; + +struct mjrFilamentConfig { + // "Loads" an asset, returning its contents in `out` and size in `out_size`. + // Returns 0 on success and non-zero to indicate an error. The caller must + // free `out`. + typedef int (*load_asset_fn)(const char* path, void* user_data, + unsigned char** out, uint64_t* out_size); + + // Used to load filament assets (e.g. materials, image-based lights, etc.). + load_asset_fn load_asset; + void* load_asset_user_data; + + // The native window handle into which we can render directly. + void* native_window; + + // The backend graphics API to use. + int graphics_api; + + // Whether or not to enable GUI rendering. + bool enable_gui; +}; + +void mjr_defaultFilamentConfig(mjrFilamentConfig* config); + +void mjr_makeFilamentContext(const mjModel* m, mjrContext* con, + const mjrFilamentConfig* config); + +void mjr_defaultContext(mjrContext* con); + +void mjr_makeContext(const mjModel* m, mjrContext* con, int fontscale); + +void mjr_freeContext(mjrContext* con); + +void mjr_render(mjrRect viewport, mjvScene* scn, const mjrContext* con); + +void mjr_uploadMesh(const mjModel* m, const mjrContext* con, int meshid); + +void mjr_uploadTexture(const mjModel* m, const mjrContext* con, int texid); + +void mjr_uploadFont(unsigned char* pixels, int width, int height, int bpp, + int id, const mjrContext* con); + +void mjr_setBuffer(int framebuffer, mjrContext* con); + +void mjr_readPixels(unsigned char* rgb, float* depth, mjrRect viewport, + const mjrContext* con); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // MUJOCO_SRC_EXPERIMENTAL_FILAMENT_RENDER_CONTEXT_FILAMENT_H_ diff --git a/src/experimental/filament/stubs.cc b/src/experimental/filament/stubs.cc new file mode 100644 index 00000000..d29a3328 --- /dev/null +++ b/src/experimental/filament/stubs.cc @@ -0,0 +1,82 @@ +// Copyright 2025 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +extern "C" { + +void mjr_setAux(int index, const mjrContext* con) { + mju_error("mjr_setAux not implemented."); +} +void mjr_restoreBuffer(const mjrContext* con) { + mju_error("mjr_restoreBuffer not implemented."); +} +void mjr_rectangle(mjrRect viewport, float r, float g, float b, float a) { + mju_error("mjr_rectangle not implemented."); +} +void mjr_blitAux(int index, mjrRect src, int left, int bottom, + const mjrContext* con) { + mju_error("mjr_blitAux not implemented."); +} +void mjr_changeFont(int fontscale, mjrContext* con) { + mju_error("mjr_changeFont not implemented."); +} +void mjr_addAux(int index, int width, int height, int samples, + mjrContext* con) { + mju_error("mjr_addAux not implemented."); +} +void mjr_resizeOffscreen(int width, int height, mjrContext* con) { + mju_error("mjr_resizeOffscreen not implemented."); +} +void mjr_drawPixels(const unsigned char* rgb, const float* depth, + mjrRect viewport, const mjrContext* con) { + mju_error("mjr_drawPixels not implemented."); +} +void mjr_blitBuffer(mjrRect src, mjrRect dst, int flg_color, int flg_depth, + const mjrContext* con) { + mju_error("mjr_blitBuffer not implemented."); +} +void mjr_text(int font, const char* txt, const mjrContext* con, float x, + float y, float r, float g, float b) { + mju_error("mjr_text not implemented."); +} +void mjr_overlay(int font, int gridpos, mjrRect viewport, const char* overlay, + const char* overlay2, const mjrContext* con) { + mju_error("mjr_overlay not implemented."); +} +void mjr_label(mjrRect viewport, int font, const char* txt, float r, float g, + float b, float a, float rt, float gt, float bt, + const mjrContext* con) { + mju_error("mjr_label not implemented."); +} +void mjr_figure(mjrRect viewport, mjvFigure* fig, const mjrContext* con) { + mju_error("mjr_figure not implemented."); +} +void mjr_finish() { + mju_error("mjr_finish not implemented."); +} +int mjr_getError() { + mju_error("mjr_getError not implemented."); + return 0; +} +mjrRect mjr_maxViewport(const mjrContext* con) { + mju_error("mjr_maxViewport not implemented."); + return mjrRect{}; +} +int mjr_findRect(int x, int y, int nrect, const mjrRect* rect) { + mju_error("mjr_findRect not implemented."); + return 0; +} + +} // extern "C" diff --git a/src/experimental/studio/app.cc b/src/experimental/studio/app.cc index af48aaaf..57210588 100644 --- a/src/experimental/studio/app.cc +++ b/src/experimental/studio/app.cc @@ -29,8 +29,8 @@ #include #include -#include "third_party/dear_imgui/imgui.h" -#include "third_party/implot/implot.h" +#include +#include #include #include "experimental/toolbox/helpers.h" #include "experimental/toolbox/imgui_widgets.h" @@ -40,9 +40,9 @@ #include "experimental/toolbox/window.h" #if defined(USE_FILAMENT_OPENGL) || defined(USE_FILAMENT_VULKAN) -#include "third_party/mujoco/google/filament/render_context_filament.h" +#include "experimental/filament/render_context_filament.h" #elif defined(USE_CLASSIC_OPENGL) -#include "third_party/dear_imgui/backends/imgui_impl_opengl2.h" +#include #else #error No rendering mode defined. #endif diff --git a/src/experimental/toolbox/imgui_widgets.cc b/src/experimental/toolbox/imgui_widgets.cc index 40395b0a..db2bd656 100644 --- a/src/experimental/toolbox/imgui_widgets.cc +++ b/src/experimental/toolbox/imgui_widgets.cc @@ -18,7 +18,7 @@ #include #include -#include "third_party/dear_imgui/imgui.h" +#include #include namespace mujoco::toolbox { diff --git a/src/experimental/toolbox/imgui_widgets.h b/src/experimental/toolbox/imgui_widgets.h index 3595cc9f..040df0c7 100644 --- a/src/experimental/toolbox/imgui_widgets.h +++ b/src/experimental/toolbox/imgui_widgets.h @@ -21,8 +21,8 @@ #include #include -#include "third_party/dear_imgui/imgui.h" -#include "third_party/dear_imgui/imgui_internal.h" // For ButtonEx and PressedOnClick +#include +#include // For ButtonEx and PressedOnClick #include namespace mujoco::toolbox { diff --git a/src/experimental/toolbox/interaction.cc b/src/experimental/toolbox/interaction.cc index 7c78bbbb..8d9b28a0 100644 --- a/src/experimental/toolbox/interaction.cc +++ b/src/experimental/toolbox/interaction.cc @@ -15,7 +15,7 @@ #include "experimental/toolbox/interaction.h" #include -#include "third_party/dear_imgui/imgui.h" +#include #include "experimental/toolbox/imgui_widgets.h" #include "experimental/toolbox/physics.h" #include "experimental/toolbox/renderer.h" diff --git a/src/experimental/toolbox/window.cc b/src/experimental/toolbox/window.cc index 1092999b..72512e77 100644 --- a/src/experimental/toolbox/window.cc +++ b/src/experimental/toolbox/window.cc @@ -17,16 +17,16 @@ #include #include -#include "third_party/SDL2/include/SDL.h" -#include "third_party/SDL2/include/SDL_error.h" -#include "third_party/SDL2/include/SDL_events.h" -#include "third_party/SDL2/include/SDL_hints.h" -#include "third_party/SDL2/include/SDL_syswm.h" -#include "third_party/SDL2/include/SDL_version.h" -#include "third_party/SDL2/include/SDL_video.h" -#include "third_party/dear_imgui/backends/imgui_impl_sdl2.h" -#include "third_party/dear_imgui/imgui.h" -#include "third_party/implot/implot.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "experimental/toolbox/helpers.h" #include diff --git a/src/experimental/toolbox/window.h b/src/experimental/toolbox/window.h index d4b8d730..2fbc8eae 100644 --- a/src/experimental/toolbox/window.h +++ b/src/experimental/toolbox/window.h @@ -19,7 +19,7 @@ #include #include "experimental/toolbox/helpers.h" -#include "third_party/SDL2/include/SDL_video.h" +#include namespace mujoco::toolbox {