diff --git a/src/experimental/filament/filament/buffer_util.cc b/src/experimental/filament/filament/buffer_util.cc index 82363b2f..17f62091 100644 --- a/src/experimental/filament/filament/buffer_util.cc +++ b/src/experimental/filament/filament/buffer_util.cc @@ -14,12 +14,311 @@ #include "experimental/filament/filament/buffer_util.h" +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include "third_party/filament/libs/filabridge/include/filament/MaterialEnums.h" +#include +#include +#include +#include +#include "experimental/filament/filament/math_util.h" +#include "experimental/filament/filament/vertex_util.h" namespace mujoco { +using filament::math::float3; +using filament::math::float4; + +static filament::VertexAttribute GetUsage(const VertexAttribute& attrib) { + switch (attrib.usage) { + case mjVERTEX_ATTRIBUTE_POSITION: + return filament::VertexAttribute::POSITION; + case mjVERTEX_ATTRIBUTE_NORMAL: + return filament::VertexAttribute::TANGENTS; + case mjVERTEX_ATTRIBUTE_TANGENTS: + return filament::VertexAttribute::TANGENTS; + case mjVERTEX_ATTRIBUTE_UV: + return filament::VertexAttribute::UV0; + case mjVERTEX_ATTRIBUTE_COLOR: + return filament::VertexAttribute::COLOR; + default: + mju_error("Unsupported vertex attribute usage: %d", attrib.usage); + return filament::VertexAttribute::POSITION; + } +} + +static filament::VertexBuffer::AttributeType GetType( + const VertexAttribute& attrib) { + switch (attrib.type) { + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT2: + return filament::VertexBuffer::AttributeType::FLOAT2; + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT3: + return filament::VertexBuffer::AttributeType::FLOAT3; + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT4: + return filament::VertexBuffer::AttributeType::FLOAT4; + case mjVERTEX_ATTRIBUTE_TYPE_UBYTE4: + return filament::VertexBuffer::AttributeType::UBYTE4; + default: + mju_error("Unsupported vertex attribute type: %d", attrib.type); + return filament::VertexBuffer::AttributeType::FLOAT3; + } +} + +int VertexAttributeTypeSize(const VertexAttribute& attrib) { + switch (attrib.type) { + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT2: + return sizeof(float) * 2; + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT3: + return sizeof(float) * 3; + case mjVERTEX_ATTRIBUTE_TYPE_FLOAT4: + return sizeof(float) * 4; + case mjVERTEX_ATTRIBUTE_TYPE_UBYTE4: + return sizeof(uint8_t) * 4; + default: + mju_error("Unsupported vertex attribute type: %d", attrib.type); + return 0; + } +} + +// Initializes the MeshData to default values. +void DefaultMeshData(MeshData* data) { + std::memset(data, 0, sizeof(MeshData)); +} + +Mesh::Mesh(filament::Engine* engine, const MeshData& data) + : engine_(engine) { + type_ = data.primitive_type == mjPRIM_TYPE_TRIANGLES + ? filament::RenderableManager::PrimitiveType::TRIANGLES + : filament::RenderableManager::PrimitiveType::LINES; + + // If the user has provided a release callback, then we need to ensure we + // call is when filament is done with the mesh data. + if (data.release_callback) { + release_callbacks_.push_back([=]() { + data.release_callback(data.user_data); + }); + } + + BuildVertexBuffer(data); + BuildIndexBuffer(data); + UpdateBounds(data); +} + +Mesh::~Mesh() { + ReleaseResources(); + if (index_buffer_) { + engine_->destroy(index_buffer_); + } + if (vertex_buffer_) { + engine_->destroy(vertex_buffer_); + } +} + +void Mesh::BuildVertexBuffer(const MeshData& data) { + if (data.nvertices == 0) { + mju_error("MeshData has no vertices."); + } + + // The filament BufferDescriptor callback for releasing the memory. We assume + // that ReleaseResources() can be called multiple times, so we assign this + // callback to each buffer descriptor. + auto callback = +[](void* buffer, size_t size, void* user) { + static_cast(user)->ReleaseResources(); + }; + + // Pointers to specific attributes in the mesh data, used for additional + // validation and processing. + const VertexAttribute* positions = nullptr; + const VertexAttribute* normals = nullptr; + const VertexAttribute* tangents = nullptr; + + // Calculate the stride of the vertex buffer. + int stride = 0; + for (int i = 0; i < data.nattributes; ++i) { + stride += VertexAttributeTypeSize(data.attributes[i]); + if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_POSITION) { + positions = &data.attributes[i]; + } else if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_NORMAL) { + normals = &data.attributes[i]; + } else if (data.attributes[i].usage == mjVERTEX_ATTRIBUTE_TANGENTS) { + tangents = &data.attributes[i]; + } + } + + if (!positions) { + mju_error("MeshData has no positions."); + } + if (data.attributes[0].usage != mjVERTEX_ATTRIBUTE_POSITION) { + mju_error("Positions must be the first attribute."); + } + if (normals && tangents) { + mju_error("MeshData has both normals and tangents."); + } + if (normals && data.interleaved) { + // We need to build orientations from normals and so we require each + // attribute to be in a separate buffer. + mju_error("Cannot support normals with interleaved vertex attributes."); + } + + // Build the vertex buffer. + filament::VertexBuffer::Builder vb_builder; + vb_builder.bufferCount(data.interleaved ? 1 : data.nattributes); + vb_builder.vertexCount(data.nvertices); + + int offset = 0; + for (int i = 0; i < data.nattributes; ++i) { + const VertexAttribute& attrib = data.attributes[i]; + const filament::VertexAttribute usage = GetUsage(attrib); + const filament::VertexBuffer::AttributeType type = GetType(attrib); + if (data.interleaved) { + vb_builder.attribute(usage, 0, type, offset, stride); + } else { + vb_builder.attribute(usage, i, type); + } + if (usage == filament::VertexAttribute::COLOR) { + vb_builder.normalized(usage); + } + offset += VertexAttributeTypeSize(attrib); + } + vertex_buffer_ = vb_builder.build(*engine_); + + if (data.interleaved) { + const VertexAttribute& attrib = data.attributes[0]; + const size_t nbytes = data.nvertices * stride; + filament::backend::BufferDescriptor desc(attrib.bytes, nbytes, callback, + this); + vertex_buffer_->setBufferAt(*engine_, 0, std::move(desc)); + } else { + for (int i = 0; i < data.nattributes; ++i) { + const VertexAttribute& attrib = data.attributes[i]; + const size_t nbytes = data.nvertices * VertexAttributeTypeSize(attrib); + if (attrib.usage == mjVERTEX_ATTRIBUTE_NORMAL) { + const float4* orientations = + BuildOrientationsFromNormals(data.nvertices, attrib); + filament::backend::BufferDescriptor desc(orientations, nbytes, callback, + this); + vertex_buffer_->setBufferAt(*engine_, i, std::move(desc)); + } else { + filament::backend::BufferDescriptor desc(attrib.bytes, nbytes, callback, + this); + vertex_buffer_->setBufferAt(*engine_, i, std::move(desc)); + } + } + } +} + +void Mesh::BuildIndexBuffer(const MeshData& data) { + if (data.nindices == 0) { + return; + } + + const int element_size = data.index_type == mjINDEX_TYPE_USHORT + ? sizeof(uint16_t) + : sizeof(uint32_t); + const int num_bytes = data.nindices * element_size; + + // If indices == 0 and nindices > 0, then the user is specifying that the + // vertices are provided "in order", i.e. the indices are 0, 1, 2, 3, ... + // In this case, we need to create the sequence of indices explicitly. + const void* indices = data.indices; + if (indices == nullptr) { + std::byte* sequence = new std::byte[num_bytes]; + release_callbacks_.push_back([=]() { + delete[] sequence; + }); + + if (data.index_type == mjINDEX_TYPE_USHORT) { + FillSequence(sequence, num_bytes); + } else { + FillSequence(sequence, num_bytes); + } + indices = sequence; + } + + filament::IndexBuffer::Builder ib_builder; + ib_builder.indexCount(data.nindices); + ib_builder.bufferType(data.index_type == mjINDEX_TYPE_USHORT + ? filament::IndexBuffer::IndexType::USHORT + : filament::IndexBuffer::IndexType::UINT); + index_buffer_ = ib_builder.build(*engine_); + // We don't worry about setting a release callback here because the release + // callback for the vertex buffer will call release_callbacks_. + filament::backend::BufferDescriptor desc(indices, num_bytes); + index_buffer_->setBuffer(*engine_, std::move(desc)); +} + +float4* Mesh::BuildOrientationsFromNormals(int nvertices, const VertexAttribute& normals) { + float4* orientations = new float4[nvertices]; + release_callbacks_.push_back([=]() { + delete[] orientations; + }); + const float* normals_ptr = reinterpret_cast(normals.bytes); + for (int i = 0; i < nvertices; ++i) { + orientations[i] = CalculateOrientation(ReadFloat3(normals_ptr, i)); + } + return orientations; +} + +void Mesh::UpdateBounds(const MeshData& data) { + float3 bounds_min = ReadFloat3(data.bounds_min); + float3 bounds_max = ReadFloat3(data.bounds_max); + if (bounds_min != bounds_max) { + bounds_.emplace().set(bounds_min, bounds_max); + } else if (data.compute_bounds) { + bounds_min = float3(FLT_MAX, FLT_MAX, FLT_MAX); + bounds_max = float3(-FLT_MAX, -FLT_MAX, -FLT_MAX); + + if (data.attributes[0].usage != mjVERTEX_ATTRIBUTE_POSITION) { + mju_error("MeshData has no positions."); + } + const float* positions = + reinterpret_cast(data.attributes[0].bytes); + + for (int i = 0; i < data.nvertices; ++i) { + const float3 position = ReadFloat3(positions, i); + bounds_min = min(bounds_min, position); + bounds_max = max(bounds_max, position); + } + bounds_.emplace().set(bounds_min, bounds_max); + } +} + +void Mesh::ReleaseResources() { + for (const auto& callback : release_callbacks_) { + callback(); + } + release_callbacks_.clear(); +} + +filament::IndexBuffer* Mesh::GetFilamentIndexBuffer() const { + return index_buffer_; +} + +filament::VertexBuffer* Mesh::GetFilamentVertexBuffer() const { + return vertex_buffer_; +} + +filament::RenderableManager::PrimitiveType Mesh::GetPrimitiveType() const { + return type_; +} + +bool Mesh::HasBounds() const { + return bounds_.has_value(); +} + +filament::Box Mesh::GetBounds() const { + return bounds_.value(); +} + filament::backend::BufferDescriptor CreateBufferDescriptor( std::size_t num_bytes, const FillBufferFn& fill) { std::byte* bytes = new std::byte[num_bytes]; @@ -30,5 +329,4 @@ filament::backend::BufferDescriptor CreateBufferDescriptor( }; return filament::backend::BufferDescriptor(bytes, num_bytes, callback, bytes); } - } // namespace mujoco diff --git a/src/experimental/filament/filament/buffer_util.h b/src/experimental/filament/filament/buffer_util.h index 5b5c576b..d72afedb 100644 --- a/src/experimental/filament/filament/buffer_util.h +++ b/src/experimental/filament/filament/buffer_util.h @@ -21,19 +21,125 @@ #include #include #include +#include + #include #include #include #include #include #include +#include // Functions for creating filament vertex and index buffers. namespace mujoco { +// The type of data stored in an index buffer. +typedef enum mjtIndexType_ { + mjINDEX_TYPE_USHORT = 0, + mjINDEX_TYPE_UINT = 1, +} mjtIndexType; + +// The type of primitive to be drawn by vertex data. +typedef enum mjtMeshPrimitiveType_ { + mjPRIM_TYPE_TRIANGLES = 0, + mjPRIM_TYPE_LINES = 1, +} mjtMeshPrimitiveType; + +// The usage/purpose of an attribute of a vertex. +typedef enum mjtVertexAttributeUsage_ { + mjVERTEX_ATTRIBUTE_POSITION = 0, + mjVERTEX_ATTRIBUTE_NORMAL = 1, + mjVERTEX_ATTRIBUTE_TANGENTS = 2, + mjVERTEX_ATTRIBUTE_UV = 3, + mjVERTEX_ATTRIBUTE_COLOR = 4, +} mjtVertexAttributeUsage; + +// The data format of an attribute of a vertex. +typedef enum mjtVertexAttributeType_ { + mjVERTEX_ATTRIBUTE_TYPE_FLOAT2 = 0, + mjVERTEX_ATTRIBUTE_TYPE_FLOAT3 = 1, + mjVERTEX_ATTRIBUTE_TYPE_FLOAT4 = 2, + mjVERTEX_ATTRIBUTE_TYPE_UBYTE4 = 3, +} mjtVertexAttributeType; + +// Information about a single attribute of a vertex. +struct VertexAttribute { + // The data for the attribute. + const void* bytes; + + // The usage/purpose of the attribute. + mjtVertexAttributeUsage usage; + + // The data format of the attribute. + mjtVertexAttributeType type; +}; + +// The binary contents of a mesh. +struct MeshData { + // The number of vertices in the mesh. Each of the vertex arrays below is + // assumed to have this number of elements. + size_t nvertices; + + // The number of attributes for each vertex in the mesh. + int nattributes; + + // Information about each attribute of a vertex in the mesh. See `interleaved` + // for more details. + VertexAttribute attributes[16]; + + // Whether the vertex attributes are interleaved or not. + // + // If true, assumes that the attributes are packed in the order specified in + // the attributes array, with no padding in-between. Additionally, the + // `data` pointer for each attribute is assumed to point to the first element + // of that type. + // + // If false, assume each attribute is stored in a separate array as defined + // by the `data` field of the attribute. + bool interleaved; + + // The number of indices in the mesh. The indices array is assumed to have + // this number of elements. + size_t nindices; + + // The indices of the mesh, stored as either ushort or uint depending on the + // index type. + const void* indices; + + // The type of data stored in the indices array. + mjtIndexType index_type; + + // The type of primitive to be drawn by vertex data. + mjtMeshPrimitiveType primitive_type; + + // Whether to compute the bounds of the mesh using the vertex positions. + bool compute_bounds; + + // The bounds of the mesh. If bounds_min == bounds_max, then we assume that + // that the bounds are not set (i.e. the bounds is empty). + float bounds_min[3]; + float bounds_max[3]; + + // Because rendering may be multithreaded, we cannot make assumptions about + // when the mesh data will finish uploading to the GPU. As such, we will use + // this callback to notify callers when it is safe to free the mesh data. + void (*release_callback)(void* user_data); + + // User data to pass to the release callback. + void* user_data; +}; + +// Initializes the MeshData to default values. +void DefaultMeshData(MeshData* data); + // Owns a Vertex and Index buffer representing a geometry mesh. class Mesh { public: + // Creates a Mesh from the given MeshData. + Mesh(filament::Engine* engine, const MeshData& data); + + // Create a Mesh directly from filament objects. Internal use only. Mesh(filament::Engine* engine, filament::IndexBuffer* index_buffer, filament::VertexBuffer* vertex_buffer, std::optional bounds = std::nullopt, @@ -45,41 +151,43 @@ class Mesh { type_(type), bounds_(bounds) {} - ~Mesh() { - if (index_buffer_) { - engine_->destroy(index_buffer_); - } - if (vertex_buffer_) { - engine_->destroy(vertex_buffer_); - } - } + ~Mesh(); - filament::IndexBuffer* GetFilamentIndexBuffer() const { - return index_buffer_; - } - filament::VertexBuffer* GetFilamentVertexBuffer() const { - return vertex_buffer_; - } - filament::RenderableManager::PrimitiveType GetPrimitiveType() const { - return type_; - } - bool HasBounds() const { - return bounds_.has_value(); - } - filament::Box GetBounds() const { - return bounds_.value(); - } + // Returns the filament IndexBuffer for the mesh. + filament::IndexBuffer* GetFilamentIndexBuffer() const; + + // Returns the filament VertexBuffer for the mesh. + filament::VertexBuffer* GetFilamentVertexBuffer() const; + + // Returns the primitive type of the mesh. + filament::RenderableManager::PrimitiveType GetPrimitiveType() const; + + // Returns whether the mesh has bounds. + bool HasBounds() const; + + // Returns the bounds of the mesh. + filament::Box GetBounds() const; Mesh(const Mesh&) = delete; Mesh& operator=(const Mesh&) = delete; private: + void BuildVertexBuffer(const MeshData& data); + void BuildIndexBuffer(const MeshData& data); + void UpdateBounds(const MeshData& data); + + filament::math::float4* BuildOrientationsFromNormals( + int nvertices, const VertexAttribute& normals); + + void ReleaseResources(); + filament::Engine* engine_ = nullptr; filament::IndexBuffer* index_buffer_ = nullptr; filament::VertexBuffer* vertex_buffer_ = nullptr; filament::RenderableManager::PrimitiveType type_ = filament::RenderableManager::PrimitiveType::TRIANGLES; std::optional bounds_; + std::vector> release_callbacks_; }; using MeshPtr = std::unique_ptr;