From 9fa9193a627baab4395d4fa949aa57bccdb958bb Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Wed, 1 Apr 2026 04:31:54 -0700 Subject: [PATCH] Split Texture creation into two structs. TextureConfig describes the properties of the texture (e.g. width, height, pixel format, etc.). TextureData contains the binary payload of the texture. PiperOrigin-RevId: 892854578 Change-Id: I604b926ac38ff1050cac8fdf9ec767b789c0798e --- .../filament/filament/model_objects.cc | 57 ++-- .../filament/filament/object_manager.cc | 65 +++- .../filament/filament/texture_util.cc | 306 ++++++++++-------- .../filament/filament/texture_util.h | 93 +++++- 4 files changed, 338 insertions(+), 183 deletions(-) diff --git a/src/experimental/filament/filament/model_objects.cc b/src/experimental/filament/filament/model_objects.cc index 0529d90e..b65bb1be 100644 --- a/src/experimental/filament/filament/model_objects.cc +++ b/src/experimental/filament/filament/model_objects.cc @@ -125,30 +125,43 @@ void ModelObjects::UploadTexture(const mjModel* model, int id) { mju_error("Invalid texture index: %d", id); } - const int width = model->tex_width[id]; - const int height = model->tex_height[id]; - const int num_channels = model->tex_nchannel[id]; - const int tex_type = model->tex_type[id]; - const mjtByte* data = model->tex_data + model->tex_adr[id]; - const mjtColorSpace color_space = (mjtColorSpace)model->tex_colorspace[id]; + TextureConfig config; + DefaultTextureConfig(&config); + config.width = model->tex_width[id]; + config.height = model->tex_height[id]; + config.target = (mjtTexture)model->tex_type[id]; + config.color_space = (mjtColorSpace)model->tex_colorspace[id]; + switch (model->tex_nchannel[id]) { + case 1: + config.format = mjPIXEL_FORMAT_R8; + break; + case 3: + config.format = mjPIXEL_FORMAT_RGB8; + break; + case 4: + config.format = mjPIXEL_FORMAT_RGBA8; + break; + default: + mju_error("Unsupported texture format: %d", model->tex_nchannel[id]); + break; + } + if (config.height == 1 && model->tex_nchannel[id] == 1) { + config.format = mjPIXEL_FORMAT_KTX; + } - const TextureType type = [&] { - if (height == 1) { - return TextureType::kKtx; - } else if (tex_type == mjTEXTURE_2D) { - return TextureType::kNormal2d; - } else if (tex_type == mjTEXTURE_CUBE) { - return TextureType::kCube; - } else if (tex_type == mjTEXTURE_SKYBOX) { - return TextureType::kCube; - } else { - mju_error("Unsupported texture type: %d", tex_type); - return TextureType::kNormal2d; - } - }(); - textures_[id] = std::make_unique(engine_, type, color_space, width, - height, num_channels, data); + TextureData payload; + DefaultTextureData(&payload); + payload.bytes = model->tex_data + model->tex_adr[id]; + payload.nbytes = + model->tex_width[id] * model->tex_height[id] * model->tex_nchannel[id]; + // We assume that the model has the same lifetime as the engine. + payload.user_data = nullptr; + payload.release_callback = nullptr; + + auto texture = std::make_unique(engine_, config); + texture->Upload(payload); + textures_[id] = std::move(texture); } void ModelObjects::UploadHeightField(const mjModel* model, int id) { diff --git a/src/experimental/filament/filament/object_manager.cc b/src/experimental/filament/filament/object_manager.cc index 7699e655..01895161 100644 --- a/src/experimental/filament/filament/object_manager.cc +++ b/src/experimental/filament/filament/object_manager.cc @@ -14,6 +14,7 @@ #include "experimental/filament/filament/object_manager.h" +#include #include #include #include @@ -87,18 +88,35 @@ ObjectManager::ObjectManager(filament::Engine* engine) materials_[kUnlitUi] = LoadMaterial("unlit_ui.filamat"); static uint8_t black_rgb[3] = {0, 0, 0}; - fallback_black_ = std::make_unique( - engine_, TextureType::kNormal2d, mjCOLORSPACE_LINEAR, 1, 1, 3, black_rgb); static uint8_t white_rgb[3] = {255, 255, 255}; - fallback_white_ = std::make_unique( - engine_, TextureType::kNormal2d, mjCOLORSPACE_LINEAR, 1, 1, 3, white_rgb); static uint8_t normal_data[3] = {128, 128, 255}; - fallback_normal_ = - std::make_unique(engine_, TextureType::kNormal2d, - mjCOLORSPACE_LINEAR, 1, 1, 3, normal_data); static uint8_t orm_data[3] = {0, 255, 0}; - fallback_orm_ = std::make_unique( - engine_, TextureType::kNormal2d, mjCOLORSPACE_LINEAR, 1, 1, 3, orm_data); + + TextureConfig config; + DefaultTextureConfig(&config); + config.width = 1; + config.height = 1; + config.target = mjTEXTURE_2D; + config.format = mjPIXEL_FORMAT_RGB8; + config.color_space = mjCOLORSPACE_LINEAR; + + auto CreateFallbackTexture = [this, &config](uint8_t color[3]) { + auto texture = std::make_unique(engine_, config); + + TextureData payload; + DefaultTextureData(&payload); + payload.bytes = color; + payload.nbytes = 3; + payload.release_callback = nullptr; + payload.user_data = nullptr; + texture->Upload(payload); + return texture; + }; + + fallback_black_ = CreateFallbackTexture(black_rgb); + fallback_white_ = CreateFallbackTexture(white_rgb); + fallback_normal_ = CreateFallbackTexture(normal_data); + fallback_orm_ = CreateFallbackTexture(orm_data); fallback_textures_[mjTEXROLE_USER] = fallback_black_.get(); fallback_textures_[mjTEXROLE_RGB] = fallback_white_.get(); @@ -149,14 +167,33 @@ void ObjectManager::LoadFallbackIndirectLight( fallback_indirect_light_ = nullptr; } - Asset asset(filename); - if (asset.size == 0) { + Asset* asset = new Asset(filename); + auto release_asset = +[](void* user_data) { + delete static_cast(user_data); + }; + if (asset->size == 0) { + release_asset(asset); return; } - fallback_indirect_light_texture_ = std::make_unique( - engine_, TextureType::kKtx, mjCOLORSPACE_AUTO, asset.size, 1, 1, - reinterpret_cast(asset.payload)); + TextureConfig config; + DefaultTextureConfig(&config); + config.width = 1; + config.height = 1; + config.target = mjTEXTURE_CUBE; + config.format = mjPIXEL_FORMAT_KTX; + config.color_space = mjCOLORSPACE_AUTO; + + fallback_indirect_light_texture_ = std::make_unique(engine_, config); + + TextureData payload; + DefaultTextureData(&payload); + payload.bytes = asset->payload; + payload.nbytes = static_cast(asset->size); + payload.release_callback = release_asset; + payload.user_data = asset; + + fallback_indirect_light_texture_->Upload(payload); if (fallback_indirect_light_texture_ == nullptr) { return; } diff --git a/src/experimental/filament/filament/texture_util.cc b/src/experimental/filament/filament/texture_util.cc index 20b63393..b711d248 100644 --- a/src/experimental/filament/filament/texture_util.cc +++ b/src/experimental/filament/filament/texture_util.cc @@ -27,61 +27,124 @@ namespace mujoco { -static filament::Texture::Format GetTextureFormat(int num_channels) { - switch (num_channels) { - case 1: +static constexpr int kNumFacesPerCube = 6; + +static bool IsCompressed(const TextureConfig& config) { + return config.format == mjPIXEL_FORMAT_KTX; +} + +static bool IsCubeMap(const TextureConfig& config) { + return config.target == mjTEXTURE_CUBE || config.target == mjTEXTURE_SKYBOX; +} + +static int GetFaceHeight(const TextureConfig& config) { + int face_height = config.height; + if (config.width != config.height) { + if (config.width * kNumFacesPerCube != config.height) { + mju_error("Cube maps must contain 6 square images."); + } + face_height = config.height / kNumFacesPerCube; + } + if (config.width != face_height) { + mju_error("Cube map faces must be square."); + } + return face_height; +} + +static int GetNumChannels(const TextureConfig& config) { + switch (config.format) { + case mjPIXEL_FORMAT_R8: + return 1; + case mjPIXEL_FORMAT_RGB8: + return 3; + case mjPIXEL_FORMAT_RGBA8: + return 4; + default: + mju_error("Unsupported format: %d", (int)config.format); + return 0; + } +} + +static filament::Texture::Format GetTextureFormat(const TextureConfig& config) { + switch (config.format) { + case mjPIXEL_FORMAT_R8: return filament::Texture::Format::R; - case 3: + case mjPIXEL_FORMAT_RGB8: return filament::Texture::Format::RGB; - case 4: + case mjPIXEL_FORMAT_RGBA8: return filament::Texture::Format::RGBA; default: - mju_error("Unsupported number of channels: %d", num_channels); + mju_error("Unsupported format: %d", (int)config.format); return filament::Texture::Format::UNUSED; } } static filament::Texture::InternalFormat GetTextureInternalFormat( - int num_channels, bool is_srgb) { - if (is_srgb) { - switch (num_channels) { - case 3: + const TextureConfig& config) { + if (config.color_space == mjCOLORSPACE_SRGB) { + switch (config.format) { + case mjPIXEL_FORMAT_RGB8: return filament::Texture::InternalFormat::SRGB8; - case 4: + case mjPIXEL_FORMAT_RGBA8: return filament::Texture::InternalFormat::SRGB8_A8; default: - mju_error("Unsupported number of channels: %d", num_channels); + mju_error("Unsupported format: %d", (int)config.format); return filament::Texture::InternalFormat::UNUSED; } } else { - switch (num_channels) { - case 1: + switch (config.format) { + case mjPIXEL_FORMAT_R8: return filament::Texture::InternalFormat::R8; - case 3: + case mjPIXEL_FORMAT_RGB8: return filament::Texture::InternalFormat::RGB8; - case 4: + case mjPIXEL_FORMAT_RGBA8: return filament::Texture::InternalFormat::RGBA8; default: - mju_error("Unsupported number of channels: %d", num_channels); + mju_error("Unsupported format: %d", (int)config.format); return filament::Texture::InternalFormat::UNUSED; } } } -Texture::Texture(filament::Engine* engine, TextureType texture_type, - mjtColorSpace color_space, int width, int height, - int num_channels, const uint8_t* data) - : engine_(engine) { - const bool is_srgb = color_space == mjCOLORSPACE_SRGB; - if (texture_type == TextureType::kCube) { - CreateCubeTexture(width, height, num_channels, data, is_srgb); - } else if (texture_type == TextureType::kNormal2d) { - Create2dTexture(width, height, num_channels, data, is_srgb); - } else if (texture_type == TextureType::kKtx) { - CreateKtxTexture(data, width * height * num_channels); - } else { - mju_error("Unsupported texture type: %d", static_cast(texture_type)); +void DefaultTextureData(TextureData* data) { + std::memset(data, 0, sizeof(TextureData)); +} + +void DefaultTextureConfig(TextureConfig* config) { + std::memset(config, 0, sizeof(TextureConfig)); +} + +Texture::Texture(filament::Engine* engine, const TextureConfig& config) + : engine_(engine), config_(config) { + if (IsCompressed(config_)) { + // We defer creation of compressed textures until Upload() is called. In + // the meantime, we don't really know anything about the texture (e.g. + // width, height, etc.). + return; } + + filament::Texture::Builder builder; + builder.width(config_.width); + builder.height(config_.height); + builder.format(GetTextureInternalFormat(config_)); + + if (IsCubeMap(config_)) { + if (config_.format != mjPIXEL_FORMAT_RGB8) { + mju_error("Only support RGB cubemaps."); + return; + } + builder.height(GetFaceHeight(config_)); + builder.sampler(filament::Texture::Sampler::SAMPLER_CUBEMAP); + } else { + builder.sampler(filament::Texture::Sampler::SAMPLER_2D); + } + + if (config_.color_space != mjCOLORSPACE_SRGB) { + builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE | + filament::Texture::Usage::SAMPLEABLE | + filament::Texture::Usage::UPLOADABLE); + } + texture_ = builder.build(*engine_); } Texture::Texture(filament::Engine* engine, RenderTargetTextureType type, @@ -117,112 +180,91 @@ Texture::Texture(filament::Engine* engine, RenderTargetTextureType type, texture_ = builder.build(*engine); } -void Texture::Create2dTexture(int width, int height, int num_channels, - const uint8_t* data, bool is_srgb) { - if (num_channels != 1 && num_channels != 3 && num_channels != 4) { - mju_error("Unsupported number of channels: %d", num_channels); - return; - } - - filament::Texture::Builder builder; - builder.width(width); - builder.height(height); - builder.format(GetTextureInternalFormat(num_channels, is_srgb)); - builder.sampler(filament::Texture::Sampler::SAMPLER_2D); - if (!is_srgb) { - builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE | - filament::Texture::Usage::SAMPLEABLE | - filament::Texture::Usage::UPLOADABLE); - } - texture_ = builder.build(*engine_); - - if (data) { - const size_t num_bytes = width * height * sizeof(uint8_t) * num_channels; - const filament::Texture::Format format = GetTextureFormat(num_channels); - texture_->setImage( - *engine_, 0, - filament::Texture::PixelBufferDescriptor( - data, num_bytes, format, filament::Texture::Type::UBYTE)); - if (!is_srgb) { - texture_->generateMipmaps(*engine_); - } - } -} - -void Texture::CreateCubeTexture(int width, int height, int num_channels, - const uint8_t* data, bool is_srgb) { - if (num_channels != 3) { - mju_error("Only support RGB cubemaps."); - return; - } - - const int kNumFacesPerCube = 6; - - int face_height = height; - if (width != height) { - if (width * kNumFacesPerCube != height) { - mju_error("Cube maps must contain 6 square images."); - } - face_height = height / kNumFacesPerCube; - } - if (width != face_height) { - mju_error("Cube map faces must be square."); - } - - filament::Texture::Builder builder; - builder.width(width); - builder.height(face_height); - builder.format(GetTextureInternalFormat(num_channels, is_srgb)); - builder.sampler(filament::Texture::Sampler::SAMPLER_CUBEMAP); - if (!is_srgb) { - builder.usage(filament::Texture::Usage::GEN_MIPMAPPABLE | - filament::Texture::Usage::SAMPLEABLE | - filament::Texture::Usage::UPLOADABLE); - } - texture_ = builder.build(*engine_); - - const int face_size = width * face_height * num_channels; - const int num_bytes = face_size * kNumFacesPerCube; - - uint8_t* buffer = new uint8_t[num_bytes]; - auto callback = +[](void* buffer, size_t size, void* user) { - delete [] reinterpret_cast(buffer); - }; - - filament::Texture::FaceOffsets offsets(face_size); - if (width == height) { - // Copy the image to all the faces. - for (int i = 0; i < kNumFacesPerCube; ++i) { - std::memcpy(buffer + (i * face_size), data, face_size); - } - } else { - // Use the cubemap as is. - std::memcpy(buffer, data, num_bytes); - } - - if (data) { - filament::Texture::PixelBufferDescriptor desc( - buffer, num_bytes, filament::Texture::Format::RGB, - filament::Texture::Type::UBYTE, callback); - texture_->setImage(*engine_, 0, std::move(desc), offsets); - if (!is_srgb) { - texture_->generateMipmaps(*engine_); - } - } -} - -void Texture::CreateKtxTexture(const uint8_t* data, int size) { - image::Ktx1Bundle* bundle = new image::Ktx1Bundle(data, size); - has_spherical_harmonics_ = true; - bundle->getSphericalHarmonics(spherical_harmonics_); - const bool is_srgb = false; - texture_ = ktxreader::Ktx1Reader::createTexture(engine_, bundle, is_srgb); -} - Texture::~Texture() { + ReleaseData(); if (texture_) { engine_->destroy(texture_); } } +void Texture::Upload(const TextureData& data) { + user_data_ = data.user_data; + release_callback_ = data.release_callback; + + if (data.bytes == nullptr || data.nbytes == 0) { + ReleaseData(); + return; + } + + if (config_.format == mjPIXEL_FORMAT_KTX) { + image::Ktx1Bundle* bundle = new image::Ktx1Bundle( + reinterpret_cast(data.bytes), data.nbytes); + has_spherical_harmonics_ = true; + bundle->getSphericalHarmonics(spherical_harmonics_); + const bool is_srgb = false; + texture_ = ktxreader::Ktx1Reader::createTexture(engine_, bundle, is_srgb); + config_.width = texture_->getWidth(); + config_.height = texture_->getHeight(); + ReleaseData(); + return; + } + + const int num_channels = GetNumChannels(config_); + const filament::Texture::Type type = filament::Texture::Type::UBYTE; + const filament::Texture::Format format = GetTextureFormat(config_); + + if (!IsCubeMap(config_)) { + if (config_.width * config_.height * num_channels != data.nbytes) { + mju_error("Texture size does not match data size."); + } + + auto callback = +[](void* buffer, size_t size, void* user) { + reinterpret_cast(user)->ReleaseData(); + }; + filament::Texture::PixelBufferDescriptor desc(data.bytes, data.nbytes, + format, type, callback, this); + texture_->setImage(*engine_, 0, std::move(desc)); + } else { + const int face_size = config_.width * GetFaceHeight(config_) * num_channels; + const int num_bytes = face_size * kNumFacesPerCube; + filament::Texture::FaceOffsets offsets(face_size); + + if (config_.width == config_.height) { + uint8_t* copy = new uint8_t[num_bytes]; + auto release_callback = +[](void* buffer, size_t size, void* user) { + delete [] reinterpret_cast(buffer); + }; + for (int i = 0; i < kNumFacesPerCube; ++i) { + std::memcpy(copy + (i * face_size), data.bytes, face_size); + } + filament::Texture::PixelBufferDescriptor desc(copy, num_bytes, format, + type, release_callback); + texture_->setImage(*engine_, 0, std::move(desc), offsets); + ReleaseData(); + } else { + if (num_bytes != data.nbytes) { + mju_error("Texture size does not match data size."); + } + auto callback = +[](void* buffer, size_t size, void* user) { + reinterpret_cast(user)->ReleaseData(); + }; + filament::Texture::PixelBufferDescriptor desc( + data.bytes, data.nbytes, format, type, callback, this); + texture_->setImage(*engine_, 0, std::move(desc), offsets); + } + } + + if (config_.color_space != mjCOLORSPACE_SRGB) { + texture_->generateMipmaps(*engine_); + } +} + +void Texture::ReleaseData() { + if (release_callback_) { + release_callback_(user_data_); + release_callback_ = nullptr; + user_data_ = nullptr; + } +} + } // namespace mujoco diff --git a/src/experimental/filament/filament/texture_util.h b/src/experimental/filament/filament/texture_util.h index 42925325..bba7b16e 100644 --- a/src/experimental/filament/filament/texture_util.h +++ b/src/experimental/filament/filament/texture_util.h @@ -15,7 +15,7 @@ #ifndef MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_ #define MUJOCO_SRC_EXPERIMENTAL_FILAMENT_FILAMENT_TEXTURE_UTIL_H_ -#include +#include #include #include @@ -25,14 +25,16 @@ // Functions for creating filament textures. namespace mujoco { -// The types of textures we can create. -enum class TextureType { +// The types of textures we can create. For internal use only. +enum class TextureTarget { + // A standard 2D image with a width and a height. kNormal2d, + // A 2D texture split up into the 6 faces of a cube. kCube, - kKtx, }; // The different types of textures we can create for a render target. +// For internal use only. enum class RenderTargetTextureType { kColor, kDepth, @@ -40,23 +42,85 @@ enum class RenderTargetTextureType { kReflectionColor, }; +// Pixel formats for textures. +typedef enum mjtPixelFormat_ { + mjPIXEL_FORMAT_UNKNOWN = 0, + mjPIXEL_FORMAT_R8, + mjPIXEL_FORMAT_RGB8, + mjPIXEL_FORMAT_RGBA8, + mjPIXEL_FORMAT_DEPTH32F, + mjPIXEL_FORMAT_KTX, +} mjtPixelFormat; + +// The binary contents of a texture. +struct TextureData { + // Pointer to the image data. If null, an empty texture will be created. + void* bytes; + + // The number of bytes in the image data. + size_t nbytes; + + // Because rendering may be multithreaded, we cannot make assumptions about + // when the image data will finish uploading to the GPU. As such, we will use + // this callback to notify callers when it is safe to free the image data. + void (*release_callback)(void* user_data); + + // User data to pass to the release callback. + void* user_data; +}; + +// Initializes the TextureData to default values. +void DefaultTextureData(TextureData* data); + +// Defines the basic properties of a texture. +struct TextureConfig { + // The width of the texture. For compressed textures (e.g. KTX), this is the + // number of bytes in the compressed data. + int width; + + // The height of the texture. For compressed textures (e.g. KTX), this should + // be 0. + int height; + + // The target of the texture (e.g. 2D, cube, etc.) + mjtTexture target; + + // The format of the pixels in the texture (e.g. RGB8, RGBA8, KTX, etc.) + mjtPixelFormat format; + + // The color space of the texture (e.g. LINEAR, sRGB, etc.) + mjtColorSpace color_space; +}; + +// Initializes the TextureConfig to default values. +void DefaultTextureConfig(TextureConfig* config); + +// Wrapper around a filament::Texture. class Texture { public: // Creates a texture with the given data. - Texture(filament::Engine* engine, TextureType texture_type, - mjtColorSpace color_space, int width, int height, int num_channels, - const uint8_t* data); + Texture(filament::Engine* engine, const TextureConfig& config); - // Creates a texture for use with a render target. + // Creates a texture for use with a render target, for internal use. Texture(filament::Engine* engine, RenderTargetTextureType type, int width, int height); ~Texture(); + // Uploads the given data to the texture. + void Upload(const TextureData& data); + + // Returns the width of the texture. + int GetWidth() const { return config_.width; } + + // Returns the height of the texture. + int GetHeight() const { return config_.height; } + + // Returns the underlying filament texture. filament::Texture* GetFilamentTexture() const { return texture_; } + // Returns any spherical harmonics data associated with the texture. using SphericalHarmonics = filament::math::float3[9]; - const SphericalHarmonics* GetSphericalHarmonics() const { return has_spherical_harmonics_ ? &spherical_harmonics_ : nullptr; } @@ -65,17 +129,16 @@ class Texture { Texture& operator=(const Texture&) = delete; private: - - void Create2dTexture(int width, int height, int num_channels, - const uint8_t* data, bool is_srgb); - void CreateCubeTexture(int width, int height, int num_channels, - const uint8_t* data, bool is_srgb); - void CreateKtxTexture(const uint8_t* data, int size); + void ReleaseData(); filament::Engine* engine_ = nullptr; filament::Texture* texture_ = nullptr; + TextureConfig config_; SphericalHarmonics spherical_harmonics_; bool has_spherical_harmonics_ = false; + + void* user_data_ = nullptr; + void (*release_callback_)(void* user_data) = nullptr; }; } // namespace mujoco