From 0ddbb46fa19506d4f4e7a38eb692bad63e1dde9e Mon Sep 17 00:00:00 2001 From: Haroon Qureshi Date: Mon, 26 Jan 2026 23:52:35 -0800 Subject: [PATCH] Refactor mjVFS and resource management. Resource operations (e.g. mju_openResource, mju_readResource, and mju_closeResource, etc.) are now all handled by a VFS instance. It is now up to the VFS to determine which provider to use in order to handle those operations. This allows us to dynamically add/remove (aka "mount") providers to a VFS to handle special requests. mj_addFileVFS and mj_addBufferVFS have been reimplemented as two such use-cases. Moreover, we expose the mounting behaviour with two new functions: mj_mountVFS and mj_unmountVFS. PiperOrigin-RevId: 861550939 Change-Id: I070eb4bcc2982466c8f368f7918005538baa5185 --- doc/APIreference/functions.rst | 19 + doc/changelog.rst | 2 + doc/includes/references.h | 5 + include/mujoco/mjplugin.h | 9 + include/mujoco/mujoco.h | 7 + python/mujoco/introspect/functions.py | 46 ++ src/user/user_resource.cc | 228 ++------- src/user/user_vfs.cc | 659 +++++++++++++++++--------- src/user/user_vfs.h | 121 ++++- test/user/user_resource_test.cc | 2 +- test/user/user_vfs_test.cc | 162 +++++++ test/xml/xml_api_test.cc | 2 +- 12 files changed, 829 insertions(+), 433 deletions(-) diff --git a/doc/APIreference/functions.rst b/doc/APIreference/functions.rst index 8ac117b3..62db7ea4 100644 --- a/doc/APIreference/functions.rst +++ b/doc/APIreference/functions.rst @@ -1357,6 +1357,25 @@ The VFS must first be allocated using :ref:`mj_defaultVFS` and must be freed wit Initialize an empty VFS, :ref:`mj_deleteVFS` must be called to deallocate the VFS. +.. _mj_mountVFS: + +`mj_mountVFS <#mj_mountVFS>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. mujoco-include:: mj_mountVFS + +Mount a ResourceProvider to handle file operations under the given path; return 0: success, +2: repeated name, -1: invalid resource provider. + +.. _mj_unmountVFS: + +`mj_unmountVFS <#mj_unmountVFS>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. mujoco-include:: mj_unmountVFS + +Unmount a previously mounted ResourceProvider; return 0: success, -1: not found in VFS. + .. _mj_addFileVFS: `mj_addFileVFS <#mj_addFileVFS>`__ diff --git a/doc/changelog.rst b/doc/changelog.rst index 877c06f8..9eb296b8 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -54,6 +54,8 @@ General See :ref:`rangefinder` for details. - Cameras now have an :ref:`output` attribute, parsed into the ``mjModel.cam_output`` bitfield. Unused by the renderer, it serves as a convenient location to store a camera's supported output types. +- Added :ref:`mj_mountVFS` and :ref:`mj_unmountVFS` functions for mounting a custom VFS provider. Mounting Allows + providers to be used to open/read/close resources dynamically at arbitrary paths. - Non-breaking ABI changes: - The type of the ``sig`` (signature) argument of :ref:`mj_stateSize` and related functions has been changed from diff --git a/doc/includes/references.h b/doc/includes/references.h index 8080355e..155a5046 100644 --- a/doc/includes/references.h +++ b/doc/includes/references.h @@ -1629,6 +1629,7 @@ typedef struct mjModel_ mjModel; struct mjResource_ { char* name; // name of resource (filename, etc) void* data; // opaque data pointer + mjVFS* vfs; // pointer to the VFS char timestamp[512]; // timestamp of the resource const struct mjpResourceProvider* provider; // pointer to the provider }; @@ -1638,6 +1639,8 @@ struct mjpResourceProvider { mjfOpenResource open; // opening callback mjfReadResource read; // reading callback mjfCloseResource close; // closing callback + mjfMountResource mount; // mounting callback (optional) + mjfUnmountResource unmount; // unmounting callback (optional) mjfResourceModified modified; // resource modified callback (optional) void* data; // opaque data pointer (resource invariant) }; @@ -3095,6 +3098,8 @@ typedef struct mjvFigure_ mjvFigure; //----------------------------- MJAPI FUNCTIONS -------------------------------- void mj_defaultVFS(mjVFS* vfs); +int mj_mountVFS(mjVFS* vfs, const char* filepath, const mjpResourceProvider* provider); +int mj_unmountVFS(mjVFS* vfs, const char* filename); int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename); int mj_addBufferVFS(mjVFS* vfs, const char* name, const void* buffer, int nbuffer); int mj_deleteFileVFS(mjVFS* vfs, const char* filename); diff --git a/include/mujoco/mjplugin.h b/include/mujoco/mjplugin.h index 880da990..93268618 100644 --- a/include/mujoco/mjplugin.h +++ b/include/mujoco/mjplugin.h @@ -27,6 +27,7 @@ struct mjResource_ { char* name; // name of resource (filename, etc) void* data; // opaque data pointer + mjVFS* vfs; // pointer to the VFS char timestamp[512]; // timestamp of the resource const struct mjpResourceProvider* provider; // pointer to the provider }; @@ -42,6 +43,12 @@ typedef int (*mjfReadResource)(mjResource* resource, const void** buffer); // callback for closing a resource (responsible for freeing any allocated memory) typedef void (*mjfCloseResource)(mjResource* resource); +// callback for mounting a resource (provider), returns zero on failure +typedef int (*mjfMountResource)(mjResource* resource); + +// callback for unmounting a resource (provider), returns zero on failure +typedef int (*mjfUnmountResource)(mjResource* resource); + // callback for checking if the current resource was modified from the time // specified by the timestamp // returns 0 if the resource's timestamp matches the provided timestamp @@ -55,6 +62,8 @@ struct mjpResourceProvider { mjfOpenResource open; // opening callback mjfReadResource read; // reading callback mjfCloseResource close; // closing callback + mjfMountResource mount; // mounting callback (optional) + mjfUnmountResource unmount; // unmounting callback (optional) mjfResourceModified modified; // resource modified callback (optional) void* data; // opaque data pointer (resource invariant) }; diff --git a/include/mujoco/mujoco.h b/include/mujoco/mujoco.h index b42d7891..197d42f1 100644 --- a/include/mujoco/mujoco.h +++ b/include/mujoco/mujoco.h @@ -78,6 +78,13 @@ MJAPI extern const char* mjRNDSTRING[mjNRNDFLAG][3]; // Initialize an empty VFS, mj_deleteVFS must be called to deallocate the VFS. MJAPI void mj_defaultVFS(mjVFS* vfs); +// Mount a ResourceProvider to handle file operations under the given path; return 0: success, +// 2: repeated name, -1: invalid resource provider. +MJAPI int mj_mountVFS(mjVFS* vfs, const char* filepath, const mjpResourceProvider* provider); + +// Unmount a previously mounted ResourceProvider; return 0: success, -1: not found in VFS. +MJAPI int mj_unmountVFS(mjVFS* vfs, const char* filename); + // Add file to VFS; return 0: success, 2: repeated name, -1: failed to load. MJAPI int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename); diff --git a/python/mujoco/introspect/functions.py b/python/mujoco/introspect/functions.py index 072a63e3..33f48e46 100644 --- a/python/mujoco/introspect/functions.py +++ b/python/mujoco/introspect/functions.py @@ -40,6 +40,52 @@ FUNCTIONS: Mapping[str, FunctionDecl] = dict([ ), doc='Initialize an empty VFS, mj_deleteVFS must be called to deallocate the VFS.', # pylint: disable=line-too-long )), + ('mj_mountVFS', + FunctionDecl( + name='mj_mountVFS', + return_type=ValueType(name='int'), + parameters=( + FunctionParameterDecl( + name='vfs', + type=PointerType( + inner_type=ValueType(name='mjVFS'), + ), + ), + FunctionParameterDecl( + name='filepath', + type=PointerType( + inner_type=ValueType(name='char', is_const=True), + ), + ), + FunctionParameterDecl( + name='provider', + type=PointerType( + inner_type=ValueType(name='mjpResourceProvider', is_const=True), # pylint: disable=line-too-long + ), + ), + ), + doc='Mount a ResourceProvider to handle file operations under the given path; return 0: success, 2: repeated name, -1: invalid resource provider.', # pylint: disable=line-too-long + )), + ('mj_unmountVFS', + FunctionDecl( + name='mj_unmountVFS', + return_type=ValueType(name='int'), + parameters=( + FunctionParameterDecl( + name='vfs', + type=PointerType( + inner_type=ValueType(name='mjVFS'), + ), + ), + FunctionParameterDecl( + name='filename', + type=PointerType( + inner_type=ValueType(name='char', is_const=True), + ), + ), + ), + doc='Unmount a previously mounted ResourceProvider; return 0: success, -1: not found in VFS.', # pylint: disable=line-too-long + )), ('mj_addFileVFS', FunctionDecl( name='mj_addFileVFS', diff --git a/src/user/user_resource.cc b/src/user/user_resource.cc index a1482b44..378daa0f 100644 --- a/src/user/user_resource.cc +++ b/src/user/user_resource.cc @@ -14,216 +14,71 @@ #include "user/user_resource.h" -#include #include #include #include -#include #include #include #include #include #include -#include #include -#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) - #include -#endif - -#ifdef _WIN32 - #define stat _stat -#endif - #include #include "engine/engine_plugin.h" -#include "engine/engine_util_misc.h" #include "user/user_util.h" #include "user/user_vfs.h" -namespace { - -using mujoco::user::FileToMemory; - -// file buffer used internally for the OS filesystem -struct FileSpec { - bool is_read; // set to nonzero if buffer was read into - std::vector buffer; // raw bytes from file - time_t mtime; // last modified time -}; - -// callback for opening a resource, returns zero on failure -int FileOpen(mjResource* resource, char* error, size_t nerror) { - resource->provider = nullptr; - resource->data = new FileSpec; - const char* name = resource->name; - FileSpec* spec = (FileSpec*) resource->data; - spec->is_read = false; - struct stat file_stat; - if (stat(name, &file_stat) == 0) { - memcpy(&spec->mtime, &file_stat.st_mtime, sizeof(time_t)); - } else { - if (error) { - snprintf(error, nerror, "Error opening file '%s': %s", name, - strerror(errno)); - } - return 0; - } - mju_encodeBase64(resource->timestamp, (uint8_t*) &spec->mtime, - sizeof(time_t)); - return 1; -} - -// OS filesystem read callback -int FileRead(mjResource* resource, const void** buffer) { - FileSpec* spec = (FileSpec*) resource->data; - - // only read once from file - if (!spec->is_read) { - spec->buffer = FileToMemory(resource->name); - spec->is_read = true; - } - *buffer = spec->buffer.data(); - return spec->buffer.size(); -} - -// OS filesystem close callback -void FileClose(mjResource* resource) { - FileSpec* spec = (FileSpec*) resource->data; - if (spec) delete spec; -} - -// OS filesystem modified callback -int FileModified(const mjResource* resource, const char*timestamp) { - if (mju_isValidBase64(timestamp) != sizeof(time_t)) { - return 1; // error (assume modified) - } - - time_t time1, time2; - mju_decodeBase64((uint8_t*) &time1, timestamp); - time2 = ((FileSpec*) resource->data)->mtime; - double diff = difftime(time2, time1); - if (diff < 0) return -1; - if (diff > 0) return 1; - return 0; -} - -} // namespace - -// open the given resource; if the name doesn't have a prefix matching with a -// resource provider, then the OS filesystem is used mjResource* mju_openResource(const char* dir, const char* name, const mjVFS* vfs, char* error, size_t nerror) { - // no error so far + // TODO: Update API to use non-const pointer. Unfortunately, while this is + // ABI stable, it will cause compiler errors in user code that is const + // correct. + mjVFS* non_const_vfs = const_cast(vfs); + + // TODO: Eventually, we should make `vfs` a required argument. In the + // meantime, when passing in a nullptr VFS, we will create a VFS dynamically + // that self-destructs when the resource is closed (or if the resource could + // not be opened). + if (non_const_vfs == nullptr) { + mjVFS* local_vfs = (mjVFS*)mju_malloc(sizeof(mjVFS)); + mj_defaultVFS(local_vfs); + mujoco::user::VFS::Upcast(local_vfs)->SetToSelfDestruct([](mjVFS* ptr) { + mj_deleteVFS(ptr); + mju_free(ptr); + }); + + non_const_vfs = local_vfs; + } + + mjResource* resource = + mujoco::user::VFS::Upcast(non_const_vfs)->Open(dir ? dir : "", name); + if (error) { - error[0] = '\0'; - } - - mjResource* resource = (mjResource*) mju_malloc(sizeof(mjResource)); - if (resource == nullptr) { - if (error) { - strncpy(error, "could not allocate memory", nerror); - error[nerror - 1] = '\0'; - } - return nullptr; - } - - // clear out resource - memset(resource, 0, sizeof(mjResource)); - - // make space for filename - std::string fullname = mjuu_combinePaths(dir, name); - std::size_t n = fullname.size(); - resource->name = (char*) mju_malloc(sizeof(char) * (n + 1)); - if (resource->name == nullptr) { - if (error) { - strncpy(error, "could not allocate memory", nerror); - error[nerror - 1] = '\0'; - } - mju_closeResource(resource); - return nullptr; - } - - // first priority is to check the VFS - if (vfs != nullptr) { - memcpy(resource->name, name, - sizeof(char) * (std::strlen(name) + 1)); - const mjpResourceProvider* provider = GetVfsResourceProvider(); - resource->data = (void*) vfs; - resource->provider = provider; - if (provider->open(resource)) { - return resource; + if (resource) { + error[0] = '\0'; + } else { + std::snprintf(error, nerror, "Error opening file '%s'", name); } } - // copy full path over - memcpy(resource->name, fullname.c_str(), sizeof(char) * (n + 1)); - - // find provider based off prefix of name - const mjpResourceProvider* provider = mjp_getResourceProvider(resource->name); - if (provider != nullptr) { - resource->provider = provider; - resource->data = nullptr; - if (provider->open(resource)) { - return resource; - } - - if (error) { - snprintf(error, nerror, "could not open '%s'" - "using a resource provider matching prefix '%s'", - resource->name, provider->prefix); - } - - mju_closeResource(resource); - return nullptr; - } - - // lastly fallback to OS filesystem - if (FileOpen(resource, error, nerror)) { - return resource; - } - mju_closeResource(resource); - return nullptr; + return resource; } - - -// close the given resource; no-op if resource is NULL void mju_closeResource(mjResource* resource) { - if (resource == nullptr) { - return; + if (resource && resource->vfs) { + mujoco::user::VFS::Upcast(resource->vfs)->Close(resource); } - - // use the resource provider close callback - const mjpResourceProvider* provider = resource->provider; - if (provider) { - if (provider->close) provider->close(resource); - } else { - FileClose(resource); // clear OS filesystem if present - } - - // free resource - if (resource->name) mju_free(resource->name); - mju_free(resource); } - - -// set buffer to bytes read from the resource and return number of bytes in -// buffer; return negative value if error int mju_readResource(mjResource* resource, const void** buffer) { - if (resource->provider) { - return resource->provider->read(resource, buffer); + if (resource && resource->vfs) { + return mujoco::user::VFS::Upcast(resource->vfs)->Read(resource, buffer); } - - // if provider is NULL, then OS filesystem is used - return FileRead(resource, buffer); + return -1; // default (error reading bytes) } - - -// get directory path of resource void mju_getResourceDir(mjResource* resource, const char** dir, int* ndir) { *dir = nullptr; *ndir = 0; @@ -247,22 +102,11 @@ void mju_getResourceDir(mjResource* resource, const char** dir, int* ndir) { } } - - -// return 0 if the resource's timestamp matches the provided timestamp -// return > 0 if the resource is younger than the given timestamp -// return < 0 if the resource is older than the given timestamp int mju_isModifiedResource(const mjResource* resource, const char* timestamp) { - // provider is not OS filesystem - if (resource->provider) { - if (resource->provider->modified) { - return resource->provider->modified(resource, timestamp); - } - return 1; // default (modified) + if (resource && resource->provider && resource->provider->modified) { + return resource->provider->modified(resource, timestamp); } - - // fallback to OS filesystem - return FileModified(resource, timestamp); + return 1; // default (assume modified) } mjSpec* mju_decodeResource(mjResource* resource, const char* content_type, const mjVFS* vfs) { diff --git a/src/user/user_vfs.cc b/src/user/user_vfs.cc index 5e8cfedb..d8d4b2e2 100644 --- a/src/user/user_vfs.cc +++ b/src/user/user_vfs.cc @@ -14,265 +14,490 @@ #include "user/user_vfs.h" +#include +#ifdef _WIN32 +#define stat _stat +#endif + +#include +#include #include -#include #include -#include -#include +#include +#include +#include +#include +#include #include +#include #include #include #include +#include #include "engine/engine_util_misc.h" #include "user/user_util.h" namespace { -using mujoco::user::FilePath; -using mujoco::user::FileToMemory; - -// internal struct for VFS files -struct VFSFile { - FilePath filename; - std::vector filedata; - std::size_t filesize; - uint64_t filestamp; +// struct for holding the contents of a file +struct ResourceFileData { + std::vector contents; + time_t modified_time = 0; + bool is_read = false; }; -// internal container class for VFS -class VFS { - public: - // returns true if the file exists in the VFS - bool HasFile(const FilePath& filename) const; - - // returns inserted mjuuVFSFile if the file was added successfully. This class - // assumes ownership of the buffer. - VFSFile* AddFile(const FilePath& filename, std::vector&& buffer, - uint64_t filestamp); - - // returns the internal file struct for the given filename - const VFSFile* GetFile(const FilePath& filename) const; - - // deletes file from VFS, return 0: success, -1: not found - int DeleteFile(const FilePath& filename); - - private: - std::unordered_map files_; -}; - -// returns the internal VFS class pointer from the VFS C struct -inline VFS* GetVFSImpl(const mjVFS* vfs) { - return vfs->impl_ ? static_cast(vfs->impl_) : nullptr; -} - -// strip path prefix from filename and make lowercase -FilePath StripPath(const char* filename) { - return FilePath(filename).StripPath().Lower(); -} - -// copies data into a buffer and produces a hash of the data -uint64_t vfs_memcpy(std::vector& dest, const void* src, size_t n) { - uint64_t hash = 0xcbf29ce484222325; // magic number - uint64_t prime = 0x100000001b3; // magic prime - const uint8_t* bytes = (uint8_t*) src; - for (size_t i = 0; i < n; i++) { - dest.push_back(bytes[i]); - - // do FNV-1 hash - hash |= bytes[i]; - hash *= prime; +time_t GetModifiedTime(const char* path) { + struct stat file_stat; + if (stat(path, &file_stat) == 0) { + return file_stat.st_mtime; } - return hash; -} - -// VFS hash function implemented using the FNV-1 hash -uint64_t vfs_hash(const std::vector& buffer) { - uint64_t hash = 0xcbf29ce484222325; // magic number - uint64_t prime = 0x100000001b3; // magic prime - const uint8_t* bytes = (uint8_t*) buffer.data(); - std::size_t n = buffer.size(); - for (std::size_t i = 0; i < n; i++) { - hash |= bytes[i]; - hash *= prime; - } - return hash; -} - -bool VFS::HasFile(const FilePath& filename) const { - return files_.find(filename.Str()) != files_.end(); -} - -VFSFile* VFS::AddFile(const FilePath& filename, std::vector&& buffer, - uint64_t filestamp) { - auto [it, inserted] = files_.insert({filename.Str(), VFSFile()}); - if (!inserted) { - return nullptr; // repeated name - } - it->second.filename = filename; - it->second.filedata = buffer; - it->second.filestamp = filestamp; - return &(it->second); -} - -const VFSFile* VFS::GetFile(const FilePath& filename) const { - auto it = files_.find(filename.Str()); - if (it == files_.end()) { - return nullptr; - } - return &it->second; -} - -int VFS::DeleteFile(const FilePath& filename) { - auto it = files_.find(filename.Str()); - if (it == files_.end()) { - return -1; - } - files_.erase(it); return 0; } -// open callback for the VFS resource provider -int Open(mjResource* resource) { - if (!resource || !resource->name || !resource->data) { - return 0; - } +int OpenFile(const char* filename, mjResource* resource) { + const time_t mtime = GetModifiedTime(filename); + if (mtime != 0) { + ResourceFileData* data = new ResourceFileData(); + resource->data = data; - const mjVFS* vfs = (const mjVFS*) resource->data; - const VFS* cvfs = GetVFSImpl(vfs); - const VFSFile* file = cvfs->GetFile(StripPath(resource->name)); - if (file == nullptr) { - file = cvfs->GetFile(FilePath(resource->name)); - if (file == nullptr) { - return 0; - } + data->modified_time = mtime; + mju_encodeBase64(resource->timestamp, (uint8_t*)&mtime, sizeof(time_t)); + return 1; } - - resource->data = (void*) file; - resource->timestamp[0] = '\0'; - if (file->filestamp) { - mju_encodeBase64(resource->timestamp, (uint8_t*) &file->filestamp, - sizeof(uint64_t)); - } - return 1; + return 0; } -// read callback for the VFS resource provider -int Read(mjResource* resource, const void** buffer) { - if (!resource || !resource->name || !resource->data) { - *buffer = nullptr; - return -1; +int ReadFile(const char* filename, mjResource* resource, const void** buffer) { + ResourceFileData* data = (ResourceFileData*)resource->data; + if (!data->is_read) { + data->contents = mujoco::user::FileToMemory(filename); + data->is_read = true; } - - const VFSFile* file = static_cast(resource->data); - if (file == nullptr) { - *buffer = nullptr; - return -1; - } - - *buffer = file->filedata.data(); - return file->filedata.size(); + *buffer = data->contents.data(); + return static_cast(data->contents.size()); } -// close callback for the VFS resource provider -void Close(mjResource* resource) { +void CloseFile(mjResource* resource) { + delete (ResourceFileData*)resource->data; + resource->data = nullptr; } -// modified callback for the VFS resource provider -// return > 0 if modified and 0 if unmodified -int Modified(const mjResource* resource, const char* timestamp) { - uint64_t filestamp; - if (mju_isValidBase64(timestamp) > sizeof(uint64_t)) { - return 2; // error (assume modified) +int FileModified(const mjResource* resource, const char* timestamp) { + if (mju_isValidBase64(timestamp) != sizeof(time_t)) { + return 1; } + time_t time; + mju_decodeBase64((uint8_t*)&time, timestamp); - mju_decodeBase64((uint8_t*) &filestamp, timestamp); - if (!filestamp) return 3; // no hash (assume modified) + ResourceFileData* data = (ResourceFileData*)resource->data; + const double diff = difftime(data->modified_time, time); - if (resource) { - const VFSFile* file = static_cast(resource->data); - if (file == nullptr) return 4; // missing file (assume modified) - if (!file->filestamp) return 5; // missing filestamp (assume modified) + if (diff < 0) return -1; + if (diff > 0) return 1; + return 0; +} - if (file->filestamp == filestamp) { - return 0; // unmodified - } +std::string StripPathAndLower(std::string path) { + std::size_t n = path.find_last_of("/\\"); + if (n != std::string::npos) { + path = path.substr(n + 1); } - return 1; // modified + std::transform(path.begin(), path.end(), path.begin(), + [](unsigned char c) { return std::tolower(c); }); + return path; } } // namespace -// initialize to empty (no deallocation) +namespace mujoco::user { + +VFS::VFS(mjVFS* vfs) : self_(vfs) { + mjp_defaultResourceProvider(&default_provider_); + default_provider_.open = [](mjResource* res) { + return OpenFile(res->name, res); + }; + default_provider_.read = [](mjResource* res, const void** buffer) { + return ReadFile(res->name, res, buffer); + }; + default_provider_.close = [](mjResource* res) { + CloseFile(res); + }; + default_provider_.modified = [](const mjResource* res, const char* time) { + return FileModified(res, time); + }; + + default_provider_.prefix = nullptr; + default_mount_.vfs = self_; + default_mount_.provider = &default_provider_; + default_mount_.data = nullptr; + default_mount_.name = nullptr; +} + +VFS::~VFS() { + if (!open_resources_.empty()) { + mju_warning( + "VFS destroyed with %zu open resources. Resources will be invalidated.", + open_resources_.size()); + } + for (auto& [ptr, res] : open_resources_) { + if (res->provider->close) { + res->provider->close(res.get()); + } + } + open_resources_.clear(); + + for (auto& [path, res] : mounts_) { + if (res->provider->unmount) { + res->provider->unmount(res.get()); + } + } + mounts_.clear(); +} + +mjResource* VFS::Open(const char* dir, const char* name) { + const std::string path = FilePath(dir, name).Str(); + + const mjResource* mount = FindMount(path); + if (!mount) { + MaybeSelfDestruct(); + return nullptr; + } + + ResourcePtr res = CreateResource(path.c_str(), mount->provider); + + // Smuggle the mounted resource provider's mount-specific data pointer in the + // requested resource's data pointer. This allows the provider to access its + // own per-mount data without any intrusive changes to the provider interface. + res->data = mount->data; + const int result = mount->provider->open(res.get()); + // If the data pointer was not modified, then that means the resource did not + // set its own data pointer. So, we need to set it back to nullptr. + if (res->data == mount->data) { + res->data = nullptr; + } + + if (result == 0) { + res.reset(); + MaybeSelfDestruct(); + return nullptr; + } + + std::lock_guard lock(mutex_); + mjResource* res_ptr = res.get(); + open_resources_.emplace(res_ptr, std::move(res)); + return res_ptr; +} + +VFS::Status VFS::Mount(const FilePath& path, + const mjpResourceProvider* provider) { + if (!provider) { + return kInvalidResourceProvider; + } + { + std::lock_guard lock(mutex_); + if (mounts_.contains(path.Str())) { + return kRepeatedName; + } + } + + ResourcePtr res = CreateResource(path.c_str(), provider); + provider->mount(res.get()); + + std::lock_guard lock(mutex_); + mounts_.emplace(path.Str(), std::move(res)); + return kSuccess; +} + +VFS::Status VFS::Close(mjResource* res) { + VFS::Status status = kInvalidResource; + bool last_resource = false; + { + std::lock_guard lock(mutex_); + if (auto it = open_resources_.find(res); it != open_resources_.end()) { + if (res->provider->close) { + res->provider->close(res); + } + open_resources_.erase(it); + last_resource = open_resources_.empty(); + status = kSuccess; + } + } + + if (status == kSuccess && last_resource) { + MaybeSelfDestruct(); + } + return status; +} + +VFS::Status VFS::Unmount(const FilePath& path) { + std::lock_guard lock(mutex_); + if (auto it = mounts_.find(path.Str()); it != mounts_.end()) { + if (it->second->provider->unmount) { + it->second->provider->unmount(it->second.get()); + } + mounts_.erase(it); + return kSuccess; + } + return kInvalidResourceProvider; +} + +int VFS::Read(mjResource* resource, const void** buffer) { + if (resource && resource->provider && resource->provider->read) { + return resource->provider->read(resource, buffer); + } + return kFailedToRead; +} + +VFS::ResourcePtr VFS::CreateResource(std::string_view name, + const mjpResourceProvider* provider) { + mjResource* res = new mjResource(); + res->vfs = self_; + res->provider = provider; + res->data = nullptr; + res->name = new char[name.size() + 1]; + std::strncpy(res->name, name.data(), name.size()); + res->name[name.size()] = 0; + res->timestamp[0] = 0; + + return ResourcePtr(res, [](mjResource* ptr) { + if (ptr->data) { + // TODO: Make this an error eventually. For now, we continue to allow + // users to free their data pointers without resetting them to nullptr. + mju_warning( + "mjResource::data is not null; did you forget to close/unmount it?"); + } + delete[] ptr->name; + delete ptr; + }); +} + +mjResource* VFS::FindMount(const std::string& fullpath) { + std::lock_guard lock(mutex_); + + std::string str = fullpath; + while (!str.empty()) { + auto it = mounts_.find(str); + if (it != mounts_.end()) { + return it->second.get(); + } + + std::size_t n = str.find_last_of("/\\"); + if (n == std::string::npos) { + str = ""; + } else { + str = str.substr(0, n); + } + } + + const mjpResourceProvider* provider = + mjp_getResourceProvider(fullpath.c_str()); + if (provider) { + if (auto it = mounts_.find(provider->prefix); it != mounts_.end()) { + return it->second.get(); + } + + ResourcePtr res = CreateResource(provider->prefix, provider); + mjResource* res_ptr = res.get(); + mounts_.emplace(provider->prefix, std::move(res)); + if (provider->mount) { + provider->mount(res_ptr); + } + return res_ptr; + } + + // Legacy use-case: match on just the case-insensitive filename. + const std::string filename = StripPathAndLower(fullpath); + for (auto& [path, res] : mounts_) { + if (StripPathAndLower(path) == filename) { + return res.get(); + } + } + + return &default_mount_; +} + +void VFS::MaybeSelfDestruct() { + if (destructor_) { + destructor_(self_); + } +} + +void VFS::SetToSelfDestruct(std::function destructor) { + destructor_ = std::move(destructor); +} + +VFS* VFS::Upcast(mjVFS* vfs) { + return vfs ? static_cast(vfs->impl_) : nullptr; +} + +const VFS* VFS::Upcast(const mjVFS* vfs) { + return vfs ? static_cast(vfs->impl_) : nullptr; +} + +} // namespace mujoco::user + void mj_defaultVFS(mjVFS* vfs) { - vfs->impl_ = new VFS(); + if (vfs == nullptr) { + mju_error("mjVFS is null."); + } else { + vfs->impl_ = new mujoco::user::VFS(vfs); + } } -// add file to VFS, return 0: success, 2: repeated name, -1: failed to load -int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename) { - VFS* cvfs = GetVFSImpl(vfs); - - // make full name - const char* dir = directory != nullptr ? directory : ""; - FilePath fullname = FilePath(dir, filename); - - // strip path - FilePath newname = StripPath(filename); - - // check beforehand for repeated name, to avoid reading file into memory - if (cvfs->HasFile(newname)) { - return 2; - } - - // allocate and read - std::vector buffer = FileToMemory(fullname.c_str()); - if (buffer.empty()) { - return -1; - } - - if (!cvfs->AddFile(newname, std::move(buffer), vfs_hash(buffer))) { - return 2; // AddFile failed, SHOULD NOT OCCUR - } - return 0; -} - -// add file from buffer into VFS -int mj_addBufferVFS(mjVFS* vfs, const char* name, const void* buffer, - int nbuffer) { - std::vector inbuffer; - VFS* cvfs = GetVFSImpl(vfs); - VFSFile* file; - if (!(file = cvfs->AddFile(FilePath(name), std::move(inbuffer), 0))) { - return 2; // AddFile failed, repeated name - } - try { - file->filedata.reserve(nbuffer); - } catch (...) { - cvfs->DeleteFile(file->filename); // delete file on error - return -1; - } - file->filestamp = vfs_memcpy(file->filedata, buffer, nbuffer); - return 0; -} - -// delete file from VFS, return 0: success, -1: not found in VFS -int mj_deleteFileVFS(mjVFS* vfs, const char* filename) { - VFS* cvfs = GetVFSImpl(vfs); - if (cvfs->DeleteFile(StripPath(filename))) { - return cvfs->DeleteFile(FilePath(filename)); - } - return 0; -} - -// delete all files from VFS void mj_deleteVFS(mjVFS* vfs) { if (vfs) { - delete GetVFSImpl(vfs); + delete mujoco::user::VFS::Upcast(vfs); + vfs->impl_ = nullptr; } } -const mjpResourceProvider* GetVfsResourceProvider() { - static mjpResourceProvider provider - = { nullptr, &Open, &Read, &Close, &Modified, nullptr }; - return &provider; +int mj_mountVFS(mjVFS* vfs, const char* filepath, + const mjpResourceProvider* provider) { + mujoco::user::VFS* impl = mujoco::user::VFS::Upcast(vfs); + if (impl == nullptr) { + mju_error("mjVFS is null."); + return mujoco::user::VFS::kInvalidVfs; + } + + if (filepath == nullptr) { + return mujoco::user::VFS::kNotFound; + } + const mujoco::user::FilePath path(filepath); + const mujoco::user::VFS::Status status = impl->Mount(path, provider); + return static_cast(status); +} + +int mj_unmountVFS(mjVFS* vfs, const char* filename) { + mujoco::user::VFS* impl = mujoco::user::VFS::Upcast(vfs); + if (impl == nullptr) { + mju_error("mjVFS is null."); + return mujoco::user::VFS::kInvalidVfs; + } + + if (filename == nullptr) { + return mujoco::user::VFS::kNotFound; + } + const mujoco::user::FilePath path(filename); + const mujoco::user::VFS::Status status = impl->Unmount(path); + return static_cast(status); +} + +namespace { + +// Custom provider for mj_addFileVFS and mj_addBufferVFS. +class BufferProvider : public mjpResourceProvider { + public: + template + static int Mount(mjVFS* vfs, Args&&... args) { + mujoco::user::VFS* impl = mujoco::user::VFS::Upcast(vfs); + if (impl == nullptr) { + mju_error("mjVFS is null."); + return -1; + } + + BufferProvider* provider = new BufferProvider(std::forward(args)...); + provider->mount = [](mjResource* res) { + return static_cast(mujoco::user::VFS::kSuccess); + }; + provider->unmount = [](mjResource* res) { + delete (BufferProvider*)res->provider; + return static_cast(mujoco::user::VFS::kSuccess); + }; + provider->open = [](mjResource* res) { + BufferProvider* self = (BufferProvider*)res->provider; + mju_encodeBase64(res->timestamp, (std::uint8_t*)&self->hash_, + sizeof(self->hash_)); + return 1; + }; + provider->read = [](mjResource* res, const void** out) { + BufferProvider* self = (BufferProvider*)res->provider; + *out = reinterpret_cast(self->contents_.data()); + return static_cast(self->contents_.size()); + }; + provider->close = [](mjResource* res) { + // no-op + }; + provider->modified = [](const mjResource* res, const char* timestamp) { + const BufferProvider* self = (const BufferProvider*)res->provider; + if (mju_isValidBase64(timestamp) > sizeof(std::uint64_t)) { + return 1; + } + std::uint64_t test = 0; + mju_decodeBase64((std::uint8_t*)&test, timestamp); + if (self->hash_ != test) { + return 1; + } + return 0; + }; + + const mujoco::user::VFS::Status status = + impl->Mount(provider->path_, provider); + if (status != mujoco::user::VFS::kSuccess) { + delete provider; + } + return static_cast(status); + } + + private: + BufferProvider(const char* dir, const char* filename) { + mjp_defaultResourceProvider(this); + + mujoco::user::FilePath file_path(dir ? dir : "", filename); + path_ = file_path.StripPath().Lower(); + contents_ = mujoco::user::FileToMemory(file_path.c_str()); + + static constexpr std::uint64_t prime = 0x100000001b3; + hash_ = contents_.empty() ? 0 : 0xcbf29ce484222325; + for (const std::uint8_t& byte : contents_) { + hash_ |= byte; + hash_ *= prime; + } + } + + BufferProvider(const char* name, const void* src, size_t n) { + mjp_defaultResourceProvider(this); + path_ = mujoco::user::FilePath(name); + + static constexpr std::uint64_t prime = 0x100000001b3; + hash_ = n ? 0xcbf29ce484222325 : 0; + + const std::uint8_t* src_bytes = static_cast(src); + contents_.reserve(n); + for (size_t i = 0; i < n; i++) { + contents_.push_back(src_bytes[i]); + hash_ |= src_bytes[i]; + hash_ *= prime; + } + } + + mujoco::user::FilePath path_; + std::vector contents_; + std::uint64_t hash_ = 0; +}; + +} // namespace + +int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename) { + // Opens the files and copies its contents into the BufferProvider, then + // mounts the provider at the given path. + return BufferProvider::Mount(vfs, directory, filename); +} + +int mj_addBufferVFS(mjVFS* vfs, const char* name, const void* buffer, + int nbuffer) { + // Copies the buffer into the BufferProvider and mounts it at the given path. + return BufferProvider::Mount(vfs, name, buffer, nbuffer); +} + +int mj_deleteFileVFS(mjVFS* vfs, const char* filename) { + if (filename == nullptr) { + return mujoco::user::VFS::kNotFound; + } + + if (mj_unmountVFS(vfs, filename) != 0) { + mujoco::user::FilePath path(filename); + return mj_unmountVFS(vfs, path.StripPath().Lower().c_str()); + } + return mujoco::user::VFS::kSuccess; } diff --git a/src/user/user_vfs.h b/src/user/user_vfs.h index 416de6f7..b8d8013f 100644 --- a/src/user/user_vfs.h +++ b/src/user/user_vfs.h @@ -17,38 +17,115 @@ #ifndef MUJOCO_SRC_USER_USER_VFS_H_ #define MUJOCO_SRC_USER_USER_VFS_H_ -#include +#include +#include +#include +#include +#include +#include -#include #include -#include +#include +#include "user/user_util.h" -#ifdef __cplusplus -extern "C" { -#endif +namespace mujoco::user { -// Initialize an empty VFS, mj_deleteVFS must be called to deallocate the VFS -MJAPI void mj_defaultVFS(mjVFS* vfs); +// Underlying Virtual File System implementation for opaque mjVFS struct. +// +// This class owns and manages all the mjResource instances that are created +// using the mju_openResource. Its main job is to find the correct +// mjpResourceProvider to handle the mju open/read/close operations. It does +// this by "mounting" mjpResourceProviders at specific paths such that any +// operation within that path will be handled by the mjpResourceProvider. +// +// Mounting can be done explicitly (using mj_mountVFS) or implicitly (using +// mjp_registerResourceProvider). If no provider is found for a given path, +// then a "default" provider is used that uses normal C file operations to +// open/read/close files. +// +// To support legacy use-cases (where the VFS is an optional argument), a +// "self-destruct" mode can be configured so that a temporary VFS instance can +// be created with a lifetime tied to the opened mjResource instance. +// +// This class itself is thread-safe, but it makes no guarantees about the +// thread-safety of the underlying mjResourceProviders. +class VFS { + public: + explicit VFS(mjVFS* vfs); + ~VFS(); -// add file to VFS, return 0: success, 2: repeated name, -1: not found on disk -MJAPI int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename); + VFS(const VFS&) = delete; + VFS& operator=(const VFS&) = delete; -// add file from buffer into VFS, return 0: success, 2: repeated name, -1: failed to load -MJAPI int mj_addBufferVFS(mjVFS* vfs, const char* filename, const void* buffer, int nbuffer); + // Status codes for VFS operations. Values are based on the mj_VFS APIs. + enum Status { + kSuccess = 0, + kFailedToLoad = -1, + kFailedToRead = -1, + kNotFound = -1, + kRepeatedName = 2, + kInvalidVfs = -1, + kInvalidResource = -1, + kInvalidResourceProvider = -1, + }; -// return file index in VFS, or -1 if not found in VFS -MJAPI int mj_findFileVFS(const mjVFS* vfs, const char* filename); + // Opens a mjResource for the given path, or nullptr on error. If successful, + // will invoke the 'open' callback for the mjpResourceProvider associated with + // the path/dir. + mjResource* Open(const char* dir, const char* name); -// delete file from VFS, return 0: success, -1: not found in VFS -MJAPI int mj_deleteFileVFS(mjVFS* vfs, const char* filename); + // Sets `buffer` to the contents of the resource and returns the number of + // bytes of the content. This is done by invoking the 'read' callback for the + // mjpResourceProvider associated with the resource. Returns -1 on error. + int Read(mjResource* resource, const void** buffer); -// delete all files from VFS -MJAPI void mj_deleteVFS(mjVFS* vfs); + // Closes the resource by invoking the 'close' callback for the + // mjpResourceProvider associated with the resource. + Status Close(mjResource* resource); -#ifdef __cplusplus -} -#endif + // Mounts a ResourceProvider at the given path. All subsequent operations + // under `path` will be delegated to the `provider` until it is unmounted. + Status Mount(const FilePath& path, const mjpResourceProvider* provider); -const mjpResourceProvider* GetVfsResourceProvider(); + // Unmounts the ResourceProvider from the given path. + Status Unmount(const FilePath& path); + + // Sets a destructor to be called when the VFS has no more open resources. + // Assumes that `destructor` will delete `this`. + // + // This is useful for when you want to create a temporary VFS instance with + // a lifetime tied to a single mjResource to be opened. The `destructor` + // should be set to `delete this` and any other cleanup that needs to happen. + void SetToSelfDestruct(std::function destructor); + + // Converts the public C-API pointer to the internal C++ class. + static VFS* Upcast(mjVFS* vfs); + static const VFS* Upcast(const mjVFS* vfs); + + private: + using ResourcePtr = std::unique_ptr; + ResourcePtr CreateResource(std::string_view name, + const mjpResourceProvider* provider); + + // Returns a mounted mjResource* that matches the given path. If no explicitly + // mounted mjResource* is found, returns a "default" mounting that uses the + // C file system. + mjResource* FindMount(const std::string& fullpath); + + // Invokes the `destructor_, but only if it has been set previously. This + // should be only called when resources_ is empty and callers should assume + // that `this` will be invalidated after this call. + void MaybeSelfDestruct(); + + mjVFS* self_; + std::mutex mutex_; // Protects open_resources_ and mounts_. + std::unordered_map open_resources_; + std::unordered_map mounts_; + mjResource default_mount_; + mjpResourceProvider default_provider_; + std::function destructor_; +}; + +} // namespace mujoco::user #endif // MUJOCO_SRC_USER_USER_VFS_H_ diff --git a/test/user/user_resource_test.cc b/test/user/user_resource_test.cc index b783761f..4017c0ae 100644 --- a/test/user/user_resource_test.cc +++ b/test/user/user_resource_test.cc @@ -301,7 +301,7 @@ TEST_F(ResourceTest, GeneralFailureTest) { error.data(), error.size()); ASSERT_THAT(resource, IsNull()); - EXPECT_THAT(error.data(), HasSubstr("could not open")); + EXPECT_THAT(error.data(), HasSubstr("Error opening file")); } TEST_F(ResourceTest, NameWithValidPrefix) { diff --git a/test/user/user_vfs_test.cc b/test/user/user_vfs_test.cc index 12ca78f2..84d750c3 100644 --- a/test/user/user_vfs_test.cc +++ b/test/user/user_vfs_test.cc @@ -26,6 +26,7 @@ namespace mujoco { namespace { +using ::testing::IsNull; using ::testing::NotNull; using UserVfsTest = MujocoTest; @@ -36,6 +37,46 @@ static bool HasFile(const mjVFS* vfs, const std::string& filename) { return result; } +struct TestProvider : public mjpResourceProvider { + // The TestProvider will increment the value at `addr` by these amounts when + // each callback is invoked. This can be used to verity that the correct + // callbacks are being invoked when expected. + enum CallbackValues { + Mounted = 100, + Unmounted = 200, + Opened = 300, + Read = 400, + Closed = 500, + }; + + explicit TestProvider(int* addr) { + mjp_defaultResourceProvider(this); + prefix = "test"; + data = addr; + + mount = [](mjResource* res) { + *(int*)res->provider->data += Mounted; + return 1; + }; + unmount = [](mjResource* res) { + *(int*)res->provider->data += Unmounted; + return 1; + }; + open = [](mjResource* res) { + *(int*)res->provider->data += Opened; + return 1; + }; + read = [](mjResource* res, const void** out) { + *(int*)res->provider->data += Read; + *out = res->provider->data; + return (int)sizeof(int); + }; + close = [](mjResource* res) { + *(int*)res->provider->data += Closed; + }; + } +}; + TEST_F(UserVfsTest, AddFile) { constexpr char path[] = "engine/testdata/actuation/"; const std::string dir = GetTestDataFilePath(path); @@ -245,5 +286,126 @@ TEST_F(UserVfsTest, Timestamps) { mj_deleteVFS(&vfs); } +TEST_F(UserVfsTest, MountUnmount) { + int test = 0; + int expect = 0; + TestProvider provider(&test); + + mjVFS vfs; + mj_defaultVFS(&vfs); + mjResource* res = mju_openResource("", "/some/path/foo", &vfs, nullptr, 0); + EXPECT_THAT(res, IsNull()); + EXPECT_EQ(test, expect); + + mj_mountVFS(&vfs, "/some/path", &provider); + expect += TestProvider::Mounted; + EXPECT_EQ(test, expect); + + res = mju_openResource("", "/some/path/foo", &vfs, nullptr, 0); + expect += TestProvider::Opened; + EXPECT_THAT(res, NotNull()); + EXPECT_EQ(test, expect); + + const void* buffer = nullptr; + const int size = mju_readResource(res, &buffer); + expect += TestProvider::Read; + EXPECT_THAT(buffer, NotNull()); + EXPECT_GT(size, 0); + EXPECT_EQ(test, expect); + + mju_closeResource(res); + expect += TestProvider::Closed; + EXPECT_EQ(test, expect); + + mj_unmountVFS(&vfs, "/some/path"); + expect += TestProvider::Unmounted; + EXPECT_EQ(test, expect); + + mj_deleteVFS(&vfs); +} + +TEST_F(UserVfsTest, AutoMountProviders) { + int test = 0; + int expect = 0; + + TestProvider provider(&test); + mjp_registerResourceProvider(&provider); + + mjVFS vfs; + mj_defaultVFS(&vfs); + + mjResource* res = mju_openResource("", "test:foo", &vfs, nullptr, 0); + expect += TestProvider::Mounted + TestProvider::Opened; + EXPECT_THAT(res, NotNull()); + + mju_closeResource(res); + expect += TestProvider::Closed; + EXPECT_EQ(test, expect); + + mj_deleteVFS(&vfs); + expect += TestProvider::Unmounted; + EXPECT_EQ(test, expect); +} + +TEST_F(UserVfsTest, StackedMounts) { + int test1 = 0; + int test2 = 1000; + int test3 = 1000000; + int expect1 = test1; + int expect2 = test2; + int expect3 = test3; + + TestProvider provider1(&test1); + TestProvider provider2(&test2); + TestProvider provider3(&test3); + + mjVFS vfs; + mj_defaultVFS(&vfs); + + mj_mountVFS(&vfs, "/some/path", &provider1); + mj_mountVFS(&vfs, "/some/path/further/down/very/deep", &provider2); + mj_mountVFS(&vfs, "/some/path/further/down", &provider3); + expect1 += TestProvider::Mounted; + expect2 += TestProvider::Mounted; + expect3 += TestProvider::Mounted; + EXPECT_EQ(test1, expect1); + EXPECT_EQ(test2, expect2); + EXPECT_EQ(test3, expect3); + + mjResource* res1 = mju_openResource("", "/some/path/foo", &vfs, nullptr, 0); + expect1 += TestProvider::Opened; + EXPECT_EQ(test1, expect1); + EXPECT_EQ(test2, expect2); + EXPECT_EQ(test3, expect3); + + mjResource* res2 = + mju_openResource("", "/some/path/further/down/foo", &vfs, nullptr, 0); + expect3 += TestProvider::Opened; + EXPECT_EQ(test1, expect1); + EXPECT_EQ(test2, expect2); + EXPECT_EQ(test3, expect3); + + mjResource* res3 = mju_openResource( + "", "/some/path/further/down/very/deep/foo", &vfs, nullptr, 0); + expect2 += TestProvider::Opened; + EXPECT_EQ(test1, expect1); + EXPECT_EQ(test2, expect2); + EXPECT_EQ(test3, expect3); + + mju_closeResource(res1); + mju_closeResource(res2); + mju_closeResource(res3); + expect1 += TestProvider::Closed; + expect2 += TestProvider::Closed; + expect3 += TestProvider::Closed; + + mj_deleteVFS(&vfs); + expect1 += TestProvider::Unmounted; + expect2 += TestProvider::Unmounted; + expect3 += TestProvider::Unmounted; + EXPECT_EQ(test1, expect1); + EXPECT_EQ(test2, expect2); + EXPECT_EQ(test3, expect3); +} } // namespace } // namespace mujoco diff --git a/test/xml/xml_api_test.cc b/test/xml/xml_api_test.cc index c985d3d0..4207dd16 100644 --- a/test/xml/xml_api_test.cc +++ b/test/xml/xml_api_test.cc @@ -126,7 +126,7 @@ TEST_F(LoadXmlTest, InvalidFileFails) { std::array error; mjSpec* spec = mj_parseXML("invalid", nullptr, error.data(), error.size()); EXPECT_THAT(spec, IsNull()) << "Expected model loading to fail."; - EXPECT_THAT(error.data(), HasSubstr("No such file or directory")); + EXPECT_THAT(error.data(), HasSubstr("Error opening file")); } TEST_F(MujocoTest, SaveXmlShortString) {