chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a new plugin / extension mechanism called a resource provider along with retrofitting VFS on top of it.

Browse Source 
A resource provider provides a mechanism for MuJoCo to read from filesystems other than the OS filesystem or the Virtual File System (VFS).

PiperOrigin-RevId: 525394983
Change-Id: I077ff5a7e2e76806b48b6defb531280aadc8b169
This commit is contained in:
Kyle Bayes
2023-04-19 03:01:33 -07:00
committed by Copybara-Service
parent b25728cc2e
commit fe3dccfd1d
27 changed files with 1467 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine/CMakeLists.txt
+2 -2
View File
@@ -33,8 +33,6 @@ set(MUJOCO_ENGINE_SRCS
engine_derivative.h
engine_derivative_fd.c
engine_derivative_fd.h
engine_file.c
engine_file.h
engine_forward.c
engine_forward.h
engine_inverse.c
@@ -50,6 +48,8 @@ set(MUJOCO_ENGINE_SRCS
engine_print.h
engine_ray.c
engine_ray.h
engine_resource.c
engine_resource.h
engine_sensor.c
engine_sensor.h
engine_setconst.c
src/engine/engine_file.c
-80
View File
@@ -1,80 +0,0 @@
// Copyright 2022 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "engine/engine_file.h"
#include <stdio.h>
#include <limits.h>
#include "engine/engine_util_errmem.h"
void* mju_fileToMemory(const char* filename, int* filesize) {
// open file
*filesize = 0;
FILE* fp = fopen(filename, "rb");
if (!fp) {
return NULL;
}
// find size
if (fseek(fp, 0, SEEK_END) != 0) {
fclose(fp);
mju_warning("Failed to calculate size for '%s'", filename);
return NULL;
}
// ensure file size fits in int
long long_filesize = ftell(fp); // NOLINT(runtime/int)
if (long_filesize > INT_MAX) {
fclose(fp);
mju_warning("File size over 2GB is not supported. File: '%s'", filename);
return NULL;
} else if (long_filesize < 0) {
fclose(fp);
mju_warning("Failed to calculate size for '%s'", filename);
return NULL;
}
*filesize = long_filesize;
// go back to start of file
if (fseek(fp, 0, SEEK_SET) != 0) {
fclose(fp);
mju_warning("Read error while reading '%s'", filename);
return NULL;
}
// allocate and read
void* buffer = mju_malloc(*filesize);
if (!buffer) {
mju_error("mjFileToMemory: could not allocate memory");
}
size_t bytes_read = fread(buffer, 1, *filesize, fp);
// check that read data matches file size
if (bytes_read != *filesize) { // SHOULD NOT OCCUR
if (ferror(fp)) {
fclose(fp);
mju_free(buffer);
*filesize = 0;
mju_warning("Read error while reading '%s'", filename);
return NULL;
} else if (feof(fp)) {
*filesize = bytes_read;
}
}
// close file, return contents
fclose(fp);
return buffer;
}
src/engine/engine_io.c
+67 -105
View File
@@ -25,6 +25,7 @@
#include <mujoco/mjplugin.h>
#include <mujoco/mjxmacro.h>
#include "engine/engine_array_safety.h"
#include "engine/engine_resource.h"
#include "engine/engine_macro.h"
#include "engine/engine_plugin.h"
#include "engine/engine_util_blas.h"
@@ -620,82 +621,63 @@ void mj_saveModel(const mjModel* m, const char* filename, void* buffer, int buff
// load model from binary MJB file
// if vfs is not NULL, look up file in vfs before reading from disk
mjModel* mj_loadModel(const char* filename, const mjVFS* vfs) {
// load model from binary MJB resource
static mjModel* _mj_loadModel(const char* filename, int default_provider) {
int header[4] = {0};
int expected_header[4] = {ID, sizeof(mjtNum), getnint(), getnptr()};
int info[2000];
int ptrbuf = 0;
mjModel *m = 0;
FILE* fp = 0;
mjResource* r = NULL;
if((r = mju_openResource(filename, default_provider)) == NULL) {
return NULL;
}
// find file in VFS if given
const void* buffer = NULL;
int buffer_sz = 0;
if (vfs) {
int i = mj_findFileVFS(vfs, filename);
if (i>=0) {
buffer_sz = vfs->filesize[i];
buffer = vfs->filedata[i];
}
int buffer_sz = mju_readResource(r, &buffer);
if (buffer_sz <= 0) {
mju_closeResource(r);
return NULL;
}
// open file for reading if no buffer
if (!buffer) {
fp = fopen(filename, "rb");
if (!fp) {
mju_warning("Could not open file '%s'", filename);
return 0;
}
if (buffer_sz < 4*sizeof(int)) {
mju_warning("Model file has an incomplete header");
mju_closeResource(r);
return NULL;
}
// read header
if (fp) {
if (fread(header, 4, sizeof(int), fp) != 4) {
mju_warning("Model file has an incomplete header");
return 0;
}
} else {
bufread(header, 4*sizeof(int), buffer_sz, buffer, &ptrbuf);
}
bufread(header, 4*sizeof(int), buffer_sz, buffer, &ptrbuf);
// check header
for (int i=0; i<4; i++) {
if (header[i]!=expected_header[i]) {
if (fp) {
fclose(fp);
}
switch (i) {
case 0:
mju_warning("Model missing header ID");
return 0;
mju_closeResource(r);
return NULL;
case 1:
mju_warning("Model and executable have different floating point precision");
return 0;
mju_closeResource(r);
return NULL;
case 2:
mju_warning("Model and executable have different number of ints in mjModel");
return 0;
mju_closeResource(r);
return NULL;
default:
mju_warning("Model and executable have different number of pointers in mjModel");
return 0;
mju_closeResource(r);
return NULL;
}
}
}
// read mjModel structure: info only
if (fp) {
if (fread(info, sizeof(int), getnint(), fp) != getnint()) {
mju_warning("Model file does not contain enough ints");
return 0;
}
} else {
bufread(info, sizeof(int)*getnint(), buffer_sz, buffer, &ptrbuf);
}
bufread(info, sizeof(int)*getnint(), buffer_sz, buffer, &ptrbuf);
// allocate new mjModel, check sizes
m = mj_makeModel(info[0], info[1], info[2], info[3], info[4], info[5], info[6],
@@ -707,90 +689,70 @@ mjModel* mj_loadModel(const char* filename, const mjVFS* vfs) {
info[42], info[43], info[44], info[45], info[46], info[47], info[48],
info[49], info[50], info[51], info[52]);
if (!m || m->nbuffer!=info[getnint()-1]) {
if (fp) {
fclose(fp);
}
mju_closeResource(r);
mju_warning("Corrupted model, wrong size parameters");
mj_deleteModel(m);
return 0;
return NULL;
}
// set info fields
memcpy(m, info, sizeof(int)*getnint());
// read options and buffer
if (fp) {
if (fread((void*)&m->opt, sizeof(mjOption), 1, fp) != 1) {
mju_warning("Model file does not have a complete mjOption");
mj_deleteModel(m);
return 0;
}
if (fread((void*)&m->vis, sizeof(mjVisual), 1, fp) != 1) {
mju_warning("Model file does not have a complete mjVisual");
mj_deleteModel(m);
return 0;
}
if (fread((void*)&m->stat, sizeof(mjStatistic), 1, fp) != 1) {
mju_warning("Model file does not have a complete mjStatistic");
mj_deleteModel(m);
return 0;
}
{
MJMODEL_POINTERS_PREAMBLE(m)
#define X(type, name, nr, nc) \
if (fread(m->name, sizeof(type), (m->nr)*(nc), fp) != (m->nr)*(nc)) { \
mju_warning("Model file does not contain a large enough buffer"); \
mj_deleteModel(m); \
return 0; \
}
MJMODEL_POINTERS
#undef X
}
} else {
bufread((void*)&m->opt, sizeof(mjOption), buffer_sz, buffer, &ptrbuf);
bufread((void*)&m->vis, sizeof(mjVisual), buffer_sz, buffer, &ptrbuf);
bufread((void*)&m->stat, sizeof(mjStatistic), buffer_sz, buffer, &ptrbuf);
{
MJMODEL_POINTERS_PREAMBLE(m)
#define X(type, name, nr, nc) \
bufread(m->name, sizeof(type)*(m->nr)*(nc), buffer_sz, buffer, &ptrbuf);
MJMODEL_POINTERS
#undef X
}
bufread((void*)&m->opt, sizeof(mjOption), buffer_sz, buffer, &ptrbuf);
bufread((void*)&m->vis, sizeof(mjVisual), buffer_sz, buffer, &ptrbuf);
bufread((void*)&m->stat, sizeof(mjStatistic), buffer_sz, buffer, &ptrbuf);
{
MJMODEL_POINTERS_PREAMBLE(m)
#define X(type, name, nr, nc) \
bufread(m->name, sizeof(type)*(m->nr)*(nc), buffer_sz, buffer, &ptrbuf);
MJMODEL_POINTERS
#undef X
}
// make sure file size is correct
if (fp) {
if (feof(fp)) {
fclose(fp);
mju_warning("Model file is too small");
mj_deleteModel(m);
return 0;
}
char dummy;
if (fread(&dummy, 1, 1, fp) || !feof(fp)) {
fclose(fp);
mju_warning("Model file is too large");
mj_deleteModel(m);
return 0;
}
// make sure buffer is the correct size
if (ptrbuf != buffer_sz) {
mju_closeResource(r);
mju_warning("Model file is too large");
mj_deleteModel(m);
return NULL;
}
const char* validationError = mj_validateReferences(m);
if (validationError) {
mju_closeResource(r);
mju_warning("%s", validationError);
mj_deleteModel(m);
return 0;
return NULL;
}
if (fp) {
fclose(fp);
}
mju_closeResource(r);
return m;
}
// load model from binary MJB file
// if vfs is not NULL, look up file in vfs before reading from disk
mjModel* mj_loadModel(const char* filename, const mjVFS* vfs) {
if (vfs == NULL) {
return _mj_loadModel(filename, 0);
}
int index = mj_registerVfsProvider(vfs);
if (index < 1) {
mju_error("mj_loadModel: could not allocate memory");
return NULL;
}
mjModel* model = _mj_loadModel(filename, index);
mjp_unregisterResourceProvider(index);
return model;
}
// de-allocate mjModel
void mj_deleteModel(mjModel* m) {
if (m) {
src/engine/engine_plugin.cc
+330 -55
View File
@@ -28,6 +28,7 @@
#include <mutex>
#include <new>
#include <shared_mutex>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
@@ -62,28 +63,33 @@ constexpr int kMaxAttributes = 255;
constexpr int kCacheLine = 64;
// vfs prefix
constexpr const char* kVfsPrefix = mjVFS_PREFIX;
// A table of registered plugins, implemented as a linked list of array "blocks".
// This is a compromise that maintains a good degree of memory locality while not invalidating
// existing pointers when growing the table. It is expected that for most users, the number of
// plugins loaded into a program will be small enough to fit in the initial block, and so the global
// table will behave like an array. Since pointers are never invalidated, we do not need to apply a
// read lock on the global table when resolving a plugin.
template<typename T>
struct alignas(kCacheLine) PluginTable {
static constexpr int kBlockSize = 15;
PluginTable() {
for (int i = 0; i < kBlockSize; ++i) {
mjp_defaultPlugin(&plugins[i]);
std::memset(&plugins[i], 0, sizeof(plugins[i]));
}
}
mjpPlugin plugins[kBlockSize];
PluginTable* next = nullptr;
T plugins[kBlockSize];
PluginTable<T>* next = nullptr;
};
static_assert(
sizeof(PluginTable) / kCacheLine ==
sizeof(PluginTable::plugins) / kCacheLine + (sizeof(PluginTable::plugins) % kCacheLine > 0),
sizeof(PluginTable<mjpPlugin>) / kCacheLine ==
sizeof(PluginTable<mjpPlugin>::plugins) / kCacheLine
+ (sizeof(PluginTable<mjpPlugin>::plugins) % kCacheLine > 0),
"PluginTable::next doesn't fit in the same cache line as the end of PluginTable::plugins");
using Mutex = std::shared_mutex;
@@ -112,12 +118,13 @@ class ReentrantWriteLock {
}
};
template<typename T>
class Global {
public:
Global() {
new(mutex_) Mutex;
}
PluginTable& table() {
PluginTable<T>& table() {
return table_;
}
std::atomic_int& count() {
@@ -132,7 +139,7 @@ class Global {
}
private:
PluginTable table_;
PluginTable<T> table_;
std::atomic_int count_;
// A mutex whose destructor is never run.
@@ -143,14 +150,60 @@ class Global {
alignas(Mutex) unsigned char mutex_[sizeof(Mutex)];
};
Global& GetGlobal() {
static Global global;
template<typename T>
Global<T>& GetGlobal() {
static Global<T> global;
static_assert(std::is_trivially_destructible_v<decltype(global)>);
return global;
}
// allocate new block for the global table
template<typename T>
PluginTable<T>* AddNewTableBlock(PluginTable<T>* table) {
char err[512];
err[0] = '\0';
#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14
// aligned nothrow new is not available until macOS 10.14
posix_memalign(reinterpret_cast<void**>(&table->next),
alignof(PluginTable<T>), sizeof(PluginTable<T>));
if (table->next) new(table->next) PluginTable<T>;
#else
table->next = new(std::nothrow) PluginTable<T>;
#endif
if (!table->next) {
std::snprintf(err, sizeof(err), "failed to allocate memory for the global plugin table");
return nullptr;
}
return table->next;
}
// look up a plugin by slot number
template<typename T>
const T* GetAtSlot(int slot, int nslot) {
if (slot < 0 || slot >= nslot) {
return nullptr;
}
Global<T>& global = GetGlobal<T>();
PluginTable<T>* table = &global.table();
// iterate over blocks in the global table until the local index is less than the block size
int local_idx = slot;
while (local_idx >= PluginTable<T>::kBlockSize) {
local_idx -= PluginTable<T>::kBlockSize;
table = table->next;
if (!table) {
return nullptr;
}
}
// local_idx is now a valid index into the current block
return &(table->plugins[local_idx]);
}
// return the length of a null-terminated string, or -1 if it is not terminated after kMaxNameLength
int strnlen(const char* s) {
int strklen(const char* s) {
for (int i = 0; i < kMaxNameLength; ++i) {
if (!s[i]) {
return i;
@@ -161,7 +214,7 @@ int strnlen(const char* s) {
// copy a null-terminated string into a new heap-allocated char array managed by a unique_ptr
std::unique_ptr<char[]> CopyName(const char* s) {
int len = strnlen(s);
int len = strklen(s);
if (len == -1) {
return nullptr;
}
@@ -212,6 +265,15 @@ bool PluginsAreIdentical(const mjpPlugin& plugin1, const mjpPlugin& plugin2) {
sizeof(mjpPlugin) - (ptr1 - reinterpret_cast<const char*>(&plugin1));
return !std::memcmp(ptr1, ptr2, remaining_size);
}
// check if two resource providers are identical
bool ResourceProvidersAreIdentical(const mjpResourceProvider* p1, const mjpResourceProvider* p2) {
return (!std::strcmp(p1->prefix, p2->prefix) &&
p1->open == p2->open &&
p1->read == p2->read &&
p1->close == p2->close &&
p1->data == p2->data);
}
} // namespace
// globally register a plugin (thread-safe), return new slot id
@@ -237,7 +299,7 @@ int mjp_registerPlugin(const mjpPlugin* plugin) {
// check and copy the plugin name
std::unique_ptr<char[]> name = CopyName(plugin->name);
if (!name) {
if (strnlen(plugin->name) == -1) {
if (strklen(plugin->name) == -1) {
std::snprintf(err, sizeof(err),
"plugin->name length exceeds the maximum limit of %d", kMaxNameLength);
} else {
@@ -253,7 +315,7 @@ int mjp_registerPlugin(const mjpPlugin* plugin) {
for (int i = 0; i < plugin->nattribute; ++i) {
std::unique_ptr<char[]> attr = CopyName(plugin->attributes[i]);
if (!attr) {
if (strnlen(plugin->attributes[i]) == -1) {
if (strklen(plugin->attributes[i]) == -1) {
std::snprintf(
err, sizeof(err),
"plugin->attributes[%d] exceeds the maximum limit of %d", i, kMaxAttributes);
@@ -266,16 +328,16 @@ int mjp_registerPlugin(const mjpPlugin* plugin) {
}
}
Global& global = GetGlobal();
Global<mjpPlugin>& global = GetGlobal<mjpPlugin>();
auto lock = global.lock_mutex_exclusively();
int count = global.count().load(std::memory_order_acquire);
int local_idx = 0;
PluginTable* table = &global.table();
PluginTable<mjpPlugin>* table = &global.table();
// check if a non-identical plugin with the same name has already been registered
for (int i = 0; i < count; ++i, ++local_idx) {
if (local_idx == PluginTable::kBlockSize) {
if (local_idx == PluginTable<mjpPlugin>::kBlockSize) {
local_idx = 0;
table = table->next;
}
@@ -291,21 +353,12 @@ int mjp_registerPlugin(const mjpPlugin* plugin) {
}
// allocate a new block of PluginTable if the last allocated block is full
if (local_idx == PluginTable::kBlockSize) {
if (local_idx == PluginTable<mjpPlugin>::kBlockSize) {
local_idx = 0;
#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14
// aligned nothrow new is not available until macOS 10.14
posix_memalign(reinterpret_cast<void**>(&table->next),
alignof(PluginTable), sizeof(PluginTable));
if (table->next) new(table->next) PluginTable;
#else
table->next = new(std::nothrow) PluginTable;
#endif
if (!table->next) {
std::snprintf(err, sizeof(err), "failed to allocate memory for the global plugin table");
table = AddNewTableBlock<mjpPlugin>(table);
if (!table) {
return -1;
}
table = table->next;
}
// release the attribute names from unique_ptr into a plain array
@@ -347,29 +400,11 @@ int mjp_registerPlugin(const mjpPlugin* plugin) {
// look up plugin by slot number, assuming that mjp_pluginCount has already been called
const mjpPlugin* mjp_getPluginAtSlotUnsafe(int slot, int nslot) {
if (slot < 0 || slot >= nslot) {
const mjpPlugin* plugin = GetAtSlot<mjpPlugin>(slot, nslot);
if (!plugin || !plugin->name) {
return nullptr;
}
Global& global = GetGlobal();
PluginTable* table = &global.table();
// iterate over blocks in the global table until the local index is less the block size
int local_idx = slot;
while (local_idx >= PluginTable::kBlockSize) {
local_idx -= PluginTable::kBlockSize;
table = table->next;
if (!table) {
return nullptr;
}
}
// local_idx is now a valid index into the current block
const mjpPlugin& plugin = table->plugins[local_idx];
if (!plugin.name) {
return nullptr;
}
return &plugin;
return plugin;
}
// look up plugin by name, assuming that mjp_pluginCount has already been called
@@ -380,12 +415,12 @@ const mjpPlugin* mjp_getPluginUnsafe(const char* name, int* slot, int nslot) {
return nullptr;
}
Global& plugins = GetGlobal();
PluginTable* table = &plugins.table();
Global<mjpPlugin>& plugins = GetGlobal<mjpPlugin>();
PluginTable<mjpPlugin>* table = &plugins.table();
int found_slot = 0;
while (table) {
for (int i = 0;
i < PluginTable::kBlockSize && found_slot < nslot;
i < PluginTable<mjpPlugin>::kBlockSize && found_slot < nslot;
++i, ++found_slot) {
const mjpPlugin& plugin = table->plugins[i];
@@ -407,10 +442,9 @@ const mjpPlugin* mjp_getPluginUnsafe(const char* name, int* slot, int nslot) {
// return the number of globally registered plugins
int mjp_pluginCount() {
return GetGlobal().count().load(std::memory_order_acquire);
return GetGlobal<mjpPlugin>().count().load(std::memory_order_acquire);
}
// look up a plugin by slot number
const mjpPlugin* mjp_getPluginAtSlot(int slot) {
const int count = mjp_pluginCount();
@@ -466,6 +500,247 @@ const char* mj_getPluginConfig(const mjModel* m, int plugin_id, const char* attr
return nullptr;
}
// set default resource provider definition
void mjp_defaultResourceProvider(mjpResourceProvider* provider) {
std::memset(provider, 0, sizeof(*provider));
}
// globally register a resource provider (thread-safe), return new slot id
int mjp_registerResourceProvider(const mjpResourceProvider* provider) {
// check against reserved prefixes
int n = std::strlen(provider->prefix),
m = std::strlen(kVfsPrefix);
// one of the prefixes is a subprefix of the other
if (!std::strncmp(kVfsPrefix, provider->prefix, n) ||
!std::strncmp(kVfsPrefix, provider->prefix, m)) {
mju_warning("provider->prefix is '%s' which is reserved", provider->prefix);
return -1;
}
return mjp_registerResourceProviderInternal(provider);
}
// internal version of mjp_registerResourceProvider without prechecks on reserved prefixes
int mjp_registerResourceProviderInternal(const mjpResourceProvider* provider) {
if (!provider->prefix || provider->prefix[0] == '\0') {
mju_warning("provider->prefix is an empty string");
return -1;
}
if (!provider->open || !provider->read || !provider->close) {
mju_warning("provider must have the open, read, and close callbacks defined");
return -1;
}
char err[512];
err[0] = '\0';
// ========= ATTENTION! ==========================================================================
// Do not handle objects with nontrivial destructors outside of this lambda.
// Do not call mju_error inside this lambda.
int slot = [&]() -> int {
bool vfs_provider = false;
std::unique_ptr<char[]> prefix;
// check if this is a VFS provider
if (!std::strcmp(mjVFS_PREFIX, provider->prefix)) {
vfs_provider = true;
}
// copy prefix
if (!vfs_provider) {
prefix = CopyName(provider->prefix);
if (!prefix) {
if (strklen(provider->prefix) == -1) {
std::snprintf(err, sizeof(err),
"provider->prefix length exceeds the maximum limit of %d", kMaxNameLength);
} else {
std::snprintf(err, sizeof(err), "failed to allocate memory for resource provider prefix");
}
return -1;
}
}
Global<mjpResourceProvider>& global = GetGlobal<mjpResourceProvider>();
auto lock = global.lock_mutex_exclusively();
int count = global.count().load(std::memory_order_acquire);
int local_idx = 0;
int free_idx = -1, free_local_idx = -1;
PluginTable<mjpResourceProvider>* table = &global.table();
PluginTable<mjpResourceProvider>* free_table = nullptr;
// check if a non-identical provider with the same name has already been registered
for (int i = 0; i < count; ++i, ++local_idx) {
if (local_idx == PluginTable<mjpResourceProvider>::kBlockSize) {
local_idx = 0;
table = table->next;
}
mjpResourceProvider& existing = table->plugins[local_idx];
// VFS providers can safely go in open slots
if (vfs_provider && existing.prefix == nullptr && free_idx == -1) {
free_table = table;
free_idx = i;
free_local_idx = local_idx;
// can skip the rest
break;
}
if (!vfs_provider) {
int n = std::strlen(provider->prefix);
int m = std::strlen(existing.prefix);
// one of the prefixes is a subprefix of the other
if (!std::strncmp(existing.prefix, provider->prefix, n) ||
!std::strncmp(existing.prefix, provider->prefix, m)) {
// if identical then return slot number
if (ResourceProvidersAreIdentical(provider, &existing)) {
return i;
} else {
std::snprintf(err, sizeof(err),
"a resource provider with prefix '%s' cannot be registered",
provider->prefix);
return -1;
}
}
}
}
// allocate a new block of PluginTable if the last allocated block is full
if (free_local_idx == -1 && local_idx == PluginTable<mjpResourceProvider>::kBlockSize) {
local_idx = 0;
table = AddNewTableBlock<mjpResourceProvider>(table);
if (!table) {
return -1;
}
}
// all checks passed, register the plugin into the global table
mjpResourceProvider& registered_provider = table->plugins[local_idx];
if (free_local_idx != -1) {
registered_provider = free_table->plugins[free_local_idx];
}
registered_provider = *provider;
registered_provider.prefix = (!vfs_provider) ? prefix.release() : kVfsPrefix;
// increment the global plugin count with a release memory barrier
if (free_idx == -1) {
free_idx = count;
global.count().store(count + 1, std::memory_order_release);
}
return free_idx;
}();
// ========= ATTENTION! ==========================================================================
// End of safe lambda, do not handle objects with non-trivial destructors beyond this point.
// plugin registration failed, throw a warning
if (slot < 0) {
err[sizeof(err) - 1] = '\0';
mju_warning("%s", err);
}
return slot+1;
}
// globally unregister resource provider (thread-safe)
// only used for VFS resource providers
void mjp_unregisterResourceProvider(int slot) {
// shift slot to zero-index
slot--;
if (slot < 0) {
return;
}
// get global table, acquire lock
Global<mjpResourceProvider>& global = GetGlobal<mjpResourceProvider>();
auto lock = global.lock_mutex_exclusively();
int count = global.count().load(std::memory_order_acquire);
if (slot >= count) {
return;
}
PluginTable<mjpResourceProvider>* table = &global.table();
// iterate over blocks in the global table until the local index is less than the block size
int local_idx = slot;
while (local_idx >= PluginTable<mjpResourceProvider>::kBlockSize) {
local_idx -= PluginTable<mjpResourceProvider>::kBlockSize;
table = table->next;
if (!table) {
return;
}
}
// local_idx is now a valid index into the current block
mjpResourceProvider& provider = table->plugins[local_idx];
// no-op for anything other than VFS resource providers
if (provider.prefix == kVfsPrefix) {
provider.prefix = nullptr;
}
}
// return the number of globally registered resource providers
int mjp_resourceProviderCount() {
return GetGlobal<mjpResourceProvider>().count().load(std::memory_order_acquire);
}
// look up a resource provider that matches its prefix against the given resource name
const mjpResourceProvider* mjp_getResourceProvider(const char* resource_name) {
const int count = mjp_resourceProviderCount();
if (!resource_name || !resource_name[0]) {
return nullptr;
}
// since multiple VFS resource providers can be registered with the same
// prefix, it doesn't make sense to try to match against them
if (!std::strncmp(kVfsPrefix, resource_name, std::strlen(kVfsPrefix))) {
return nullptr;
}
Global<mjpResourceProvider>& global = GetGlobal<mjpResourceProvider>();
PluginTable<mjpResourceProvider>* table = &global.table();
int found_slot = 0;
while (table) {
for (int i = 0;
i < PluginTable<mjpPlugin>::kBlockSize && found_slot < count;
++i, ++found_slot) {
const mjpResourceProvider& provider = table->plugins[i];
const char *prefix = provider.prefix;
if (prefix != nullptr &&
!std::strncmp(prefix, resource_name, std::strlen(prefix))) {
return &provider;
}
}
table = table->next;
}
return nullptr;
}
// look up a resource provider by slot number
const mjpResourceProvider* mjp_getResourceProviderAtSlot(int slot) {
// mjp_resourceProviderCount uses memory_order_acquire which acts as a barrier
// that guarantees that all providers up to `count` have been completely inserted
const int count = mjp_resourceProviderCount();
// shift slot to be zero-indexed
const mjpResourceProvider* provider = GetAtSlot<mjpResourceProvider>(slot - 1, count);
if (!provider || provider->prefix[0] == '\0') {
return nullptr;
}
return provider;
}
// load plugins from a dynamic library
void mj_loadPluginLibrary(const char* path) {
#if defined(_WIN32) || defined(__CYGWIN__)
@@ -483,7 +758,7 @@ void mj_loadAllPluginLibraries(const char* directory,
int nplugin_before;
int nplugin_after;
Global& global = GetGlobal();
Global<mjpPlugin>& global = GetGlobal<mjpPlugin>();
{
auto lock = global.lock_mutex_exclusively();
nplugin_before = mjp_pluginCount();
src/engine/engine_plugin.h
+23 -2
View File
@@ -28,15 +28,33 @@ MJAPI void mjp_defaultPlugin(mjpPlugin* plugin);
// globally register a plugin (thread-safe), return new slot id
MJAPI int mjp_registerPlugin(const mjpPlugin* plugin);
// globally register a resource provider (thread-safe), return new slot id
MJAPI int mjp_registerResourceProvider(const mjpResourceProvider* provider);
// globally unregister a resource provider (thread-safe)
MJAPI void mjp_unregisterResourceProvider(int slot);
// return the number of globally registered plugins
MJAPI int mjp_pluginCount();
// return the number of globally registered resource providers
MJAPI int mjp_resourceProviderCount();
// look up a plugin by name, optionally also get its registered slot number
MJAPI const mjpPlugin* mjp_getPlugin(const char* name, int* slot);
// set default resource provider definition
MJAPI void mjp_defaultResourceProvider(mjpResourceProvider* provider);
// look up a resource provider that matches its prefix against the given resource name
MJAPI const mjpResourceProvider* mjp_getResourceProvider(const char* resource_name);
// look up a plugin by slot number
MJAPI const mjpPlugin* mjp_getPluginAtSlot(int slot);
// look up a resource provider by slot number
MJAPI const mjpResourceProvider* mjp_getResourceProviderAtSlot(int slot);
// return a config attribute of a plugin instance
// NULL: invalid plugin instance ID or attribute name
MJAPI const char* mj_getPluginConfig(const mjModel* m, int plugin_id, const char* attrib);
@@ -51,10 +69,13 @@ MJAPI void mj_loadAllPluginLibraries(const char* directory, mjfPluginLibraryLoad
// MuJoCo-internal functions beyond this point.
// "Unsafe" suffix indicates that improper use of these functions may result in data races.
//
// The unsafe functions assume that called mjp_pluginCount has already been called, and that it is
// safe to assume that all plugins up to `count` have been completely written into the global table.
// The unsafe functions assume that mjp_pluginCount has already been called, and that all plugins
// up to `count` have been completely written into the global table.
// =================================================================================================
// internal version of mjp_registerResourceProvider without prechecks on reserved prefixes
MJAPI int mjp_registerResourceProviderInternal(const mjpResourceProvider* provider);
// look up a plugin by name, assuming that mjp_pluginCount has already been called
const mjpPlugin* mjp_getPluginUnsafe(const char* name, int* slot, int nslot);
src/engine/engine_resource.c
+215
View File
@@ -0,0 +1,215 @@
// Copyright 2022 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "engine/engine_resource.h"
#include <limits.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
#include "engine/engine_plugin.h"
#include "engine/engine_util_errmem.h"
// file buffer used internally for the OS filesystem
typedef struct {
void* buffer;
int nbuffer;
} file_buffer;
// open the given resource; if the name doesn't have a prefix matching with a
// resource provider, then the default_provider is used
// if default_provider non-positive, then the OS filesystem is used
mjResource* mju_openResource(const char* name, int default_provider) {
mjResource* resource = (mjResource*) mju_malloc(sizeof(mjResource));
const mjpResourceProvider* provider = NULL;
if (resource == NULL) {
mju_error("mju_openResource: could not allocate memory");
return NULL;
}
// copy name
resource->name = mju_malloc(sizeof(char) * (strlen(name) + 1));
if (resource->name == NULL) {
mju_free(resource);
mju_error("mju_openResource: could not allocate memory");
return NULL;
}
strcpy(resource->name, name);
// find provider based off prefix of name
provider = mjp_getResourceProvider(name);
if (provider != NULL) {
resource->read = provider->read;
resource->close = provider->close;
resource->provider_data = provider->data;
if (provider->open(resource)) {
return resource;
}
mju_warning("mju_openResource: could not open resource '%s", name);
mju_free(resource->name);
mju_free(resource);
return NULL;
}
// fallback to default provider
if (default_provider > 0) {
provider = mjp_getResourceProviderAtSlot(default_provider);
if (provider == NULL) {
mju_warning("mju_openResource: unknown resource provider");
mju_free(resource->name);
mju_free(resource);
return NULL;
}
resource->read = provider->read;
resource->close = provider->close;
resource->provider_data = provider->data;
if (provider->open(resource)) {
return resource;
}
mju_warning("mju_openResource: could not open resource '%s", name);
mju_free(resource->name);
mju_free(resource);
return NULL;
}
// lastly fallback to OS filesystem
else {
resource->read = NULL;
resource->close = NULL;
resource->provider_data = NULL;
resource->data = mju_malloc(sizeof(file_buffer));
file_buffer* fb = (file_buffer*) resource->data;
fb->buffer = mju_fileToMemory(name, &(fb->nbuffer));
if (fb->buffer == NULL) {
mju_warning("mju_openResource: unknown file '%s'", name);
mju_free(fb);
mju_free(resource->name);
mju_free(resource);
return NULL;
}
}
return resource;
}
// close the given resource; no-op if resource is NULL
void mju_closeResource(mjResource* resource) {
if (resource == NULL) {
return;
}
// use the resource provider to close resource
if (resource->close) {
resource->close(resource);
}
// if provider is NULL, then OS filesystem is used
else {
file_buffer* fb = (file_buffer*) resource->data;
mju_free(fb->buffer);
mju_free(fb);
}
// free name and resource
mju_free(resource->name);
mju_free(resource);
}
// set buffer to bytes read from the resource and return number of bytes in buffer;
// return negative value if error
int mju_readResource(mjResource* resource, const void** buffer) {
if (resource == NULL) {
return 0;
}
if (resource->read) {
return resource->read(resource, buffer);
}
// if provider is NULL, then OS filesystem is used
const file_buffer* fb = (file_buffer*) resource->data;
*buffer = fb->buffer;
return fb->nbuffer;
}
// read file into memory buffer (allocated here with mju_malloc)
void* mju_fileToMemory(const char* filename, int* filesize) {
// open file
*filesize = 0;
FILE* fp = fopen(filename, "rb");
if (!fp) {
return NULL;
}
// find size
if (fseek(fp, 0, SEEK_END) != 0) {
fclose(fp);
mju_warning("Failed to calculate size for '%s'", filename);
return NULL;
}
// ensure file size fits in int
long long_filesize = ftell(fp); // NOLINT(runtime/int)
if (long_filesize > INT_MAX) {
fclose(fp);
mju_warning("File size over 2GB is not supported. File: '%s'", filename);
return NULL;
} else if (long_filesize < 0) {
fclose(fp);
mju_warning("Failed to calculate size for '%s'", filename);
return NULL;
}
*filesize = long_filesize;
// go back to start of file
if (fseek(fp, 0, SEEK_SET) != 0) {
fclose(fp);
mju_warning("Read error while reading '%s'", filename);
return NULL;
}
// allocate and read
void* buffer = mju_malloc(*filesize);
if (!buffer) {
mju_error("mjFileToMemory: could not allocate memory");
}
size_t bytes_read = fread(buffer, 1, *filesize, fp);
// check that read data matches file size
if (bytes_read != *filesize) { // SHOULD NOT OCCUR
if (ferror(fp)) {
fclose(fp);
mju_free(buffer);
*filesize = 0;
mju_warning("Read error while reading '%s'", filename);
return NULL;
} else if (feof(fp)) {
*filesize = bytes_read;
}
}
// close file, return contents
fclose(fp);
return buffer;
}
src/engine/engine_file.h → src/engine/engine_resource.h
+18 -3
View File
@@ -12,13 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MUJOCO_SRC_ENGINE_ENGINE_FILE_H_
#define MUJOCO_SRC_ENGINE_ENGINE_FILE_H_
#ifndef MUJOCO_SRC_ENGINE_ENGINE_RESOURCE_H_
#define MUJOCO_SRC_ENGINE_ENGINE_RESOURCE_H_
#include <mujoco/mjexport.h>
#include "engine/engine_plugin.h"
#ifdef __cplusplus
extern "C" {
#endif
// open the given resource; if the name doesn't have a prefix matching with a
// resource provider, then the default_provider is used
// if default_provider non-positive, then the OS filesystem is used
MJAPI mjResource* mju_openResource(const char* name, int default_provider);
// close the given resource; no-op if resource is NULL
MJAPI void mju_closeResource(mjResource* resource);
// set buffer to bytes read from the resource and return number of bytes in buffer;
// return negative value if error
MJAPI int mju_readResource(mjResource* resource, const void** buffer);
// read file into memory buffer (allocated here with mju_malloc)
void* mju_fileToMemory(const char* filename, int* filesize);
@@ -26,4 +41,4 @@ void* mju_fileToMemory(const char* filename, int* filesize);
}
#endif
#endif // MUJOCO_SRC_ENGINE_ENGINE_FILE_H_
#endif // MUJOCO_SRC_ENGINE_ENGINE_RESOURCE_H_
src/engine/engine_vfs.c
+55 -2
View File
@@ -18,7 +18,8 @@
#include <stdlib.h>
#include "engine/engine_array_safety.h"
#include "engine/engine_file.h"
#include "engine/engine_plugin.h"
#include "engine/engine_resource.h"
#include "engine/engine_util_errmem.h"
#include "engine/engine_util_misc.h"
@@ -154,7 +155,6 @@ int mj_findFileVFS(const mjVFS* vfs, const char* filename) {
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, filename);
// find specific file
for (int i=0; i<vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME)==0) {
@@ -210,3 +210,56 @@ void mj_deleteVFS(mjVFS* vfs) {
memset(vfs, 0, sizeof(mjVFS));
}
// open callback for the VFS resource provider
static int vfs_open_callback(mjResource* resource) {
if (!resource || !resource->provider_data || !resource->name) {
return 0;
}
const mjVFS* vfs = (const mjVFS*) resource->provider_data;
return mj_findFileVFS(vfs, resource->name) >= 0;
}
// read callback for the VFS resource provider
static int vfs_read_callback(mjResource* resource, const void** buffer) {
if (!resource || !resource->provider_data) {
*buffer = NULL;
return -1;
}
const mjVFS* vfs = (const mjVFS*) resource->provider_data;
int i = mj_findFileVFS(vfs, resource->name);
if (i < 0) {
*buffer = NULL;
return -1;
}
*buffer = vfs->filedata[i];
return vfs->filesize[i];
}
// close callback for the VFS resource provider
static void vfs_close_callback(mjResource* resource) {
}
// registers a VFS resource provider; returns the index of the provider
int mj_registerVfsProvider(const mjVFS* vfs) {
mjpResourceProvider provider = {
.prefix = mjVFS_PREFIX,
.open = &vfs_open_callback,
.read = &vfs_read_callback,
.close = &vfs_close_callback,
.data = (void*) vfs
};
return mjp_registerResourceProviderInternal(&provider);
}
src/engine/engine_vfs.h
+3
View File
@@ -40,6 +40,9 @@ MJAPI int mj_deleteFileVFS(mjVFS* vfs, const char* filename);
// delete all files from VFS
MJAPI void mj_deleteVFS(mjVFS* vfs);
// registers a VFS resource provider; returns the index of the provider
MJAPI int mj_registerVfsProvider(const mjVFS* vfs);
#ifdef __cplusplus
}
#endif