chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-implement VFS internals in C++, removing the constraints of the previous implementation.

Browse Source 
PiperOrigin-RevId: 649433087
Change-Id: Icaa3e6b7f2ef14f56b04fef2f4360b46d8b5f022
This commit is contained in:
Kyle Bayes
2024-07-04 09:21:18 -07:00
committed by Copybara-Service
parent 118b6810b3
commit 57e6760ec9
19 changed files with 513 additions and 717 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine/engine_util_misc.c
-45
View File
@@ -1407,51 +1407,6 @@ char* mju_strncpy(char *dst, const char *src, int n) {
// assemble full filename from directory and filename, return 0 on success
int mju_makefullname(char* full, size_t nfull, const char* dir, const char* file) {
int dirlen = (!dir) ? 0 : strlen(dir);
int filelen = (!file) ? 0 : strlen(file);
char* filepos = full + dirlen;
// missing filename
if (!filelen) {
return -1;
}
// no directory then just copy filename over
if (!dirlen) {
// make sure full has space
if (filelen >= nfull) {
return -1;
}
strcpy(full, file);
return 0;
}
// make sure full has space
if (dirlen + filelen >= nfull) {
return -1;
}
// dir doesn't end with a slash
if (dir[dirlen - 1] != '\\' && dir[dirlen - 1] != '/') {
// need extra space for forward slash
if ((dirlen + filelen + 1) >= nfull) {
return -1;
}
// add forward slash
*filepos++ = '/';
}
// copy directory and file over
memcpy(full, dir, sizeof(char) * dirlen);
strcpy(filepos, file);
return 0;
}
// sigmoid function over 0<=x<=1 using quintic polynomial
mjtNum mju_sigmoid(mjtNum x) {
// fast return
src/engine/engine_util_misc.h
-4
View File
@@ -164,10 +164,6 @@ MJAPI mjtNum mju_Halton(int index, int base);
// call strncpy, then set dst[n-1] = 0
MJAPI char* mju_strncpy(char *dst, const char *src, int n);
// assemble full filename from directory and filename, return 0 on success
MJAPI int mju_makefullname(char* full, size_t nfull,
const char* dir, const char* file);
// sigmoid function over 0<=x<=1 using quintic polynomial
MJAPI mjtNum mju_sigmoid(mjtNum x);
src/user/CMakeLists.txt
+1 -1
View File
@@ -29,7 +29,7 @@ set(MUJOCO_USER_SRCS
user_objects.h
user_util.cc
user_util.h
user_vfs.c
user_vfs.cc
user_vfs.h
)
src/user/user_vfs.c
-420
View File
@@ -1,420 +0,0 @@
// Copyright 2021 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "user/user_vfs.h"
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include "engine/engine_array_safety.h"
#include "engine/engine_resource.h"
#include "engine/engine_util_errmem.h"
#include "engine/engine_util_misc.h"
// strip path prefix from filename
static void vfs_strippath(char* newname, const char* oldname) {
int sz = strlen(oldname);
// find last delimiter
int i;
for (i=sz-1; i >= 0; i--) {
if (oldname[i] == '\\' || oldname[i] == '/') {
break;
}
}
// check resulting length
if (sz-(i+1) >= mjMAXVFSNAME) {
mjERROR("filename too long");
}
if (sz-(i+1) <= 0) {
mjERROR("empty filename");
}
// copy
mju_strncpy(newname, oldname+i+1, mjMAXVFSNAME);
// make lowercase
for (int j=strlen(newname)-1; j >= 0; j--) {
if (newname[j] >= 'A' && newname[j] <= 'Z') {
newname[j] = (char)(((int)newname[j]) +'a' - 'A');
}
}
}
// copies data into a buffer and produces a hash of the data
static uint64_t vfs_memcpy(void* dest, const void* restrict src, size_t n) {
uint64_t hash = 0xcbf29ce484222325; // magic number
uint64_t prime = 0x100000001b3; // magic prime
const uint8_t* bytes = (uint8_t*) src;
uint8_t* buffer = (uint8_t*) dest;
for (size_t i = 0; i < n; i++) {
buffer[i] = bytes[i];
// do FNV-1 hash
hash |= bytes[i];
hash *= prime;
}
return hash;
}
// VFS hash function implemented using the FNV-1 hash
static uint64_t vfs_hash(const void* restrict buffer, size_t n) {
uint64_t hash = 0xcbf29ce484222325; // magic number
uint64_t prime = 0x100000001b3; // magic prime
const uint8_t* bytes = (uint8_t*) buffer;
for (size_t i = 0; i < n; i++) {
hash |= bytes[i];
hash *= prime;
}
return hash;
}
// initialize to empty (no deallocation)
void mj_defaultVFS(mjVFS* vfs) {
memset(vfs, 0, sizeof(mjVFS));
}
// add file to VFS, return 0: success, 1: full, 2: repeated name, -1: failed to load
int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename) {
// check vfs size
if (vfs->nfile >= mjMAXVFS-1) {
return 1;
}
// make full name
char fullname[1000];
if (mju_makefullname(fullname, sizeof(fullname), directory, filename)) {
return -1;
}
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, filename);
// check for repeated name
for (int i=0; i < vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME) == 0) {
return 2;
}
}
// assign name
mjSTRNCPY(vfs->filename[vfs->nfile], newname);
// allocate and read
size_t filesize = 0;
vfs->filedata[vfs->nfile] = mju_fileToMemory(fullname, &filesize);
if (!vfs->filedata[vfs->nfile]) {
return -1;
}
// assign size, count, and checksum
vfs->filestamp[vfs->nfile] = vfs_hash(vfs->filedata[vfs->nfile], filesize);
vfs->filesize[vfs->nfile] = filesize;
vfs->nfile++;
return 0;
}
// make empty file in VFS, return 0: success, 1: full, 2: repeated name
int mj_makeEmptyFileVFS(mjVFS* vfs, const char* filename, int filesize) {
// check vfs size
if (vfs->nfile >= mjMAXVFS-1) {
return 1;
}
// check filesize
if (filesize <= 0) {
mjERROR("expects positive filesize");
}
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, filename);
// check for repeated name
for (int i=0; i < vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME) == 0) {
return 2;
}
}
// assign name
mjSTRNCPY(vfs->filename[vfs->nfile], newname);
// allocate and clear
vfs->filedata[vfs->nfile] = mju_malloc(filesize);
if (!vfs->filedata[vfs->nfile]) {
mjERROR("could not allocate memory");
}
memset(vfs->filedata[vfs->nfile], 0, filesize);
vfs->filestamp[vfs->nfile] = 0;
// assign size and count
vfs->filesize[vfs->nfile] = filesize;
vfs->nfile++;
return 0;
}
// add file from buffer into VFS
int mj_addBufferVFS(mjVFS* vfs, const char* name, const void* buffer, int nbuffer) {
if (!vfs || !buffer || !name) {
mjERROR("null pointer");
}
if (vfs->nfile >= mjMAXVFS-1) {
return 1;
}
// check buffer size
if (nbuffer <= 0) {
mjERROR("expects positive buffer size");
}
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, name);
// check for repeated name
for (int i=0; i < vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME) == 0) {
return 2;
}
}
// assign name
mjSTRNCPY(vfs->filename[vfs->nfile], newname);
// allocate and clear
vfs->filedata[vfs->nfile] = mju_malloc(nbuffer);
if (!vfs->filedata[vfs->nfile]) {
mjERROR("could not allocate memory");
}
vfs->filestamp[vfs->nfile] = vfs_memcpy(vfs->filedata[vfs->nfile], buffer, nbuffer);
// assign size and count
vfs->filesize[vfs->nfile] = nbuffer;
vfs->nfile++;
return 0;
}
// return file index in VFS, or -1 if not found in VFS
int mj_findFileVFS(const mjVFS* vfs, const char* filename) {
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, filename);
// find specific file
for (int i=0; i < vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME) == 0) {
return i;
}
}
return -1;
}
// delete file from VFS, return 0: success, -1: not found in VFS
int mj_deleteFileVFS(mjVFS* vfs, const char* filename) {
// strip path
char newname[mjMAXVFSNAME];
vfs_strippath(newname, filename);
// find specified file
for (int i=0; i < vfs->nfile; i++) {
if (strncmp(newname, vfs->filename[i], mjMAXVFSNAME) == 0) {
// free buffer
mju_free(vfs->filedata[i]);
// scroll remaining files forward
for (int j=i; j < vfs->nfile-1; j++) {
mjSTRNCPY(vfs->filename[j], vfs->filename[j+1]);
vfs->filesize[j] = vfs->filesize[j+1];
vfs->filedata[j] = vfs->filedata[j+1];
}
// set last to 0, for style
vfs->filename[vfs->nfile-1][0] = 0;
vfs->filesize[vfs->nfile-1] = 0;
vfs->filedata[vfs->nfile-1] = NULL;
// decrease counter
vfs->nfile--;
return 0;
}
}
return -1;
}
// delete all files from VFS
void mj_deleteVFS(mjVFS* vfs) {
for (int i=0; i < vfs->nfile; i++) {
mju_free(vfs->filedata[i]);
}
memset(vfs, 0, sizeof(mjVFS));
}
// open callback for the VFS resource provider
static int vfs_open_callback(mjResource* resource) {
if (!resource || !resource->name || !resource->data) {
return 0;
}
const mjVFS* vfs = (const mjVFS*) resource->data;
int i = mj_findFileVFS(vfs, resource->name);
resource->timestamp[0] = '\0';
if (i >= 0 && vfs->filestamp[i]) {
mju_encodeBase64(resource->timestamp, (uint8_t*) &vfs->filestamp[i],
sizeof(uint64_t));
}
return i >= 0;
}
// read callback for the VFS resource provider
static int vfs_read_callback(mjResource* resource, const void** buffer) {
if (!resource || !resource->name || !resource->data) {
*buffer = NULL;
return -1;
}
const mjVFS* vfs = (const mjVFS*) resource->data;
int i = mj_findFileVFS(vfs, resource->name);
if (i < 0) {
*buffer = NULL;
return -1;
}
*buffer = vfs->filedata[i];
return vfs->filesize[i];
}
// close callback for the VFS resource provider
static void vfs_close_callback(mjResource* resource) {
}
// getdir callback for the VFS resource provider
static void vfs_getdir_callback(mjResource* resource, const char** dir, int* ndir) {
if (resource) {
*dir = resource->name;
*ndir = mju_dirnamelen(resource->name);
} else {
*dir = NULL;
*ndir = 0;
}
}
// modified callback for the VFS resource provider
// return > 0 if modified and 0 if unmodified
static int vfs_modified_callback(const mjResource* resource, const char* timestamp) {
uint64_t filestamp;
if (mju_isValidBase64(timestamp) > sizeof(uint64_t)) {
return 2; // error (assume modified)
}
mju_decodeBase64((uint8_t*) &filestamp, timestamp);
if (!filestamp) return 3; // no hash (assume modified)
if (resource) {
const mjVFS* vfs = (const mjVFS*) resource->data;
int i = mj_findFileVFS(vfs, resource->name);
if (i < 0) return 4; // missing file (assume modified)
if (!vfs->filestamp[i]) return 5; // missing filestamp (assume modified)
if (vfs->filestamp[i] == filestamp) {
return 0; // unmodified
}
}
return 1; // modified
}
// open VFS resource
mjResource* mju_openVfsResource(const char* name, const mjVFS* vfs) {
if (vfs == NULL) {
return NULL;
}
// VFS provider
static struct mjpResourceProvider provider = {
.prefix = NULL,
.data = NULL,
.open = &vfs_open_callback,
.read = &vfs_read_callback,
.close = &vfs_close_callback,
.getdir = &vfs_getdir_callback,
.modified = &vfs_modified_callback,
};
// create resource
mjResource* resource = (mjResource*) mju_malloc(sizeof(mjResource));
if (resource == NULL) {
mjERROR("could not allocate memory");
return NULL;
}
// clear out resource
memset(resource, 0, sizeof(mjResource));
// copy name
resource->name = mju_malloc(sizeof(char) * (strlen(name) + 1));
if (resource->name == NULL) {
mju_closeResource(resource);
mjERROR("could not allocate memory");
return NULL;
}
memcpy(resource->name, name, sizeof(char) * (strlen(name) + 1));
resource->data = (void*) vfs;
// open resource
resource->provider = &provider;
if (provider.open(resource)) {
return resource;
}
// not found in VFS
mju_closeResource(resource);
return NULL;
}
src/user/user_vfs.cc
+322
View File
@@ -0,0 +1,322 @@
// Copyright 2021 DeepMind Technologies Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "user/user_vfs.h"
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <cstring>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include "engine/engine_resource.h"
#include "engine/engine_util_errmem.h"
#include "engine/engine_util_misc.h"
#include "user/user_util.h"
namespace {
// internal struct for VFS files
struct VFSFile {
std::string filename;
std::unique_ptr<void, std::function<void(void*)>> filedata;
std::size_t filesize;
uint64_t filestamp;
};
// internal container class for VFS
class VFS {
public:
// returns true if the file exists in the VFS
bool HasFile(const std::string& filename) const;
// returns inserted mjuuVFSFile if the file was added successfully. This class
// assumes ownership of the buffer and will free it when the VFS is deleted.
VFSFile* AddFile(const std::string& filename, void* buffer,
std::size_t nbuffer, uint64_t filestamp);
// returns the internal file struct for the given filename
const VFSFile* GetFile(const std::string& filename) const;
// deletes file from VFS, return 0: success, -1: not found
int DeleteFile(const std::string& filename);
private:
std::unordered_map<std::string, VFSFile> files_;
};
// returns the internal VFS class pointer from the VFS C struct
inline VFS* GetVFSImpl(const mjVFS* vfs) {
return vfs->impl_ ? static_cast<VFS*>(vfs->impl_) : nullptr;
}
// strip path prefix from filename and make lowercase
std::string StripPath(const char* name) {
std::string newname = mjuu_strippath(name);
// make lowercase
std::transform(newname.begin(), newname.end(), newname.begin(),
[](unsigned char c) { return std::tolower(c); });
return newname;
}
// copies data into a buffer and produces a hash of the data
uint64_t vfs_memcpy(void* dest, const void* src, size_t n) {
uint64_t hash = 0xcbf29ce484222325; // magic number
uint64_t prime = 0x100000001b3; // magic prime
const uint8_t* bytes = (uint8_t*) src;
uint8_t* buffer = (uint8_t*) dest;
for (size_t i = 0; i < n; i++) {
buffer[i] = bytes[i];
// do FNV-1 hash
hash |= bytes[i];
hash *= prime;
}
return hash;
}
// VFS hash function implemented using the FNV-1 hash
uint64_t vfs_hash(const void* buffer, size_t n) {
uint64_t hash = 0xcbf29ce484222325; // magic number
uint64_t prime = 0x100000001b3; // magic prime
const uint8_t* bytes = (uint8_t*) buffer;
for (size_t i = 0; i < n; i++) {
hash |= bytes[i];
hash *= prime;
}
return hash;
}
bool VFS::HasFile(const std::string& filename) const {
return files_.find(filename) != files_.end();
}
VFSFile* VFS::AddFile(const std::string& filename, void* buffer,
std::size_t nbuffer, uint64_t filestamp) {
auto [it, inserted] = files_.insert({filename, VFSFile()});
if (!inserted) {
return nullptr; // repeated name
}
it->second.filename = filename;
it->second.filedata = std::unique_ptr<void, void(*)(void*)>(
buffer, [](void* b) { mju_free(b); }); // corresponding to mju_malloc
it->second.filesize = nbuffer;
it->second.filestamp = filestamp;
return &(it->second);
}
const VFSFile* VFS::GetFile(const std::string& filename) const {
auto it = files_.find(filename);
if (it == files_.end()) {
return nullptr;
}
return &it->second;
}
int VFS::DeleteFile(const std::string& filename) {
auto it = files_.find(filename);
if (it == files_.end()) {
return -1;
}
files_.erase(it);
return 0;
}
// open callback for the VFS resource provider
int Open(mjResource* resource) {
if (!resource || !resource->name || !resource->data) {
return 0;
}
const mjVFS* vfs = (const mjVFS*) resource->data;
const VFS* cvfs = GetVFSImpl(vfs);
const VFSFile* file = cvfs->GetFile(StripPath(resource->name));
if (file == nullptr) {
return 0;
}
resource->timestamp[0] = '\0';
if (file->filestamp) {
mju_encodeBase64(resource->timestamp, (uint8_t*) &file->filestamp,
sizeof(uint64_t));
}
return 1;
}
// read callback for the VFS resource provider
int Read(mjResource* resource, const void** buffer) {
if (!resource || !resource->name || !resource->data) {
*buffer = nullptr;
return -1;
}
const VFS* vfs = GetVFSImpl(static_cast<const mjVFS*>(resource->data));
const VFSFile* file = vfs->GetFile(StripPath(resource->name));
if (file == nullptr) {
*buffer = nullptr;
return -1;
}
*buffer = file->filedata.get();
return file->filesize;
}
// close callback for the VFS resource provider
void Close(mjResource* resource) {
}
// getdir callback for the VFS resource provider
void GetDir(mjResource* resource, const char** dir, int* ndir) {
*dir = (resource) ? resource->name : nullptr;
*ndir = (resource) ? mju_dirnamelen(resource->name) : 0;
}
// modified callback for the VFS resource provider
// return > 0 if modified and 0 if unmodified
int Modified(const mjResource* resource, const char* timestamp) {
uint64_t filestamp;
if (mju_isValidBase64(timestamp) > sizeof(uint64_t)) {
return 2; // error (assume modified)
}
mju_decodeBase64((uint8_t*) &filestamp, timestamp);
if (!filestamp) return 3; // no hash (assume modified)
if (resource) {
const VFS* cvfs = GetVFSImpl(static_cast<const mjVFS*>(resource->data));
const VFSFile* file = cvfs->GetFile(StripPath(resource->name));
if (file == nullptr) return 4; // missing file (assume modified)
if (!file->filestamp) return 5; // missing filestamp (assume modified)
if (file->filestamp == filestamp) {
return 0; // unmodified
}
}
return 1; // modified
}
} // namespace
// initialize to empty (no deallocation)
void mj_defaultVFS(mjVFS* vfs) {
vfs->impl_ = new VFS();
}
// add file to VFS, return 0: success, 2: repeated name, -1: failed to load
int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename) {
VFS* cvfs = GetVFSImpl(vfs);
// make full name
std::string fullname = mjuu_combinePaths(directory, filename);
// strip path
std::string newname = StripPath(filename);
// check beforehand for repeated name, to avoid reading file into memory
if (cvfs->HasFile(newname)) {
return 2;
}
// allocate and read
size_t nbuffer = 0;
void* buffer = mju_fileToMemory(fullname.c_str(), &nbuffer);
if (buffer == nullptr) {
return -1;
}
if (!cvfs->AddFile(newname, buffer, nbuffer, vfs_hash(buffer, nbuffer))) {
mju_free(buffer);
return 2; // AddFile failed, SHOULD NOT OCCUR
}
return 0;
}
// add file from buffer into VFS
int mj_addBufferVFS(mjVFS* vfs, const char* name, const void* buffer,
int nbuffer) {
VFS* cvfs = GetVFSImpl(vfs);
// allocate and clear
void* inbuffer = mju_malloc(nbuffer);
if (buffer == nullptr) {
mjERROR("could not allocate memory");
}
VFSFile* file;
if (!(file = cvfs->AddFile(StripPath(name), inbuffer, nbuffer, 0))) {
mju_free(inbuffer);
return 2; // AddFile failed, repeated name
}
file->filestamp = vfs_memcpy(inbuffer, buffer, nbuffer);
return 0;
}
// delete file from VFS, return 0: success, -1: not found in VFS
int mj_deleteFileVFS(mjVFS* vfs, const char* filename) {
VFS* cvfs = GetVFSImpl(vfs);
return cvfs->DeleteFile(StripPath(filename));
}
// delete all files from VFS
void mj_deleteVFS(mjVFS* vfs) {
if (vfs) {
delete GetVFSImpl(vfs);
}
}
// open VFS resource
mjResource* mju_openVfsResource(const char* name, const mjVFS* vfs) {
if (vfs == nullptr) {
return nullptr;
}
// VFS provider
static struct mjpResourceProvider provider = { nullptr, &Open, &Read, &Close,
&GetDir, &Modified, nullptr };
// create resource
mjResource* resource = (mjResource*) mju_malloc(sizeof(mjResource));
if (resource == nullptr) {
mjERROR("could not allocate memory");
return nullptr;
}
// clear out resource
memset(resource, 0, sizeof(mjResource));
// copy name
std::size_t n = std::strlen(name);
resource->name = (char*) mju_malloc(sizeof(char) * (n + 1));
if (resource->name == nullptr) {
mju_closeResource(resource);
mjERROR("could not allocate memory");
return nullptr;
}
std::memcpy(resource->name, name, sizeof(char) * (n + 1));
resource->data = (void*) vfs;
// open resource
resource->provider = &provider;
if (provider.open(resource)) {
return resource;
}
// not found in VFS
mju_closeResource(resource);
return nullptr;
}
src/user/user_vfs.h
+2 -2
View File
@@ -30,13 +30,13 @@ extern "C" {
// Initialize an empty VFS, mj_deleteVFS must be called to deallocate the VFS
MJAPI void mj_defaultVFS(mjVFS* vfs);
// add file to VFS, return 0: success, 1: full, 2: repeated name, -1: not found on disk
// add file to VFS, return 0: success, 2: repeated name, -1: not found on disk
MJAPI int mj_addFileVFS(mjVFS* vfs, const char* directory, const char* filename);
// deprecated: use mj_addBufferVFS
MJAPI int mj_makeEmptyFileVFS(mjVFS* vfs, const char* filename, int filesize);
// add file from buffer into VFS, return 0: success, 1: full, 2: repeated name, -1: failed to load
// add file from buffer into VFS, return 0: success, 2: repeated name, -1: failed to load
MJAPI int mj_addBufferVFS(mjVFS* vfs, const char* filename, const void* buffer, int nbuffer);
// return file index in VFS, or -1 if not found in VFS