chenlin/Mujoco_WASM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial implementation of MuJoCo Live.

Browse Source 
This is a combination of minor tweaks to the existing Studio WASM application, CMake cleanup to make it buildable, and integration with GitHub Actions for deployment.

The new features added are drag-and-drop, model specification through `?model=` URL parameter, HTTP and HTTPS resource provider, and a user-visible loading message while the model is loading.

PiperOrigin-RevId: 904594610
Change-Id: I3a73b1ca0fcd6fc9ba192469942b2dab010a9308
This commit is contained in:
Saran Tunyasuvunakool
2026-04-23 12:55:24 -07:00
committed by Copybara-Service
parent 4eba85093a
commit 1465d8b6ce
14 changed files with 573 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/experimental/studio/CMakeLists.txt
+105 -68
View File
@@ -14,87 +14,124 @@
cmake_minimum_required(VERSION 3.16)
set(MUJOCO_STUDIO_TARGET_NAME mujoco_studio)
add_executable(${MUJOCO_STUDIO_TARGET_NAME})
target_sources(${MUJOCO_STUDIO_TARGET_NAME}
PRIVATE
app.cc
app.h
main.cc
)
target_include_directories(${MUJOCO_STUDIO_TARGET_NAME}
PUBLIC
${PROJECT_SOURCE_DIR}/include
${PROJECT_SOURCE_DIR}/src
)
if (WIN32)
target_compile_definitions(${MUJOCO_STUDIO_TARGET_NAME}
PRIVATE
-D_USE_MATH_DEFINES
)
set_target_properties(${MUJOCO_STUDIO_TARGET_NAME}
PROPERTIES
VS_DEBUGGER_WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}"
)
endif()
include(third_party_deps/dear_imgui)
include(third_party_deps/implot)
include(third_party_deps/opensans)
include(third_party_deps/font_awesome)
target_link_libraries(${MUJOCO_STUDIO_TARGET_NAME}
PRIVATE
absl::flags
absl::flags_parse
dear_imgui
implot
mujoco::mujoco
mujoco::platform
)
# TODO: re-enable mjz support on Windows builds once DllMain issue is resolved.
if (NOT WIN32)
target_link_libraries(${MUJOCO_STUDIO_TARGET_NAME}
# Common configuration shared between mujoco_studio and mujoco_live.
function(configure_studio_target TARGET_NAME)
target_sources(${TARGET_NAME}
PRIVATE
mujoco::mjz
app.cc
app.h
)
endif()
file(MAKE_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets)
target_include_directories(${TARGET_NAME}
PUBLIC
${PROJECT_SOURCE_DIR}/include
${PROJECT_SOURCE_DIR}/src
)
add_custom_command(
TARGET ${MUJOCO_STUDIO_TARGET_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND}
-E copy
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../_deps/opensans-src/fonts/ttf/OpenSans-Regular.ttf
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets
COMMENT "Copying OpenSans-Regular.ttf assets to build directory"
)
add_custom_command(
TARGET ${MUJOCO_STUDIO_TARGET_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND}
-E copy
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../_deps/font_awesome-src/fonts/fontawesome-webfont.ttf
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets
COMMENT "Copying fontawesome-webfont.ttf to build directory"
)
if (WIN32)
target_compile_definitions(${TARGET_NAME}
PRIVATE
-D_USE_MATH_DEFINES
)
set_target_properties(${TARGET_NAME}
PROPERTIES
VS_DEBUGGER_WORKING_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}"
)
endif()
target_link_libraries(${TARGET_NAME}
PRIVATE
absl::flags
absl::flags_parse
dear_imgui
implot
mujoco::mujoco
mujoco::platform
)
# TODO: re-enable mjz support on Windows builds once DllMain issue is resolved.
if (NOT WIN32)
target_link_libraries(${TARGET_NAME}
PRIVATE
mujoco::mjz
)
endif()
file(MAKE_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets)
# Filament backend requires additional files to be copied into an "assets" folder.
if(MUJOCO_USE_FILAMENT)
add_custom_command(
TARGET ${MUJOCO_STUDIO_TARGET_NAME}
TARGET ${TARGET_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND}
-E copy_directory
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../src/experimental/filament/assets
-E copy
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../_deps/opensans-src/fonts/ttf/OpenSans-Regular.ttf
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets
COMMENT "Copying Filament assets to build directory"
COMMENT "Copying OpenSans-Regular.ttf assets to build directory"
)
add_custom_command(
TARGET ${TARGET_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND}
-E copy
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../_deps/font_awesome-src/fonts/fontawesome-webfont.ttf
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets
COMMENT "Copying fontawesome-webfont.ttf to build directory"
)
# Filament backend requires additional files to be copied into an "assets" folder.
if(MUJOCO_USE_FILAMENT)
add_custom_command(
TARGET ${TARGET_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND}
-E copy_directory
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/../src/experimental/filament/assets
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/assets
COMMENT "Copying Filament assets to build directory"
)
endif()
endfunction()
# Desktop Studio target.
if(NOT EMSCRIPTEN)
add_executable(mujoco_studio)
target_sources(mujoco_studio PRIVATE main.cc)
configure_studio_target(mujoco_studio)
endif()
# WASM Live target.
if(EMSCRIPTEN)
add_executable(mujoco_live)
target_sources(mujoco_live PRIVATE wasm.cc)
target_compile_options(mujoco_live PRIVATE -g)
configure_studio_target(mujoco_live)
target_link_libraries(mujoco_live PRIVATE mujoco::render_noop)
# Ensure resource decoder plugins are linked into the WASM binary.
target_link_options(mujoco_live PRIVATE
-Wl,--whole-archive $<TARGET_FILE:mujoco> -Wl,--no-whole-archive
)
# Filament's OpenGL backend requires WebGL2 / full ES3 bindings.
target_link_options(mujoco_live PRIVATE
--bind
-sUSE_WEBGL2=1
-sFULL_ES3
-sMIN_WEBGL_VERSION=2
-sMAX_WEBGL_VERSION=2
-sALLOW_MEMORY_GROWTH=1
-sASYNCIFY=1
-sFETCH=1
-sGL_PREINITIALIZED_CONTEXT=1
-sSTACK_SIZE=512mb
-sINITIAL_MEMORY=1024mb
-sASSERTIONS=1
-fexceptions
-g
)
endif()
src/experimental/studio/index.html
+143 -18
View File
@@ -1,8 +1,8 @@
<!doctype html>
<html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>MuJoCo Studio!</title>
<title>MuJoCo Live</title>
</head>
<body style="margin: 0; overflow: hidden">
<div style="position: absolute; top: 3px; right: 3px; z-index: 1;">
@@ -16,6 +16,28 @@
style="width: 100vw; height: 100vh; display: block"
></canvas>
<script>
// --- Loading overlay (shown while model is compiling) ---
var loadingOverlay = document.createElement('div');
loadingOverlay.id = 'loadingOverlay';
loadingOverlay.style.cssText =
'position:fixed;top:0;left:0;width:100%;height:100%;' +
'background:rgba(0,0,0,0.6);z-index:10000;display:none;' +
'align-items:center;justify-content:center;';
loadingOverlay.innerHTML =
'<div style="color:#fff;font-size:24px;font-family:sans-serif;' +
'text-align:center;">' +
'<div style="margin-bottom:16px;">Loading model\u2026</div>' +
'<div style="font-size:14px;opacity:0.7;">This may take a moment ' +
'for large models.</div></div>';
document.body.appendChild(loadingOverlay);
function showLoading() {
loadingOverlay.style.display = 'flex';
}
function hideLoading() {
loadingOverlay.style.display = 'none';
}
var Module = {
preRun: [],
postRun: [],
@@ -46,8 +68,30 @@
totalDependencies: 0,
monitorRunDependencies(left) {},
onRuntimeInitialized: () => {
// Define assets to prefetch. These paths are relative to the wasm_binary/ directory.
// Define assets to prefetch. These paths are relative to the mujoco_live_wasm/ directory.
const assetsToPrefetch = [
"assets/fontawesome-webfont.ttf",
"assets/ibl.ktx",
"assets/OpenSans-Regular.ttf",
"assets/pbr.filamat",
"assets/pbr_packed.filamat",
"assets/phong_2d_fade.filamat",
"assets/phong_2d.filamat",
"assets/phong_2d_reflect.filamat",
"assets/phong_2d_uv_fade.filamat",
"assets/phong_2d_uv.filamat",
"assets/phong_2d_uv_reflect.filamat",
"assets/phong_color_fade.filamat",
"assets/phong_color.filamat",
"assets/phong_color_reflect.filamat",
"assets/phong_cube_fade.filamat",
"assets/phong_cube.filamat",
"assets/phong_cube_reflect.filamat",
"assets/unlit_decor.filamat",
"assets/unlit_depth.filamat",
"assets/unlit_line.filamat",
"assets/unlit_segmentation.filamat",
"assets/unlit_ui.filamat"
];
const assetPromises = assetsToPrefetch.map(async (relativePath) => {
@@ -71,7 +115,31 @@
.then(() => {
try {
Module.init();
requestAnimationFrame(Module.animate);
// Check for a ?model= URL parameter and load from URL.
const params = new URLSearchParams(window.location.search);
const modelUrl = params.get('model');
if (modelUrl) {
// loadUrl uses ASYNCIFY (via EM_ASYNC_JS fetch), which
// suspends the WASM module. We must not start the animation
// loop until the load completes, otherwise renderFrame will
// hit "Cannot have multiple async operations in flight".
showLoading();
requestAnimationFrame(() => {
requestAnimationFrame(async () => {
try {
await Module.loadUrl(modelUrl);
} catch (error) {
console.error('Failed to load model from URL:', error);
} finally {
hideLoading();
requestAnimationFrame(Module.animate);
}
});
});
} else {
requestAnimationFrame(Module.animate);
}
} catch (error) {
console.error('Failed to initialize app.', error);
}
@@ -95,6 +163,31 @@
Module.canvas.style.height = window.innerHeight + "px";
});
function handleFile(file) {
const reader = new FileReader();
reader.onload = (e) => {
const buffer = e.target.result;
// Show the loading overlay, then defer the blocking WASM call by two
// animation frames so the browser has a chance to paint the overlay.
showLoading();
requestAnimationFrame(() => {
requestAnimationFrame(() => {
try {
Module.loadFile(file.name, buffer);
} catch (error) {
console.error('Failed to load model from file:', error);
} finally {
hideLoading();
}
});
});
};
reader.onerror = (e) => {
console.error('Error reading file:', e);
};
reader.readAsArrayBuffer(file);
}
document.addEventListener('DOMContentLoaded', () => {
const uploadButton = document.getElementById('uploadButton');
const fileInput = document.getElementById('fileInput');
@@ -108,23 +201,55 @@
if (!file) {
return;
}
handleFile(file);
});
const reader = new FileReader();
reader.onload = (e) => {
const buffer = e.target.result;
try {
Module.loadFile(file.name, buffer);
} catch (error) {
console.error('Failed to load model from file:', error);
}
};
reader.onerror = (e) => {
console.error('Error reading file:', e);
};
reader.readAsArrayBuffer(file);
// --- Drag-and-drop support ---
const dropOverlay = document.createElement('div');
dropOverlay.id = 'dropOverlay';
dropOverlay.style.cssText =
'position:fixed;top:0;left:0;width:100%;height:100%;' +
'background:rgba(0,0,0,0.5);z-index:9999;display:none;' +
'pointer-events:none;align-items:center;justify-content:center;';
dropOverlay.innerHTML =
'<div style="color:#fff;font-size:24px;font-family:sans-serif;' +
'padding:32px 48px;border:3px dashed #fff;border-radius:16px;">' +
'Drop model file here</div>';
document.body.appendChild(dropOverlay);
let dragCounter = 0;
document.addEventListener('dragenter', (e) => {
e.preventDefault();
dragCounter++;
if (dragCounter === 1) {
dropOverlay.style.display = 'flex';
}
});
document.addEventListener('dragleave', (e) => {
e.preventDefault();
dragCounter--;
if (dragCounter === 0) {
dropOverlay.style.display = 'none';
}
});
document.addEventListener('dragover', (e) => {
e.preventDefault();
});
document.addEventListener('drop', (e) => {
e.preventDefault();
dragCounter = 0;
dropOverlay.style.display = 'none';
const files = e.dataTransfer.files;
for (let i = 0; i < files.length; i++) {
handleFile(files[i]);
}
});
});
</script>
<script async src="/bin/wasm.js"></script>
<script async src="bin/mujoco_live.js"></script>
</body>
</html>
src/experimental/studio/wasm.cc
+115
View File
@@ -18,7 +18,11 @@
#include <emscripten/bind.h>
#include <emscripten/val.h>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
@@ -59,6 +63,84 @@ class AssetRegistry {
std::unordered_map<std::string, std::string> assets_;
};
// ---------------------------------------------------------------------------
// HTTP/HTTPS resource fetching via the JS fetch API (uses ASYNCIFY to yield).
// ---------------------------------------------------------------------------
// Fetches a URL using the JS fetch API. Returns a malloc'd buffer and its size.
// The caller is responsible for freeing the buffer. Returns 0 on failure.
EM_ASYNC_JS(int, FetchUrl,
(const char* url, char** out_data, std::int32_t* out_size), {
try {
const urlStr = UTF8ToString(url);
const response = await fetch(urlStr);
if (!response.ok) {
console.error('Fetch failed: ' + response.status + ' ' +
urlStr);
return 0;
}
const buffer = await response.arrayBuffer();
const bytes = new Uint8Array(buffer);
const ptr = _malloc(bytes.length);
HEAPU8.set(bytes, ptr);
setValue(out_data, ptr, '*');
setValue(out_size, bytes.length, 'i32');
return 1;
} catch (e) {
console.error('Fetch error:', e);
return 0;
}
});
// Cache for data fetched via HTTP/HTTPS. Stores the downloaded bytes keyed by
// the resource name (URL) so that read() can return a pointer to the data.
class FetchCache {
public:
static FetchCache& Instance() {
static FetchCache instance;
return instance;
}
// Fetches the URL and stores the result. Returns the size (>0) on success.
int Fetch(const char* url) {
char* data = nullptr;
std::int32_t size = 0;
if (!FetchUrl(url, &data, &size)) {
return 0;
}
entries_[url] = Entry{UniquePtrWasm<char[]>(data), size};
return size;
}
// Returns pointer and size for a previously fetched URL.
int Read(const char* url, const void** buffer) {
auto it = entries_.find(url);
if (it == entries_.end()) {
return -1;
}
*buffer = it->second.data.get();
return it->second.size;
}
// Frees the data for a URL.
void Close(const char* url) { entries_.erase(url); }
private:
struct FreeDeleter {
void operator()(void* p) const { std::free(p); }
};
template <typename T>
using UniquePtrWasm = std::unique_ptr<T, FreeDeleter>;
struct Entry {
UniquePtrWasm<char[]> data;
int size;
};
std::unordered_map<std::string, Entry> entries_;
};
// ---------------------------------------------------------------------------
// Javascript-facing function to register an asset.
void RegisterAsset(std::string filename, std::string contents) {
AssetRegistry::Instance().RegisterAsset(std::move(filename),
@@ -92,6 +174,27 @@ void Init() {
resource_provider.prefix = "filament";
mjp_registerResourceProvider(&resource_provider);
// Register HTTP/HTTPS resource providers so that models loaded from URLs
// can automatically fetch referenced assets (meshes, textures, etc.) over
// the network.
mjpResourceProvider http_provider;
mjp_defaultResourceProvider(&http_provider);
http_provider.open = [](mjResource* resource) {
return FetchCache::Instance().Fetch(resource->name);
};
http_provider.read = [](mjResource* resource, const void** buffer) {
return FetchCache::Instance().Read(resource->name, buffer);
};
http_provider.close = [](mjResource* resource) {
FetchCache::Instance().Close(resource->name);
};
http_provider.prefix = "http";
mjp_registerResourceProvider(&http_provider);
http_provider.prefix = "https";
mjp_registerResourceProvider(&http_provider);
g_app = new mujoco::studio::App({
.width = width,
.height = height,
@@ -124,6 +227,17 @@ void LoadFile(const std::string& filename, const std::string& data) {
g_app->LoadModelFromBuffer({ptr, ptr + data.size()}, content_type, filename);
}
// Javascript-facing function to load a model from a URL.
// The URL is passed directly to LoadModelFromFile, which will use the
// registered HTTP/HTTPS resource providers to fetch the model and any
// referenced assets.
void LoadUrl(const std::string& url) {
if (!g_app) {
return;
}
g_app->LoadModelFromFile(url);
}
// Javascript-facing function to render a single frame.
void RenderFrame() {
if (g_app) {
@@ -144,6 +258,7 @@ EMSCRIPTEN_BINDINGS(studio_bindings) {
emscripten::function("registerAsset", &RegisterAsset);
emscripten::function("init", &Init);
emscripten::function("loadFile", &LoadFile);
emscripten::function("loadUrl", &LoadUrl);
emscripten::function("renderFrame", &RenderFrame);
emscripten::function("deinit", &Deinit);
}