diff --git a/doc/includes/references.h b/doc/includes/references.h index c996f46d..c0d5de37 100644 --- a/doc/includes/references.h +++ b/doc/includes/references.h @@ -1318,7 +1318,9 @@ typedef enum mjtEvent_ { // mouse and keyboard event type mjEVENT_RELEASE, // mouse button release mjEVENT_SCROLL, // scroll mjEVENT_KEY, // key press - mjEVENT_RESIZE // resize + mjEVENT_RESIZE, // resize + mjEVENT_REDRAW, // redraw + mjEVENT_FILESDROP // files drop } mjtEvent; typedef enum mjtItem_ { // UI item type mjITEM_END = -2, // end of definition list (not an item) @@ -1377,6 +1379,10 @@ struct mjuiState_ { // mouse and keyboard state int mouserect; // which rectangle contains mouse int dragrect; // which rectangle is dragged with mouse int dragbutton; // which button started drag (mjtButton) + + // files dropping (only valid when type == mjEVENT_FILESDROP) + int dropcount; // number of files dropped + const char** droppaths; // paths to files dropped }; typedef struct mjuiState_ mjuiState; struct mjuiThemeSpacing_ { // UI visualization theme spacing diff --git a/include/mujoco/mjui.h b/include/mujoco/mjui.h index 92ec8dca..e9a2b0fe 100644 --- a/include/mujoco/mjui.h +++ b/include/mujoco/mjui.h @@ -74,7 +74,9 @@ typedef enum mjtEvent_ { // mouse and keyboard event type mjEVENT_RELEASE, // mouse button release mjEVENT_SCROLL, // scroll mjEVENT_KEY, // key press - mjEVENT_RESIZE // resize + mjEVENT_RESIZE, // resize + mjEVENT_REDRAW, // redraw + mjEVENT_FILESDROP // files drop } mjtEvent; @@ -143,6 +145,10 @@ struct mjuiState_ { // mouse and keyboard state int mouserect; // which rectangle contains mouse int dragrect; // which rectangle is dragged with mouse int dragbutton; // which button started drag (mjtButton) + + // files dropping (only valid when type == mjEVENT_FILESDROP) + int dropcount; // number of files dropped + const char** droppaths; // paths to files dropped }; typedef struct mjuiState_ mjuiState; diff --git a/introspect/enums.py b/introspect/enums.py index 5d72b063..1d6ef75b 100755 --- a/introspect/enums.py +++ b/introspect/enums.py @@ -627,6 +627,8 @@ ENUMS: Mapping[str, EnumDecl] = dict([ ('mjEVENT_SCROLL', 4), ('mjEVENT_KEY', 5), ('mjEVENT_RESIZE', 6), + ('mjEVENT_REDRAW', 7), + ('mjEVENT_FILESDROP', 8), ]), )), ('mjtItem', diff --git a/python/mujoco/simulate.cc b/python/mujoco/simulate.cc index 841e1dd7..45f84b81 100644 --- a/python/mujoco/simulate.cc +++ b/python/mujoco/simulate.cc @@ -14,8 +14,11 @@ #include #include +#include #include +#include +#include #include #include "structs.h" #include @@ -38,7 +41,10 @@ PYBIND11_MODULE(_simulate, pymodule) { py::call_guard()); py::class_(pymodule, "Simulate") - .def(py::init<>()) + .def(py::init([]() { + return std::make_unique( + std::make_unique()); + })) .def( "renderloop", [](mujoco::Simulate& simulate) { simulate.renderloop(); }, @@ -134,7 +140,7 @@ PYBIND11_MODULE(_simulate, pymodule) { }); pymodule.def("setglfwdlhandle", [](std::uintptr_t dlhandle) { - mujoco::setglfwdlhandle(reinterpret_cast(dlhandle)); + mujoco::Glfw(reinterpret_cast(dlhandle)); }); } diff --git a/simulate/CMakeLists.txt b/simulate/CMakeLists.txt index 1e6cc1f5..a39c6226 100644 --- a/simulate/CMakeLists.txt +++ b/simulate/CMakeLists.txt @@ -101,24 +101,42 @@ if(NOT TARGET lodepng) endif() # Simulate library -add_library(libsimulate STATIC) +add_library(platform_ui_adapter OBJECT) +target_sources( + platform_ui_adapter + PUBLIC glfw_adapter.h + glfw_dispatch.h + platform_ui_adapter.h + PRIVATE glfw_adapter.cc + glfw_dispatch.cc + platform_ui_adapter.cc +) +target_compile_options(platform_ui_adapter PUBLIC ${MUJOCO_SIMULATE_COMPILE_OPTIONS}) +target_include_directories( + platform_ui_adapter + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + $ +) +target_link_libraries(platform_ui_adapter PUBLIC mujoco::mujoco) +if(SIMULATE_GLFW_DYNAMIC_SYMBOLS) + target_compile_definitions(platform_ui_adapter PUBLIC mjGLFW_DYNAMIC_SYMBOLS) +endif() +add_library(mujoco::platform_ui_adapter ALIAS platform_ui_adapter) + +add_library(libsimulate STATIC $) +set_target_properties(libsimulate PROPERTIES OUTPUT_NAME simulate) add_library(mujoco::libsimulate ALIAS libsimulate) target_sources( libsimulate PUBLIC simulate.h - array_safety.h - glfw_dispatch.h - glfw_dispatch.cc - simulate.cc - uitools.h - uitools.cc + PRIVATE simulate.cc + array_safety.h ) target_include_directories(libsimulate PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_compile_definitions(libsimulate PUBLIC MJSIMULATE_STATIC) target_compile_options(libsimulate PUBLIC ${MUJOCO_SIMULATE_COMPILE_OPTIONS}) -target_link_libraries(libsimulate PUBLIC lodepng mujoco::mujoco) -target_include_directories(libsimulate PUBLIC $) +target_link_libraries(libsimulate PUBLIC lodepng mujoco::platform_ui_adapter mujoco::mujoco) target_link_options(libsimulate PRIVATE ${MUJOCO_SIMULATE_LINK_OPTIONS}) if(APPLE) @@ -126,10 +144,6 @@ if(APPLE) target_link_libraries(libsimulate PUBLIC "-framework Cocoa") endif() -if(SIMULATE_GLFW_DYNAMIC_SYMBOLS) - target_compile_definitions(libsimulate PUBLIC mjGLFW_DYNAMIC_SYMBOLS) -endif() - # Build simulate executable if(SIMULATE_BUILD_EXECUTABLE) if(APPLE) diff --git a/simulate/glfw_adapter.cc b/simulate/glfw_adapter.cc new file mode 100644 index 00000000..2651af27 --- /dev/null +++ b/simulate/glfw_adapter.cc @@ -0,0 +1,223 @@ +// Copyright 2023 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "glfw_adapter.h" + +#include +#include + +#include +#include +#include +#include "glfw_dispatch.h" + +namespace mujoco { +namespace { +int MaybeGlfwInit() { + static const int is_initialized = []() { + auto success = Glfw().glfwInit(); + if (success == GLFW_TRUE) { + std::atexit(Glfw().glfwTerminate); + } + return success; + }(); + return is_initialized; +} + +GlfwAdapter& GlfwAdapterFromWindow(GLFWwindow* window) { + return *static_cast(Glfw().glfwGetWindowUserPointer(window)); +} +} // namespace + +GlfwAdapter::GlfwAdapter() { + if (MaybeGlfwInit() != GLFW_TRUE) { + mju_error("could not initialize GLFW"); + } + + // multisampling + Glfw().glfwWindowHint(GLFW_SAMPLES, 4); + Glfw().glfwWindowHint(GLFW_VISIBLE, 1); + + // get video mode and save + vidmode_ = *Glfw().glfwGetVideoMode(Glfw().glfwGetPrimaryMonitor()); + + // create window + window_ = Glfw().glfwCreateWindow((2 * vidmode_.width) / 3, + (2 * vidmode_.height) / 3, + "MuJoCo", nullptr, nullptr); + if (!window_) { + mju_error("could not create window"); + } + + // save window position and size + Glfw().glfwGetWindowPos(window_, &window_pos_.first, &window_pos_.second); + Glfw().glfwGetWindowSize(window_, &window_size_.first, &window_size_.second); + + // set callbacks + Glfw().glfwSetWindowUserPointer(window_, this); + Glfw().glfwSetDropCallback( + window_, +[](GLFWwindow* window, int count, const char** paths) { + GlfwAdapterFromWindow(window).OnFilesDrop(count, paths); + }); + Glfw().glfwSetKeyCallback( + window_, +[](GLFWwindow* window, int key, int scancode, int act, int mods) { + GlfwAdapterFromWindow(window).OnKey(key, scancode, act); + }); + Glfw().glfwSetMouseButtonCallback( + window_, +[](GLFWwindow* window, int button, int act, int mods) { + GlfwAdapterFromWindow(window).OnMouseButton(button, act); + }); + Glfw().glfwSetCursorPosCallback( + window_, +[](GLFWwindow* window, double x, double y) { + GlfwAdapterFromWindow(window).OnMouseMove(x, y); + }); + Glfw().glfwSetScrollCallback( + window_, +[](GLFWwindow* window, double xoffset, double yoffset) { + GlfwAdapterFromWindow(window).OnScroll(xoffset, yoffset); + }); + Glfw().glfwSetWindowRefreshCallback( + window_, +[](GLFWwindow* window) { + GlfwAdapterFromWindow(window).OnWindowRefresh(); + }); + Glfw().glfwSetWindowSizeCallback( + window_, +[](GLFWwindow* window, int width, int height) { + GlfwAdapterFromWindow(window).OnWindowResize(width, height); + }); + + // make context current + Glfw().glfwMakeContextCurrent(window_); +} + +GlfwAdapter::~GlfwAdapter() { + Glfw().glfwMakeContextCurrent(nullptr); + Glfw().glfwDestroyWindow(window_); +} + +std::pair GlfwAdapter::GetCursorPosition() const { + double x, y; + Glfw().glfwGetCursorPos(window_, &x, &y); + return {x, y}; +} + +double GlfwAdapter::GetDisplayPixelsPerInch() const { + int width_mm, height_mm; + Glfw().glfwGetMonitorPhysicalSize( + Glfw().glfwGetPrimaryMonitor(), &width_mm, &height_mm); + return 25.4 * vidmode_.width / width_mm; +} + +std::pair GlfwAdapter::GetFramebufferSize() const { + int width, height; + Glfw().glfwGetFramebufferSize(window_, &width, &height); + return {width, height}; +} + +std::pair GlfwAdapter::GetWindowSize() const { + int width, height; + Glfw().glfwGetWindowSize(window_, &width, &height); + return {width, height}; +} + +void GlfwAdapter::PollEvents() { + Glfw().glfwPollEvents(); +} + +void GlfwAdapter::SetClipboardString(const char* text) { + Glfw().glfwSetClipboardString(window_, text); +} + +void GlfwAdapter::SetVSync(bool enabled){ + Glfw().glfwSwapInterval(enabled); +} + +void GlfwAdapter::SetWindowTitle(const char* title) { + Glfw().glfwSetWindowTitle(window_, title); +} + +bool GlfwAdapter::ShouldCloseWindow() const { + return Glfw().glfwWindowShouldClose(window_); +} + +void GlfwAdapter::SwapBuffers() { + Glfw().glfwSwapBuffers(window_); +} + +void GlfwAdapter::ToggleFullscreen() { + // currently full screen: switch to windowed + if (Glfw().glfwGetWindowMonitor(window_)) { + // restore window from saved data + Glfw().glfwSetWindowMonitor(window_, nullptr, window_pos_.first, window_pos_.second, + window_size_.first, window_size_.second, 0); + } + + // currently windowed: switch to full screen + else { + // save window data + Glfw().glfwGetWindowPos(window_, &window_pos_.first, &window_pos_.second); + Glfw().glfwGetWindowSize(window_, &window_size_.first, + &window_size_.second); + + // switch + Glfw().glfwSetWindowMonitor(window_, Glfw().glfwGetPrimaryMonitor(), 0, + 0, vidmode_.width, vidmode_.height, + vidmode_.refreshRate); + } +} + +bool GlfwAdapter::IsLeftMouseButtonPressed() const { + return Glfw().glfwGetMouseButton(window_, GLFW_MOUSE_BUTTON_LEFT) == GLFW_PRESS; +} + +bool GlfwAdapter::IsMiddleMouseButtonPressed() const { + return Glfw().glfwGetMouseButton(window_, GLFW_MOUSE_BUTTON_MIDDLE) == GLFW_PRESS; +} + +bool GlfwAdapter::IsRightMouseButtonPressed() const { + return Glfw().glfwGetMouseButton(window_, GLFW_MOUSE_BUTTON_RIGHT) == GLFW_PRESS; +} + +bool GlfwAdapter::IsAltKeyPressed() const { + return Glfw().glfwGetKey(window_, GLFW_KEY_LEFT_ALT) == GLFW_PRESS || + Glfw().glfwGetKey(window_, GLFW_KEY_RIGHT_ALT) == GLFW_PRESS; +} + +bool GlfwAdapter::IsCtrlKeyPressed() const { + return Glfw().glfwGetKey(window_, GLFW_KEY_LEFT_CONTROL) == GLFW_PRESS || + Glfw().glfwGetKey(window_, GLFW_KEY_RIGHT_CONTROL) == GLFW_PRESS; +} + +bool GlfwAdapter::IsShiftKeyPressed() const { + return Glfw().glfwGetKey(window_, GLFW_KEY_LEFT_SHIFT) == GLFW_PRESS || + Glfw().glfwGetKey(window_, GLFW_KEY_RIGHT_SHIFT) == GLFW_PRESS; +} + +bool GlfwAdapter::IsMouseButtonDownEvent(int act) const { + return act == GLFW_PRESS; +} + +bool GlfwAdapter::IsKeyDownEvent(int act) const { return act == GLFW_PRESS; } + +int GlfwAdapter::TranslateKeyCode(int key) const { return key; } + +mjtButton GlfwAdapter::TranslateMouseButton(int button) const { + if (button == GLFW_MOUSE_BUTTON_LEFT) { + return mjBUTTON_LEFT; + } else if (button == GLFW_MOUSE_BUTTON_RIGHT) { + return mjBUTTON_RIGHT; + } else if (button == GLFW_MOUSE_BUTTON_MIDDLE) { + return mjBUTTON_MIDDLE; + } + return mjBUTTON_NONE; +} +} // namespace mujoco diff --git a/simulate/glfw_adapter.h b/simulate/glfw_adapter.h new file mode 100644 index 00000000..188091df --- /dev/null +++ b/simulate/glfw_adapter.h @@ -0,0 +1,66 @@ +// Copyright 2023 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SIMULATE_GLFW_ADAPTER_H_ +#define MUJOCO_SIMULATE_GLFW_ADAPTER_H_ + +#include + +#include +#include +#include "platform_ui_adapter.h" + +namespace mujoco { +class GlfwAdapter : public PlatformUIAdapter { + public: + GlfwAdapter(); + ~GlfwAdapter() override; + + std::pair GetCursorPosition() const override; + double GetDisplayPixelsPerInch() const override; + std::pair GetFramebufferSize() const override; + std::pair GetWindowSize() const override; + void PollEvents() override; + void SetClipboardString(const char* text) override; + void SetVSync(bool enabled) override; + void SetWindowTitle(const char* title) override; + bool ShouldCloseWindow() const override; + void SwapBuffers() override; + void ToggleFullscreen() override; + + bool IsLeftMouseButtonPressed() const override; + bool IsMiddleMouseButtonPressed() const override; + bool IsRightMouseButtonPressed() const override; + + bool IsAltKeyPressed() const override; + bool IsCtrlKeyPressed() const override; + bool IsShiftKeyPressed() const override; + + bool IsMouseButtonDownEvent(int act) const override; + bool IsKeyDownEvent(int act) const override; + + int TranslateKeyCode(int key) const override; + mjtButton TranslateMouseButton(int button) const override; + + private: + GLFWvidmode vidmode_; + GLFWwindow* window_; + + // store last window information when going to full screen + std::pair window_pos_; + std::pair window_size_; +}; +} // namespace mujoco + +#endif // MUJOCO_SIMULATE_GLFW_ADAPTER_H_ diff --git a/simulate/main.cc b/simulate/main.cc index d26481e4..3e558d9d 100644 --- a/simulate/main.cc +++ b/simulate/main.cc @@ -26,7 +26,7 @@ #include #include -#include "glfw_dispatch.h" +#include "glfw_adapter.h" #include "simulate.h" #include "array_safety.h" @@ -48,8 +48,6 @@ namespace { namespace mj = ::mujoco; namespace mju = ::mujoco::sample_util; -using ::mujoco::Glfw; - // constants const double syncMisalign = 0.1; // maximum mis-alignment before re-sync (simulation seconds) const double simRefreshFraction = 0.7; // fraction of refresh available for simulation @@ -62,6 +60,7 @@ mjData* d = nullptr; // control noise variables mjtNum* ctrlnoise = nullptr; +using Seconds = std::chrono::duration; //---------------------------------------- plugin handling ----------------------------------------- @@ -254,7 +253,7 @@ mjModel* LoadModel(const char* file, mj::Simulate& sim) { // simulate in background thread (while rendering in main thread) void PhysicsLoop(mj::Simulate& sim) { // cpu-sim syncronization point - double syncCPU = 0; + std::chrono::time_point syncCPU; mjtNum syncSim = 0; // run until asked to exit @@ -321,10 +320,10 @@ void PhysicsLoop(mj::Simulate& sim) { // running if (sim.run) { // record cpu time at start of iteration - double startCPU = Glfw().glfwGetTime(); + const auto startCPU = mj::Simulate::Clock::now(); // elapsed CPU and simulation time since last sync - double elapsedCPU = startCPU - syncCPU; + const auto elapsedCPU = startCPU - syncCPU; double elapsedSim = d->time - syncSim; // inject noise @@ -346,10 +345,12 @@ void PhysicsLoop(mj::Simulate& sim) { double slowdown = 100 / sim.percentRealTime[sim.realTimeIndex]; // misalignment condition: distance from target sim time is bigger than syncmisalign - bool misaligned = mju_abs(elapsedCPU/slowdown - elapsedSim) > syncMisalign; + bool misaligned = + mju_abs(Seconds(elapsedCPU).count()/slowdown - elapsedSim) > syncMisalign; // out-of-sync (for any reason): reset sync times, step - if (elapsedSim < 0 || elapsedCPU < 0 || syncCPU == 0 || misaligned || sim.speedChanged) { + if (elapsedSim < 0 || elapsedCPU.count() < 0 || syncCPU.time_since_epoch().count() == 0 || + misaligned || sim.speedChanged) { // re-sync syncCPU = startCPU; syncSim = d->time; @@ -368,14 +369,16 @@ void PhysicsLoop(mj::Simulate& sim) { else { bool measured = false; mjtNum prevSim = d->time; + double refreshTime = simRefreshFraction/sim.refreshRate; // step while sim lags behind cpu and within refreshTime - while ((d->time - syncSim)*slowdown < (Glfw().glfwGetTime()-syncCPU) && - (Glfw().glfwGetTime()-startCPU) < refreshTime) { + while (Seconds((d->time - syncSim)*slowdown) < mj::Simulate::Clock::now() - syncCPU && + mj::Simulate::Clock::now() - startCPU < Seconds(refreshTime)) { // measure slowdown before first step if (!measured && elapsedSim) { - sim.measuredSlowdown = elapsedCPU / elapsedSim; + sim.measuredSlowdown = + std::chrono::duration(elapsedCPU).count() / elapsedSim; measured = true; } @@ -466,12 +469,7 @@ int main(int argc, const char** argv) { scanPluginLibraries(); // simulate object encapsulates the UI - auto sim = std::make_unique(); - - // init GLFW - if (!Glfw().glfwInit()) { - mju_error("could not initialize GLFW"); - } + mj::Simulate sim(std::make_unique()); const char* filename = nullptr; if (argc > 1) { @@ -479,16 +477,11 @@ int main(int argc, const char** argv) { } // start physics thread - std::thread physicsthreadhandle = std::thread(&PhysicsThread, sim.get(), filename); + std::thread physicsthreadhandle = std::thread(&PhysicsThread, &sim, filename); // start simulation UI loop (blocking call) - sim->renderloop(); + sim.renderloop(); physicsthreadhandle.join(); - // terminate GLFW (crashes with Linux NVidia drivers) -#if defined(__APPLE__) || defined(_WIN32) - Glfw().glfwTerminate(); -#endif - return 0; } diff --git a/simulate/platform_ui_adapter.cc b/simulate/platform_ui_adapter.cc new file mode 100644 index 00000000..0eacf072 --- /dev/null +++ b/simulate/platform_ui_adapter.cc @@ -0,0 +1,242 @@ +// Copyright 2023 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "platform_ui_adapter.h" + +#include + +namespace mujoco { +PlatformUIAdapter::PlatformUIAdapter() { + mjr_defaultContext(&con_); +} + +PlatformUIAdapter::~PlatformUIAdapter() { + mjr_freeContext(&con_); +} + +bool PlatformUIAdapter::RefreshMjrContext(const mjModel* m, int fontscale) { + if (m != last_model_ || fontscale != last_fontscale_) { + mjr_makeContext(m, &con_, fontscale); + last_model_ = m; + last_fontscale_ = fontscale; + return true; + } + return false; +} + +void PlatformUIAdapter::OnFilesDrop(int count, const char** paths) { + state_.type = mjEVENT_FILESDROP; + state_.dropcount = count; + state_.droppaths = paths; + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } + + // remove paths pointer from mjuiState since we don't own it + state_.dropcount = 0; + state_.droppaths = nullptr; +} + +void PlatformUIAdapter::OnKey(int key, int scancode, int act) { + // translate API-specific key code + int mj_key = TranslateKeyCode(key); + + // release: nothing to do + if (!IsKeyDownEvent(act)) { + return; + } + + // update state + UpdateMjuiState(); + + // set key info + state_.type = mjEVENT_KEY; + state_.key = mj_key; + state_.keytime = std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()).count(); + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } +} + +void PlatformUIAdapter::OnMouseButton(int button, int act) { + // translate API-specific mouse button code + mjtButton mj_button = TranslateMouseButton(button); + + // update state + UpdateMjuiState(); + + // swap left and right if Alt + if (state_.alt) { + if (mj_button == mjBUTTON_LEFT) { + mj_button = mjBUTTON_RIGHT; + } else if (mj_button == mjBUTTON_RIGHT) { + mj_button = mjBUTTON_LEFT; + } + } + + // press + if (IsMouseButtonDownEvent(act)) { + double now = std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()).count(); + + // detect doubleclick: 250 ms + if (mj_button == state_.button && now - state_.buttontime < 0.25) { + state_.doubleclick = 1; + } else { + state_.doubleclick = 0; + } + + // set info + state_.type = mjEVENT_PRESS; + state_.button = mj_button; + state_.buttontime = now; + + // start dragging + if (state_.mouserect) { + state_.dragbutton = state_.button; + state_.dragrect = state_.mouserect; + } + } + + // release + else { + state_.type = mjEVENT_RELEASE; + } + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } + + // stop dragging after application processing + if (state_.type == mjEVENT_RELEASE) { + state_.dragrect = 0; + state_.dragbutton = 0; + } +} + +void PlatformUIAdapter::OnMouseMove(double x, double y) { + // no buttons down: nothing to do + if (!state_.left && !state_.right && !state_.middle) { + return; + } + + // update state + UpdateMjuiState(); + + // set move info + state_.type = mjEVENT_MOVE; + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } +} + +void PlatformUIAdapter::OnScroll(double xoffset, double yoffset) { + // update state + UpdateMjuiState(); + + // set scroll info, scale by buffer-to-window ratio + const double buffer_window_ratio = + static_cast(GetFramebufferSize().first) / GetWindowSize().first; + state_.type = mjEVENT_SCROLL; + state_.sx = xoffset * buffer_window_ratio; + state_.sy = yoffset * buffer_window_ratio; + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } +} + +void PlatformUIAdapter::OnWindowRefresh() { + state_.type = mjEVENT_REDRAW; + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } +} + +void PlatformUIAdapter::OnWindowResize(int width, int height) { + auto [buf_width, buf_height] = GetFramebufferSize(); + state_.rect[0].width = buf_width; + state_.rect[0].height = buf_height; + if (state_.nrect < 1) state_.nrect = 1; + + // update window layout + if (layout_callback_) { + layout_callback_(&state_); + } + + // update state + UpdateMjuiState(); + + // set resize info + state_.type = mjEVENT_RESIZE; + + // stop dragging + state_.dragbutton = 0; + state_.dragrect = 0; + + // application-specific processing + if (event_callback_) { + event_callback_(&state_); + } +} + +void PlatformUIAdapter::UpdateMjuiState() { + // mouse buttons + state_.left = IsLeftMouseButtonPressed(); + state_.right = IsRightMouseButtonPressed(); + state_.middle = IsMiddleMouseButtonPressed(); + + // keyboard modifiers + state_.control = IsCtrlKeyPressed(); + state_.shift = IsShiftKeyPressed(); + state_.alt = IsAltKeyPressed(); + + // swap left and right if Alt + if (state_.alt) { + int tmp = state_.left; + state_.left = state_.right; + state_.right = tmp; + } + + // get mouse position, scale by buffer-to-window ratio + auto [x, y] = GetCursorPosition(); + const double buffer_window_ratio = + static_cast(GetFramebufferSize().first) / GetWindowSize().first; + x *= buffer_window_ratio; + y *= buffer_window_ratio; + + // invert y to match OpenGL convention + y = state_.rect[0].height - y; + + // save + state_.dx = x - state_.x; + state_.dy = y - state_.y; + state_.x = x; + state_.y = y; + + // find mouse rectangle + state_.mouserect = mjr_findRect(mju_round(x), mju_round(y), state_.nrect-1, state_.rect+1) + 1; +} +} // namespace mujoco diff --git a/simulate/platform_ui_adapter.h b/simulate/platform_ui_adapter.h new file mode 100644 index 00000000..e1810986 --- /dev/null +++ b/simulate/platform_ui_adapter.h @@ -0,0 +1,96 @@ +// Copyright 2023 DeepMind Technologies Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MUJOCO_SIMULATE_PLATFORM_UI_ADAPTER_H_ +#define MUJOCO_SIMULATE_PLATFORM_UI_ADAPTER_H_ + +#include + +#include + +namespace mujoco { +class PlatformUIAdapter { + public: + virtual ~PlatformUIAdapter(); + + inline mjuiState& state() { return state_; } + inline const mjuiState& state() const { return state_; } + + inline mjrContext& mjr_context() { return con_; } + inline const mjrContext& mjr_context() const { return con_; } + + inline void SetEventCallback(void (*event_callback)(mjuiState*)) { + event_callback_ = event_callback; + } + + inline void SetLayoutCallback(void (*layout_callback)(mjuiState*)) { + layout_callback_ = layout_callback; + } + + // Optionally overrideable function to (re)create an mjrContext for an mjModel + virtual bool RefreshMjrContext(const mjModel* m, int fontscale); + + // Pure virtual functions to be implemented by individual adapters + virtual std::pair GetCursorPosition() const = 0; + virtual double GetDisplayPixelsPerInch() const = 0; + virtual std::pair GetFramebufferSize() const = 0; + virtual std::pair GetWindowSize() const = 0; + virtual void PollEvents() = 0; + virtual void SetClipboardString(const char* text) = 0; + virtual void SetVSync(bool enabled) = 0; + virtual void SetWindowTitle(const char* title) = 0; + virtual bool ShouldCloseWindow() const = 0; + virtual void SwapBuffers() = 0; + virtual void ToggleFullscreen() = 0; + + virtual bool IsLeftMouseButtonPressed() const = 0; + virtual bool IsMiddleMouseButtonPressed() const = 0; + virtual bool IsRightMouseButtonPressed() const = 0; + + virtual bool IsAltKeyPressed() const = 0; + virtual bool IsCtrlKeyPressed() const = 0; + virtual bool IsShiftKeyPressed() const = 0; + + virtual bool IsMouseButtonDownEvent(int act) const = 0; + virtual bool IsKeyDownEvent(int act) const = 0; + + virtual int TranslateKeyCode(int key) const = 0; + virtual mjtButton TranslateMouseButton(int button) const = 0; + + protected: + PlatformUIAdapter(); + + // Event handlers + void OnFilesDrop(int count, const char** paths); + void OnKey(int key, int scancode, int act); + void OnMouseButton(int button, int act); + void OnMouseMove(double x, double y); + void OnScroll(double xoffset, double yoffset); + void OnWindowRefresh(); + void OnWindowResize(int width, int height); + + mjuiState state_; + void (*event_callback_)(mjuiState*); + void (*layout_callback_)(mjuiState*); + + mjrContext con_; + const mjModel* last_model_ = nullptr; + int last_fontscale_ = -1; + + private: + void UpdateMjuiState(); +}; +} // namespace mujoco + +#endif // MUJOCO_SIMULATE_PLATFORM_UI_ADAPTER_H_ diff --git a/simulate/simulate.cc b/simulate/simulate.cc index b20d2350..f4de7e8d 100644 --- a/simulate/simulate.cc +++ b/simulate/simulate.cc @@ -20,18 +20,15 @@ #include #include #include +#include #include #include +#include -#include #include "lodepng.h" -#include -#include -#include #include #include -#include "glfw_dispatch.h" -#include "uitools.h" +#include "platform_ui_adapter.h" #include "array_safety.h" // When launched via an App Bundle on macOS, the working directory is the path to the App Bundle's @@ -51,7 +48,8 @@ namespace { namespace mj = ::mujoco; namespace mju = ::mujoco::sample_util; -using ::mujoco::Glfw; +using Seconds = std::chrono::duration; +using Milliseconds = std::chrono::duration; //------------------------------------------- global ----------------------------------------------- @@ -350,13 +348,13 @@ void profilershow(mj::Simulate* sim, mjrRect rect) { rect.width/4, rect.height/4 }; - mjr_figure(viewport, &sim->figtimer, &sim->con); + mjr_figure(viewport, &sim->figtimer, &sim->platform_ui->mjr_context()); viewport.bottom += rect.height/4; - mjr_figure(viewport, &sim->figsize, &sim->con); + mjr_figure(viewport, &sim->figsize, &sim->platform_ui->mjr_context()); viewport.bottom += rect.height/4; - mjr_figure(viewport, &sim->figcost, &sim->con); + mjr_figure(viewport, &sim->figcost, &sim->platform_ui->mjr_context()); viewport.bottom += rect.height/4; - mjr_figure(viewport, &sim->figconstraint, &sim->con); + mjr_figure(viewport, &sim->figconstraint, &sim->platform_ui->mjr_context()); } @@ -453,7 +451,7 @@ void sensorshow(mj::Simulate* sim, mjrRect rect) { width, rect.height/3 }; - mjr_figure(viewport, &sim->figsensor, &sim->con); + mjr_figure(viewport, &sim->figsensor, &sim->platform_ui->mjr_context()); } // prepare info text @@ -935,12 +933,12 @@ void copykey(mj::Simulate* sim) { mju::strcat_arr(clipboard, "'/>"); // copy to clipboard - Glfw().glfwSetClipboardString(sim->window, clipboard); + sim->platform_ui->SetClipboardString(clipboard); } // millisecond timer, for MuJoCo built-in profiler mjtNum timer() { - return 1000*Glfw().glfwGetTime(); + return Milliseconds(mj::Simulate::Clock::now().time_since_epoch()).count(); } // clear all times @@ -975,7 +973,7 @@ void copycamera(mj::Simulate* sim) { camera[0].up[0], camera[0].up[1], camera[0].up[2]); // copy spec into clipboard - Glfw().glfwSetClipboardString(sim->window, clipboard); + sim->platform_ui->SetClipboardString(clipboard); } // update UI 0 when MuJoCo structures change (except for joint sliders) @@ -1000,11 +998,36 @@ void updatesettings(mj::Simulate* sim) { } // update UI - mjui_update(-1, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(-1, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); +} + +// Compute suitable font scale. +int ComputeFontScale(const mj::PlatformUIAdapter& platform_ui) { + // compute framebuffer-to-window ratio + auto [buf_width, buf_height] = platform_ui.GetFramebufferSize(); + auto [win_width, win_height] = platform_ui.GetWindowSize(); + double b2w = static_cast(buf_width) / win_width; + + // compute PPI + double PPI = b2w * platform_ui.GetDisplayPixelsPerInch(); + + // estimate font scaling, guard against unrealistic PPI + int fs; + if (buf_width > win_width) { + fs = mju_round(b2w * 100); + } else if (PPI>50 && PPI<350) { + fs = mju_round(PPI); + } else { + fs = 150; + } + fs = mju_round(fs * 0.02) * 50; + fs = mjMIN(300, mjMAX(100, fs)); + + return fs; } -//---------------------------------- UI hooks (for uitools.c) -------------------------------------- +//---------------------------------- UI handlers --------------------------------------------------- // determine enable/disable item state given category int uiPredicate(int category, void* userdata) { @@ -1033,11 +1056,6 @@ void uiLayout(mjuiState* state) { // set number of rectangles state->nrect = 4; - // rect 0: entire framebuffer - rect[0].left = 0; - rect[0].bottom = 0; - Glfw().glfwGetFramebufferSize(sim->window, &rect[0].width, &rect[0].height); - // rect 1: UI 0 rect[1].left = 0; rect[1].width = sim->ui0_enable ? sim->ui0.width : 0; @@ -1057,6 +1075,13 @@ void uiLayout(mjuiState* state) { rect[3].height = rect[0].height; } +void uiModify(mjUI* ui, mjuiState* state, mjrContext* con) { + mjui_resize(ui, con); + mjr_addAux(ui->auxid, ui->width, ui->maxheight, ui->spacing.samples, con); + uiLayout(state); + mjui_update(-1, -1, ui, state, con); +} + // handle UI event void uiEvent(mjuiState* state) { mj::Simulate* sim = static_cast(state->userdata); @@ -1070,7 +1095,7 @@ void uiEvent(mjuiState* state) { (state->dragrect==0 && state->mouserect==sim->ui0.rectid) || state->type==mjEVENT_KEY) { // process UI event - mjuiItem* it = mjui_event(&sim->ui0, state, &sim->con); + mjuiItem* it = mjui_event(&sim->ui0, state, &sim->platform_ui->mjr_context()); // file section if (it && it->sectionid==SECT_FILE) { @@ -1125,39 +1150,21 @@ void uiEvent(mjuiState* state) { break; case 2: // Font - mjr_changeFont(50*(sim->font+1), &sim->con); + mjr_changeFont(50*(sim->font+1), &sim->platform_ui->mjr_context()); break; case 9: // Full screen - if (Glfw().glfwGetWindowMonitor(sim->window)) { - // restore window from saved data - Glfw().glfwSetWindowMonitor(sim->window, nullptr, sim->windowpos[0], sim->windowpos[1], - sim->windowsize[0], sim->windowsize[1], 0); - } - - // currently windowed: switch to full screen - else { - // save window data - Glfw().glfwGetWindowPos(sim->window, sim->windowpos, sim->windowpos+1); - Glfw().glfwGetWindowSize(sim->window, sim->windowsize, sim->windowsize+1); - - // switch - Glfw().glfwSetWindowMonitor(sim->window, Glfw().glfwGetPrimaryMonitor(), 0, 0, - sim->vmode.width, sim->vmode.height, sim->vmode.refreshRate); - } - - // reinstante vsync, just in case - Glfw().glfwSwapInterval(sim->vsync); + sim->platform_ui->ToggleFullscreen(); break; case 10: // Vertical sync - Glfw().glfwSwapInterval(sim->vsync); + sim->platform_ui->SetVSync(sim->vsync); break; } // modify UI - uiModify(sim->window, &sim->ui0, state, &sim->con); - uiModify(sim->window, &sim->ui1, state, &sim->con); + uiModify(&sim->ui0, state, &sim->platform_ui->mjr_context()); + uiModify(&sim->ui1, state, &sim->platform_ui->mjr_context()); } // simulation section @@ -1245,7 +1252,8 @@ void uiEvent(mjuiState* state) { } else { sim->cam.type = mjCAMERA_FREE; sim->camera = 0; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, + &sim->platform_ui->mjr_context()); } } else { sim->cam.type = mjCAMERA_FIXED; @@ -1264,7 +1272,7 @@ void uiEvent(mjuiState* state) { sim->ui1.nsect = SECT_JOINT; makejoint(sim, sim->ui1.sect[SECT_JOINT].state); sim->ui1.nsect = NSECT1; - uiModify(sim->window, &sim->ui1, state, &sim->con); + uiModify(&sim->ui1, state, &sim->platform_ui->mjr_context()); } // remake control section if actuator group changed @@ -1272,7 +1280,7 @@ void uiEvent(mjuiState* state) { sim->ui1.nsect = SECT_CONTROL; makecontrol(sim, sim->ui1.sect[SECT_CONTROL].state); sim->ui1.nsect = NSECT1; - uiModify(sim->window, &sim->ui1, state, &sim->con); + uiModify(&sim->ui1, state, &sim->platform_ui->mjr_context()); } } @@ -1287,14 +1295,14 @@ void uiEvent(mjuiState* state) { (state->dragrect==0 && state->mouserect==sim->ui1.rectid) || state->type==mjEVENT_KEY) { // process UI event - mjuiItem* it = mjui_event(&sim->ui1, state, &sim->con); + mjuiItem* it = mjui_event(&sim->ui1, state, &sim->platform_ui->mjr_context()); // control section if (it && it->sectionid==SECT_CONTROL) { // clear controls if (it->itemid==0) { mju_zero(d->ctrl, m->nu); - mjui_update(SECT_CONTROL, -1, &sim->ui1, &sim->uistate, &sim->con); + mjui_update(SECT_CONTROL, -1, &sim->ui1, &sim->uistate, &sim->platform_ui->mjr_context()); } } @@ -1311,7 +1319,7 @@ void uiEvent(mjuiState* state) { if (m) { sim->run = 1 - sim->run; sim->pert.active = 0; - mjui_update(-1, -1, &sim->ui0, state, &sim->con); + mjui_update(-1, -1, &sim->ui0, state, &sim->platform_ui->mjr_context()); } break; @@ -1348,7 +1356,7 @@ void uiEvent(mjuiState* state) { sim->camera += 1; } sim->cam.fixedcamid = sim->camera - 2; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); } break; @@ -1362,28 +1370,28 @@ void uiEvent(mjuiState* state) { sim->camera -= 1; } sim->cam.fixedcamid = sim->camera - 2; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); } break; case mjKEY_F6: // cycle frame visualisation if (m) { sim->vopt.frame = (sim->vopt.frame + 1) % mjNFRAME; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); } break; case mjKEY_F7: // cycle label visualisation if (m) { sim->vopt.label = (sim->vopt.label + 1) % mjNLABEL; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); } break; case mjKEY_ESCAPE: // free camera sim->cam.type = mjCAMERA_FREE; sim->camera = 0; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->platform_ui->mjr_context()); break; case '-': // slow down @@ -1472,7 +1480,8 @@ void uiEvent(mjuiState* state) { // UI camera sim->camera = 1; - mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, &sim->con); + mjui_update(SECT_RENDERING, -1, &sim->ui0, &sim->uistate, + &sim->platform_ui->mjr_context()); } } @@ -1531,33 +1540,30 @@ void uiEvent(mjuiState* state) { return; } -} -void uiRender(mjuiState* state) { - mj::Simulate* sim = static_cast(state->userdata); - sim->render(); -} - -// drop file callback -void drop(mj::Simulate* sim, int count, const char** paths) { - // make sure list is non-empty - if (count>0) { + // Dropped files + if (state->type == mjEVENT_FILESDROP && state->dropcount > 0) { while (sim->droploadrequest.load()) {} - mju::strcpy_arr(sim->dropfilename, paths[0]); + mju::strcpy_arr(sim->dropfilename, state->droppaths[0]); sim->droploadrequest.store(true); + return; + } + + // Redraw + if (state->type == mjEVENT_REDRAW) { + sim->render(); + return; } } - -void uiDrop(mjuiState* state, int count, const char** paths) { - mj::Simulate* simulate = static_cast(state->userdata); - drop(simulate, count, paths); -} - } // namespace namespace mujoco { namespace mju = ::mujoco::sample_util; +Simulate::Simulate(std::unique_ptr platform_ui) + : platform_ui(std::move(platform_ui)), + uistate(this->platform_ui->state()) {} + //------------------------------------ apply pose perturbations ------------------------------------ void Simulate::applyposepertubations(int flg_paused) { if (this->m != nullptr) { @@ -1598,7 +1604,7 @@ void Simulate::loadmodel() { // re-create scene and context mjv_makeScene(this->m, &this->scn, maxgeom); - mjr_makeContext(this->m, &this->con, 50*(this->font+1)); + this->platform_ui->RefreshMjrContext(this->m, 50*(this->font+1)); // clear perturbation state this->pert.active = 0; @@ -1616,10 +1622,10 @@ void Simulate::loadmodel() { mjv_updateScene(this->m, this->d, &this->vopt, &this->pert, &this->cam, mjCAT_ALL, &this->scn); // set window title to model name - if (this->window && this->m->names) { - char title[200] = "Simulate : "; + if (this->m->names) { + char title[200] = "MuJoCo : "; mju::strcat_arr(title, this->m->names); - Glfw().glfwSetWindowTitle(this->window, title); + platform_ui->SetWindowTitle(title); } // set keyframe range and divisions @@ -1631,8 +1637,8 @@ void Simulate::loadmodel() { makesections(this); // full ui update - uiModify(this->window, &this->ui0, &this->uistate, &this->con); - uiModify(this->window, &this->ui1, &this->uistate, &this->con); + uiModify(&this->ui0, &this->uistate, &this->platform_ui->mjr_context()); + uiModify(&this->ui1, &this->uistate, &this->platform_ui->mjr_context()); updatesettings(this); // clear request @@ -1659,11 +1665,11 @@ void Simulate::loadmodel() { // prepare to render void Simulate::prepare() { // data for FPS calculation - static double lastupdatetm = 0; + static std::chrono::time_point lastupdatetm; // update interval, save update time - double tmnow = Glfw().glfwGetTime(); - double interval = tmnow - lastupdatetm; + auto tmnow = Clock::now(); + double interval = Seconds(tmnow - lastupdatetm).count(); interval = mjMIN(1, mjMAX(0.0001, interval)); lastupdatetm = tmnow; @@ -1678,12 +1684,12 @@ void Simulate::prepare() { // update watch if (this->ui0_enable && this->ui0.sect[SECT_WATCH].state) { watch(this); - mjui_update(SECT_WATCH, -1, &this->ui0, &this->uistate, &this->con); + mjui_update(SECT_WATCH, -1, &this->ui0, &this->uistate, &this->platform_ui->mjr_context()); } // update joint if (this->ui1_enable && this->ui1.sect[SECT_JOINT].state) { - mjui_update(SECT_JOINT, -1, &this->ui1, &this->uistate, &this->con); + mjui_update(SECT_JOINT, -1, &this->ui1, &this->uistate, &this->platform_ui->mjr_context()); } // update info text @@ -1693,7 +1699,7 @@ void Simulate::prepare() { // update control if (this->ui1_enable && this->ui1.sect[SECT_CONTROL].state) { - mjui_update(SECT_CONTROL, -1, &this->ui1, &this->uistate, &this->con); + mjui_update(SECT_CONTROL, -1, &this->ui1, &this->uistate, &this->platform_ui->mjr_context()); } // update profiler @@ -1712,6 +1718,11 @@ void Simulate::prepare() { // render the ui to the window void Simulate::render() { + if (this->platform_ui->RefreshMjrContext(this->m, 50*(this->font+1))) { + uiModify(&this->ui0, &this->uistate, &this->platform_ui->mjr_context()); + uiModify(&this->ui1, &this->uistate, &this->platform_ui->mjr_context()); + } + // get 3D rectangle and reduced for profiler mjrRect rect = this->uistate.rect[3]; mjrRect smallrect = rect; @@ -1726,39 +1737,43 @@ void Simulate::render() { // label if (this->loadrequest) { - mjr_overlay(mjFONT_BIG, mjGRID_TOPRIGHT, smallrect, "loading", nullptr, &this->con); + mjr_overlay(mjFONT_BIG, mjGRID_TOPRIGHT, smallrect, "loading", nullptr, + &this->platform_ui->mjr_context()); } else { char intro_message[Simulate::kMaxFilenameLength]; mju::sprintf_arr(intro_message, "MuJoCo version %s\nDrag-and-drop model file here", mj_versionString()); - mjr_overlay(mjFONT_NORMAL, mjGRID_TOPLEFT, rect, intro_message, 0, &this->con); + mjr_overlay(mjFONT_NORMAL, mjGRID_TOPLEFT, rect, intro_message, 0, + &this->platform_ui->mjr_context()); } // show last loading error if (this->loadError[0]) { - mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->loadError, 0, &this->con); + mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->loadError, 0, + &this->platform_ui->mjr_context()); } // render uis if (this->ui0_enable) { - mjui_render(&this->ui0, &this->uistate, &this->con); + mjui_render(&this->ui0, &this->uistate, &this->platform_ui->mjr_context()); } if (this->ui1_enable) { - mjui_render(&this->ui1, &this->uistate, &this->con); + mjui_render(&this->ui1, &this->uistate, &this->platform_ui->mjr_context()); } // finalize - Glfw().glfwSwapBuffers(this->window); + this->platform_ui->SwapBuffers(); return; } // render scene - mjr_render(rect, &this->scn, &this->con); + mjr_render(rect, &this->scn, &this->platform_ui->mjr_context()); // show last loading error if (this->loadError[0]) { - mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->loadError, 0, &this->con); + mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->loadError, 0, + &this->platform_ui->mjr_context()); } // make pause/loading label @@ -1794,28 +1809,30 @@ void Simulate::render() { std::string newline = !pauseloadlabel.empty() && rtlabel[0] ? "\n" : ""; std::string topleftlabel = rtlabel + newline + pauseloadlabel; mjr_overlay(mjFONT_BIG, mjGRID_TOPLEFT, smallrect, - topleftlabel.c_str(), nullptr, &this->con); + topleftlabel.c_str(), nullptr, &this->platform_ui->mjr_context()); } // show ui 0 if (this->ui0_enable) { - mjui_render(&this->ui0, &this->uistate, &this->con); + mjui_render(&this->ui0, &this->uistate, &this->platform_ui->mjr_context()); } // show ui 1 if (this->ui1_enable) { - mjui_render(&this->ui1, &this->uistate, &this->con); + mjui_render(&this->ui1, &this->uistate, &this->platform_ui->mjr_context()); } // show help if (this->help) { - mjr_overlay(mjFONT_NORMAL, mjGRID_TOPLEFT, rect, help_title, help_content, &this->con); + mjr_overlay(mjFONT_NORMAL, mjGRID_TOPLEFT, rect, help_title, help_content, + &this->platform_ui->mjr_context()); } // show info if (this->info) { - mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->info_title, this->info_content, &this->con); + mjr_overlay(mjFONT_NORMAL, mjGRID_BOTTOMLEFT, rect, this->info_title, this->info_content, + &this->platform_ui->mjr_context()); } // show profiler @@ -1836,7 +1853,7 @@ void Simulate::render() { if (!rgb) { mju_error("could not allocate buffer for screenshot"); } - mjr_readPixels(rgb.get(), nullptr, uistate.rect[0], &con); + mjr_readPixels(rgb.get(), nullptr, uistate.rect[0], &this->platform_ui->mjr_context()); // flip up-down for (int r = 0; r < h/2; ++r) { @@ -1861,45 +1878,15 @@ void Simulate::render() { } // finalize - Glfw().glfwSwapBuffers(this->window); + this->platform_ui->SwapBuffers(); } -// clear callbacks registered in external structures -void Simulate::clearcallback() { - uiClearCallback(this->window); -} void Simulate::renderloop() { // Set timer callback (milliseconds) mjcb_time = timer; - // multisampling - Glfw().glfwWindowHint(GLFW_SAMPLES, 4); - Glfw().glfwWindowHint(GLFW_VISIBLE, 1); - - // get videomode and save - this->vmode = *Glfw().glfwGetVideoMode(Glfw().glfwGetPrimaryMonitor()); - - // use videomode refreshrate if nonzero - if (this->vmode.refreshRate) this->refreshRate = this->vmode.refreshRate; - - // create window - this->window = Glfw().glfwCreateWindow((2*this->vmode.width)/3, (2*this->vmode.height)/3, - "Simulate", nullptr, nullptr); - if (!this->window) { - Glfw().glfwTerminate(); - mju_error("could not create window"); - } - - // save window position and size - Glfw().glfwGetWindowPos(this->window, this->windowpos, this->windowpos+1); - Glfw().glfwGetWindowSize(this->window, this->windowsize, this->windowsize+1); - - // make context current, set v-sync - Glfw().glfwMakeContextCurrent(this->window); - Glfw().glfwSwapInterval(this->vsync); - // init abstract visualization mjv_defaultCamera(&this->cam); mjv_defaultOption(&this->vopt); @@ -1911,31 +1898,38 @@ void Simulate::renderloop() { mjv_makeScene(nullptr, &this->scn, maxgeom); // select default font - int fontscale = uiFontScale(this->window); + int fontscale = ComputeFontScale(*this->platform_ui); this->font = fontscale/50 - 1; // make empty context - mjr_defaultContext(&this->con); - mjr_makeContext(nullptr, &this->con, fontscale); + this->platform_ui->RefreshMjrContext(nullptr, fontscale); // init state and uis std::memset(&this->uistate, 0, sizeof(mjuiState)); std::memset(&this->ui0, 0, sizeof(mjUI)); std::memset(&this->ui1, 0, sizeof(mjUI)); + + auto [buf_width, buf_height] = this->platform_ui->GetFramebufferSize(); + this->uistate.nrect = 1; + this->uistate.rect[0].width = buf_width; + this->uistate.rect[0].height = buf_height; + this->ui0.spacing = mjui_themeSpacing(this->spacing); this->ui0.color = mjui_themeColor(this->color); this->ui0.predicate = uiPredicate; this->ui0.rectid = 1; this->ui0.auxid = 0; + this->ui1.spacing = mjui_themeSpacing(this->spacing); this->ui1.color = mjui_themeColor(this->color); this->ui1.predicate = uiPredicate; this->ui1.rectid = 2; this->ui1.auxid = 1; - // set GLFW callbacks + // set GUI adapter callbacks this->uistate.userdata = this; - uiSetCallback(this->window, &this->uistate, uiEvent, uiLayout, uiRender, uiDrop); + this->platform_ui->SetEventCallback(uiEvent); + this->platform_ui->SetLayoutCallback(uiLayout); // populate uis with standard sections this->ui0.userdata = this; @@ -1944,11 +1938,11 @@ void Simulate::renderloop() { mjui_add(&this->ui0, this->defOption); mjui_add(&this->ui0, this->defSimulation); mjui_add(&this->ui0, this->defWatch); - uiModify(this->window, &this->ui0, &this->uistate, &this->con); - uiModify(this->window, &this->ui1, &this->uistate, &this->con); + uiModify(&this->ui0, &this->uistate, &this->platform_ui->mjr_context()); + uiModify(&this->ui1, &this->uistate, &this->platform_ui->mjr_context()); // run event loop - while (!Glfw().glfwWindowShouldClose(this->window) && !this->exitrequest.load()) { + while (!this->platform_ui->ShouldCloseWindow() && !this->exitrequest.load()) { { const std::lock_guard lock(this->mtx); @@ -1959,8 +1953,8 @@ void Simulate::renderloop() { this->loadrequest = 1; } - // handle events (calls all callbacks) - Glfw().glfwPollEvents(); + // poll and handle events + this->platform_ui->PollEvents(); // prepare to render this->prepare(); @@ -1972,16 +1966,6 @@ void Simulate::renderloop() { this->exitrequest.store(true); - this->clearcallback(); mjv_freeScene(&this->scn); - mjr_freeContext(&this->con); - - Glfw().glfwDestroyWindow(this->window); } - -//------------------------------------ setup the glfw dispatch table ------------------------------- -void setglfwdlhandle(void* dlhandle) { - Glfw(dlhandle); -} - } // namespace mujoco diff --git a/simulate/simulate.h b/simulate/simulate.h index 5b4a3120..8f44b3be 100644 --- a/simulate/simulate.h +++ b/simulate/simulate.h @@ -16,35 +16,28 @@ #define MUJOCO_SIMULATE_SIMULATE_H_ #include +#include #include +#include #include +#include #include -#include #include - -#ifdef MJSIMULATE_STATIC - // static library - #define MJSIMULATEAPI - #define MJSIMULATELOCAL -#else - #ifdef MJSIMULATE_DLL_EXPORTS - #define MJSIMULATEAPI MUJOCO_HELPER_DLL_EXPORT - #else - #define MJSIMULATEAPI MUJOCO_HELPER_DLL_IMPORT - #endif - #define MJSIMULATELOCAL MUJOCO_HELPER_DLL_LOCAL -#endif +#include "platform_ui_adapter.h" namespace mujoco { //-------------------------------- global ----------------------------------------------- // Simulate states not contained in MuJoCo structures -class MJSIMULATEAPI Simulate { +class Simulate { public: + using Clock = std::chrono::steady_clock; + static_assert(std::ratio_less_equal_v); + // create object and initialize the simulate ui - Simulate() = default; + Simulate(std::unique_ptr platform_ui_adapter); // Apply UI pose perturbations to model and data void applyposepertubations(int flg_paused); @@ -66,11 +59,7 @@ class MJSIMULATEAPI Simulate { // render the ui to the window void render(); - // clear callbacks registered in external structures - void clearcallback(); - // loop to render the UI (must be called from main thread because of MacOS) - // https://discourse.glfw.org/t/multithreading-glfw/573/5 void renderloop(); // constants @@ -162,13 +151,11 @@ class MJSIMULATEAPI Simulate { mjvFigure figsensor = {}; // OpenGL rendering and UI - GLFWvidmode vmode = {}; int refreshRate = 60; int windowpos[2] = {0}; int windowsize[2] = {0}; - mjrContext con = {}; - GLFWwindow* window = nullptr; - mjuiState uistate = {}; + std::unique_ptr platform_ui; + mjuiState& uistate; mjUI ui0 = {}; mjUI ui1 = {}; @@ -226,11 +213,6 @@ class MJSIMULATEAPI Simulate { char info_title[Simulate::kMaxFilenameLength] = {0}; char info_content[Simulate::kMaxFilenameLength] = {0}; }; - -// setup the glfw dispatch table -// if set, must be called prior to other Simulate functions -MJSIMULATEAPI void setglfwdlhandle(void* dlhandle); - } // namespace mujoco #endif diff --git a/simulate/uitools.cc b/simulate/uitools.cc deleted file mode 100644 index 362fd544..00000000 --- a/simulate/uitools.cc +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright 2021 DeepMind Technologies Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "uitools.h" - -#include -#include - -#include -#include "glfw_dispatch.h" - -namespace { -using ::mujoco::Glfw; -} - -//------------------------------------ internal GLFW callbacks ------------------------------------- - -// update state -static void uiUpdateState(GLFWwindow* wnd) { - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // mouse buttons - state->left = (Glfw().glfwGetMouseButton(wnd, GLFW_MOUSE_BUTTON_LEFT)==GLFW_PRESS); - state->right = (Glfw().glfwGetMouseButton(wnd, GLFW_MOUSE_BUTTON_RIGHT)==GLFW_PRESS); - state->middle = (Glfw().glfwGetMouseButton(wnd, GLFW_MOUSE_BUTTON_MIDDLE)==GLFW_PRESS); - - // keyboard modifiers - state->control = (Glfw().glfwGetKey(wnd, GLFW_KEY_LEFT_CONTROL)==GLFW_PRESS || - Glfw().glfwGetKey(wnd, GLFW_KEY_RIGHT_CONTROL)==GLFW_PRESS); - state->shift = (Glfw().glfwGetKey(wnd, GLFW_KEY_LEFT_SHIFT)==GLFW_PRESS || - Glfw().glfwGetKey(wnd, GLFW_KEY_RIGHT_SHIFT)==GLFW_PRESS); - state->alt = (Glfw().glfwGetKey(wnd, GLFW_KEY_LEFT_ALT)==GLFW_PRESS || - Glfw().glfwGetKey(wnd, GLFW_KEY_RIGHT_ALT)==GLFW_PRESS); - - // swap left and right if Alt - if (state->alt) { - int tmp = state->left; - state->left = state->right; - state->right = tmp; - } - - // get mouse position, scale by buffer-to-window ratio - double x, y; - Glfw().glfwGetCursorPos(wnd, &x, &y); - x *= ptr->buffer2window; - y *= ptr->buffer2window; - - // invert y to match OpenGL convention - y = state->rect[0].height - y; - - // save - state->dx = x - state->x; - state->dy = y - state->y; - state->x = x; - state->y = y; - - // find mouse rectangle - state->mouserect = mjr_findRect(mju_round(x), mju_round(y), state->nrect-1, state->rect+1) + 1; -} - - - -// keyboard -static void uiKeyboard(GLFWwindow* wnd, int key, int scancode, int act, int mods) { - // release: nothing to do - if (act==GLFW_RELEASE) { - return; - } - - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // update state - uiUpdateState(wnd); - - // set key info - state->type = mjEVENT_KEY; - state->key = key; - state->keytime = Glfw().glfwGetTime(); - - // application-specific processing - ptr->uiEvent(state); -} - - - -// mouse button -static void uiMouseButton(GLFWwindow* wnd, int button, int act, int mods) { - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // update state - uiUpdateState(wnd); - - // translate button - if (button==GLFW_MOUSE_BUTTON_LEFT) { - button = mjBUTTON_LEFT; - } else if (button==GLFW_MOUSE_BUTTON_RIGHT) { - button = mjBUTTON_RIGHT; - } else { - button = mjBUTTON_MIDDLE; - } - - // swap left and right if Alt - if (Glfw().glfwGetKey(wnd, GLFW_KEY_LEFT_ALT)==GLFW_PRESS || - Glfw().glfwGetKey(wnd, GLFW_KEY_RIGHT_ALT)==GLFW_PRESS) { - if (button==mjBUTTON_LEFT) { - button = mjBUTTON_RIGHT; - } else if (button==mjBUTTON_RIGHT) { - button = mjBUTTON_LEFT; - } - } - - // press - if (act==GLFW_PRESS) { - // detect doubleclick: 250 ms - if (button==state->button && Glfw().glfwGetTime()-state->buttontime<0.25) { - state->doubleclick = 1; - } else { - state->doubleclick = 0; - } - - // set info - state->type = mjEVENT_PRESS; - state->button = button; - state->buttontime = Glfw().glfwGetTime(); - - // start dragging - if (state->mouserect) { - state->dragbutton = state->button; - state->dragrect = state->mouserect; - } - } - - // release - else { - state->type = mjEVENT_RELEASE; - } - - // application-specific processing - ptr->uiEvent(state); - - // stop dragging after application processing - if (state->type==mjEVENT_RELEASE) { - state->dragrect = 0; - state->dragbutton = 0; - } -} - - - -// mouse move -static void uiMouseMove(GLFWwindow* wnd, double xpos, double ypos) { - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // no buttons down: nothing to do - if (!state->left && !state->right && !state->middle) { - return; - } - - // update state - uiUpdateState(wnd); - - // set move info - state->type = mjEVENT_MOVE; - - // application-specific processing - ptr->uiEvent(state); -} - - - -// scroll -static void uiScroll(GLFWwindow* wnd, double xoffset, double yoffset) { - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // update state - uiUpdateState(wnd); - - // set scroll info, scale by buffer-to-window ratio - state->type = mjEVENT_SCROLL; - state->sx = xoffset * ptr->buffer2window; - state->sy = yoffset * ptr->buffer2window; - - // application-specific processing - ptr->uiEvent(state); -} - - - -// resize -static void uiResize(GLFWwindow* wnd, int width, int height) { - // extract data from user pointer - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - - // set layout - ptr->uiLayout(state); - - // update state - uiUpdateState(wnd); - - // set resize info - state->type = mjEVENT_RESIZE; - - // stop dragging - state->dragbutton = 0; - state->dragrect = 0; - - // application-specific processing (unless called with 0,0 from uiModify) - if (width && height) { - ptr->uiEvent(state); - } -} - - -static void uiRender(GLFWwindow* wnd) { - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - ptr->uiRender(state); -} - -static void uiDrop(GLFWwindow* wnd, int count, const char** paths) { - uiUserPointer* ptr = (uiUserPointer*)Glfw().glfwGetWindowUserPointer(wnd); - mjuiState* state = ptr->state; - ptr->uiDrop(state, count, paths); -} - -//------------------------------------------- public API ------------------------------------------- - -// Compute suitable font scale. -int uiFontScale(GLFWwindow* wnd) { - // compute framebuffer-to-window ratio - int width_win, width_buf, height; - Glfw().glfwGetWindowSize(wnd, &width_win, &height); - Glfw().glfwGetFramebufferSize(wnd, &width_buf, &height); - double b2w = (double)width_buf / (double)width_win; - - // compute PPI - int width_MM, height_MM; - Glfw().glfwGetMonitorPhysicalSize(Glfw().glfwGetPrimaryMonitor(), &width_MM, &height_MM); - int width_vmode = Glfw().glfwGetVideoMode(Glfw().glfwGetPrimaryMonitor())->width; - double PPI = 25.4 * b2w * (double)width_vmode / (double)width_MM; - - // estimate font scaling, guard against unrealistic PPI - int fs; - if (width_buf>width_win) { - fs = mju_round(b2w * 100); - } else if (PPI>50 && PPI<350) { - fs = mju_round(PPI); - } else { - fs = 150; - } - fs = mju_round(fs * 0.02) * 50; - fs = mjMIN(300, mjMAX(100, fs)); - - return fs; -} - - - -// Set internal and user-supplied UI callbacks in GLFW window. -void uiSetCallback(GLFWwindow* wnd, mjuiState* state, - uiEventFn uiEvent, uiLayoutFn uiLayout, - uiRenderFn uiUserRender, uiDropFn uiUserDrop) { - // make container with user-supplied objects and set window pointer - uiUserPointer* ptr = (uiUserPointer*) mju_malloc(sizeof(uiUserPointer)); - ptr->state = state; - ptr->uiEvent = uiEvent; - ptr->uiLayout = uiLayout; - ptr->uiRender = uiUserRender; - ptr->uiDrop = uiUserDrop; - Glfw().glfwSetWindowUserPointer(wnd, ptr); - - // compute framebuffer-to-window pixel ratio - int width_win, width_buf, height; - Glfw().glfwGetWindowSize(wnd, &width_win, &height); - Glfw().glfwGetFramebufferSize(wnd, &width_buf, &height); - ptr->buffer2window = (double)width_buf / (double)width_win; - - // set internal callbacks - Glfw().glfwSetKeyCallback(wnd, uiKeyboard); - Glfw().glfwSetCursorPosCallback(wnd, uiMouseMove); - Glfw().glfwSetMouseButtonCallback(wnd, uiMouseButton); - Glfw().glfwSetScrollCallback(wnd, uiScroll); - Glfw().glfwSetWindowSizeCallback(wnd, uiResize); - Glfw().glfwSetWindowRefreshCallback(wnd, uiRender); - Glfw().glfwSetDropCallback(wnd, uiDrop); -} - - - -// Clear UI callbacks in GLFW window. -void uiClearCallback(GLFWwindow* wnd) { - // clear container - if (Glfw().glfwGetWindowUserPointer(wnd)) { - mju_free(Glfw().glfwGetWindowUserPointer(wnd)); - Glfw().glfwSetWindowUserPointer(wnd, nullptr); - } - - // clear internal callbacks - Glfw().glfwSetKeyCallback(wnd, nullptr); - Glfw().glfwSetCursorPosCallback(wnd, nullptr); - Glfw().glfwSetMouseButtonCallback(wnd, nullptr); - Glfw().glfwSetScrollCallback(wnd, nullptr); - Glfw().glfwSetWindowSizeCallback(wnd, nullptr); - Glfw().glfwSetWindowRefreshCallback(wnd, nullptr); - Glfw().glfwSetDropCallback(wnd, nullptr); -} - - - -// Modify UI structure. -void uiModify(GLFWwindow* wnd, mjUI* ui, mjuiState* state, mjrContext* con) { - mjui_resize(ui, con); - mjr_addAux(ui->auxid, ui->width, ui->maxheight, ui->spacing.samples, con); - uiResize(wnd, 0, 0); - mjui_update(-1, -1, ui, state, con); -} diff --git a/simulate/uitools.h b/simulate/uitools.h deleted file mode 100644 index 92720221..00000000 --- a/simulate/uitools.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2021 DeepMind Technologies Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MUJOCO_SIMULATE_UITOOLS_H_ -#define MUJOCO_SIMULATE_UITOOLS_H_ - -#include -#include - -// this is a C-API -#if defined(__cplusplus) -extern "C" { -#endif - - -// User-supplied callback function types. -typedef void (*uiEventFn)(mjuiState* state); -typedef void (*uiLayoutFn)(mjuiState* state); -typedef void (*uiRenderFn)(mjuiState* state); -typedef void (*uiDropFn) (mjuiState* state, int count, const char** paths); - -// Container for GLFW window pointer. -struct _uiUserPointer { - mjuiState* state; - uiEventFn uiEvent; - uiLayoutFn uiLayout; - uiRenderFn uiRender; - uiDropFn uiDrop; - double buffer2window; -}; -typedef struct _uiUserPointer uiUserPointer; - -// Set internal and user-supplied UI callbacks in GLFW window. -void uiSetCallback(GLFWwindow* wnd, mjuiState* state, - uiEventFn uiEvent, uiLayoutFn uiLayout, - uiRenderFn uiUserRender, uiDropFn uiUserDrop); - -// Clear UI callbacks in GLFW window. -void uiClearCallback(GLFWwindow* wnd); - -// Compute suitable font scale. -int uiFontScale(GLFWwindow* wnd); - -// Modify UI structure. -void uiModify(GLFWwindow* wnd, mjUI* ui, mjuiState* state, mjrContext* con); - - -#if defined(__cplusplus) -} -#endif - -#endif // MUJOCO_UITOOLS_H_ diff --git a/unity/Runtime/Bindings/MjBindings.cs b/unity/Runtime/Bindings/MjBindings.cs index bd2c547f..51f88e24 100644 --- a/unity/Runtime/Bindings/MjBindings.cs +++ b/unity/Runtime/Bindings/MjBindings.cs @@ -404,6 +404,8 @@ public enum mjtEvent : int{ mjEVENT_SCROLL = 4, mjEVENT_KEY = 5, mjEVENT_RESIZE = 6, + mjEVENT_REDRAW = 7, + mjEVENT_FILESDROP = 8, } public enum mjtCatBit : int{ mjCAT_STATIC = 1, @@ -2332,6 +2334,8 @@ public unsafe struct mjuiState_ { public int mouserect; public int dragrect; public int dragbutton; + public int dropcount; + public char** droppaths; } [StructLayout(LayoutKind.Sequential)]