Various UX improvements including:

- Add Toolbar and StatusBar.
- Reconfigure sections into Settings and Inspector groupings.
- Use docking for Settings/Inspector panels.
- Rename "Free" camera to "Tumble"; add WASD-based free camera.
- More dynamic layout of table-based elements.
- Remove "non-classic" UX mode.
- Switch to light mode by default.

PiperOrigin-RevId: 835174573
Change-Id: I1855caa79f00002e44ed2516631675727bdcb9f6

src/experimental/studio/app.h

@@ -24,8 +24,10 @@
 #include <unordered_map>
 #include <vector>
 
+#include <imgui.h>
 #include <mujoco/mujoco.h>
 #include "experimental/toolbox/helpers.h"
+#include "experimental/toolbox/interaction.h"
 #include "experimental/toolbox/physics.h"
 #include "experimental/toolbox/renderer.h"
 #include "experimental/toolbox/sim_profiler.h"
@@ -61,28 +63,19 @@ class App {
   static int LoadAssetCallback(const char* path, void* user_data,
                                unsigned char** out, std::uint64_t* out_size);
 
+  enum Style {
+    kLight,
+    kDark,
+  };
+
   // UI state that is persisted across application runs
   struct UiState {
-    bool classic_ui = true;
-
     char watch_field[1000] = "qpos";
     int watch_index = 0;
-    int camera_idx = 0;
+    int camera_idx = toolbox::kTumbleCameraIdx;
     int key_idx = 0;
     int scrub_idx = 0;
-    bool dark_mode = true;
+    Style style = kLight;
-
-    // UI visibility.
-    bool simulation = false;
-    bool physics = false;
-    bool rendering = false;
-    bool watch = false;
-    bool visualization = false;
-    bool groups = false;
-    bool joints = false;
-    bool controls = false;
-    bool profiler = false;
-    bool sensor = false;
 
     using Dict = std::unordered_map<std::string, std::string>;
     Dict ToDict() const;
@@ -90,13 +83,40 @@ class App {
   };
 
   // UI state that is transient and only needed while the application runs
-  // TODO(matijak): Combine this with UiState and identify the list of transient
-  // variables with a comment
   struct UiTempState {
+    bool should_exit = false;
+
+    // Windows.
     bool help = false;
     bool info = false;
+    bool chart_cpu_time = false;
+    bool chart_dimensions = false;
+    bool chart_counts = false;
+    bool chart_convergence = false;
+    bool settings_panel = true;
+    bool inspector_panel = true;
+    bool style_editor = false;
+    bool imgui_demo = false;
+    bool implot_demo = false;
     bool modal_open = false;
     bool load_popup = false;
+
+    // Controls.
+    bool perturb_active = false;
+    int speed_index = 0;
+    float cam_speed = 0.0f;
+
+    // Cached data.
+    float expected_label_width = 0;
+    std::vector<std::string> camera_names;
+    std::vector<std::string> speed_names;
+
+    // State.
+    int state_sig = 0;
+    std::vector<mjtNum> state;
+
+    // File dialogs.
+    char filename[1000] = "";
     std::string last_load_file;
     bool save_xml_popup = false;
     std::string last_save_xml_file;
@@ -108,26 +128,6 @@ class App {
     std::string last_print_model_file;
     bool print_data_popup = false;
     std::string last_print_data_file;
-    bool should_exit = false;
-    bool style_editor = false;
-    bool imgui_demo = false;
-    bool perturb_active = false;
-
-    int speed_index = 0;
-
-    // Visibility, position and size of the left and right UI panels
-    bool show_ui_lhs = true;
-    bool show_ui_rhs = true;
-    float pos_ui_lhs[2];
-    float pos_ui_rhs[2];
-    float size_ui_lhs[2];
-    float size_ui_rhs[2];
-
-    // Data for StateGui
-    int state_sig = 0;
-    std::vector<mjtNum> state;
-
-    char filename[1000] = "";
   };
 
   void OnModelLoaded(std::string_view model_file);
@@ -135,28 +135,31 @@ class App {
   void LoadSettings();
   void SaveSettings();
 
-  void SetCamera(int idx);
+  void SetSpeedIndex(int idx);
-
-  void ChangeSpeed(int delta);
 
   void HandleMouseEvents();
   void HandleKeyboardEvents();
+  void MoveCamera(toolbox::CameraMotion motion, mjtNum reldx, mjtNum reldy);
 
-  void BuildGuiWithWindows();
+  void SetupStyle(Style style);
-  void BuildGuiWithSections();
+  ImVec4 ConfigureDockingLayout();
+  void MainMenuGui();
+  void ToolBarGui();
+  void SettingsGui();
+  void InspectorGui();
+  void StatusBarGui();
 
   void InfoGui();
   void HelpGui();
-  void MainMenuGui();
   void FileDialogGui();
-  void SimulationGui();
+  void PlaybackGui();
   void PhysicsGui();
+  void NoiseGui();
   void RenderingGui();
   void VisualizationGui();
   void GroupsGui();
   void WatchGui();
   void SensorGui();
-  void ProfilerGui();
   void StateGui();
   void JointsGui();
   void ControlsGui();
@@ -164,6 +167,10 @@ class App {
   mjModel* Model() { return physics_->GetModel(); };
   mjData* Data() { return physics_->GetData(); };
 
+  float GetExpectedLabelWidth();
+  std::vector<const char*> GetCameraNames();
+  std::vector<const char*> GetSpeedNames();
+
   std::string ini_path_;
   std::string model_file_;
 

src/experimental/studio/main.cc

@@ -26,7 +26,7 @@
 #include "experimental/studio/app.h"
 
 ABSL_FLAG(int, window_width, 1400, "Window width");
-ABSL_FLAG(int, window_height, 700, "Window height");
+ABSL_FLAG(int, window_height, 720, "Window height");
 ABSL_FLAG(std::string, model_file, "", "MuJoCo model file.");
 
 static std::vector<std::byte> LoadAsset(std::string_view path) {

src/experimental/toolbox/imgui_widgets.h

@@ -15,8 +15,8 @@
 #ifndef MUJOCO_SRC_EXPERIMENTAL_TOOLBOX_IMGUI_WIDGETS_H_
 #define MUJOCO_SRC_EXPERIMENTAL_TOOLBOX_IMGUI_WIDGETS_H_
 
-#include <cstring>
 #include <optional>
+#include <utility>
 #include <string>
 #include <type_traits>
 #include <unordered_map>
@@ -37,6 +37,53 @@ void AppendIniSection(std::string& ini, const std::string& section,
 KeyValues ReadIniSection(const std::string& contents,
                          const std::string& section);
 
+// Helper class for setting ImGui style options; automatically resets the
+// styles when going out of scope.
+struct ScopedStyle {
+  ScopedStyle() = default;
+  ~ScopedStyle() {
+    Reset();
+  }
+
+  ScopedStyle(const ScopedStyle&) = delete;
+  ScopedStyle& operator=(const ScopedStyle&) = delete;
+  ScopedStyle(ScopedStyle&& other) { Swap(other); }
+  ScopedStyle& operator=(ScopedStyle&& other) { Swap(other); return *this; }
+
+  void Swap(ScopedStyle& other) {
+    std::swap(num_colors, other.num_colors);
+    std::swap(num_vars, other.num_vars);
+  }
+
+  ScopedStyle& Color(ImGuiCol col, ImColor color) {
+    ImGui::PushStyleColor(col, (ImU32)color);
+    ++num_colors;
+    return *this;
+  }
+
+  ScopedStyle& Var(ImGuiStyleVar var, float value) {
+    ImGui::PushStyleVar(var, value);
+    ++num_vars;
+    return *this;
+  }
+
+  ScopedStyle& Var(ImGuiStyleVar var, const ImVec2& value) {
+    ImGui::PushStyleVar(var, value);
+    ++num_vars;
+    return *this;
+  }
+
+  void Reset() {
+    ImGui::PopStyleVar(num_vars);
+    ImGui::PopStyleColor(num_colors);
+    num_colors = 0;
+    num_vars = 0;
+  }
+
+  int num_colors = 0;
+  int num_vars = 0;
+};
+
 // ImGui file dialog.
 bool ImGui_FileDialog(char* buf, int len);
 
@@ -54,108 +101,46 @@ bool ImGui_Checkbox(const char* name, T& value) {
   return res;
 }
 
-enum class ToggleKind {
-  // Solid when ON and transparent when OFF.
-  kButton,
-
-  // Slider which is right when ON and left when OFF.
-  kSlider,
-};
-
 template <typename T>
-bool Toggle(const char* label, T& boolean,
+bool ImGui_ButtonToggle(const char* label, T* boolean,
-            ToggleKind kind = ToggleKind::kButton, bool set_width = true) {
+                        const ImVec2& size = ImVec2(0, 0)) {
   static_assert(std::is_integral_v<T>, "Toggle only supports integral types.");
 
-  // Compute this width once and cache it. Only used when set_width is true.
+  ScopedStyle style;
-  static int toggle_width = []() {
+  if (!(*boolean)) {
-    int longest = 0;
+    ImColor button = ImGui::GetStyle().Colors[ImGuiCol_Button];
-    const char* longest_label = "";
+    button.Value.w = 0.0f;
-    for (int i = 0; i < mjNVISFLAG; ++i) {
+    style.Color(ImGuiCol_Button, button);
-      int length = static_cast<int>(strlen(mjVISSTRING[i][0]));
-      if (length > longest) {
-        longest_label = mjVISSTRING[i][0];
-        longest = length;
-      }
-    }
-    return ImGui::CalcTextSize(longest_label).x + 5;
-  }();
-
-  ImGui::PushID(label);
-  bool changed = false;
-  switch (kind) {
-    case ToggleKind::kButton: {
-      bool b = (boolean != 0);
-      bool transparent = !b;
-
-      // NOTE(matijak): Its nice to have the button trigger on click but this
-      // requires using the currently internal PressedOnClick flag and ButtonEx
-      // function. It looks like this API has been stable for a long time, but
-      // in case it changes in a way which breaks and is annoying to maintain we
-      // can revert to the else clause and remove the imgui_internal.h include.
-      // Note that the else clause overrides different style colors since the UI
-      // is more intuitive with different settings.
-      if constexpr (true) {
-        ImColor button = ImGui::GetStyle().Colors[ImGuiCol_Button];
-        if (transparent) button.Value.w = 0.0f;
-        ImGui::PushStyleColor(ImGuiCol_Button, (ImU32)button);
-        ImGui::PushStyleColor(ImGuiCol_ButtonActive, (ImU32)button);
-        ImGui::PushStyleColor(ImGuiCol_ButtonHovered, (ImU32)button);
-
-        // Button width is set via an explicit size parameter, not via the
-        // SetNextItemWidth function.
-        ImVec2 size = set_width ? ImVec2(toggle_width, 0) : ImVec2(0, 0);
-        changed = ImGui::ButtonEx(label, size, ImGuiButtonFlags_PressedOnClick);
-
-        if (changed) {
-          b = !b;
-        }
-        boolean = b;
-
-        ImGui::PopStyleColor(3);
-      } else {
-        if (transparent) {
-          ImColor button = ImGui::GetStyle().Colors[ImGuiCol_Button];
-          button.Value.w = 0.0f;
-          ImGui::PushStyleColor(ImGuiCol_Button, (ImU32)button);
-        }
-
-        // Button width is set via an explicit size parameter, not via the
-        // SetNextItemWidth function.
-        ImVec2 size = set_width ? ImVec2(toggle_width, 0) : ImVec2(0, 0);
-        changed = ImGui::Button(label, size);
-
-        if (changed) {
-          b = !b;
-        }
-        boolean = b;
-
-        if (transparent) {
-          ImGui::PopStyleColor(1);
-        }
-      }
-    } break;
-
-    case ToggleKind::kSlider: {
-      int i = (int)boolean;
-      const char* labels[2] = {label, label};
-      const ImGuiSliderFlags flags = ImGuiSliderFlags_NoInput;
-      if (set_width) ImGui::SetNextItemWidth(toggle_width);
-      changed = ImGui::SliderInt("", &i, 0, 1, labels[i], flags);
-      boolean = (i != 0);
-    } break;
   }
-  ImGui::PopID();
+
+  if (ImGui::Button(label, size)) {
+    *boolean = !(*boolean);
+    return true;
+  }
+  return false;
+}
+
+template <typename T>
+bool ImGui_SwitchToggle(const char* label, T* boolean,
+                        const ImVec2& size = ImVec2(0, 0)) {
+  static_assert(std::is_integral_v<T>, "Toggle only supports integral types.");
+
+  int i = static_cast<int>(*boolean);
+  const ImGuiSliderFlags flags = ImGuiSliderFlags_NoInput;
+  if (size.x > 0) {
+    ImGui::SetNextItemWidth(size.x);
+  }
+  const bool changed = ImGui::SliderInt(label, &i, 0, 1, label, flags);
+  *boolean = (i != 0);
   return changed;
 }
 
-inline bool ToggleBit(const char* label, int& flags, int flags_value,
+inline bool ImGui_BitToggle(const char* label, int* flags, int flags_value,
-                      ToggleKind kind = ToggleKind::kButton,
+                            const ImVec2& size = ImVec2(0, 0)) {
-                      bool set_width = true) {
+  bool boolean = (*flags) & flags_value;
-  bool boolean = flags & flags_value;
+  const bool changed = ImGui_ButtonToggle(label, &boolean, size);
-  bool changed = Toggle(label, boolean, kind, set_width);
   if (changed) {
-    flags = boolean ? (flags | flags_value) : (flags & ~flags_value);
+    *flags = boolean ? ((*flags) | flags_value) : ((*flags) & ~flags_value);
   }
   return changed;
 }

src/experimental/toolbox/interaction.cc

@@ -213,7 +213,7 @@ void MovePerturb(const mjModel* m, const mjData* d, const mjvCamera* cam,
 }
 
 void MoveCamera(const mjModel* m, const mjData* d, mjvCamera* cam,
-                mjtMouse action, mjtNum reldx, mjtNum reldy) {
+                CameraMotion motion, mjtNum dx, mjtNum dy) {
   if (cam->type == mjCAMERA_FIXED) {
     return;
   }
@@ -221,15 +221,51 @@ void MoveCamera(const mjModel* m, const mjData* d, mjvCamera* cam,
   mjtNum headpos[3], forward[3], up[3], right[3];
   mjtNum vec[3], dif[3], scl;
 
-  switch (action) {
+  switch (motion) {
-    case mjMOUSE_ROTATE_V:
+    case CameraMotion::ZOOM:
-    case mjMOUSE_ROTATE_H:
+      // Zoom the camera towards the target by adjusting its distance to the
-      cam->azimuth -= reldx * 180.0;
+      // target.
-      cam->elevation -= reldy * 180.0;
+      cam->distance -= mju_log(1 + cam->distance / m->stat.extent / 3) * dy *
+                       9 * m->stat.extent;
       break;
 
-    case mjMOUSE_MOVE_V:
+    case CameraMotion::ORBIT:
-    case mjMOUSE_MOVE_H:
+      cam->azimuth -= dx * 180.0;
+      cam->elevation -= dy * 180.0;
+      break;
+
+    case CameraMotion::TRUCK_PEDESTAL:
+    case CameraMotion::TRUCK_DOLLY:
+      if (cam->type == mjCAMERA_TRACKING) {
+        return;
+      }
+
+      mjv_cameraFrame(headpos, forward, up, nullptr, d, cam);
+      mju_cross(right, forward, up);
+
+      // y movement: either dolly (forward/back) or pedestal (up/down)
+      mju_addToScl3(cam->lookat,
+                    (motion == CameraMotion::TRUCK_PEDESTAL) ? up : forward, dy);
+
+      // x movement: camera truck (left/right)
+      mju_addToScl3(cam->lookat, right, dx);
+
+      break;
+
+    case CameraMotion::PAN_TILT:
+      if (cam->type == mjCAMERA_TRACKING) {
+        return;
+      }
+
+      mjv_cameraFrame(headpos, forward, nullptr, nullptr, d, cam);
+      cam->azimuth -= dx * 180.0;
+      cam->elevation -= dy * 180.0;
+      mjv_cameraFrame(nullptr, forward, nullptr, nullptr, d, cam);
+      mju_addScl3(cam->lookat, headpos, forward, cam->distance);
+      break;
+
+    case CameraMotion::PLANAR_MOVE_V:
+    case CameraMotion::PLANAR_MOVE_H:
       // do not move lookat point of tracking camera
       if (cam->type == mjCAMERA_TRACKING) {
         return;
@@ -237,7 +273,10 @@ void MoveCamera(const mjModel* m, const mjData* d, mjvCamera* cam,
 
       // get camera info and align
       mjv_cameraFrame(headpos, forward, nullptr, nullptr, d, cam);
-      AlignToCamera(vec, action, reldx, reldy, forward);
+      AlignToCamera(vec,
+                    (motion == CameraMotion::PLANAR_MOVE_V) ? mjMOUSE_MOVE_V
+                                                            : mjMOUSE_MOVE_H,
+                    dx, dy, forward);
 
       // compute scaling: rendered lookat displacement = mouse displacement
       mju_sub3(dif, cam->lookat, headpos);
@@ -246,34 +285,6 @@ void MoveCamera(const mjModel* m, const mjData* d, mjvCamera* cam,
       // move lookat point in opposite direction
       mju_addToScl3(cam->lookat, vec, -scl);
       break;
-
-    case mjMOUSE_ZOOM:
-      cam->distance -= mju_log(1 + cam->distance / m->stat.extent / 3) * reldy *
-                       9 * m->stat.extent;
-      break;
-
-    case mjMOUSE_MOVE_V_REL:
-    case mjMOUSE_MOVE_H_REL:
-      // do not move lookat point of tracking camera
-      if (cam->type == mjCAMERA_TRACKING) {
-        return;
-      }
-
-      mjv_cameraFrame(headpos, forward, up, nullptr, d, cam);
-      mju_cross(right, forward, up);
-
-      // y-axis movement moves forward/backward (ie. camera dolly) on horizontal
-      // plane or up/down (ie. camera pedestal) on vertical plane
-      mju_addToScl3(cam->lookat, (action == mjMOUSE_MOVE_V_REL) ? up : forward,
-                    reldy);
-
-      // x-axis movement strafes left/right (ie. camera truck)
-      mju_addToScl3(cam->lookat, right, reldx);
-
-      break;
-
-    default:
-      mjERROR("unexpected action %d", action);
   }
 
   // clamp camera parameters
@@ -589,20 +600,26 @@ PickResult Pick(const mjModel* m, const mjData* d, const mjvCamera* camera,
 }
 
 int SetCamera(const mjModel* m, mjvCamera* camera, int request_idx) {
-  // 0 = free, 1 = tracking, 2+ = fixed
+  const int ncam = m ? m->ncam : 0;
-  int camera_idx = std::clamp(request_idx, 0, std::max(m->ncam + 1, 0));
+  const int camera_idx = std::clamp(request_idx, kTumbleCameraIdx, ncam - 1);
-  if (camera_idx == 0) {
+
+  if (camera_idx == kTumbleCameraIdx) {
     camera->type = mjCAMERA_FREE;
-  } else if (camera_idx == 1) {
+    camera->fixedcamid = -1;
+  } else if (camera_idx == kFreeCameraIdx) {
+    camera->type = mjCAMERA_FREE;
+    camera->distance = 2.0f;
+    camera->fixedcamid = -1;
+  } else if (camera_idx == kTrackingCameraIdx) {
     if (camera->trackbodyid >= 0) {
       camera->type = mjCAMERA_TRACKING;
-      camera->fixedcamid = -1;
     } else {
       camera->type = mjCAMERA_FREE;
     }
+    camera->fixedcamid = -1;
   } else {
     camera->type = mjCAMERA_FIXED;
-    camera->fixedcamid = camera_idx - 2;
+    camera->fixedcamid = camera_idx;
   }
 
   return camera_idx;

src/experimental/toolbox/interaction.h

@@ -34,21 +34,44 @@ PickResult Pick(const mjModel* m, const mjData* d, const mjvCamera* camera,
                 float x, float y, float aspect_ratio,
                 const mjvOption* vis_options);
 
-// Updates the camera according to the requested index using this convention:
+// Indices for cameras that are not defined in the model.
+static constexpr int kTumbleCameraIdx = -3;
+static constexpr int kFreeCameraIdx = -2;
+static constexpr int kTrackingCameraIdx = -1;
+
+// Updates the camera according to the requested index.
 //
-//  0  : selects the free camera (not defined in the model)
+// The function returns the new index of the camera which may differ from the
-//  1  : selects the tracking camera (also not defined in the model)
+// request if the request was invalid (e.g. request was out of range).
-//  2+ : selects a camera in the model; e.g. index 2 => model.cam[0];
-//
-// The function returns the index of the used camera following the same
-// convention. Note the returned index may differ from the request if the
-// request was invalid (index was out of range or tracking camera was not
-// available).
 int SetCamera(const mjModel* m, mjvCamera* camera, int request_idx);
 
-// Moves the camera according to the mouse action and relative displacement.
+// Camera motions are either relative to a target or the camera itself.
+//
+// We use the following camera nomenclature:
+//   - Truck: moves the camera left/right along a horizontal plane.
+//   - Pedestal: moves the camera up/down along a vertical plane.
+//   - Dolly: moves the camera forward/backward along a horizontal plane.
+//   - Pan: turns the camera left/right.
+//   - Tilt: turns the camera upwards/downwards.
+//   - Zoom: moves the camera closer to or away from the target. This is
+//           different from dolly in that the movement is relative to the
+//           target. (It's also not actually a camera zoom, which is an
+//           action of lens of the camera, rather than the camera itself.)
+//   - Orbit: moves the camera around the target.
+//   - Planer: creates a horizontal or vertical plane based on the cameras
+//             position and orientation, then moves the camera along that plane.
+enum class CameraMotion {
+  ZOOM,
+  ORBIT,
+  TRUCK_PEDESTAL,
+  TRUCK_DOLLY,
+  PAN_TILT,
+  PLANAR_MOVE_H,
+  PLANAR_MOVE_V,
+};
+
 void MoveCamera(const mjModel* m, const mjData* d, mjvCamera* cam,
-                mjtMouse action, mjtNum reldx, mjtNum reldy);
+                CameraMotion motion, mjtNum dx, mjtNum dy);
 
 void InitPerturb(const mjModel* m, const mjData* d, const mjvCamera* cam,
                  mjvPerturb* pert, mjtPertBit active);

src/experimental/toolbox/sim_profiler.cc

@@ -20,6 +20,10 @@
 
 namespace mujoco::toolbox {
 
+// TO DO:
+// - convergence profiler
+// - solver iteration profiler
+
 SimProfiler::SimProfiler() {
   Clear();
 }
@@ -118,9 +122,8 @@ void SimProfiler::Update(const mjModel* model, const mjData* data) {
 }
 
 
-void SimProfiler::Gui() {
+void SimProfiler::CpuTimeGraph() {
-  const int plot_flags = 0;
+  if (ImPlot::BeginPlot("CPU Time", ImVec2(-1, 0))) {
-  if (ImPlot::BeginPlot("CPU Time", ImVec2(-1, 0), plot_flags)) {
     ImPlot::SetupAxis(ImAxis_X1, "frame", ImPlotAxisFlags_AutoFit);
     ImPlot::SetupAxis(ImAxis_Y1, "msec", ImPlotAxisFlags_AutoFit);
     ImPlot::SetupAxisFormat(ImAxis_Y1, "%.2f");
@@ -139,8 +142,10 @@ void SimProfiler::Gui() {
                      -(int)cpu_other_.size());
     ImPlot::EndPlot();
   }
+}
 
-  if (ImPlot::BeginPlot("Dimensions", ImVec2(-1, 0), plot_flags)) {
+void SimProfiler::DimensionsGraph() {
+  if (ImPlot::BeginPlot("Dimensions", ImVec2(-1, 0))) {
     ImPlot::SetupAxis(ImAxis_X1, "frame", ImPlotAxisFlags_AutoFit);
     ImPlot::SetupAxis(ImAxis_Y1, "count", ImPlotAxisFlags_AutoFit);
     ImPlot::SetupAxisFormat(ImAxis_Y1, "%.0f");

src/experimental/toolbox/sim_profiler.h

@@ -33,7 +33,8 @@ class SimProfiler {
   void Update(const mjModel* model, const mjData* data);
 
   // Displays the profiling data using ImPlot.
-  void Gui();
+  void CpuTimeGraph();
+  void DimensionsGraph();
 
  private:
   std::vector<float> cpu_total_;