From 692265980b3d46d4f6f88cd4c41ab74ddbabcfa0 Mon Sep 17 00:00:00 2001 From: Colin Faulkingham Date: Sun, 25 Oct 2026 01:25:31 +0000 Subject: [PATCH] Identify commands beside the animated agent cursor (#622) (#6026) * claim: #622 * fix(ui): identify commands during agent cursor animation --- crates/ui-egui/src/agent_cursor.rs | 90 ++++++++++++++++++++++++++---- docs/control-protocol.md | 2 +- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/crates/ui-egui/src/agent_cursor.rs b/crates/ui-egui/src/agent_cursor.rs index 4163308d1..028284f12 100644 --- a/crates/ui-egui/src/agent_cursor.rs +++ b/crates/ui-egui/src/agent_cursor.rs @@ -1,6 +1,7 @@ //! The agent cursor (#35): a second, clearly marked pointer that acts out what an agent does //! over the control channel or MCP. Before an `engine.execute` runs, the pointer glides to the //! command's toolbar button (or its panel when the button sits in a drop-down) and clicks it, +//! with the command's name beside the pointer so a closed group still identifies the tool, //! then glides to each pick in the parameters (faces, edges and points projected to the screen, //! sketch points, the plane of a new sketch), and only then does the command run. The reply //! waits for the animation, so an agent's timing stays honest. With "follow camera" on, a new @@ -21,9 +22,7 @@ use std::collections::VecDeque; use std::sync::mpsc::Sender; -#[cfg(test)] -use egui::vec2; -use egui::{Pos2, Stroke, pos2}; +use egui::{Pos2, Stroke, pos2, vec2}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use solidcraft_engine::geom::{Vec2, Vec3}; @@ -133,7 +132,8 @@ pub struct Job { method: String, params: Value, reply: Sender, - stops: VecDeque, + /// Screen position and registry label, kept together even when several commands share a panel. + stops: VecDeque<(Pos2, &'static str)>, /// Seconds per glide and per click for this request (within [`TIME_BUDGET`] in all). glide: f64, click: f64, @@ -253,13 +253,17 @@ pub fn stops(app: &SolveApp, command: &str, params: &Value) -> Vec { /// Where the pointer goes for an `engine.batch`: the toolbar buttons of its distinct commands, in /// the order they first come, at most [`MAX_BATCH_STOPS`] of them. pub fn batch_stops(steps: &[Value]) -> Vec { + batch_targets(steps).into_iter().map(|(at, _)| at).collect() +} + +fn batch_targets(steps: &[Value]) -> Vec<(Pos2, &'static str)> { let mut seen: Vec<&str> = Vec::new(); for c in steps.iter().filter_map(crate::control::batch_step_command) { if !seen.contains(&c) { seen.push(c); } } - let buttons: Vec = seen.into_iter().filter_map(button).collect(); + let buttons: Vec<_> = seen.into_iter().filter_map(|c| Some((button(c)?, solidcraft_engine::find_command(c)?.label))).collect(); summarise(&buttons, MAX_BATCH_STOPS) } @@ -330,22 +334,23 @@ pub fn intercept(app: &mut SolveApp, method: &str, params: &Value, reply: &Sende if !s.show || params.get("animate").and_then(Value::as_bool) == Some(false) { return false; } - let stops: VecDeque = match method { + let stops: VecDeque<_> = match method { "engine.execute" | "command" => { let Some(cmd) = params.get("command").or(params.get("id")).and_then(Value::as_str) else { return false }; let inner = params.get("params").cloned().unwrap_or(json!({})); - stops(app, cmd, &inner).into() + let label = solidcraft_engine::find_command(cmd).map_or("", |c| c.label); + stops(app, cmd, &inner).into_iter().map(|at| (at, label)).collect() } // A batch is acted out as one request; one the app refuses is answered at once. "engine.batch" => match params.get("commands").and_then(Value::as_array) { - Some(steps) if crate::control::batch_refusal(steps).is_none() => batch_stops(steps).into(), + Some(steps) if crate::control::batch_refusal(steps).is_none() => batch_targets(steps).into(), _ => return false, }, _ => return false, }; if s.speed == Speed::Instant { // No animation: the command runs now; the pointer jumps to where it acted. - if let Some(last) = stops.back() { + if let Some((last, _)) = stops.back() { app.agent.pos = Some(*last); app.agent.ripples.push((*last, app.now)); } @@ -395,7 +400,7 @@ pub fn step(app: &mut SolveApp, ctx: &egui::Context) { let Some(job) = app.agent.job.as_mut() else { return }; let (glide, click) = (job.glide, job.click); ctx.request_repaint(); - let Some(&to) = job.stops.front() else { + let Some(&(to, _)) = job.stops.front() else { if job.ahead.as_ref().is_none_or(crate::control_job::Ahead::finished) { finish(app, ctx); } @@ -468,6 +473,22 @@ pub fn paint(app: &SolveApp, ctx: &egui::Context) { painter.circle_stroke(*at, 6.0 + 22.0 * k, Stroke::new(3.0 * (1.0 - k) + 0.5, agent().gamma_multiply(1.0 - k))); } crate::icons::pointer::paint(&painter, p); + if let Some((_, label)) = app.agent.job.as_ref().and_then(|job| job.stops.front()).filter(|(_, label)| !label.is_empty()) { + use crate::ui_kit::{Container, Surface}; + let t = crate::theme::Tokens::get(); + Container::new(ctx, Surface::Popup) + .area(egui::Id::new("sc_agent_command")) + .order(egui::Order::Debug) + .fixed_pos(p + vec2(15.0, 42.0)) + .constrain(true) + .interactable(false) + .fade_in(false) + .show(ctx, |ui| { + Container::popup_frame(ui.style()).fill(t.overlay).show(ui, |ui| { + ui.label(egui::RichText::new(crate::i18n::tr_label(label)).size(t.font_label).color(t.text)); + }); + }); + } } #[cfg(test)] @@ -579,6 +600,55 @@ mod tests { assert!(frames <= 3, "{frames}"); } + #[test] + fn a_command_in_a_closed_group_is_named_while_the_pointer_acts_it_out() { + let mut h = crate::scenario::Harness::new(); + h.call("ui.agent_cursor", json!({"show": true, "speed": 1})); + assert!(crate::scenario::handle_at("toolbar:solid.box").is_none()); + assert!(button("solid.box").is_some(), "the closed group is the pointer's target"); + let (tx, rx) = std::sync::mpsc::channel(); + assert!(intercept(&mut h.app, "engine.execute", &json!({"command":"solid.box", "params":{"length":10,"width":10,"height":10}}), &tx)); + h.frames(3); + assert!(h.widget("Box").is_some(), "the command must be identified before it executes"); + assert!(rx.try_recv().is_err()); + assert!(h.app.dialog.is_none(), "programmatic commands do not open interactive tools"); + h.frames(120); + assert_eq!(rx.try_recv().unwrap()["ok"], true); + assert_eq!(h.app.session.world_state().bodies.len(), 1); + assert!(!h.app.agent.busy()); + assert!(h.widget("Box").is_none(), "the command label ends with the animation"); + } + + #[test] + fn a_batch_names_each_command_even_when_they_share_a_closed_group() { + let mut h = crate::scenario::Harness::new(); + h.call("ui.agent_cursor", json!({"show": true, "speed": 1})); + assert_eq!(button("solid.box"), button("solid.sphere")); + let (tx, rx) = std::sync::mpsc::channel(); + let params = json!({"commands":[ + {"command":"solid.box","params":{"length":10,"width":10,"height":10}}, + {"command":"solid.sphere","params":{"center":[30,0,0],"radius":3}} + ]}); + assert!(intercept(&mut h.app, "engine.batch", ¶ms, &tx)); + let mut seen = Vec::new(); + for _ in 0..120 { + h.frame(); + for label in ["Box", "Sphere"] { + if h.widget(label).is_some() && seen.last() != Some(&label) { + seen.push(label); + } + } + if !h.app.agent.busy() { + break; + } + } + assert_eq!(seen, ["Box", "Sphere"]); + let reply = rx.try_recv().unwrap(); + assert_eq!(reply["result"]["completed"], 2, "{reply}"); + assert!(h.app.dialog.is_none()); + assert!(h.widget("Box").is_none() && h.widget("Sphere").is_none()); + } + #[test] fn the_pointer_follows_the_agents_own_mouse_input() { let mut h = crate::scenario::Harness::new(); diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 2a16e4e29..2e789cd51 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -248,7 +248,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox. | `ui.prefs` | any of `default_units`, `nav` (`fusion`, `solidworks`, `inventor`, `alias`, `blender`, `tinkercad`, `openscad`, `touchpad`), `zoom_reverse`, `pan_reverse_x` (default false: horizontal scrolling pans the other way), `zoom_to_cursor`, `smooth_wheel_zoom` (default true), `trackpad_pan` (default true: two-finger scroll pans, except Touchpad always zooms; pinch zooms), `orbit_around` (`view_centre` (default: what Pan moves and Fit frames), `origin` (the world origin stays still on screen), `cursor` (the model point under the pointer when the drag began); the older `orbit_cursor` boolean still sets `cursor` when true), `look_at_anim`, `look_at_speed`, `middle_double_click` (`home` (default), `fit`, `off`), `msaa`, `show_frame_rate` (default false: frames per second in the view's bottom-right corner, counting the frames the view draws anyway; it reads 0 FPS once the view is idle), `window_system` (Linux: `auto` (default: XWayland whenever there is an X display, native Wayland on niri), `wayland`, `x11`; saved in `graphics.json`, applies at the next start; `SOLIDCRAFT_WAYLAND` overrides it), `frame_rate_details` (default false: with the frame rate, a frame-time graph of the frames drawn with the triangles over it, a frame-time histogram and what the GPU drew; clicking the readout switches it), `length_decimals`, `angle_decimals`, `theme` (`dark`, `light`, `system` or a theme file's id: a built-in one such as `gruvbox-dark`, or an imported one; the reply's `theme_ids` lists them), `theme_import` (a `solidcraft-theme/1` file's JSON or text: adds the theme and turns it on; the reply's `theme_imported` is its id), `theme_remove` (an imported theme's id), `theme_export: true` (the reply's `theme_file` is the theme on now as a complete file; see [colour themes](book/src/guide/ui-tour.md#colour-themes)), `language` (`en`, `ja`, `de`, `fr`, `ko`, `ms`, `ru`, `uk`, `zh`; see [localization](localization.md)), `ui_scale`, `ui_font` (an installed font family or a font file for the interface; empty for the built-in font), `up_axis` (`z`, `y`), `offer_stand_up`, `extrude_auto_to_object` (default false: in the Extrude dialog, with a profile picked, a face, vertex or parallel plane picked off its plane switches the extent to To Object; it applies to Extrude dialogs opened afterwards), `hole_snaps` (default false: a Single hole snaps to its face's middle and round edges' centres), `hole_moves` (default false: a placed Single hole has a drag handle (`hole:`), and a pick on a face that has a hole moves it there), `color_vision` (`off` (default), `protan`, `deutan`, `tritan`: colour vision modes, #778), `check_updates` (default true; [updates](book/src/guide/updates.md)); `open` (a page: General, Interface, Design, Navigation, Keyboard Shortcuts, Sketch, Graphics, AI, Plugins; Shortcuts, Assistant and Sketch & selection are aliases, and the former Selection page opens Sketch) opens the Preferences dialog there; `button`: `ok`, `cancel`, `apply`, `defaults`; `stand_up: true` stands up a just-opened Y-up file, `stand_up: false` leaves it as it is ("Leave it": the offer closes and `offer_stand_up` stays on); the offer's buttons are the handles `stand_up:stand`, `stand_up:leave` and `stand_up:never` | sets preferences (kept between runs), returns them all plus `dialog` (the open page or null), `dark` and `stand_up_offer` | | `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits | | `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first | -| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. It also follows the pointer input sent with `ui.move`, `ui.click`, `ui.drag` and `ui.scroll`, as each event reaches the app, and a press leaves a ripple. `"animate": false` on a call skips it | +| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. The pointer shows the command's localized name, including commands inside a closed toolbar group; an animated batch names each command it visits. It also follows the pointer input sent with `ui.move`, `ui.click`, `ui.drag` and `ui.scroll`, as each event reaches the app, and a press leaves a ripple. `"animate": false` on a call skips it | | `engine.execute` → `view.layout` | `mode`: `floating` or `compact` | Apply and save the workspace layout without opening a dialog, changing the design, resetting the camera or cancelling its active tool. General preferences expose the same choice; `ui.prefs {"layout":"compact"}` also saves it. | | `engine.execute` → `browser.expand_all` | `path?`: occurrence ids; `component?`: id; omit both for the whole tree | Open every Browser row under that scope, including folders that start closed (Origin, Named Views, Features). Display only: not saved in the design, not an undo step. | | `engine.execute` → `browser.collapse_all` | same | Close those rows. The design row stays open when the whole tree is collapsed. |