diff --git a/crates/mcp/src/tools.rs b/crates/mcp/src/tools.rs index 231e2b67e..8104a29fc 100644 --- a/crates/mcp/src/tools.rs +++ b/crates/mcp/src/tools.rs @@ -92,7 +92,8 @@ pub fn tool_definitions() -> Vec { created: {points, curves, constraints, dimensions}: the ids of what they added, stable and not reused \ after a delete (undo rewinds the counters with the design), usable right away \ in sketch.dimension, sketch.constraint.* and sketch.delete. Feature commands return the feature's id \ - and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies).", + and name, every body (bodies) and the bodies the feature created, changed and removed (created_bodies, changed_bodies, removed_bodies). \ + view.pick {pixels: [[x,y]…]} answers the body, face, exact world point, normal and depth under pixels of a screenshot (read-only; docs/control-protocol.md).", obj( json!({ "command": string("Command id (see list_commands)"), diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 7dad4493b..9685ec85c 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -117,6 +117,13 @@ fn key_from(name: &str) -> Option { /// Run one command as `engine.execute` does: through `app.run`, then the first-body fit and the /// camera the command asked for. fn execute(app: &mut SolveApp, id: &str, params: Value) -> Result { + // `view.pick` is answered by the viewport, not the engine: pixels and rays need the + // window's camera and pick index, and the command must not touch the document, the + // selection or the undo stack. Intercepted here so `engine.execute` and `engine.batch` + // (and through them MCP `execute` and `batch`) reach it like any other command. + if id == "view.pick" { + return crate::view_pick::pick_command(app, ¶ms); + } let r = app.run(id, params); if r.is_ok() && !matches!( @@ -604,6 +611,7 @@ fn dispatch(app: &mut SolveApp, ctx: &egui::Context, method: &str, p: &Value) -> let list = crate::scenario::handles(s("prefix").unwrap_or("")); ok(json!(list.into_iter().map(|(n, p)| json!({"name": n, "x": p.x, "y": p.y})).collect::>())) } + "ui.pick" => wrap(crate::view_pick::pick_command(app, p)), "ui.at" => { if let Some(path) = p.get("occurrence").and_then(Value::as_array) { let ids: Option> = path.iter().map(Value::as_u64).collect(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index ff4429f5b..cda7d7fbb 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -258,6 +258,7 @@ pub mod ui_kit; #[cfg(test)] mod ui_kit_guard_tests; pub mod update; +pub mod view_pick; pub mod viewport; mod viewport_layout; pub mod workspace; diff --git a/crates/ui-egui/src/view_pick.rs b/crates/ui-egui/src/view_pick.rs new file mode 100644 index 000000000..3da1ee6e6 --- /dev/null +++ b/crates/ui-egui/src/view_pick.rs @@ -0,0 +1,519 @@ +//! `view.pick` (control: `ui.pick`): screenshot pixels or world rays → the exact geometry +//! under each one (#6304). +//! +//! An agent that has just looked at a screenshot sends back the pixels it saw and gets the +//! body, face index, world point, surface normal and depth under each — the inputs +//! `solid.fillet`, `inspect.measure` and `construct.point` take. The command is read-only: +//! it never changes the selection, the camera or the document, and it answers from the same +//! per-body pick index the mouse uses, so a pick here matches what a click on the same +//! pixel would pick. +//! +//! Frames: `window` pixels are physical pixels of a window screenshot (divided by +//! `pixels_per_point`, offset by the viewport rect); `model` pixels are pixels of a +//! `ui.render` of the same size, so an agent can render, look, and pick without a window. + +use std::collections::HashMap; + +use egui::{Pos2, Rect, pos2, vec2}; +use serde_json::{Value, json}; + +use solidcraft_engine::doc::{apply_point, is_identity}; +use solidcraft_engine::geom::{Mesh, Vec3}; + +use crate::SolveApp; + +/// At most this many pixels + rays per call: a batch of picks is one request, but it still +/// runs on the window thread. +const MAX_PICKS: usize = 1024; + +/// The default model-render size, as the MCP `screenshot` tool renders by default. +const DEFAULT_MODEL_W: f64 = 960.0; +const DEFAULT_MODEL_H: f64 = 600.0; + +/// How far past the section plane a face point may lie and still count as kept (the renderer +/// clips at `n·p <= n·o`; this is the same hairline `clip_segment` allows). +const SECTION_EPS: f64 = 1e-6; + +/// Crossings one body may contribute before the section-cap walk gives up on it. +const MAX_CROSSINGS: usize = 64; + +/// One input: the raw request (echoed back), its screen pixel when it came from one, and +/// the world ray it maps to. +struct Input<'a> { + raw: &'a Value, + pixel: Option, + o: Vec3, + d: Vec3, +} + +/// The nearest triangle of one body along one ray, refined onto its face. +struct Cand { + body: String, + face: usize, + point: Vec3, + normal: Vec3, + t: f64, + ghosted: bool, +} + +/// A body's outermost placement: (occurrence path, occurrence name). +struct Placed { + path: Vec, + name: String, +} + +/// `view.pick` / `ui.pick`: `pixels` (screenshot pixels) and/or `rays` (world-space +/// `{origin, dir}`) → one entry per input, in order; `null` where nothing is hit. +pub fn pick_command(app: &mut SolveApp, p: &Value) -> Result { + let f = |k: &str| p.get(k).and_then(Value::as_f64).filter(|x| x.is_finite()); + let frame = match p.get("frame").and_then(Value::as_str) { + Some("window") => true, + Some("model") => false, + None => app.viewport.rect.is_some(), + Some(other) => return Err(format!(r#"view.pick: frame must be "window" or "model", not {other:?}"#)), + }; + let edges_on = p.get("edges").and_then(Value::as_bool).unwrap_or(true); + let edge_px = f("edge_px").filter(|x| *x > 0.0).unwrap_or(6.0) as f32; + let all = p.get("all").and_then(Value::as_bool).unwrap_or(false); + let include_ghosted = p.get("include_ghosted").and_then(Value::as_bool).unwrap_or(false); + + // The projection the pixels live in: the window's, or a model render's own w×h. + let (proj, ppp) = if frame { + let rect = app.viewport.rect.ok_or(r#"view.pick: frame "window" needs a drawn window; headless callers use frame "model""#)?; + (crate::viewport::projection(app, rect), app.viewport.pixels_per_point()) + } else { + let w = f("width").unwrap_or(DEFAULT_MODEL_W).clamp(16.0, 8192.0); + let h = f("height").unwrap_or(DEFAULT_MODEL_H).clamp(16.0, 8192.0); + let rect = Rect::from_min_size(Pos2::ZERO, vec2(w as f32, h as f32)); + let cam = app.cam; + (crate::viewport::Proj { cam, vp: cam.view_proj(w / h, crate::viewport::scene_radius(app)), rect }, 1.0) + }; + + let mut inputs: Vec = Vec::new(); + if let Some(px) = p.get("pixels") { + for q in px.as_array().ok_or("view.pick: `pixels` must be an array of [x, y]".to_string())? { + let mut xy = [0.0f64; 2]; + for (i, v) in xy.iter_mut().enumerate() { + *v = q.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or("view.pick: a pixel must be [x, y] numbers")?; + } + let (x, y) = (xy[0], xy[1]); + // Window screenshots are whole-window, in physical pixels; `Proj` speaks + // absolute window points, so only the HiDPI scale comes off. + let pos = if frame { pos2((x / ppp as f64) as f32, (y / ppp as f64) as f32) } else { pos2(x as f32, y as f32) }; + let (o, d) = proj.ray(pos); + inputs.push(Input { raw: q, pixel: Some(pos), o, d }); + } + } + if let Some(rs) = p.get("rays") { + for r in rs.as_array().ok_or("view.pick: `rays` must be an array of {origin, dir}".to_string())? { + let triple = |key: &str| -> Result { + let v = r.get(key).ok_or_else(|| format!("view.pick: a ray needs `{key}` [x, y, z]"))?; + let mut xyz = [0.0f64; 3]; + for (i, x) in xyz.iter_mut().enumerate() { + *x = v.get(i).and_then(Value::as_f64).filter(|x| x.is_finite()).ok_or(format!("view.pick: `{key}` must be [x, y, z] numbers"))?; + } + Ok(Vec3::new(xyz[0], xyz[1], xyz[2])) + }; + let d = triple("dir")?.normalized().ok_or("view.pick: a ray needs a non-zero `dir`")?; + inputs.push(Input { raw: r, pixel: None, o: triple("origin")?, d }); + } + } + if inputs.is_empty() { + return Err("view.pick: give `pixels` and/or `rays`".to_string()); + } + if inputs.len() > MAX_PICKS { + return Err(format!("view.pick: at most {MAX_PICKS} pixels+rays per call")); + } + + // What the viewport shows now, and what a pick may not answer for. + let st = crate::open_meshing::shown_world(app); + let bounds = crate::viewport::body_bounds(app, &st); + let active_bodies = app.session.active_occurrence_path.as_ref().and_then(|q| app.session.occurrence_bodies(q, true).ok()); + let replaced: &[String] = app.preview.effective_replaced(); + let section = crate::viewport::section_plane(app); + let wireframe = app.ui.visual_style == 2; + let slack = crate::viewport::scene_radius(app) * 0.02; + let occ = occurrence_map(app); + let comps = component_names(app); + + let mut hits: Vec = Vec::with_capacity(inputs.len()); + for input in &inputs { + let mut cands = face_candidates(app, &st, &bounds, input, &active_bodies, replaced, include_ghosted); + cands.sort_by(|a, b| a.t.total_cmp(&b.t)); + // A section hides the material its normal points away from: faces on the removed + // side do not stop a pick, and the cut cap answers where the plane crosses material. + let kept: Vec<&Cand> = match section { + Some((so, sn)) => cands.iter().filter(|c| sn.dot(c.point) <= sn.dot(so) + SECTION_EPS).collect(), + None => cands.iter().collect(), + }; + let cap = section + .and_then(|(so, sn)| section_cap(app, &st, &bounds, input, so, sn, kept.first().map(|c| c.t), include_ghosted, replaced, &active_bodies)); + let face_t = kept.first().map(|c| c.t).or(cap.as_ref().map(|(t, _, _)| *t)); + let (edge, vertex) = match (edges_on, input.pixel) { + (true, Some(pos)) => { + near_edges(app, &st, &bounds, &proj, input, pos, face_t, edge_px, &active_bodies, replaced, include_ghosted, wireframe, slack) + } + _ => (None, None), + }; + hits.push(match (all, cap, kept) { + (true, cap, kept) => { + let mut row: Vec = Vec::new(); + if let Some((t, body, sn)) = cap { + row.push(cap_json(input, &body, t, input.o + input.d * t, sn)); + } + row.extend(kept.iter().map(|c| hit_json(input, c, &occ, &comps, None, None))); + Value::Array(row) + } + (false, Some((t, body, sn)), kept) => match kept.first() { + // The cap is nearer than any kept face: the pick lands on the cut. + Some(c) if c.t < t => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()), + _ => cap_json(input, &body, t, input.o + input.d * t, sn), + }, + (false, None, kept) => match kept.first() { + Some(c) => hit_json(input, c, &occ, &comps, edge.as_ref(), vertex.as_ref()), + None => match (&edge, &vertex) { + (None, None) => Value::Null, + _ => json!({ + "pixel": input.raw, "body": null, "face": null, + "edge": edge.as_ref().map(edge_json), "vertex": vertex.as_ref().map(vertex_json), + }), + }, + }, + }); + } + Ok(json!({ + "frame": if frame { "window" } else { "model" }, + "camera": { + "projection": if proj.cam.fov > 0.0 { "perspective" } else { "orthographic" }, + "eye": v3(proj.cam.eye()), + "target": v3(proj.cam.target), + }, + "hits": hits, + })) +} + +/// The nearest face of every pickable body along the ray, nearest first. Bodies the +/// viewport hides, ghosts (unless asked for) or stands in for with a preview are skipped, +/// exactly as the mouse picker skips them. +fn face_candidates( + app: &SolveApp, + st: &solidcraft_engine::doc::ModelState, + bounds: &[(String, solidcraft_engine::geom::Aabb3)], + input: &Input, + active_bodies: &Option>, + replaced: &[String], + include_ghosted: bool, +) -> Vec { + let mut out = Vec::new(); + for (i, b) in st.bodies.iter().enumerate() { + if !app.session.body_visible(&b.name) || replaced.contains(&b.name) { + continue; + } + let ghosted = crate::viewport::body_faded(app, b, active_bodies); + if ghosted && !include_ghosted { + continue; + } + if let Some((_, bb)) = bounds.get(i) { + let pad = bb.diagonal() * 1e-6 + 1e-9; + if crate::viewport::ray_box(input.o, input.d, bb, pad).is_none() { + continue; + } + } + // A placed part is picked in its own frame, as the mouse picker does: the ray moves + // there (a rigid move keeps distances along it), so no moved copy of the mesh is made. + let (m, place) = b.mesh_in_place(); + let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue }; + let Some((t, ti)) = crate::pick_index::raycast(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld) else { continue }; + let Some((point, normal)) = refine(&m, ti, t, lo, ld, &place) else { continue }; + out.push(Cand { body: b.name.clone(), face: m.tri_face.get(ti).copied().unwrap_or(0) as usize, point, normal, t, ghosted }); + } + out +} + +/// The hit point and outward surface normal in world space. A flat face (constant mesh +/// normals) is solved exactly against its plane, so a box picks to the last digit; a curved +/// face keeps the mesh hit (within the display tolerance) and takes its normal interpolated +/// from the mesh's per-vertex surface normals. +fn refine(m: &Mesh, ti: usize, t: f64, lo: Vec3, ld: Vec3, place: &solidcraft_engine::doc::Mat) -> Option<(Vec3, Vec3)> { + let [a, b, c] = *m.triangles.get(ti)?; + let (p0, p1, p2) = (m.positions.get(a as usize)?, m.positions.get(b as usize)?, m.positions.get(c as usize)?); + let lp = lo + ld * t; + let mesh_normal = || (*p1 - *p0).cross(*p2 - *p0).normalized().or_else(|| (ld * -1.0).normalized()); + let (point_local, n_local) = match (m.normals.get(a as usize), m.normals.get(b as usize), m.normals.get(c as usize)) { + (Some(na), Some(nb), Some(nc)) if nb.dist(*na) < 1e-6 && nc.dist(*na) < 1e-6 && nc.dist(*nb) < 1e-6 => { + // Flat: the exact ray-plane intersection, on the face's own plane. + let n = na.normalized()?; + let den = n.dot(ld); + let tp = if den.abs() < 1e-12 { t } else { (n.dot(*p0 - lo) / den).max(0.0) }; + (lo + ld * tp, n) + } + (Some(na), Some(nb), Some(nc)) => { + let (w0, w1, w2) = barycentric(lp, *p0, *p1, *p2); + let n = (Vec3::new(na.x * w0 + nb.x * w1 + nc.x * w2, na.y * w0 + nb.y * w1 + nc.y * w2, na.z * w0 + nb.z * w1 + nc.z * w2)) + .normalized() + .or_else(mesh_normal)?; + (lp, n) + } + _ => (lp, mesh_normal()?), + }; + let point = apply_point(place, point_local); + let normal = apply_dir(place, point_local, n_local); + Some((point, normal)) +} + +/// A placement is rigid, so a direction rides along it as a point difference. +fn apply_dir(place: &solidcraft_engine::doc::Mat, at: Vec3, n: Vec3) -> Vec3 { + if is_identity(place) { + return n; + } + (apply_point(place, at + n) - apply_point(place, at)).normalized().unwrap_or(n) +} + +/// Affine weights of `p` in triangle `a b c`. +fn barycentric(p: Vec3, a: Vec3, b: Vec3, c: Vec3) -> (f64, f64, f64) { + let (e0, e1, e2) = (b - a, c - a, p - a); + let (d00, d01, d11) = (e0.dot(e0), e0.dot(e1), e1.dot(e1)); + let (d20, d21) = (e2.dot(e0), e2.dot(e1)); + let den = d00 * d11 - d01 * d01; + if !(den.is_finite() && den.abs() > 1e-24) { + return (1.0, 0.0, 0.0); + } + let v = (d11 * d20 - d01 * d21) / den; + let w = (d00 * d21 - d01 * d20) / den; + (1.0 - v - w, v, w) +} + +/// Whether the section plane crosses material at the ray (where a cut cap answers): the +/// plane crossing lies inside a body, told by an odd number of mesh crossings before it. +/// The walk re-raycasts each body past every crossing it finds, bounded per body. +#[allow(clippy::too_many_arguments)] +fn section_cap( + app: &SolveApp, + st: &solidcraft_engine::doc::ModelState, + bounds: &[(String, solidcraft_engine::geom::Aabb3)], + input: &Input, + so: Vec3, + sn: Vec3, + first_kept: Option, + include_ghosted: bool, + replaced: &[String], + active_bodies: &Option>, +) -> Option<(f64, String, Vec3)> { + let den = sn.dot(input.d); + if den.abs() < 1e-12 { + return None; + } + let t_cap = (sn.dot(so) - sn.dot(input.o)) / den; + if !(t_cap > 0.0) || t_cap >= first_kept.unwrap_or(f64::INFINITY) { + return None; + } + let walk_eps = 1e-6 * (1.0 + crate::viewport::scene_radius(app)); + for (i, b) in st.bodies.iter().enumerate() { + if !app.session.body_visible(&b.name) || replaced.contains(&b.name) { + continue; + } + if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) { + continue; + } + if let Some((_, bb)) = bounds.get(i) { + let pad = bb.diagonal() * 1e-6 + 1e-9; + match crate::viewport::ray_box(input.o, input.d, bb, pad) { + None => continue, + Some(t0) if t0 > t_cap => continue, + _ => {} + } + } + let (m, place) = b.mesh_in_place(); + let index = app.viewport.pick_index.of(&m); + let (mut t, mut crossings) = (0.0f64, 0usize); + while let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o + input.d * t, input.d) { + let Some((dt, _)) = crate::pick_index::raycast(index.as_deref(), &m, lo, ld) else { break }; + let tt = t + dt; + if tt >= t_cap - SECTION_EPS { + break; + } + crossings += 1; + t = tt + walk_eps; + if crossings > MAX_CROSSINGS { + break; + } + } + if crossings % 2 == 1 { + return Some((t_cap, b.name.clone(), sn)); + } + } + None +} + +/// The edge and vertex nearest the pixel, within `edge_px`, hidden geometry excluded the way +/// the mouse picker excludes it. Rays without a pixel get neither (they have no screen reach). +#[allow(clippy::too_many_arguments)] +fn near_edges( + app: &SolveApp, + st: &solidcraft_engine::doc::ModelState, + bounds: &[(String, solidcraft_engine::geom::Aabb3)], + proj: &crate::viewport::Proj, + input: &Input, + pos: Pos2, + face_t: Option, + edge_px: f32, + active_bodies: &Option>, + replaced: &[String], + include_ghosted: bool, + wireframe: bool, + slack: f64, +) -> (Option, Option) { + let reach = crate::pick_index::reach(proj, pos, edge_px.max(6.0)); + let visible = |p: Vec3| wireframe || face_t.is_none_or(|t| (p - input.o).dot(input.d) <= t + slack); + let mut best_e: Option = None; + let mut best_v: Option = None; + for (i, b) in st.bodies.iter().enumerate() { + if !app.session.body_visible(&b.name) || replaced.contains(&b.name) { + continue; + } + if !include_ghosted && crate::viewport::body_faded(app, b, active_bodies) { + continue; + } + if let Some((_, bb)) = bounds.get(i) + && crate::viewport::screen_box(proj, bb, edge_px).is_some_and(|r| !r.contains(pos)) + { + continue; + } + let (m, place) = b.mesh_in_place(); + let Some((lo, ld)) = crate::viewport::local_ray(&place, input.o, input.d) else { continue }; + let moved = !is_identity(&place); + let at = |p: Vec3| if moved { apply_point(&place, p) } else { p }; + let near = crate::pick_index::segments_near(app.viewport.pick_index.of(&m).as_deref(), &m, lo, ld, reach); + let mut last_edge = None; + for (ei, si) in near { + let Some(e) = m.edges.get(ei) else { continue }; + if !m.edge_pickable(ei) { + continue; + } + if last_edge != Some(ei) { + last_edge = Some(ei); + for v in [e.first(), e.last()].into_iter().flatten().map(|v| at(*v)) { + if let Some(sp) = proj.to_screen(v) { + let dd = sp.distance(pos); + if dd <= edge_px + 1.0 && visible(v) && best_v.as_ref().is_none_or(|x| dd < x.dist_px) { + best_v = Some(VertexHit { dist_px: dd, point: v }); + } + } + } + } + let (Some(p0), Some(p1)) = (e.get(si).map(|p| at(*p)), e.get(si + 1).map(|p| at(*p))) else { continue }; + let (Some(a), Some(c)) = (proj.to_screen(p0), proj.to_screen(p1)) else { continue }; + let dd = crate::viewport::seg_dist(pos, a, c); + if dd <= edge_px + && visible(crate::viewport::ray_segment_closest(input.o, input.d, p0, p1)) + && best_e.as_ref().is_none_or(|x| dd < x.dist_px) + { + let k = seg_param(pos, a, c); + best_e = Some(EdgeHit { index: ei, dist_px: dd, point: p0 + (p1 - p0) * f64::from(k) }); + } + } + } + (best_e, best_v) +} + +struct EdgeHit { + index: usize, + dist_px: f32, + point: Vec3, +} + +struct VertexHit { + dist_px: f32, + point: Vec3, +} + +/// Where on the screen segment `a`–`b` the pixel `p` lies closest, as a 0..1 parameter. +fn seg_param(p: Pos2, a: Pos2, b: Pos2) -> f32 { + let ab = b - a; + let l2 = ab.length_sq(); + if l2 < 1e-9 { 0.0 } else { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) } +} + +/// Body world name → outermost placement, walked breadth-first so a shallower placement +/// wins; bounded like the engine's own occurrence expansion. +fn occurrence_map(app: &SolveApp) -> HashMap { + let mut out: HashMap = HashMap::new(); + let mut queue: std::collections::VecDeque> = std::collections::VecDeque::from([vec![]]); + let mut budget = 10_000usize; + while let Some(path) = queue.pop_front() { + if budget == 0 { + break; + } + let Ok(children) = app.session.occurrence_children(&path) else { continue }; + for placement in children { + budget -= 1; + for b in &placement.bodies { + out.entry(b.world.clone()).or_insert_with(|| Placed { path: placement.path.clone(), name: placement.name.clone() }); + } + queue.push_back(placement.path.clone()); + } + } + out +} + +/// Body name → the component that owns it, by name (the root component names nothing). +fn component_names(app: &SolveApp) -> HashMap { + let doc = &app.session.doc; + let names: HashMap = doc.components.iter().map(|c| (c.id, c.name.clone())).collect(); + doc.body_components.iter().filter_map(|(body, id)| names.get(id).map(|n| (body.clone(), n.clone()))).collect() +} + +fn v3(v: Vec3) -> [f64; 3] { + [v.x, v.y, v.z] +} + +fn hit_json( + input: &Input, + c: &Cand, + occ: &HashMap, + comps: &HashMap, + edge: Option<&EdgeHit>, + vertex: Option<&VertexHit>, +) -> Value { + let mut j = json!({ + "pixel": input.raw, + "body": c.body, + "face": c.face, + "point": v3(c.point), + "normal": v3(c.normal), + "depth": c.t, + }); + if let Some(p) = occ.get(&c.body) { + j["occurrence"] = json!(p.path); + j["occurrence_name"] = json!(p.name); + } + if let Some(name) = comps.get(&c.body) { + j["component"] = json!(name); + } + if c.ghosted { + j["ghosted"] = json!(true); + } + if let Some(e) = edge { + j["edge"] = edge_json(e); + } + if let Some(v) = vertex { + j["vertex"] = vertex_json(v); + } + j +} + +fn cap_json(input: &Input, body: &str, t: f64, point: Vec3, normal: Vec3) -> Value { + json!({ "pixel": input.raw, "body": body, "face": null, "section_cap": true, "point": v3(point), "normal": v3(normal), "depth": t }) +} + +fn edge_json(e: &EdgeHit) -> Value { + json!({ "index": e.index, "point": v3(e.point), "distance_px": e.dist_px }) +} + +fn vertex_json(v: &VertexHit) -> Value { + json!({ "point": v3(v.point), "distance_px": v.dist_px }) +} + +#[cfg(test)] +#[path = "view_pick_tests.rs"] +mod tests; diff --git a/crates/ui-egui/src/view_pick_tests.rs b/crates/ui-egui/src/view_pick_tests.rs new file mode 100644 index 000000000..1f5967ed6 --- /dev/null +++ b/crates/ui-egui/src/view_pick_tests.rs @@ -0,0 +1,203 @@ +//! `view.pick` / `ui.pick` (#6304): pixels and rays answer with the geometry under them, +//! read-only, from both frames, section-aware. + +use serde_json::json; + +use solidcraft_engine::geom::Vec3; +use solidcraft_engine::render::Camera; + +use crate::scenario::Harness; + +/// A 10×10×10 box with an orthographic camera straight over its top face: the centre pixel +/// of a 960×600 model render looks down the +Z face at (5, 5, 10), 95 mm from the eye. +fn top_view() -> Harness { + let mut h = Harness::new(); + h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}})); + h.app.cam = Camera { + target: Vec3::new(5.0, 5.0, 5.0), + yaw: 0.0, + pitch: std::f64::consts::FRAC_PI_2, + roll: 0.0, + distance: 100.0, + fov: 0.0, + up: Default::default(), + }; + h +} + +fn pick(h: &mut Harness, mut params: serde_json::Value) -> serde_json::Value { + params.as_object_mut().unwrap().entry("frame").or_insert(json!("model")); + let v = h.call("ui.pick", params); + assert_eq!(v["ok"], true, "{v}"); + v["result"].clone() +} + +/// The centre pixel of the top face answers with the exact face, point, normal and depth; +/// a pixel off the box answers with nothing. +#[test] +fn the_centre_pixel_of_a_box_hits_its_facing_face() { + let mut h = top_view(); + let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]]})); + assert_eq!(r["frame"], "model"); + assert_eq!(r["camera"]["projection"], "orthographic"); + let hit = &r["hits"][0]; + assert!(hit["body"].is_string(), "{hit}"); + assert!(hit["face"].as_u64().is_some_and(|f| f < 6), "{hit}"); + let [x, y, z] = hit["point"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect::>().try_into().unwrap(); + assert!((x - 5.0).abs() < 1e-9 && (y - 5.0).abs() < 1e-9 && (z - 10.0).abs() < 1e-9, "{hit}"); + let n = hit["normal"].as_array().unwrap(); + assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}"); + assert!((hit["depth"].as_f64().unwrap() - 395.0).abs() < 1e-9, "orthographic depth runs from the ray origin, 4 distances ahead: {hit}"); + assert_eq!(r["hits"][1], serde_json::Value::Null); +} + +/// Perspective and orthographic agree on the centre pixel's face and point. +#[test] +fn perspective_and_orthographic_agree_on_the_centre_pixel() { + let mut h = top_view(); + let flat = pick(&mut h, json!({"pixels": [[480, 300]]})); + h.app.cam.fov = 0.6; + let round = pick(&mut h, json!({"pixels": [[480, 300]]})); + assert_eq!(round["camera"]["projection"], "perspective"); + assert_eq!(round["hits"][0]["face"], flat["hits"][0]["face"]); + let (a, b) = (round["hits"][0]["point"].as_array().unwrap(), flat["hits"][0]["point"].as_array().unwrap()); + for i in 0..3 { + assert!((a[i].as_f64().unwrap() - b[i].as_f64().unwrap()).abs() < 1e-6, "{round} vs {flat}"); + } +} + +/// A world ray answers without any pixel, and a zero direction is refused. +#[test] +fn world_rays_pick_without_a_pixel() { + let mut h = top_view(); + let r = pick(&mut h, json!({"rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]})); + assert!((r["hits"][0]["depth"].as_f64().unwrap() - 40.0).abs() < 1e-9, "{}", r["hits"][0]); + assert!((r["hits"][0]["point"][2].as_f64().unwrap() - 10.0).abs() < 1e-9); + let v = h.call("ui.pick", json!({"rays": [{"origin": [0.0, 0.0, 0.0], "dir": [0.0, 0.0, 0.0]}]})); + assert_eq!(v["ok"], false); +} + +/// A hidden body answers with nothing, as the viewport does not show it. +#[test] +fn a_hidden_body_is_not_picked() { + let mut h = top_view(); + let name = h.app.session.world_state().bodies[0].name.clone(); + h.app.session.visibility.hidden_bodies.push(name); + let r = pick(&mut h, json!({"pixels": [[480, 300]]})); + assert_eq!(r["hits"][0], serde_json::Value::Null); +} + +/// A section hides the material its normal points at: over the removed half the pick +/// lands on the cut cap (or the far side), never on the removed face. +#[test] +fn a_section_pick_lands_on_the_cap_not_the_removed_face() { + let mut h = top_view(); + h.call("engine.execute", json!({"command": "inspect.section", "params": {"plane": {"origin": [0, 0, 5], "normal": [0, 0, 1]}}})); + let r = pick(&mut h, json!({"pixels": [[480, 300]]})); + let hit = &r["hits"][0]; + assert_eq!(hit["section_cap"], true, "{hit}"); + assert!(hit["face"].is_null(), "{hit}"); + assert!((hit["point"][2].as_f64().unwrap() - 5.0).abs() < 1e-9, "{hit}"); + assert!((hit["normal"][2].as_f64().unwrap() - 1.0).abs() < 1e-9, "{hit}"); + // Off the sectioned column (the plane cuts the whole box here, so a side pixel that + // missed the box entirely still answers with nothing). + assert_eq!(r["hits"].as_array().map(Vec::len), Some(1)); +} + +/// A pixel near a silhouette edge reports that edge with its distance; a pixel in the +/// middle of the face reports none. +#[test] +fn edge_proximity_follows_the_pixel() { + let mut h = top_view(); + // The front edge of the top face lies at y = 0: a pixel two points off it (0.2 mm at + // this zoom is well inside 6 px) reports the edge. + let near = pick(&mut h, json!({"pixels": [[480, 335.5]]})); + assert!(near["hits"][0]["edge"].get("index").is_some(), "{}", near["hits"][0]); + let middle = pick(&mut h, json!({"pixels": [[480, 300]]})); + assert!(middle["hits"][0].get("edge").is_none(), "{}", middle["hits"][0]); +} + +/// `ui.at` (world → screen) and `ui.pick` (screen → world) round-trip through the window. +#[test] +fn window_picks_round_trip_through_ui_at() { + let mut h = Harness::new(); + h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}})); + h.frames(60); // the first-body fit animates; the camera must hold still between the two calls + let at = h.call("ui.at", json!({"world": [5.0, 5.0, 10.0]})); + assert_eq!(at["ok"], true, "{at}"); + let xy = at["result"].as_array().unwrap(); + let r = pick(&mut h, json!({"frame": "window", "pixels": [[xy[0], xy[1]]]})); + assert_eq!(r["frame"], "window"); + let hit = &r["hits"][0]; + let n = hit["normal"].as_array().unwrap(); + assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the pick must land on the top face: {hit}"); + let p = hit["point"].as_array().unwrap(); + for (got, want) in p.iter().zip([5.0, 5.0, 10.0]) { + assert!((got.as_f64().unwrap() - want).abs() < 1e-6, "{hit}"); + } +} + +/// The same round-trip at a 2× HiDPI scale: window pixels are physical, so they double. +/// Driven through `control::handle` directly: a harness frame would reset the scale factor. +#[test] +fn window_picks_divide_physical_pixels_at_hidpi() { + let mut h = Harness::new(); + h.call("engine.execute", json!({"command": "solid.box", "params": {"length": 10, "width": 10, "height": 10}})); + h.app.viewport.set_pixels_per_point(2.0); + let at = raw_call(&mut h, "ui.at", json!({"world": [5.0, 5.0, 10.0]})); + let xy = at["result"].as_array().unwrap(); + let doubled: Vec = xy.iter().map(|v| v.as_f64().unwrap() * 2.0).collect(); + let v = raw_call(&mut h, "ui.pick", json!({"frame": "window", "pixels": [doubled]})); + assert_eq!(v["ok"], true, "{v}"); + let hit = &v["result"]["hits"][0]; + let n = hit["normal"].as_array().unwrap(); + assert!((n[2].as_f64().unwrap() - 1.0).abs() < 1e-6, "the HiDPI pick must land on the same face: {hit}"); +} + +/// One control request without the harness settling frames (which would reset the scale). +fn raw_call(h: &mut Harness, method: &str, params: serde_json::Value) -> serde_json::Value { + let (req, _rx) = crate::control::ControlRequest::new(method, params); + match crate::control::handle(&mut h.app, &h.ctx, &req) { + crate::control::Outcome::Done(v) => v, + crate::control::Outcome::Screenshot { .. } => json!({"ok": true}), + } +} + +/// The command is read-only: selection, camera, revision and undo stand exactly where they were. +#[test] +fn picking_changes_nothing() { + let mut h = top_view(); + let (cam, undo, revision) = (h.app.cam, h.app.session.undo.len(), h.app.session.revision); + let r = pick(&mut h, json!({"pixels": [[480, 300], [40, 40]], "rays": [{"origin": [5.0, 5.0, 50.0], "dir": [0.0, 0.0, -1.0]}]})); + assert_eq!(r["hits"].as_array().map(Vec::len), Some(3)); + assert_eq!((h.app.cam.distance, h.app.cam.target.z), (cam.distance, cam.target.z)); + assert_eq!(h.app.session.undo.len(), undo); + assert_eq!(h.app.session.revision, revision); +} + +/// MCP reaches the command through `engine.execute` and `engine.batch` like any command. +#[test] +fn the_command_runs_through_execute_and_batch() { + let mut h = top_view(); + let params = json!({"frame": "model", "pixels": [[480, 300]]}); + let v = h.call("engine.execute", json!({"command": "view.pick", "params": params})); + assert_eq!(v["ok"], true, "{v}"); + assert!(v["result"]["hits"][0]["face"].is_number(), "{v}"); + let v = h.call("engine.batch", json!({"commands": [{"command": "view.pick", "params": params}]})); + assert_eq!(v["ok"], true, "{v}"); + assert!(v["result"]["results"][0]["hits"][0]["face"].is_number(), "{v}"); +} + +/// `all` lists every body the ray meets, nearest first; the input cap is enforced. +#[test] +fn all_lists_the_faces_along_the_ray_and_the_cap_is_enforced() { + let mut h = top_view(); + let r = pick(&mut h, json!({"pixels": [[480, 300]], "all": true})); + let row = r["hits"][0].as_array().unwrap(); + assert_eq!(row.len(), 1, "one body under the ray: {r}"); + assert!(row[0]["face"].is_number()); + let pixels: Vec> = (0..1025).map(|i| vec![i as f64 % 960.0, 300.0]).collect(); + let v = h.call("ui.pick", json!({"pixels": pixels})); + assert_eq!(v["ok"], false); + assert!(v["error"].as_str().is_some_and(|e| e.contains("1024")), "{v}"); +} diff --git a/crates/ui-egui/src/viewport.rs b/crates/ui-egui/src/viewport.rs index cc830df04..1cad06641 100644 --- a/crates/ui-egui/src/viewport.rs +++ b/crates/ui-egui/src/viewport.rs @@ -440,7 +440,7 @@ fn sketch_color(t: &Tokens, c: Rgb) -> [u8; 4] { } /// With a component active, bodies outside it are drawn faded. -fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option>) -> bool { +pub(crate) fn body_faded(app: &SolveApp, b: &solidcraft_engine::doc::ModelBody, active_bodies: &Option>) -> bool { let s = &app.session; if let Some(names) = active_bodies { !names.contains(&b.name) @@ -1314,7 +1314,7 @@ pub fn construction_quads(app: &SolveApp) -> Vec<(String, Vec3, [Vec3; 4])> { /// Pick what is under `pos`: sketch points, sketch curves, vertices, edges and axes first, then /// the surfaces along the ray (profiles, faces, planes) nearest first. /// Bounding boxes of the shown model's bodies (cached until the model changes). -fn body_bounds( +pub(crate) fn body_bounds( app: &SolveApp, st: &std::sync::Arc, ) -> std::sync::Arc> { @@ -1393,7 +1393,7 @@ fn sketch_bounds(app: &SolveApp, st: &std::sync::Arc Option { +pub(crate) fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Option { let (mut t0, mut t1) = (f64::NEG_INFINITY, f64::INFINITY); for (oo, dd, lo, hi) in [(o.x, d.x, b.min.x, b.max.x), (o.y, d.y, b.min.y, b.max.y), (o.z, d.z, b.min.z, b.max.z)] { let (lo, hi) = (lo - pad, hi + pad); @@ -1411,7 +1411,7 @@ fn ray_box(o: Vec3, d: Vec3, b: &solidcraft_engine::geom::Aabb3, pad: f64) -> Op } /// The point of segment `a`–`b` nearest the ray from `o` along `d`. -fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 { +pub(crate) fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 { let u = b - a; let w = a - o; let (dd, du, uu) = (d.dot(d), d.dot(u), u.dot(u)); @@ -1428,7 +1428,7 @@ fn ray_segment_closest(o: Vec3, d: Vec3, a: Vec3, b: Vec3) -> Vec3 { } /// A box's outline on screen, grown by `px` (None: behind the eye). -fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option { +pub(crate) fn screen_box(proj: &Proj, b: &solidcraft_engine::geom::Aabb3, px: f32) -> Option { let mut r = Rect::NOTHING; for i in 0..8 { let p = Vec3::new( @@ -2887,7 +2887,7 @@ fn body_rgb(app: &SolveApp, name: &str) -> (u8, u8, u8) { /// The ray `o` + t `d` in the frame of a mesh that `place` (a rigid move) puts in the world, with /// the same t along it; `None` for a move with no inverse. -fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> { +pub(crate) fn local_ray(place: &solidcraft_engine::doc::Mat, o: Vec3, d: Vec3) -> Option<(Vec3, Vec3)> { if solidcraft_engine::doc::is_identity(place) { return Some((o, d)); } @@ -2908,7 +2908,7 @@ pub(crate) fn polyline_mid(pts: &[Vec3]) -> Vec3 { pts.first().copied().unwrap_or_default() } -fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 { +pub(crate) fn seg_dist(p: Pos2, a: Pos2, b: Pos2) -> f32 { let ab = b - a; let l2 = ab.length_sq(); let t = if l2 > 0.0 { ((p - a).dot(ab) / l2).clamp(0.0, 1.0) } else { 0.0 }; diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 4773f65e6..2a16e4e29 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -230,6 +230,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox. | `ui.sketchToScreen` | `points`: `[[x, y]…]` in active-sketch coordinates | their screen points (null when behind the camera) | | `ui.dialogInput` | `index` | what clicking an input's Select chip does: picks go to that input | | `ui.at` | `world: [x,y,z]` \| `sketch: [x,y]` \| `plane: "XY"` \| `axis: "Z"` \| `dimension: "d1"` (its value text) \| `handle: "arrow"` (a manipulator handle, see `scenario::publish_handle`) | the screen point (a scenario step's target; see `crates/ui-egui/src/scenario.rs`) | +| `ui.pick` | `pixels?: [[x,y]…]`, `rays?: [{origin, dir}…]`, `frame?: "window"\|"model"`, `width?`, `height?`, `edges?` (default true), `edge_px?` (default 6), `all?`, `include_ghosted?` | read-only geometry under each pixel or ray (#6304): one entry per input, `null` where nothing is hit — body, face index, exact world `point` and outward `normal`, `depth`, `occurrence`/`component` for placed parts, `edge`/`vertex` within `edge_px` of a pixel, `section_cap: true` on a cut face. `window` pixels are physical pixels of a window screenshot; `model` pixels are pixels of a `ui.render` of `width`×`height` (default 960×600), so it works headless. As `view.pick` through `engine.execute`/`engine.batch`. See `crates/ui-egui/src/view_pick.rs` | | `ui.handles` | `prefix?` | the handles drawn last frame, `[{name, x, y}]`: toolbar buttons and drop-down items `toolbar:` (the button alone: `button:`; a drop-down row: `dropdown:`, highlighted `dropdown_hot:`, its pin `dropdown_pin:`; Customize Toolbar's check boxes `customize:`), ribbon tabs `tab:` (MODEL, SURFACE, ASSEMBLE, INSERT, INSPECT, MANAGE, VIEW; contextual SKETCH and SHEET METAL; workspace RENDER), panel labels `panel::` (also `ribbon:section::
`), document overflow list `tab:overflow`, workspace column `ribbon:workspace`, command-search launcher `ribbon:search`, browser rows `row: