diff --git a/Cargo.lock b/Cargo.lock index f17283e16..9ce727f1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -865,6 +865,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.8" @@ -1665,6 +1671,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gl_generator" version = "0.14.0" @@ -2077,6 +2093,9 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", + "color_quant", + "gif", + "image-webp", "moxcms", "num-traits", "png", @@ -2085,6 +2104,16 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -4244,6 +4273,7 @@ version = "0.15.0" name = "solidcraft-ui-egui" version = "0.15.0" dependencies = [ + "arboard", "egui", "egui-wgpu", "image", diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rs index bf10c345c..014a58428 100644 --- a/apps/solidcraft/src/main.rs +++ b/apps/solidcraft/src/main.rs @@ -116,6 +116,7 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services { let parent = rfd::AsyncFileDialog::new().set_parent(cc); let open_parent = parent.clone(); let program_parent = parent.clone(); + let file_parent = parent.clone(); Services { pick_open: Some(Box::new(move |first: &[&str]| { // The file type the command reads comes first, so the picker opens on it (#1801). @@ -166,6 +167,21 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services { .pick_file(); Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) }) })), + // One kind of file (the agent chat's attachments): only the types it names, as given and in + // lower and upper case (Linux dialogs match case; macOS allows just the listed types). + pick_file: Some(Box::new(move |label: &str, exts: &[&str]| { + let mut all: Vec = Vec::new(); + for e in exts { + for v in [e.to_string(), e.to_ascii_lowercase(), e.to_ascii_uppercase()] { + if !all.contains(&v) { + all.push(v); + } + } + } + let all: Vec<&str> = all.iter().map(String::as_str).collect(); + let future = file_parent.clone().add_filter(label, &all).pick_file(); + Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) }) + })), pick_program: Some(Box::new(move || { let dialog = program_parent.clone().set_title("Choose your slicer"); let dialog = if cfg!(target_os = "windows") { dialog.add_filter("Programs", &["exe"]) } else { dialog }; diff --git a/assets/locales/en.json b/assets/locales/en.json index 838e21663..2fa7d927e 100644 --- a/assets/locales/en.json +++ b/assets/locales/en.json @@ -841,12 +841,15 @@ "ui.activate": "Activate", "ui.active_component": "Active component", "ui.agent_chat": "Agent chat", + "ui.agent_chat_add": "Attach files", + "ui.agent_chat_add_file": "Choose a file…", "ui.agent_chat_allow": "Allow", "ui.agent_chat_allowed": "Allowed", "ui.agent_chat_answered": "Answer: {option}", "ui.agent_chat_approval": "Allow the agent to do this?", "ui.agent_chat_attach_view": "Attach view", "ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message", + "ui.agent_chat_attachable": "Pictures, text and CAD files", "ui.agent_chat_auth_agent": "Sign in with {name}", "ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.", "ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.", @@ -869,9 +872,30 @@ "ui.agent_chat_dock": "Dock in the side panel", "ui.agent_chat_dock_left": "Dock on the left", "ui.agent_chat_dock_right": "Dock on the right", + "ui.agent_chat_drop_hint": "Drop to attach to your next message", "ui.agent_chat_edit": "Edit", "ui.agent_chat_exited": "The agent stopped.", "ui.agent_chat_failed_start": "The agent could not start: {error}", + "ui.agent_chat_file_cad_not_read": "{name}: a CAD file is imported into the design, not read", + "ui.agent_chat_file_empty_picture": "{name}: an empty picture", + "ui.agent_chat_file_error": "{name}: {error}", + "ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first", + "ui.agent_chat_file_importing": "Importing into the design…", + "ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached", + "ui.agent_chat_file_loading": "Reading the file…", + "ui.agent_chat_file_not_a_file": "{name}: not a file", + "ui.agent_chat_file_not_a_picture": "{name}: not a picture", + "ui.agent_chat_file_not_utf8": "{name}: not UTF-8 text", + "ui.agent_chat_file_picture_too_big": "{name}: a picture can be at most {max} MB", + "ui.agent_chat_file_text_too_big": "{name}: a text file can be at most {max} KB", + "ui.agent_chat_file_too_large": "{name}: too large", + "ui.agent_chat_file_too_large_to_send": "{name}: too large to send, even scaled down", + "ui.agent_chat_file_unreadable_picture": "{name}: not a picture this can read", + "ui.agent_chat_files": "Files: {files}", + "ui.agent_chat_files_left_out": "At most {max} files go with one message; left out: {files}", + "ui.agent_chat_files_loading": "Waiting for the files to be read", + "ui.agent_chat_files_max": "At most {max} files on one message", + "ui.agent_chat_import_failed": "{name}: the import failed: {error}", "ui.agent_chat_intro": "Chat with a coding agent you already use, or a model endpoint you choose. It works on the open design through SolidCraft's tools: its changes show as it makes them and undo as one step per message. What it may do without asking is set in Preferences > AI.", "ui.agent_chat_key_in_env": "key set", "ui.agent_chat_left": "Left", @@ -884,11 +908,13 @@ "ui.agent_chat_no_cli": "solidcraft-cli was not found next to the app, so the agent can't reach the design.", "ui.agent_chat_no_control": "The agent chat's endpoint did not start, so the agent can't reach the design.", "ui.agent_chat_no_images": "This agent doesn't take pictures", + "ui.agent_chat_no_picture": "There is no picture on the clipboard", "ui.agent_chat_none_found": "No coding agent found. Install one of these, then press Refresh:", "ui.agent_chat_none_found_short": "No agent found", "ui.agent_chat_not_installed": "Not installed", "ui.agent_chat_not_signed_in": "not signed in", "ui.agent_chat_own_window": "Own window", + "ui.agent_chat_paste_picture": "Paste a picture", "ui.agent_chat_permission": "The agent asks to run: {title}", "ui.agent_chat_pick": "Agent", "ui.agent_chat_placeholder": "Ask the agent to change the design…", @@ -903,6 +929,8 @@ "ui.agent_chat_refresh": "Refresh", "ui.agent_chat_refused": "Not allowed for the agent: {reason}", "ui.agent_chat_remove": "Remove from the queue", + "ui.agent_chat_remove_file": "Take it off this message", + "ui.agent_chat_remove_imported": "Take it off this message; the part stays in the design (Undo removes it)", "ui.agent_chat_right": "Right", "ui.agent_chat_save": "Save", "ui.agent_chat_send": "Send", diff --git a/assets/locales/ja.json b/assets/locales/ja.json index 4fb2e4384..8685df9e9 100644 --- a/assets/locales/ja.json +++ b/assets/locales/ja.json @@ -841,12 +841,15 @@ "ui.activate": "アクティブ化", "ui.active_component": "アクティブなコンポーネント", "ui.agent_chat": "エージェントチャット", + "ui.agent_chat_add": "ファイルを添付", + "ui.agent_chat_add_file": "ファイルを選択…", "ui.agent_chat_allow": "許可", "ui.agent_chat_allowed": "許可しました", "ui.agent_chat_answered": "回答: {option}", "ui.agent_chat_approval": "エージェントにこの操作を許可しますか?", "ui.agent_chat_attach_view": "ビューを添付", "ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります", + "ui.agent_chat_attachable": "画像・テキスト・CAD ファイル", "ui.agent_chat_auth_agent": "{name} でサインイン", "ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。", "ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。", @@ -869,9 +872,30 @@ "ui.agent_chat_dock": "サイドパネルに戻す", "ui.agent_chat_dock_left": "左にドッキング", "ui.agent_chat_dock_right": "右にドッキング", + "ui.agent_chat_drop_hint": "ドロップして次のメッセージに添付", "ui.agent_chat_edit": "編集", "ui.agent_chat_exited": "エージェントが停止しました。", "ui.agent_chat_failed_start": "エージェントを起動できませんでした: {error}", + "ui.agent_chat_file_cad_not_read": "{name}: CAD ファイルは読み込まず、デザインにインポートします", + "ui.agent_chat_file_empty_picture": "{name}: 空の画像です", + "ui.agent_chat_file_error": "{name}: {error}", + "ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください", + "ui.agent_chat_file_importing": "デザインにインポート中…", + "ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です", + "ui.agent_chat_file_loading": "ファイルを読み込み中…", + "ui.agent_chat_file_not_a_file": "{name}: ファイルではありません", + "ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません", + "ui.agent_chat_file_not_utf8": "{name}: UTF-8 のテキストではありません", + "ui.agent_chat_file_picture_too_big": "{name}: 画像は {max} MB までです", + "ui.agent_chat_file_text_too_big": "{name}: テキストファイルは {max} KB までです", + "ui.agent_chat_file_too_large": "{name}: 大きすぎます", + "ui.agent_chat_file_too_large_to_send": "{name}: 縮小しても大きすぎて送れません", + "ui.agent_chat_file_unreadable_picture": "{name}: 読み込めない画像です", + "ui.agent_chat_files": "ファイル: {files}", + "ui.agent_chat_files_left_out": "1 つのメッセージで送れるファイルは {max} 個までです。送らなかったファイル: {files}", + "ui.agent_chat_files_loading": "ファイルの読み込みを待っています", + "ui.agent_chat_files_max": "1 つのメッセージに添付できるのは {max} 個までです", + "ui.agent_chat_import_failed": "{name}: インポートに失敗しました: {error}", "ui.agent_chat_intro": "普段使っているコーディングエージェント、または選んだモデルのエンドポイントとチャットできます。SolidCraft のツールで開いているデザインを操作し、変更はその場で表示され、メッセージごとに1回で元に戻せます。確認なしで許可する範囲は「環境設定 > AI」で設定します。", "ui.agent_chat_key_in_env": "キー設定済み", "ui.agent_chat_left": "左", @@ -884,11 +908,13 @@ "ui.agent_chat_no_cli": "アプリの隣に solidcraft-cli が見つからないため、エージェントはデザインにアクセスできません。", "ui.agent_chat_no_control": "エージェントチャットのエンドポイントが起動しなかったため、エージェントはデザインにアクセスできません。", "ui.agent_chat_no_images": "このエージェントは画像を受け付けません", + "ui.agent_chat_no_picture": "クリップボードに画像がありません", "ui.agent_chat_none_found": "コーディングエージェントが見つかりません。次のいずれかをインストールしてから「再検索」を押してください:", "ui.agent_chat_none_found_short": "エージェントが見つかりません", "ui.agent_chat_not_installed": "未インストール", "ui.agent_chat_not_signed_in": "未サインイン", "ui.agent_chat_own_window": "独立したウィンドウ", + "ui.agent_chat_paste_picture": "画像を貼り付け", "ui.agent_chat_permission": "エージェントが実行の許可を求めています: {title}", "ui.agent_chat_pick": "エージェント", "ui.agent_chat_placeholder": "デザインの変更をエージェントに依頼…", @@ -903,6 +929,8 @@ "ui.agent_chat_refresh": "再検索", "ui.agent_chat_refused": "エージェントには許可されていません: {reason}", "ui.agent_chat_remove": "待機から削除", + "ui.agent_chat_remove_file": "このメッセージから外す", + "ui.agent_chat_remove_imported": "このメッセージから外します。パーツはデザインに残ります (元に戻すで削除できます)", "ui.agent_chat_right": "右", "ui.agent_chat_save": "保存", "ui.agent_chat_send": "送信", diff --git a/assets/locales/zh.json b/assets/locales/zh.json index 26c7d424e..c2abf2aa2 100644 --- a/assets/locales/zh.json +++ b/assets/locales/zh.json @@ -841,12 +841,15 @@ "ui.activate": "激活", "ui.active_component": "活动组件", "ui.agent_chat": "智能体聊天", + "ui.agent_chat_add": "附加文件", + "ui.agent_chat_add_file": "选择文件…", "ui.agent_chat_allow": "允许", "ui.agent_chat_allowed": "已允许", "ui.agent_chat_answered": "回答:{option}", "ui.agent_chat_approval": "允许智能体执行此操作吗?", "ui.agent_chat_attach_view": "附加视图", "ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片", + "ui.agent_chat_attachable": "图片、文本和 CAD 文件", "ui.agent_chat_auth_agent": "使用 {name} 登录", "ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。", "ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。", @@ -869,9 +872,30 @@ "ui.agent_chat_dock": "停靠到侧面板", "ui.agent_chat_dock_left": "停靠在左侧", "ui.agent_chat_dock_right": "停靠在右侧", + "ui.agent_chat_drop_hint": "拖放到此处,附加到下一条消息", "ui.agent_chat_edit": "编辑", "ui.agent_chat_exited": "智能体已停止。", "ui.agent_chat_failed_start": "智能体无法启动:{error}", + "ui.agent_chat_file_cad_not_read": "{name}:CAD 文件会导入设计,而不是读取", + "ui.agent_chat_file_empty_picture": "{name}:空图片", + "ui.agent_chat_file_error": "{name}:{error}", + "ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG", + "ui.agent_chat_file_importing": "正在导入设计…", + "ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)", + "ui.agent_chat_file_loading": "正在读取文件…", + "ui.agent_chat_file_not_a_file": "{name}:不是文件", + "ui.agent_chat_file_not_a_picture": "{name}:不是图片", + "ui.agent_chat_file_not_utf8": "{name}:不是 UTF-8 文本", + "ui.agent_chat_file_picture_too_big": "{name}:图片最大 {max} MB", + "ui.agent_chat_file_text_too_big": "{name}:文本文件最大 {max} KB", + "ui.agent_chat_file_too_large": "{name}:太大", + "ui.agent_chat_file_too_large_to_send": "{name}:即使缩小也太大,无法发送", + "ui.agent_chat_file_unreadable_picture": "{name}:无法读取的图片", + "ui.agent_chat_files": "文件:{files}", + "ui.agent_chat_files_left_out": "一条消息最多发送 {max} 个文件;未发送:{files}", + "ui.agent_chat_files_loading": "正在等待文件读取完成", + "ui.agent_chat_files_max": "一条消息最多附加 {max} 个文件", + "ui.agent_chat_import_failed": "{name}:导入失败:{error}", "ui.agent_chat_intro": "与你已在使用的编程智能体或你选择的模型端点聊天。它通过 SolidCraft 的工具处理打开的设计:更改即时显示,每条消息的更改可一步撤销。无需询问即可执行的范围在“首选项 > AI”中设置。", "ui.agent_chat_key_in_env": "已设置密钥", "ui.agent_chat_left": "左侧", @@ -884,11 +908,13 @@ "ui.agent_chat_no_cli": "在应用旁边未找到 solidcraft-cli,因此智能体无法访问设计。", "ui.agent_chat_no_control": "智能体聊天的端点未能启动,因此智能体无法访问设计。", "ui.agent_chat_no_images": "该智能体不接收图片", + "ui.agent_chat_no_picture": "剪贴板上没有图片", "ui.agent_chat_none_found": "未找到编程智能体。请安装以下之一,然后点击刷新:", "ui.agent_chat_none_found_short": "未找到智能体", "ui.agent_chat_not_installed": "未安装", "ui.agent_chat_not_signed_in": "未登录", "ui.agent_chat_own_window": "独立窗口", + "ui.agent_chat_paste_picture": "粘贴图片", "ui.agent_chat_permission": "智能体请求运行:{title}", "ui.agent_chat_pick": "智能体", "ui.agent_chat_placeholder": "让智能体修改设计…", @@ -903,6 +929,8 @@ "ui.agent_chat_refresh": "刷新", "ui.agent_chat_refused": "不允许智能体执行:{reason}", "ui.agent_chat_remove": "从队列中移除", + "ui.agent_chat_remove_file": "从这条消息中移除", + "ui.agent_chat_remove_imported": "从这条消息中移除;零件仍保留在设计中(撤销可将其删除)", "ui.agent_chat_right": "右侧", "ui.agent_chat_save": "保存", "ui.agent_chat_send": "发送", diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 37563db23..70e1c9749 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -17,7 +17,9 @@ serde = { workspace = true } serde_json = { workspace = true } log = { workspace = true } web-time = { workspace = true } -image = { workspace = true, features = ["jpeg", "ico"] } +# gif, webp, bmp, tiff: pictures attached to an agent chat message (GIF and WebP go as they are, +# BMP and TIFF as PNG). +image = { workspace = true, features = ["jpeg", "ico", "gif", "webp", "bmp", "tiff"] } ttf-parser = { workspace = true } semver = { workspace = true } # Updates: the downloaded package's SHA-256 (against the release's SHA256SUMS.txt), its optional @@ -33,6 +35,11 @@ tiny-skia = { workspace = true } # updated by its installer (MIT; already in the tree as a build dependency). winreg = "0.56" +# Pictures pasted into the agent chat; already in the tree, with the same features, through +# egui-winit's clipboard. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +arboard = { version = "3.6", default-features = false, features = ["image-data"] } + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] # The offscreen viewport render test opens a device without a window. pollster = { workspace = true } diff --git a/crates/ui-egui/src/agent_chat/attach.rs b/crates/ui-egui/src/agent_chat/attach.rs new file mode 100644 index 000000000..5926e8cae --- /dev/null +++ b/crates/ui-egui/src/agent_chat/attach.rs @@ -0,0 +1,493 @@ +//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the +//! agent with it, text files go in as text, and CAD files are imported into the design, the agent +//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says +//! why. + +use std::path::Path; +use std::sync::Arc; + +use super::backend::{self, PromptPart}; +use crate::i18n::format as trf; + +/// A refusal in the app's language: the file's name, then why. +fn refused(key: &str, name: &str, args: &[(&str, &str)]) -> String { + let mut all = vec![("name", name)]; + all.extend_from_slice(args); + trf(key, &all) +} + +/// A refusal carrying the system's or the decoder's own words. +fn failed(name: &str, e: impl std::fmt::Display) -> String { + refused("ui.agent_chat_file_error", name, &[("error", &e.to_string())]) +} + +/// Most files on one message. +pub const MAX_FILES: usize = 8; +/// Largest picture file read, before it is scaled down. +const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024; +/// Largest text file sent in full. +const MAX_TEXT_BYTES: u64 = 256 * 1024; +/// A picture's longer side is scaled down to this before it is sent. +pub const MAX_SIDE: u32 = 2048; +/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB, +/// so eight of them still fit in a prompt. +pub const MAX_SENT_BYTES: usize = 3_750_000; +/// A picture is not halved below this to fit; past it, it is refused. +const MIN_SIDE: u32 = 64; +/// Largest picture decoded (pixels on a side), so a hostile header can't ask for gigabytes. +const MAX_DECODE_SIDE: u32 = 20_000; +/// The chip's picture. +const THUMB: u32 = 64; + +/// Pictures the agents read as they are; BMP and TIFF are sent as PNG. +const PICTURE_EXTS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"]; +/// Text sent in full; SVG goes as its source, which the agents read well. +const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv", "yaml", "yml", "toml", "xml", "log", "ini", "cfg"]; +/// CAD files and meshes, imported into the design (the file picker's list; [`kind`] asks the +/// importers themselves). +const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"]; + +/// Every extension that can be attached, pictures first (the file picker's list). +pub fn extensions() -> Vec<&'static str> { + PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect() +} + +/// What a file is, by its extension. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Picture, + Text, + Cad, + Other, +} + +pub fn kind(path: &Path) -> Kind { + let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default(); + match ext.as_str() { + e if PICTURE_EXTS.contains(&e) => Kind::Picture, + e if TEXT_EXTS.contains(&e) => Kind::Text, + _ => { + let p = path.to_string_lossy(); + if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other } + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Content { + /// The picture's bytes as sent (PNG, JPEG, GIF or WebP), and its size in pixels. + Picture { + mime: &'static str, + bytes: Arc>, + size: [u32; 2], + }, + Text(Arc), + /// A CAD file imported into the design as its own component. + Imported, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Attachment { + /// Its own on the message, never reused (set when it is added there; 0 until then): the + /// chip's thumbnail is kept by it. + pub id: u64, + pub name: String, + pub content: Content, + /// A small copy of a picture for its chip. + pub thumb: Option>, +} + +impl Attachment { + pub fn imported(name: String) -> Self { + Attachment { id: 0, name, content: Content::Imported, thumb: None } + } + pub fn is_picture(&self) -> bool { + matches!(self.content, Content::Picture { .. }) + } +} + +pub(crate) fn name_of(path: &Path) -> String { + path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string_lossy().to_string()) +} + +/// Read a picture or text file for a message. CAD files are imported by the caller, not read here. +pub fn read(path: &Path) -> Result { + let name = name_of(path); + let meta = std::fs::metadata(path).map_err(|e| failed(&name, e))?; + if !meta.is_file() { + return Err(refused("ui.agent_chat_file_not_a_file", &name, &[])); + } + match kind(path) { + Kind::Picture => { + if meta.len() > MAX_PICTURE_BYTES { + return Err(refused("ui.agent_chat_file_picture_too_big", &name, &[("max", &(MAX_PICTURE_BYTES / 1024 / 1024).to_string())])); + } + let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?; + picture(name, &bytes) + } + Kind::Text => { + if meta.len() > MAX_TEXT_BYTES { + return Err(refused("ui.agent_chat_file_text_too_big", &name, &[("max", &(MAX_TEXT_BYTES / 1024).to_string())])); + } + let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?; + let text = String::from_utf8(bytes).map_err(|_| refused("ui.agent_chat_file_not_utf8", &name, &[]))?; + Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None }) + } + Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])), + Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => { + Err(refused("ui.agent_chat_file_heic", &name, &[])) + } + Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])), + } +} + +/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read +/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most +/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]). +pub fn picture(name: String, bytes: &[u8]) -> Result { + picture_within(name, bytes, MAX_SENT_BYTES) +} + +fn picture_within(name: String, bytes: &[u8], limit: usize) -> Result { + let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().map_err(|e| failed(&name, e))?; + let format = reader.format(); + let mime = match format { + Some(image::ImageFormat::Png) => Some("image/png"), + Some(image::ImageFormat::Jpeg) => Some("image/jpeg"), + Some(image::ImageFormat::Gif) => Some("image/gif"), + Some(image::ImageFormat::WebP) => Some("image/webp"), + Some(image::ImageFormat::Bmp | image::ImageFormat::Tiff) => None, + _ => return Err(refused("ui.agent_chat_file_unreadable_picture", &name, &[])), + }; + let mut limits = image::Limits::default(); + limits.max_image_width = Some(MAX_DECODE_SIDE); + limits.max_image_height = Some(MAX_DECODE_SIDE); + limits.max_alloc = Some(1024 * 1024 * 1024); + reader.limits(limits); + let img = reader.decode().map_err(|e| failed(&name, e))?; + if img.width() == 0 || img.height() == 0 { + return Err(refused("ui.agent_chat_file_empty_picture", &name, &[])); + } + if let Some(mime) = mime + && img.width().max(img.height()) <= MAX_SIDE + && bytes.len() <= limit + { + let thumb = thumbnail(&img); + return Ok(Attachment { + id: 0, + name, + content: Content::Picture { mime, bytes: Arc::new(bytes.to_vec()), size: [img.width(), img.height()] }, + thumb, + }); + } + encoded(name, img, limit) +} + +/// A picture copied to the clipboard (RGBA rows from the system), sent as PNG. +pub fn from_rgba(name: String, width: usize, height: usize, rgba: &[u8]) -> Result { + let (w, h) = (u32::try_from(width).ok(), u32::try_from(height).ok()); + let (Some(w), Some(h)) = (w, h) else { return Err(refused("ui.agent_chat_file_too_large", &name, &[])) }; + if w == 0 || h == 0 || w > MAX_DECODE_SIDE || h > MAX_DECODE_SIDE || width.checked_mul(height).and_then(|n| n.checked_mul(4)) != Some(rgba.len()) + { + return Err(refused("ui.agent_chat_file_not_a_picture", &name, &[])); + } + let img = image::RgbaImage::from_raw(w, h, rgba.to_vec()).ok_or_else(|| refused("ui.agent_chat_file_not_a_picture", &name, &[]))?; + encoded(name, image::DynamicImage::ImageRgba8(img), MAX_SENT_BYTES) +} + +/// Scale down to [`MAX_SIDE`] and encode within `limit` bytes: as PNG when that fits; else as JPEG +/// when the picture has no transparency; else halved and tried again. A GIF arrives here as its +/// first frame. +fn encoded(name: String, img: image::DynamicImage, limit: usize) -> Result { + let mut img = if img.width().max(img.height()) > MAX_SIDE { img.resize(MAX_SIDE, MAX_SIDE, image::imageops::FilterType::Triangle) } else { img }; + let opaque = opaque(&img); + loop { + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).map_err(|e| failed(&name, e))?; + let mut sent = (png.len() <= limit).then_some(("image/png", png)); + if sent.is_none() && opaque { + let mut jpeg = Vec::new(); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 85).encode_image(&img.to_rgb8()).map_err(|e| failed(&name, e))?; + sent = (jpeg.len() <= limit).then_some(("image/jpeg", jpeg)); + } + if let Some((mime, bytes)) = sent { + let thumb = thumbnail(&img); + return Ok(Attachment { + id: 0, + name, + content: Content::Picture { mime, bytes: Arc::new(bytes), size: [img.width(), img.height()] }, + thumb, + }); + } + if img.width().max(img.height()) <= MIN_SIDE { + return Err(refused("ui.agent_chat_file_too_large_to_send", &name, &[])); + } + img = img.resize((img.width() / 2).max(1), (img.height() / 2).max(1), image::imageops::FilterType::Triangle); + } +} + +/// No pixel is see-through (a screenshot or a photo, even when stored with an alpha channel). +fn opaque(img: &image::DynamicImage) -> bool { + !img.color().has_alpha() || img.to_rgba8().pixels().all(|p| p.0[3] == 255) +} + +fn thumbnail(img: &image::DynamicImage) -> Option> { + let t = img.thumbnail(THUMB, THUMB).to_rgba8(); + let size = [usize::try_from(t.width()).ok()?, usize::try_from(t.height()).ok()?]; + Some(Arc::new(egui::ColorImage::from_rgba_unmultiplied(size, t.as_raw()))) +} + +/// What the message says about its files, after the person's words: the text files' text, and the +/// pictures and imported parts by name (the agent sees the pictures themselves as images). +/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters +/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget +/// is named only, so the message stays within the prompt cap. +pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String { + let mut out = String::new(); + let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect(); + if !pictures.is_empty() { + let list = pictures.join(", "); + out += &if pictures_sent { + format!("\n\nAttached pictures: {list}.") + } else { + format!("\n\n(Pictures attached but not sent, this agent takes none: {list}.)") + }; + } + let mut left = budget; + for f in files { + match &f.content { + Content::Text(t) => { + let t = t.trim_end(); + let len = t.chars().count(); + if left == 0 { + out += &format!("\n\nAttached file `{}`: left out, the message is at its size limit.", f.name); + } else if len <= left { + out += &format!("\n\nAttached file `{}`:\n```\n{t}\n```", f.name); + left -= len; + } else { + let kept: String = t.chars().take(left).collect(); + out += &format!("\n\nAttached file `{}` (cut: {} more characters left out):\n```\n{kept}\n```", f.name, len - left); + left = 0; + } + } + Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name), + Content::Picture { .. } => {} + } + } + out +} + +/// The pictures as image parts of the message. +pub fn parts(files: &[Attachment]) -> Vec { + files + .iter() + .filter_map(|f| match &f.content { + Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }), + _ => None, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("solidcraft-attach-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn png(path: &Path, w: u32, h: u32) { + image::RgbaImage::from_pixel(w, h, image::Rgba([200, 30, 30, 255])).save(path).unwrap(); + } + + #[test] + fn pictures_are_kept_or_scaled_down_and_text_goes_in_whole() { + let d = dir("read"); + let small = d.join("board.png"); + png(&small, 1600, 1120); + let a = read(&small).unwrap(); + let Content::Picture { mime, size, bytes } = &a.content else { panic!("a picture") }; + assert_eq!((*mime, *size), ("image/png", [1600, 1120])); + assert_eq!(bytes.as_slice(), std::fs::read(&small).unwrap().as_slice(), "small enough: sent as it came"); + assert!(a.thumb.as_ref().is_some_and(|t| t.size[0] <= 64 && t.size[1] <= 64)); + let big = d.join("photo.JPG"); + image::RgbImage::from_pixel(4000, 1000, image::Rgb([10, 10, 10])).save_with_format(&big, image::ImageFormat::Jpeg).unwrap(); + let b = read(&big).unwrap(); + let Content::Picture { mime, size, .. } = &b.content else { panic!("a picture") }; + assert_eq!((*mime, *size), ("image/png", [2048, 512]), "scaled to 2048 on the long side"); + let notes = d.join("brief.md"); + std::fs::write(¬es, "# Brief\nwall 2 mm\n").unwrap(); + let c = read(¬es).unwrap(); + assert!(matches!(&c.content, Content::Text(t) if t.contains("wall 2 mm"))); + let _ = std::fs::remove_dir_all(&d); + } + + /// Pixels that don't compress, so a small picture can be made larger than a small limit. + fn noise(w: u32, h: u32, alpha: bool) -> image::DynamicImage { + let mut seed = 0x2545_f491_u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + (seed & 0xff) as u8 + }; + image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(w, h, |_, _| { + image::Rgba([next(), next(), next(), if alpha { next() } else { 255 }]) + })) + } + + fn encode(img: &image::DynamicImage, format: image::ImageFormat) -> Vec { + let mut out = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut out), format).unwrap(); + out + } + + /// Review of #1744: a picture up to 2048 px went byte for byte up to the 40 MB file cap, and + /// eight of those don't fit in a prompt. Over the limit a picture is re-encoded: PNG when that + /// fits, JPEG when it has no transparency, halved until it fits; a GIF goes as its first frame. + #[test] + fn pictures_over_the_send_limit_are_reencoded_to_fit() { + let limit = 40_000; + let photo = encode(&noise(300, 200, false), image::ImageFormat::Png); + assert!(photo.len() > limit, "the test picture is over the limit"); + let a = picture_within("photo.png".into(), &photo, limit).unwrap(); + let Content::Picture { mime, bytes, size } = &a.content else { panic!("a picture") }; + assert!(bytes.len() <= limit, "{} bytes", bytes.len()); + assert_eq!(*mime, "image/jpeg", "an opaque picture goes as JPEG when PNG is too big"); + assert!(size[0] <= 300 && size[1] <= 200 && size[0] * 2 == size[1] * 3, "{size:?}: smaller, same shape"); + let clear = encode(&noise(300, 200, true), image::ImageFormat::Png); + let b = picture_within("overlay.png".into(), &clear, limit).unwrap(); + let Content::Picture { mime, bytes, size } = &b.content else { panic!("a picture") }; + assert_eq!(*mime, "image/png", "transparency stays PNG"); + assert!(bytes.len() <= limit && size[0] < 300, "{} bytes, {size:?}: halved to fit", bytes.len()); + let gif = encode( + &image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(400, 300, image::Rgba([40, 90, 200, 255]))), + image::ImageFormat::Gif, + ); + let c = picture_within("loop.gif".into(), &gif, gif.len() - 1).unwrap(); + let Content::Picture { mime, size, bytes } = &c.content else { panic!("a picture") }; + assert_eq!(*mime, "image/png", "a GIF over the limit goes as its first frame, as PNG"); + assert!(bytes.len() < gif.len() && size[0] * 3 == size[1] * 4, "{size:?}: within the limit, same shape"); + assert!(a.thumb.is_some() && b.thumb.is_some() && c.thumb.is_some()); + let under = encode(&noise(60, 40, false), image::ImageFormat::Png); + let d = picture_within("small.png".into(), &under, limit).unwrap(); + assert!(matches!(&d.content, Content::Picture { bytes, .. } if bytes.as_slice() == under.as_slice()), "under the limit: as it came"); + assert_eq!(MAX_SENT_BYTES, 3_750_000, "its base64 stays under 5 MB"); + } + + #[test] + fn bad_files_are_refused_with_a_reason() { + let d = dir("bad"); + let fake = d.join("fake.png"); + std::fs::write(&fake, b"not a picture at all").unwrap(); + assert!(read(&fake).unwrap_err().contains("fake.png")); + let text = d.join("big.txt"); + std::fs::write(&text, vec![b'a'; 300 * 1024]).unwrap(); + assert!(read(&text).unwrap_err().contains("at most")); + let binary = d.join("bin.json"); + std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap(); + assert!(read(&binary).unwrap_err().contains("UTF-8")); + let other = d.join("movie.mov"); + std::fs::write(&other, b"x").unwrap(); + assert!(read(&other).unwrap_err().contains("can be attached")); + assert!(read(&d.join("missing.png")).is_err()); + assert_eq!(kind(Path::new("part.STEP")), Kind::Cad); + assert_eq!(kind(Path::new("mount.stl")), Kind::Cad); + assert_eq!(kind(Path::new("frame.3dm")), Kind::Cad); + assert!(from_rgba("x".into(), 2, 2, &[0; 15]).is_err(), "wrong length"); + assert!(from_rgba("x".into(), 0, 2, &[]).is_err()); + assert!(from_rgba("x".into(), usize::MAX, 2, &[]).is_err()); + let ok = from_rgba("pasted.png".into(), 3, 2, &[255; 24]).unwrap(); + assert!(matches!(ok.content, Content::Picture { size: [3, 2], .. })); + let _ = std::fs::remove_dir_all(&d); + } + + /// Review of #1744: the refusals were English whatever the app's language. + #[test] + fn refusals_speak_the_apps_language() { + let d = dir("lang"); + let binary = d.join("bin.json"); + std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap(); + let other = d.join("movie.mov"); + std::fs::write(&other, b"x").unwrap(); + { + let _ja = crate::i18n::scope("ja"); + assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません")); + assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは")); + } + { + let _zh = crate::i18n::scope("zh"); + assert!(read(&binary).unwrap_err().contains("不是 UTF-8 文本")); + } + assert!(read(&binary).unwrap_err().contains("not UTF-8 text"), "English again outside the scope"); + let _ = std::fs::remove_dir_all(&d); + } + + /// Every picture format the agents read goes as it is (PNG, JPEG, GIF, WebP); BMP and TIFF are + /// sent as PNG; SVG goes as its source text; HEIC says why not. + #[test] + fn each_picture_format_goes_the_way_the_agent_reads_it() { + let d = dir("formats"); + let img = image::RgbaImage::from_pixel(30, 20, image::Rgba([20, 120, 220, 255])); + for (file, format, mime) in [ + ("a.gif", image::ImageFormat::Gif, "image/gif"), + ("a.webp", image::ImageFormat::WebP, "image/webp"), + ("a.bmp", image::ImageFormat::Bmp, "image/png"), + ("a.tiff", image::ImageFormat::Tiff, "image/png"), + ] { + let path = d.join(file); + image::DynamicImage::ImageRgba8(img.clone()).save_with_format(&path, format).unwrap(); + let a = read(&path).unwrap_or_else(|e| panic!("{file}: {e}")); + let Content::Picture { mime: m, size, bytes } = &a.content else { panic!("{file}: a picture") }; + assert_eq!((*m, *size), (mime, [30, 20]), "{file}"); + if mime != "image/png" { + assert_eq!(bytes.as_slice(), std::fs::read(&path).unwrap().as_slice(), "{file}: sent as it came"); + } + assert!(a.thumb.is_some(), "{file}: a thumbnail"); + } + let svg = d.join("sketch.svg"); + std::fs::write(&svg, "").unwrap(); + assert!(matches!(&read(&svg).unwrap().content, Content::Text(t) if t.contains(", }, Agent { thought: bool, @@ -152,6 +157,8 @@ struct Turn { pub struct Pending { pub text: String, pub image: bool, + /// The files attached to it (they go with it). + pub files: Vec, } /// A request held for Allow or Deny. @@ -165,6 +172,30 @@ struct Held { /// it): requests arriving there go to the sender; the port it listens on. pub type ControlStarter = Box) -> Option>; +/// A file not yet on the next message: a picture being read off the window thread (so a large one +/// doesn't stop the window), or a CAD file being imported into the design. Its chip shows +/// meanwhile; it keeps the id it was given when attached, so the files stay in the order they +/// were attached in. +pub(crate) struct Loading { + pub id: u64, + pub name: String, + what: Reading, +} + +enum Reading { + /// A picture decoding, or a pasted one encoding: the file, or why not. + File(Receiver>), + /// A CAD file's import into the design: its answer, as a control reply. + Import(Receiver), +} + +impl Loading { + /// A CAD file being imported (its part lands in the design whatever the message does). + pub fn importing(&self) -> bool { + matches!(self.what, Reading::Import(_)) + } +} + #[derive(Default)] pub struct AgentChat { /// The panel is shown (View menu, the application bar's chat button, Ctrl+Shift+A). @@ -178,8 +209,8 @@ pub struct AgentChat { pub attach_view: bool, auth: Vec, auth_status: Option, - /// A message typed before the agent was ready. - queued: Option<(String, bool)>, + /// A message typed before the agent was ready, with its files. + queued: Option<(String, bool, Vec)>, /// Messages typed while the agent works, in order. They go as one prompt when the turn ends /// (or at once with Send now); after Stop they wait for Send. pub pending: Vec, @@ -199,8 +230,21 @@ pub struct AgentChat { pub confirm_new: bool, /// Where the panel was drawn last (for its shortcuts). pub panel_rect: Option, - /// The last message sent, kept to send again after a sign-in. - last: Option<(String, bool)>, + /// The last message sent, kept to send again after a sign-in (its files too). + last: Option<(String, bool, Vec)>, + /// Files for the next message (pictures, text files, imported parts), shown as chips. + pub(crate) attachments: Vec, + /// Pictures pasted so far, to name the next one. + pasted: usize, + /// The chips' thumbnails, by the picture they show. + thumbs: std::collections::HashMap, + /// The last id given to a file added to a message. + last_file_id: u64, + /// Files still being read for the next message. + pub(crate) loading: Vec, + /// Where the chat was drawn last frame (the panel, or its folded strip), for drops; `None` + /// when it isn't drawn. + pub(crate) drop_rect: Option, /// The command the running agent was started with. argv: Vec, /// The agent picked for this conversation (the header's switcher); `None`: the default. @@ -246,6 +290,54 @@ impl Drop for AgentChat { } impl AgentChat { + /// Put a file on the next message with an id of its own; its name back. + fn add_file(&mut self, file: attach::Attachment) -> String { + let id = self.next_file_id(); + self.place(id, file) + } + fn next_file_id(&mut self) -> u64 { + self.last_file_id += 1; + self.last_file_id + } + /// A file goes where its id says: the order the files were attached in. + fn place(&mut self, id: u64, mut file: attach::Attachment) -> String { + file.id = id; + let name = file.name.clone(); + let at = self.attachments.partition_point(|f| f.id < id); + self.attachments.insert(at, file); + name + } + /// Files on the next message, read or still being read. + fn files(&self) -> usize { + self.attachments.len() + self.loading.len() + } + /// Read a file for the next message on a worker thread, in the window's language. + #[cfg(not(target_arch = "wasm32"))] + fn load( + &mut self, + name: String, + job: impl FnOnce() -> std::result::Result + Send + 'static, + ) -> std::result::Result { + let (tx, rx) = std::sync::mpsc::channel(); + let language = crate::i18n::language(); + std::thread::Builder::new() + .name("chat-file".into()) + .spawn(move || { + let _language = crate::i18n::scope(language); + let _ = tx.send(job()); + }) + .map_err(|e| trf("ui.agent_chat_file_error", &[("name", &name), ("error", &e.to_string())]))?; + let id = self.next_file_id(); + self.loading.push(Loading { id, name: name.clone(), what: Reading::File(rx) }); + Ok(name) + } + /// A CAD file whose import was asked for: it joins the message once the import is in. + fn importing(&mut self, name: String, rx: Receiver) -> String { + let id = self.next_file_id(); + self.loading.push(Loading { id, name: name.clone(), what: Reading::Import(rx) }); + name + } + pub fn state(&self) -> State { self.state.unwrap_or(State::Off) } @@ -573,37 +665,50 @@ const INTERRUPT_NOTE: &str = "[The user interrupted your previous step.]"; /// Send what the user typed: now if the agent is ready, queued behind the running message if it /// is working, else once it is ready (starting it if needed). pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) { - let text: String = text.chars().take(MAX_PROMPT).collect(); - if text.trim().is_empty() { + // A message waits for its files: the words go back in the box until they are read. + if !app.agent_chat.loading.is_empty() { + if app.agent_chat.input.trim().is_empty() { + app.agent_chat.input = text; + } + app.agent_chat.notice(tr("ui.agent_chat_files_loading"), false); return; } + let text: String = text.chars().take(MAX_PROMPT).collect(); + // Files alone make a message. + if text.trim().is_empty() && app.agent_chat.attachments.is_empty() { + return; + } + let text = if text.trim().is_empty() { String::new() } else { text }; let image = app.agent_chat.attach_view && app.agent_chat.images; if app.agent_chat.state() == State::Busy { let chat = &mut app.agent_chat; if chat.pending.len() >= MAX_PENDING { + // The files stay on the chips for when there is room. chat.notice(tr("ui.agent_chat_queue_full"), false); } else { - chat.pending.push(Pending { text, image }); + let files = std::mem::take(&mut chat.attachments); + chat.pending.push(Pending { text, image, files }); } return; } + let files = std::mem::take(&mut app.agent_chat.attachments); // Messages left waiting by Stop go first, in one prompt with this one. if !app.agent_chat.pending.is_empty() { - app.agent_chat.pending.push(Pending { text, image }); + app.agent_chat.pending.push(Pending { text, image, files }); send_queue(app, ctx, false); return; } - submit(app, ctx, text, image, false); + submit(app, ctx, text, image, files, false); } -/// Put `text` in the conversation and send it (`interrupted`: the agent is told it cut in on its -/// previous step). -fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, interrupted: bool) { - app.agent_chat.push(Item::User { text: text.clone(), image }); +/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it +/// cut in on its previous step). +fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec, interrupted: bool) { + app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() }); match app.agent_chat.state() { - State::Ready => start_turn_with(app, text, image, interrupted.then_some(INTERRUPT_NOTE)), + State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)), s => { - app.agent_chat.queued = Some((text, image)); + app.agent_chat.queued = Some((text, image, files)); #[cfg(not(target_arch = "wasm32"))] if !s.running() { connect(app, ctx); @@ -623,9 +728,23 @@ fn send_queue(app: &mut SolveApp, ctx: &egui::Context, interrupted: bool) { } let all = std::mem::take(&mut chat.pending); let image = all.iter().any(|p| p.image); - let text = all.into_iter().map(|p| p.text).collect::>().join("\n\n"); + let mut files = Vec::new(); + let mut texts = Vec::new(); + for p in all { + files.extend(p.files); + if !p.text.trim().is_empty() { + texts.push(p.text); + } + } + // One message carries at most MAX_FILES files; the rest are named, not sent. + if files.len() > attach::MAX_FILES { + let left: Vec = files.split_off(attach::MAX_FILES).into_iter().map(|f| f.name).collect(); + let max = attach::MAX_FILES.to_string(); + chat.notice(trf("ui.agent_chat_files_left_out", &[("max", &max), ("files", &left.join(", "))]), true); + } + let text = texts.join("\n\n"); let text: String = text.chars().take(MAX_PROMPT).collect(); - submit(app, ctx, text, image, interrupted); + submit(app, ctx, text, image, files, interrupted); } /// Send now: stop the running step and send the queue as the next prompt once the agent has said @@ -657,6 +776,8 @@ pub fn new_chat(app: &mut SolveApp) { chat.confirm_new = false; chat.items.clear(); chat.pending.clear(); + chat.attachments.clear(); + chat.loading.clear(); chat.pending_edit = None; chat.queued = None; chat.last = None; @@ -675,7 +796,7 @@ pub fn new_chat(app: &mut SolveApp) { /// Whether New chat would throw something away that the user may want (so it asks first). pub fn new_chat_needs_confirm(app: &SolveApp) -> bool { - app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty() + app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty() || app.agent_chat.files() > 0 } /// Take a queued message out of the queue. @@ -702,16 +823,23 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) { } } -fn start_turn(app: &mut SolveApp, text: String, image: bool) { - start_turn_with(app, text, image, None); +fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec) { + start_turn_with(app, text, image, files, None); } -/// Start a message; `note` goes before the text for the agent, not into the conversation. -fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&str>) { - let sent = match note { +/// Start a message; `note` goes before the text for the agent, not into the conversation. The +/// files are named after the text (text files in full), and pictures go as images to an agent +/// that takes them. +fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec, note: Option<&str>) { + let pictures = app.agent_chat.images; + let lead = match note { Some(n) => format!("{n}\n\n{text}"), None => text.clone(), }; + // The whole message stays within the cap: the files' text gets what the words leave. + let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM); + let about_files = attach::note(&files, pictures, budget); + let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect(); let mut parts = vec![PromptPart::Text(sent)]; if image { match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) { @@ -719,10 +847,15 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<& Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"), } } - let label: String = text.lines().next().unwrap_or("").chars().take(48).collect(); + if pictures { + parts.extend(attach::parts(&files)); + } + // The undo step is named after the message, or its first file when it has no words. + let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or(""); + let label: String = first_line.chars().take(48).collect(); app.agent_chat.turn = Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label }); - app.agent_chat.last = Some((text, image)); + app.agent_chat.last = Some((text, image, files)); app.agent_chat.state = Some(State::Busy); app.agent_chat.stopped = false; app.agent_chat.steering = false; @@ -732,6 +865,100 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<& } } +/// Add a file to the next message. A picture or text file is read and kept (see [`attach`]); a CAD +/// file is imported into the design at once, as its own component, and the message says so. +pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result { + if app.agent_chat.files() >= attach::MAX_FILES { + return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())])); + } + let p = std::path::Path::new(path); + let file = if attach::kind(p) == attach::Kind::Cad { + if !p.is_file() { + return Err(trf("ui.agent_chat_file_not_a_file", &[("name", path)])); + } + // Imported now, as a dropped file is; it joins the message once the import is in, so the + // agent is told it was added only when it was. A full queue is refused at once. + let name = attach::name_of(p); + let (tx, rx) = std::sync::mpsc::channel(); + let mut request = crate::file_load::Request::new(path, crate::file_load::Kind::Insert); + request.reply = Some(tx); + crate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?; + return Ok(app.agent_chat.importing(name, rx)); + } else { + // A picture is decoded off the window thread: a large one takes long enough to stop it. + #[cfg(not(target_arch = "wasm32"))] + if attach::kind(p) == attach::Kind::Picture { + // A missing file is said at once; reading and decoding wait for the worker. + if !p.is_file() { + return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))])); + } + let path = p.to_path_buf(); + return app.agent_chat.load(attach::name_of(p), move || attach::read(&path)); + } + attach::read(p)? + }; + Ok(app.agent_chat.add_file(file)) +} + +/// Add a picture from the clipboard (RGBA rows, as the system gives them) to the next message. +pub fn attach_rgba(app: &mut SolveApp, width: usize, height: usize, rgba: &[u8]) -> std::result::Result { + if app.agent_chat.files() >= attach::MAX_FILES { + return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())])); + } + let n = app.agent_chat.pasted + 1; + let name = format!("pasted picture {n}.png"); + app.agent_chat.pasted = n; + // Encoded off the window thread, as a decoded picture is. + #[cfg(not(target_arch = "wasm32"))] + { + let rgba = rgba.to_vec(); + let named = name.clone(); + app.agent_chat.load(name, move || attach::from_rgba(named, width, height, &rgba)) + } + #[cfg(target_arch = "wasm32")] + { + let file = attach::from_rgba(name, width, height, rgba)?; + Ok(app.agent_chat.add_file(file)) + } +} + +/// Attach the picture on the clipboard, if it holds one. +#[cfg(not(target_arch = "wasm32"))] +pub fn paste_picture(app: &mut SolveApp) -> std::result::Result { + let image = arboard::Clipboard::new().and_then(|mut c| c.get_image()).map_err(|_| tr("ui.agent_chat_no_picture").to_string())?; + attach_rgba(app, image.width, image.height, &image.bytes) +} + +/// Files dropped on the chat attach to the next message; called before the app handles drops, so +/// those taken aren't opened too. A drop is the chat's when the pointer was last seen over it: +/// the platform gives no position during a drag (winit 0.30), so that is the best the chat can +/// know; the + button always works. +pub fn take_drops(app: &mut SolveApp, ctx: &egui::Context) { + if !drop_is_ours(app, ctx) || !ctx.input(|i| !i.raw.dropped_files.is_empty()) { + return; + } + let files = ctx.input_mut(|i| std::mem::take(&mut i.raw.dropped_files)); + attach_dropped(app, files.iter().map(|f| f.path().to_path_buf())); +} + +/// Whether files dropped now would be the chat's: the pointer was last seen over it. +pub(crate) fn drop_is_ours(app: &SolveApp, ctx: &egui::Context) -> bool { + let Some(rect) = app.agent_chat.drop_rect else { return false }; + ctx.input(|i| i.pointer.latest_pos()).is_some_and(|p| rect.contains(p)) +} + +/// Attach dropped files; a refused one says why in the conversation. +pub(crate) fn attach_dropped(app: &mut SolveApp, paths: impl Iterator) { + for p in paths { + if p.as_os_str().is_empty() { + continue; + } + if let Err(e) = attach_path(app, &p.to_string_lossy()) { + app.agent_chat.notice(e, true); + } + } +} + /// Stop: ask the agent to stop the message it is on; requests waiting for approval are declined. /// Messages queued meanwhile stay queued (they wait for Send). pub fn cancel(app: &mut SolveApp) { @@ -958,6 +1185,48 @@ fn pump_tools(app: &mut SolveApp) { } /// Read what the agent sent since the last frame, its tool requests, and the agent search. +/// Files read off the window thread join the message as they are done, where their ids put them; +/// a refused one says why. +pub(crate) fn take_loaded(app: &mut SolveApp, ctx: &egui::Context) { + use std::sync::mpsc::TryRecvError; + let chat = &mut app.agent_chat; + let mut i = 0; + while i < chat.loading.len() { + let name = chat.loading[i].name.clone(); + let done = match &chat.loading[i].what { + Reading::File(rx) => match rx.try_recv() { + Ok(read) => Some(read), + Err(TryRecvError::Empty) => None, + Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_file_error", &[("name", &name), ("error", "stopped")]))), + }, + Reading::Import(rx) => match rx.try_recv() { + Ok(answer) if answer["ok"].as_bool() == Some(true) => Some(Ok(attach::Attachment::imported(name))), + Ok(answer) => { + let why = answer["error"].as_str().unwrap_or_default().to_string(); + Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &why)]))) + } + Err(TryRecvError::Empty) => None, + Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", "stopped")]))), + }, + }; + match done { + None => i += 1, + Some(read) => { + let l = chat.loading.remove(i); + match read { + Ok(file) => { + chat.place(l.id, file); + } + Err(e) => chat.notice(e, true), + } + } + } + } + if !chat.loading.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } +} + pub fn poll(app: &mut SolveApp, ctx: &egui::Context) { #[cfg(not(target_arch = "wasm32"))] if let Some(rx) = &app.agent_chat.detecting @@ -974,6 +1243,7 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) { } return; } + take_loaded(app, ctx); let events = app.agent_chat.backend.as_mut().map(|b| b.poll()).unwrap_or_default(); for e in events { let ready = matches!(e, Event::Ready { .. }); @@ -988,8 +1258,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) { if ready { apply_saved_model(app); } - if ready && let Some((text, image)) = app.agent_chat.queued.take() { - start_turn(app, text, image); + if ready && let Some((text, image, files)) = app.agent_chat.queued.take() { + start_turn(app, text, image, files); } } pump_tools(app); @@ -1055,8 +1325,9 @@ pub fn set_mode(app: &mut SolveApp, mode: &str) { } /// `ui.agent_chat` on the control channel (tests and screenshots): `enabled`, `command`, `open`, -/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `send`, `attach_view`, -/// `permission` (an option id, or `"cancel"`), `approve` (`true`/`false`, the oldest held +/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `attach` (a file's path, for the +/// next message), `remove_file` (index), `send`, `attach_view`, `permission` (an option id, or +/// `"cancel"`), `approve` (`true`/`false`, the oldest held /// request), `cancel` (Stop), `send_now` (steer: stop, then send the queue), `remove_pending` (index), `undo` (the newest message's changes); returns the panel's state. pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result { if let Some(e) = p.get("features").and_then(Value::as_bool) { @@ -1140,6 +1411,13 @@ pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result Value { .items .iter() .map(|i| match i { - Item::User { text, image } => json!({"user": text, "image": image}), + Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}), Item::Agent { thought, text } => json!({"agent": text, "thought": thought}), Item::Tool { id, title, kind, status, output } => { json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output}) @@ -1249,7 +1527,9 @@ pub fn status(app: &SolveApp) -> Value { "toolsPort": c.tools_port, "pid": pid, "waiting": c.waiting(), - "pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image})).collect::>(), + "pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image, "files": p.files.iter().map(|f| f.name.clone()).collect::>()})).collect::>(), + "attachments": c.attachments.iter().map(|f| f.name.clone()).collect::>(), + "loading": c.loading.iter().map(|f| f.name.clone()).collect::>(), "stopped": c.stopped, "turn": c.turn.as_ref().map(|t| json!({"changes": t.changes, "requests": t.requests})), "detection": detection, diff --git a/crates/ui-egui/src/agent_chat/panel.rs b/crates/ui-egui/src/agent_chat/panel.rs index 325c596e3..0b75e283d 100644 --- a/crates/ui-egui/src/agent_chat/panel.rs +++ b/crates/ui-egui/src/agent_chat/panel.rs @@ -50,6 +50,8 @@ pub fn tip() -> String { pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) { let ctx = ui.ctx().clone(); poll(app, &ctx); + // Set again below if the chat is drawn this frame. + app.agent_chat.drop_rect = None; // With AI features off nothing of the chat is drawn, and the shortcut does nothing. if !shown(app) { return; @@ -85,7 +87,13 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) { let left = app.preferences.agent_chat_side == "left"; let side = |id: Id| if left { egui::Panel::left(id) } else { egui::Panel::right(id) }; if app.preferences.agent_chat_minimized { - side(Id::new("sc_agent_chat_strip")).exact_size(STRIP).resizable(false).frame(egui::Frame::NONE.fill(t.panel)).show(ui, |ui| strip(app, ui)); + let r = side(Id::new("sc_agent_chat_strip")) + .exact_size(STRIP) + .resizable(false) + .frame(egui::Frame::NONE.fill(t.panel)) + .show(ui, |ui| strip(app, ui)); + app.agent_chat.drop_rect = Some(r.response.rect); + show_drop_hint(app, &ctx); return; } let width = app.preferences.agent_chat_width.clamp(MIN_WIDTH, MAX_WIDTH); @@ -97,12 +105,41 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) { .show(ui, |ui| body(app, ui, left)); crate::scenario::publish_rect("agent_chat:panel", r.response.rect); app.agent_chat.panel_rect = Some(r.response.rect); + app.agent_chat.drop_rect = Some(r.response.rect); + show_drop_hint(app, &ctx); let w = r.response.rect.width(); if w.is_finite() && (MIN_WIDTH..=MAX_WIDTH).contains(&w) { app.preferences.agent_chat_width = w; } } +/// While files are dragged over the window: if they would drop on the chat, say so over it. +fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) { + #[cfg(not(target_arch = "wasm32"))] + if let Some(rect) = app.agent_chat.drop_rect + && ctx.input(|i| !i.raw.hovered_files.is_empty()) + && drop_is_ours(app, ctx) + { + drop_hint(ctx, rect); + } + #[cfg(target_arch = "wasm32")] + let _ = (app, ctx); +} + +/// Where dragged files will go: a tinted zone over the chat with a line of text. +#[cfg(not(target_arch = "wasm32"))] +fn drop_hint(ctx: &egui::Context, area: Rect) { + let t = Tokens::get(); + let painter = ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, Id::new("sc_agent_chat_drop_hint"))); + let zone = area.shrink(6.0); + painter.rect(zone, 10.0, t.accent.gamma_multiply(0.16), Stroke::new(2.0, t.accent), egui::StrokeKind::Inside); + let font = egui::FontId::proportional(14.0); + let text = painter.layout(tr("ui.agent_chat_drop_hint").to_string(), font, t.text, (zone.width() - 24.0).max(40.0)); + let at = zone.center() - text.size() / 2.0; + painter.galley(at, text, t.text); + crate::scenario::publish_rect("agent_chat:drop_hint", zone); +} + /// New chat (when the chat has focus). pub const NEW_CHAT_SHORTCUT: egui::KeyboardShortcut = egui::KeyboardShortcut::new(egui::Modifiers::COMMAND.plus(egui::Modifiers::SHIFT), egui::Key::L); @@ -431,14 +468,16 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) { auth_box(app, ui); } ui.separator(); - // The input sits at the bottom; the conversation takes the rest. - let input_h = 92.0; + // The input sits at the bottom (with a row per line of file chips); the conversation takes + // the rest. + let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0; let avail = (ui.available_height() - input_h).max(80.0); egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| { conversation(app, ui); }); ui.separator(); let busy = state == State::Busy; + attachment_chips(app, ui); let edit = egui::TextEdit::multiline(&mut app.agent_chat.input) .hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" })) .desired_rows(2) @@ -466,9 +505,18 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) { cancel(app); } ui.horizontal(|ui| { - let send_b = ui.button(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })); + add_menu(app, ui); + // A message waits for files still being read. + let reading = !app.agent_chat.loading.is_empty(); + let send_b = ui + .add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" }))) + .on_disabled_hover_text(tr("ui.agent_chat_files_loading")); crate::scenario::publish_handle("agent_chat:send", send_b.rect.center()); - if (send_b.clicked() || enter || steer) && !app.agent_chat.input.trim().is_empty() { + // Files alone make a message. + let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty(); + if reading { + // Enter waits with the button. + } else if (send_b.clicked() || enter || steer) && something { let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string(); send(app, ui.ctx(), text); if steer { @@ -744,14 +792,17 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) { let current_doc = app.session.doc.clone(); for (n, item) in app.agent_chat.items.iter().enumerate() { match item { - Item::User { text, image } => { + Item::User { text, image, files } => { ui.add_space(6.0); egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| { ui.set_width(ui.available_width()); - ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap()); + if !text.is_empty() { + ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap()); + } if *image { ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0)); } + files_line(ui, files.iter().map(String::as_str)); }); } Item::Agent { thought, text } => { @@ -900,10 +951,13 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) { }); } _ => { - ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap()); + if !p.text.is_empty() { + ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap()); + } if p.image { ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0)); } + files_line(ui, p.files.iter().map(|f| f.name.as_str())); } } }); @@ -960,6 +1014,199 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) { } } +/// The files that went (or will go) with a message, by name. +#[cfg(not(target_arch = "wasm32"))] +fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator) { + let list = names.collect::>().join(", "); + if !list.is_empty() { + let t = Tokens::get(); + ui.add(egui::Label::new(RichText::new(trf("ui.agent_chat_files", &[("files", &list)])).color(t.text_dim).size(11.0)).wrap()); + } +} + +/// The widest a file chip gets; a narrower chat makes them narrower. +const CHIP_W: f32 = 170.0; + +/// How many rows `n` chips take in a chat `avail` wide (`gap` between them). +fn chip_rows(n: usize, avail: f32, gap: f32) -> usize { + let avail = avail.max(1.0); + let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1); + n.div_ceil(per_row) +} + +/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or +/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui +/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it. +#[cfg(not(target_arch = "wasm32"))] +fn attachment_chips(app: &mut SolveApp, ui: &mut egui::Ui) { + let chat = &mut app.agent_chat; + // Thumbnails of files no longer on the message are let go. + let keep: Vec = chat.attachments.iter().filter(|f| f.thumb.is_some()).map(|f| f.id).collect(); + chat.thumbs.retain(|k, _| keep.contains(k)); + if chat.files() == 0 { + return; + } + let gap = ui.spacing().item_spacing.x; + let avail = ui.available_width().max(1.0); + let w = CHIP_W.min(avail); + let per_row = (((avail + gap) / (w + gap)).floor() as usize).max(1); + let (mut remove, mut stop) = (None, None); + // The files read, then those still being read (each takes its place once it is done). + let read = chat.attachments.len(); + let indices: Vec = (0..chat.files()).collect(); + for row in indices.chunks(per_row) { + ui.horizontal(|ui| { + for &i in row { + if let Some(f) = chat.attachments.get(i) { + if chip(ui, &mut chat.thumbs, f, i, w) { + remove = Some(i); + } + } else if let Some(l) = chat.loading.get(i - read) + && loading_chip(ui, &l.name, l.importing(), i - read, w) + { + stop = Some(i - read); + } + } + }); + } + if let Some(i) = remove + && i < chat.attachments.len() + { + chat.attachments.remove(i); + } + // Taken off while it is read: its reader's answer is dropped with it. + if let Some(i) = stop + && i < chat.loading.len() + { + chat.loading.remove(i); + } +} + +/// What a chip's × says: an imported part stays in the design when it comes off the message. +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn remove_tip(f: &attach::Attachment) -> &'static str { + tr(if matches!(f.content, attach::Content::Imported) { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" }) +} + +/// The chip of a file still being read or imported: a spinner, the name, and a × that takes it +/// off (an import goes on: its part lands in the design). +#[cfg(not(target_arch = "wasm32"))] +fn loading_chip(ui: &mut egui::Ui, name: &str, importing: bool, i: usize, w: f32) -> bool { + let t = Tokens::get(); + let mut clicked = false; + let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show( + ui, + |ui| { + let inner = (w - 10.0).max(40.0); + ui.set_width(inner); + ui.horizontal(|ui| { + ui.spacing_mut().item_spacing.x = 4.0; + let doing = tr(if importing { "ui.agent_chat_file_importing" } else { "ui.agent_chat_file_loading" }); + let mark = ui.add(egui::Spinner::new().size(16.0)).on_hover_text(doing).rect.width(); + let name_w = (inner - mark - 18.0 - 8.0).max(8.0); + ui.allocate_ui(vec2(name_w, 20.0), |ui| { + ui.set_width(name_w); + ui.add(egui::Label::new(RichText::new(name).size(11.5).color(t.text_dim)).truncate()) + }) + .inner + .on_hover_text(name); + let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click()); + let c = r.center(); + let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim }); + ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s); + ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s); + crate::scenario::publish_handle(&format!("agent_chat:file_loading_remove:{i}"), c); + clicked = x.on_hover_text(tr(if importing { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" })).clicked(); + }); + }, + ); + crate::scenario::publish_handle(&format!("agent_chat:file_loading:{i}"), r.response.rect.center()); + clicked +} + +/// One file chip, `w` wide in all; true when its × was clicked. +#[cfg(not(target_arch = "wasm32"))] +fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap, f: &attach::Attachment, i: usize, w: f32) -> bool { + let t = Tokens::get(); + let mut clicked = false; + let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show( + ui, + |ui| { + let inner = (w - 10.0).max(40.0); + ui.set_width(inner); + ui.horizontal(|ui| { + ui.spacing_mut().item_spacing.x = 4.0; + let mark = match &f.thumb { + Some(img) => { + let tex = thumbs + .entry(f.id) + .or_insert_with(|| ui.ctx().load_texture(format!("sc_attach_{}", f.id), (**img).clone(), egui::TextureOptions::LINEAR)); + let size = tex.size_vec2(); + ui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width() + } + None => { + let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover()); + ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside); + r.width() + } + }; + let name_w = (inner - mark - 18.0 - 8.0).max(8.0); + ui.allocate_ui(vec2(name_w, 20.0), |ui| { + ui.set_width(name_w); + ui.add(egui::Label::new(RichText::new(&f.name).size(11.5)).truncate()) + }) + .inner + .on_hover_text(&f.name); + let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click()); + let c = r.center(); + let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim }); + ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s); + ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s); + crate::scenario::publish_handle(&format!("agent_chat:file_remove:{i}"), c); + clicked = x.on_hover_text(remove_tip(f)).clicked(); + }); + }, + ); + crate::scenario::publish_rect(&format!("agent_chat:file:{i}"), r.response.rect); + crate::scenario::publish_handle(&format!("agent_chat:file:{i}"), r.response.rect.center()); + clicked +} + +/// The + button: choose a file with the app's picker (pictures, text and CAD files), or paste +/// the clipboard's picture. +#[cfg(not(target_arch = "wasm32"))] +fn add_menu(app: &mut SolveApp, ui: &mut egui::Ui) { + let (mut choose, mut paste) = (false, false); + let r = ui.menu_button(RichText::new("+").strong().size(15.0), |ui| { + let a = ui.button(tr("ui.agent_chat_add_file")); + crate::scenario::publish_handle("agent_chat:add_file", a.rect.center()); + let b = ui.button(tr("ui.agent_chat_paste_picture")); + crate::scenario::publish_handle("agent_chat:paste_picture", b.rect.center()); + if a.clicked() { + choose = true; + ui.close(); + } + if b.clicked() { + paste = true; + ui.close(); + } + }); + crate::scenario::publish_handle("agent_chat:add", r.response.rect.center()); + r.response.on_hover_text(tr("ui.agent_chat_add")); + if choose { + app.pick_file(tr("ui.agent_chat_attachable"), &attach::extensions(), |app, path| { + if let Some(path) = path + && let Err(e) = attach_path(app, &path) + { + app.agent_chat.notice(e, true); + } + }); + } + if paste && let Err(e) = paste_picture(app) { + app.agent_chat.notice(e, true); + } +} + /// A tool call as a card: its state, name and kind on one line; a click unfolds what it returned. #[cfg(not(target_arch = "wasm32"))] fn tool_card(ui: &mut egui::Ui, n: usize, title: &str, kind: &str, status: Option, output: Option<&str>) { diff --git a/crates/ui-egui/src/agent_chat/tests.rs b/crates/ui-egui/src/agent_chat/tests.rs index aa9fc49b4..9da7d9372 100644 --- a/crates/ui-egui/src/agent_chat/tests.rs +++ b/crates/ui-egui/src/agent_chat/tests.rs @@ -26,18 +26,20 @@ fn streamed_text_joins_and_tool_updates_change_their_call() { #[test] fn a_sign_in_request_keeps_the_question_for_after_the_login() { let mut c = chat(); - c.last = Some(("make a cube".into(), false)); + c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())])); c.state = Some(State::Busy); assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some()); c.on_event(Event::AuthRequired { methods: vec![] }); assert_eq!(c.state(), State::NeedsAuth); - assert_eq!(c.queued, Some(("make a cube".into(), false))); + let (text, image, files) = c.queued.clone().expect("queued for after the login"); + assert_eq!((text.as_str(), image), ("make a cube", false)); + assert_eq!(files.iter().map(|f| f.name.as_str()).collect::>(), ["motor.step"], "its files go again too"); } #[test] fn a_plan_update_replaces_the_turns_plan() { let mut c = chat(); - c.push(Item::User { text: "go".into(), image: false }); + c.push(Item::User { text: "go".into(), image: false, files: Vec::new() }); c.on_event(Event::Plan(vec![("a".into(), "pending".into())])); c.on_event(Event::Text { thought: false, text: "working".into() }); c.on_event(Event::Plan(vec![("a".into(), "completed".into())])); @@ -66,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender) { let (tx, rx) = channel(); app.agent_chat.tools_rx = Some(rx); app.agent_chat.state = Some(State::Busy); - start_turn(&mut app, "two boxes".into(), false); + start_turn(&mut app, "two boxes".into(), false, Vec::new()); (app, tx) } @@ -416,12 +418,17 @@ use std::sync::Mutex; struct Script { calls: Arc>>, events: Arc>>, + /// Every prompt in full (its text and pictures). + prompts: Arc>>>, } impl Script { fn calls(&self) -> Vec { self.calls.lock().unwrap().clone() } + fn prompts(&self) -> Vec> { + self.prompts.lock().unwrap().clone() + } fn report(&self, e: Event) { self.events.lock().unwrap().push(e); } @@ -434,6 +441,7 @@ impl ChatBackend for Script { fn prompt(&mut self, parts: &[PromptPart]) { let text = parts.iter().find_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).unwrap_or_default(); self.calls.lock().unwrap().push(format!("prompt: {text}")); + self.prompts.lock().unwrap().push(parts.to_vec()); } fn cancel(&mut self) { self.calls.lock().unwrap().push("cancel".into()); @@ -860,6 +868,500 @@ done let _ = std::fs::remove_dir_all(&dir); } +// ---- Files attached to a message ---- + +/// A folder of files to attach: a picture and a text file. +fn attachable(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("solidcraft-chat-files-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + image::RgbaImage::from_pixel(40, 30, image::Rgba([250, 250, 250, 255])).save(d.join("target.png")).unwrap(); + std::fs::write(d.join("brief.md"), "wall 2 mm, Pi 5 inside\n").unwrap(); + d +} + +fn at(d: &std::path::Path, f: &str) -> String { + d.join(f).to_string_lossy().to_string() +} + +/// Wait for the files read off the window thread to join the message. +fn settle_files(app: &mut SolveApp, ctx: &egui::Context) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline { + take_loaded(app, ctx); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + take_loaded(app, ctx); +} + +/// The same with the whole app running frames. +fn settle_chat(h: &mut crate::scenario::Harness) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !h.app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline { + h.frames(1); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + h.frames(2); +} + +fn text_of(parts: &[PromptPart]) -> String { + parts.iter().filter_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).collect() +} + +fn pictures_in(parts: &[PromptPart]) -> usize { + parts.iter().filter(|p| matches!(p, PromptPart::Image { .. })).count() +} + +/// Files attached to a message go with it: the pictures as images, the text files in its text, +/// and every file by name; the message lists them in the conversation. +#[test] +fn a_message_carries_its_files() { + let d = attachable("carry"); + let (mut app, script, ctx) = app_with_script(); + app.agent_chat.images = true; + control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap(); + control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap(); + settle_files(&mut app, &ctx); + assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"])); + assert!(control(&mut app, &ctx, &json!({"attach": at(&d, "missing.png")})).is_err()); + send(&mut app, &ctx, "make the case like this".into()); + let sent = script.prompts(); + assert_eq!(sent.len(), 1); + let text = text_of(&sent[0]); + assert!(text.starts_with("make the case like this"), "{text}"); + assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm, Pi 5 inside"), "{text}"); + assert_eq!(sent[0].iter().filter(|p| matches!(p, PromptPart::Image { mime, .. } if mime == "image/png")).count(), 1); + assert!(app.agent_chat.attachments.is_empty(), "the files went with the message"); + assert_eq!(status(&app)["items"][0]["files"], json!(["target.png", "brief.md"])); + let _ = std::fs::remove_dir_all(&d); +} + +/// Review of #1744: a picture was decoded on the window thread, so a large one froze the window. +/// It is read off it now: the message waits for it (Send waits too), the files keep the order they +/// were attached in, and a refused picture says why later, in the app's language. +#[test] +fn pictures_are_read_off_the_window_thread() { + let d = attachable("worker"); + std::fs::write(d.join("fake.png"), b"not a picture at all").unwrap(); + let (mut app, script, ctx) = app_with_script(); + app.agent_chat.images = true; + control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap(); + control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap(); + assert_eq!(status(&app)["loading"], json!(["target.png"]), "the picture is still being read"); + send(&mut app, &ctx, "like this".into()); + assert!(script.prompts().is_empty(), "Send waits for the files"); + assert_eq!(app.agent_chat.input, "like this", "and keeps the words"); + settle_files(&mut app, &ctx); + assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"]), "in the order they were attached"); + let words = std::mem::take(&mut app.agent_chat.input); + send(&mut app, &ctx, words); + assert_eq!(script.prompts().len(), 1); + { + let _ja = crate::i18n::scope("ja"); + control(&mut app, &ctx, &json!({"attach": at(&d, "fake.png")})).unwrap(); + } + settle_files(&mut app, &ctx); + assert!( + matches!(app.agent_chat.items.last(), Some(Item::Notice { text, error: true }) if text == "fake.png: 読み込めない画像です"), + "a refused picture says why, in the language it was attached in" + ); + assert!(app.agent_chat.attachments.is_empty()); + let _ = std::fs::remove_dir_all(&d); +} + +/// Review of #1744: the prompt cap applied to the typed text only, so eight 256 KB text files +/// took a message to about 2 MB. The whole message stays within it now, every file still named. +#[test] +fn a_message_with_large_text_files_stays_within_the_prompt_cap() { + let d = attachable("cap"); + let (mut app, script, ctx) = app_with_script(); + for i in 0..attach::MAX_FILES { + let path = d.join(format!("notes{i}.txt")); + std::fs::write(&path, "y".repeat(250 * 1024)).unwrap(); + control(&mut app, &ctx, &json!({"attach": path.to_string_lossy()})).unwrap(); + } + send(&mut app, &ctx, "read these".into()); + let text = text_of(&script.prompts()[0]); + assert!(text.chars().count() <= MAX_PROMPT, "{} characters", text.chars().count()); + assert!((0..attach::MAX_FILES).all(|i| text.contains(&format!("`notes{i}.txt`"))), "every file is named"); + let _ = std::fs::remove_dir_all(&d); +} + +/// A message sent before the agent is ready keeps its files until it goes; an agent that takes +/// no pictures gets none, and the message says which were left out. Files alone make a message, +/// and its undo step is named after the first file. +#[test] +fn files_wait_with_their_message_and_pictures_skip_an_agent_that_takes_none() { + let d = attachable("wait"); + let (mut app, script, ctx) = app_with_script(); + app.agent_chat.state = Some(State::Starting); + attach_path(&mut app, &at(&d, "target.png")).unwrap(); + settle_files(&mut app, &ctx); + send(&mut app, &ctx, " ".into()); + assert!(script.prompts().is_empty(), "not sent before the agent is ready"); + assert_eq!(status(&app)["items"][0]["user"], "", "no words, just the file"); + script.report(Event::Ready { agent: "Test".into(), images: false }); + poll(&mut app, &ctx); + let sent = script.prompts(); + assert_eq!(sent.len(), 1, "sent once the agent is ready"); + assert_eq!(pictures_in(&sent[0]), 0, "no pictures to an agent that takes none"); + assert!(text_of(&sent[0]).contains("not sent"), "{}", text_of(&sent[0])); + assert_eq!(app.agent_chat.turn.as_ref().map(|t| t.label.as_str()), Some("target.png")); + let _ = std::fs::remove_dir_all(&d); +} + +#[test] +fn at_most_eight_files_on_a_message() { + let d = attachable("cap"); + let mut app = SolveApp::new(solidcraft_engine::Session::default(), crate::Services::default()); + for _ in 0..attach::MAX_FILES { + attach_path(&mut app, &at(&d, "brief.md")).unwrap(); + } + assert!(attach_path(&mut app, &at(&d, "brief.md")).unwrap_err().contains('8')); + assert!(attach_rgba(&mut app, 2, 2, &[9; 16]).is_err()); + assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES); + let ctx = egui::Context::default(); + assert!(control(&mut app, &ctx, &json!({"remove_file": 8})).is_err(), "no ninth file to take off"); + control(&mut app, &ctx, &json!({"remove_file": 0})).unwrap(); + assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES - 1); + let _ = std::fs::remove_dir_all(&d); +} + +/// While the agent works, a message queues with its files: the queue shows them, Stop keeps +/// them, and they go with the queue. +#[test] +fn a_queued_message_keeps_its_files_through_stop_and_sends_them() { + let d = attachable("queue"); + let (mut app, script, ctx) = app_with_script(); + app.agent_chat.images = true; + send(&mut app, &ctx, "make a box".into()); + attach_path(&mut app, &at(&d, "target.png")).unwrap(); + settle_files(&mut app, &ctx); + send(&mut app, &ctx, "like this".into()); + attach_path(&mut app, &at(&d, "brief.md")).unwrap(); + send(&mut app, &ctx, String::new()); + assert!(app.agent_chat.attachments.is_empty(), "the files moved to the queue"); + assert_eq!(status(&app)["pending"][0]["files"], json!(["target.png"])); + assert_eq!(status(&app)["pending"][1]["files"], json!(["brief.md"])); + cancel(&mut app); + ended(&script, "cancelled"); + poll(&mut app, &ctx); + assert_eq!(app.agent_chat.pending.len(), 2, "Stop keeps the queue and its files"); + send_now(&mut app, &ctx); + let sent = script.prompts(); + let last = sent.last().expect("the queue went"); + let text = text_of(last); + assert!(text.starts_with("like this") && !text.contains("\n\n\n\n"), "a message of files alone adds no empty paragraph: {text:?}"); + assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm"), "{text}"); + assert_eq!(pictures_in(last), 1); + assert_eq!(status(&app)["items"].as_array().and_then(|i| i.last()).map(|i| i["files"].clone()), Some(json!(["target.png", "brief.md"]))); + let _ = std::fs::remove_dir_all(&d); +} + +/// Queued messages go as one prompt with at most eight files; the rest are named, not sent. +#[test] +fn the_queue_sends_at_most_eight_files_and_names_the_rest() { + let d = attachable("queue-cap"); + let (mut app, script, ctx) = app_with_script(); + send(&mut app, &ctx, "first".into()); + for n in 0..3 { + for _ in 0..4 { + attach_path(&mut app, &at(&d, "brief.md")).unwrap(); + } + send(&mut app, &ctx, format!("batch {n}")); + } + ended(&script, ""); + poll(&mut app, &ctx); + let sent = script.prompts(); + assert_eq!(sent.len(), 2); + assert_eq!(text_of(&sent[1]).matches("Attached file `brief.md`").count(), attach::MAX_FILES); + let told = app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, .. } if text.contains("brief.md") && text.contains('8'))); + assert!(told, "the files left out are named"); + let _ = std::fs::remove_dir_all(&d); +} + +/// New chat clears the files waiting on the chips too, and asks first when there are some. +#[test] +fn new_chat_clears_the_files_waiting_and_asks_first() { + let d = attachable("new"); + let (mut app, _script, _ctx) = app_with_script(); + assert!(!new_chat_needs_confirm(&app)); + attach_path(&mut app, &at(&d, "brief.md")).unwrap(); + assert!(new_chat_needs_confirm(&app), "files would be lost"); + new_chat(&mut app); + assert!(app.agent_chat.attachments.is_empty()); + let _ = std::fs::remove_dir_all(&d); +} + +/// A harness with the chat open at its narrowest and nothing to look for. +fn narrow_chat() -> crate::scenario::Harness { + let mut h = crate::scenario::Harness::new(); + h.app.preferences.agent_chat_width = panel::MIN_WIDTH; + h.app.agent_chat.open = true; + h.app.agent_chat.detection = Some(Default::default()); + h +} + +/// Every chip and the text box sit inside the panel. +fn assert_inside_the_panel(chips: usize) { + use crate::scenario::rect_of; + let panel = rect_of("agent_chat:panel").expect("the panel"); + let mut rects: Vec<(String, egui::Rect)> = + (0..chips).map(|i| (format!("file:{i}"), rect_of(&format!("agent_chat:file:{i}")).expect("a chip"))).collect(); + rects.push(("input".into(), rect_of("agent_chat:input").expect("the text box"))); + for (name, r) in rects { + assert!(r.left() >= panel.left() && r.right() <= panel.right(), "{name} {r:?} in {panel:?}"); + } +} + +/// Files for the next message show as chips above the text box (a picture's thumbnail, the name, +/// a × that takes it off), inside the chat, and Send sends files with no text. +#[test] +fn attached_files_show_as_chips_that_can_be_taken_off_and_sent_alone() { + use crate::scenario::handle_at; + let d = attachable("chips"); + let mut h = narrow_chat(); + let script = Script::default(); + h.app.agent_chat.backend = Some(Box::new(script.clone())); + h.app.agent_chat.state = Some(State::Ready); + h.app.agent_chat.images = true; + h.frames(2); + h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")})); + h.call("ui.agent_chat", json!({"attach": at(&d, "brief.md")})); + settle_chat(&mut h); + assert!(handle_at("agent_chat:file:0").is_some() && handle_at("agent_chat:file:1").is_some(), "a chip per file"); + assert_inside_the_panel(2); + let x = handle_at("agent_chat:file_remove:0").expect("a × on the chip"); + h.call("ui.click", json!({"x": x.x, "y": x.y})); + h.frames(2); + assert_eq!(status(&h.app)["attachments"], json!(["brief.md"])); + let send = handle_at("agent_chat:send").expect("Send"); + h.call("ui.click", json!({"x": send.x, "y": send.y})); + h.frames(2); + assert_eq!(script.prompts().len(), 1, "files alone make a message"); + assert!(h.app.agent_chat.attachments.is_empty() && handle_at("agent_chat:file:0").is_none()); + let _ = std::fs::remove_dir_all(&d); +} + +/// Review of #1744: chips' thumbnails were cached by the picture's address, so a new picture at a +/// freed address could show an old one. Each file has an id of its own, never reused, and the +/// cache follows the files on the message. +#[test] +fn each_file_has_its_own_id_and_thumbnail() { + let d = attachable("ids"); + image::RgbaImage::from_pixel(20, 20, image::Rgba([10, 200, 10, 255])).save(d.join("green.png")).unwrap(); + let mut h = narrow_chat(); + h.frames(2); + h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")})); + h.call("ui.agent_chat", json!({"attach": at(&d, "green.png")})); + settle_chat(&mut h); + let first: Vec = h.app.agent_chat.attachments.iter().map(|f| f.id).collect(); + assert!(first.len() == 2 && first[0] != first[1], "{first:?}"); + let mut cached: Vec = h.app.agent_chat.thumbs.keys().copied().collect(); + cached.sort_unstable(); + assert_eq!(cached, first, "a thumbnail per file, by its id"); + h.call("ui.agent_chat", json!({"remove_file": 0})); + h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")})); + settle_chat(&mut h); + let now: Vec = h.app.agent_chat.attachments.iter().map(|f| f.id).collect(); + assert!(!now.contains(&first[0]), "a removed file's id is not used again: {now:?}"); + let mut cached: Vec = h.app.agent_chat.thumbs.keys().copied().collect(); + cached.sort_unstable(); + let mut want = now.clone(); + want.sort_unstable(); + assert_eq!(cached, want, "the cache follows the files on the message"); + let _ = std::fs::remove_dir_all(&d); +} + +/// The + button: "Choose a file…" asks the app's file picker for the types that can be attached +/// (not the design types Open shows) and attaches what it answers; "Paste a picture" attaches +/// the clipboard's picture (here given as RGBA, as the system does). +#[test] +fn the_plus_button_attaches_a_chosen_file_or_a_pasted_picture() { + use crate::scenario::handle_at; + let d = attachable("plus"); + let mut h = narrow_chat(); + let chosen = at(&d, "brief.md"); + let asked = std::rc::Rc::new(std::cell::RefCell::new(Vec::::new())); + let seen = asked.clone(); + h.app.services.pick_file = Some(Box::new(move |_label: &str, exts: &[&str]| { + *seen.borrow_mut() = exts.iter().map(|e| e.to_string()).collect(); + let p = chosen.clone(); + Box::pin(async move { Some(p) }) + })); + h.frames(2); + let plus = handle_at("agent_chat:add").expect("a + button"); + h.call("ui.click", json!({"x": plus.x, "y": plus.y})); + h.frames(2); + let choose = handle_at("agent_chat:add_file").expect("Choose a file… in its menu"); + assert!(handle_at("agent_chat:paste_picture").is_some(), "Paste a picture in its menu"); + h.call("ui.click", json!({"x": choose.x, "y": choose.y})); + h.frames(4); + assert_eq!(status(&h.app)["attachments"], json!(["brief.md"])); + let exts = asked.borrow().clone(); + for ext in ["png", "jpg", "gif", "webp", "svg", "md", "step", "stl"] { + assert!(exts.iter().any(|e| e == ext), "{ext} in {exts:?}"); + } + assert!(!exts.iter().any(|e| e == "solidcraft")); + attach_rgba(&mut h.app, 4, 3, &[200; 48]).unwrap(); + attach_rgba(&mut h.app, 4, 3, &[200; 47]).unwrap(); + settle_chat(&mut h); + assert!( + h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("pasted picture 2.png:"))), + "not a picture: it says so once it has been read" + ); + assert_eq!(status(&h.app)["attachments"], json!(["brief.md", "pasted picture 1.png"])); + let _ = std::fs::remove_dir_all(&d); +} + +#[derive(Debug)] +struct DroppedPath(std::path::PathBuf); + +impl egui::DroppedFile for DroppedPath { + fn path(&self) -> &std::path::Path { + &self.0 + } + fn bytes(&self) -> Result, String> { + std::fs::read(&self.0).map_err(|e| e.to_string()) + } +} + +fn drop_files(h: &mut crate::scenario::Harness, at: egui::Pos2, files: &[std::path::PathBuf]) { + h.drop_files(at, files.iter().map(|p| Arc::new(DroppedPath(p.clone())) as egui::DroppedFileHandle).collect()); +} + +/// A tetrahedron, as a small ASCII STL. +const TETRA_STL: &str = "solid t +facet normal 0 0 -1 + outer loop + vertex 0 0 0 + vertex 0 10 0 + vertex 10 0 0 + endloop +endfacet +facet normal 0 -1 0 + outer loop + vertex 0 0 0 + vertex 10 0 0 + vertex 0 0 10 + endloop +endfacet +facet normal -1 0 0 + outer loop + vertex 0 0 0 + vertex 0 0 10 + vertex 0 10 0 + endloop +endfacet +facet normal 1 1 1 + outer loop + vertex 10 0 0 + vertex 0 10 0 + vertex 0 0 10 + endloop +endfacet +endsolid t +"; + +/// Review of #1744: a CAD file's import wasn't waited on, so a failed or queued-out import still +/// told the agent "I added x to the design". The file joins the message once its import is in; +/// a failed import says why and the agent is told nothing of it. +#[test] +fn a_cad_file_joins_the_message_once_its_import_is_in() { + let d = attachable("cad"); + std::fs::write(d.join("block.stl"), TETRA_STL).unwrap(); + std::fs::write(d.join("broken.stl"), [0u8, 1, 2]).unwrap(); + let mut h = narrow_chat(); + let script = Script::default(); + h.app.agent_chat.backend = Some(Box::new(script.clone())); + h.app.agent_chat.state = Some(State::Ready); + h.frames(2); + attach_path(&mut h.app, &at(&d, "block.stl")).unwrap(); + assert_eq!(status(&h.app)["loading"], json!(["block.stl"]), "importing, not yet on the message"); + assert_eq!(status(&h.app)["attachments"], json!([])); + settle_chat(&mut h); + assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "in, and on the message"); + h.call("ui.agent_chat", json!({"attach": at(&d, "broken.stl")})); + settle_chat(&mut h); + assert!( + h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("broken.stl:"))), + "a failed import says why" + ); + assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "and is not on the message"); + h.call("ui.agent_chat", json!({"send": "fit around this"})); + h.frames(2); + let text = text_of(&script.prompts()[0]); + assert!(text.contains("`block.stl` to the design") && !text.contains("broken.stl"), "{text}"); + let _ = std::fs::remove_dir_all(&d); +} + +/// Review of #1744: taking an imported part's chip off leaves the part in the design, and its × +/// says so; any other file's × just takes it off. +#[test] +fn an_imported_parts_chip_says_the_part_stays() { + assert_eq!(panel::remove_tip(&attach::Attachment::imported("motor.step".into())), tr("ui.agent_chat_remove_imported")); + assert!(tr("ui.agent_chat_remove_imported").contains("stays in the design")); + let picture = attach::from_rgba("a.png".into(), 2, 2, &[9; 16]).unwrap(); + assert_eq!(panel::remove_tip(&picture), tr("ui.agent_chat_remove_file")); +} + +/// Files dropped on the chat attach to the next message, and the app doesn't open them as well +/// (while they are dragged over it, the chat says so); dropped on the 3D view they are not the +/// chat's (the app handles them as before), nor when the chat is closed. +#[test] +fn files_dropped_on_the_chat_attach_and_the_app_does_not_open_them() { + use crate::scenario::rect_of; + let d = attachable("drop"); + let mut h = narrow_chat(); + h.call("ui.resize", json!({"width": 1600, "height": 1000})); + h.frames(3); + let panel = rect_of("agent_chat:panel").expect("docked"); + // While files are dragged over the chat, it says they will attach; over the 3D view, not. + let hovered = || vec![egui::HoveredFile { path: Some(d.join("target.png")), mime: String::new() }]; + h.drag_files(panel.center(), hovered(), Vec::new()); + assert!(rect_of("agent_chat:drop_hint").is_some_and(|r| panel.contains(r.center())), "the hint over the chat"); + h.drag_files(egui::pos2(300.0, 500.0), hovered(), Vec::new()); + assert!(rect_of("agent_chat:drop_hint").is_none(), "no hint over the 3D view"); + drop_files(&mut h, panel.center(), &[d.join("target.png"), d.join("brief.md")]); + settle_chat(&mut h); + assert_eq!(status(&h.app)["attachments"], json!(["target.png", "brief.md"])); + assert!(!h.app.file_load.busy(), "the app doesn't open them as well"); + drop_files(&mut h, egui::pos2(300.0, 500.0), &[d.join("brief.md")]); + assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(2), "over the 3D view: not the chat's"); + // Folded to its strip, a drop on the strip is the chat's too. + h.call("ui.agent_chat", json!({"minimized": true})); + h.frames(3); + let strip = h.app.agent_chat.drop_rect.expect("the strip takes drops"); + assert!(strip.width() < 60.0, "{strip:?}"); + drop_files(&mut h, strip.center(), &[d.join("brief.md")]); + assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3)); + // Closed: nothing is the chat's, wherever the pointer was. + h.app.agent_chat.open = false; + h.frames(2); + assert!(h.app.agent_chat.drop_rect.is_none()); + drop_files(&mut h, strip.center(), &[d.join("brief.md")]); + assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3)); + let _ = std::fs::remove_dir_all(&d); +} + +/// Many files and a long message in a narrow chat: the chips wrap onto more rows and the text box +/// keeps to the chat (a row of chips wider than the chat would widen everything under it). +#[test] +fn chips_and_a_long_message_wrap_inside_a_narrow_chat() { + use crate::scenario::handle_at; + let d = attachable("wrap"); + let mut h = narrow_chat(); + for _ in 0..6 { + h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")})); + } + h.app.agent_chat.input = "a separate design next to the robot body, the main version, with these measures for the second one".into(); + settle_chat(&mut h); + assert_inside_the_panel(6); + let (first, second) = (handle_at("agent_chat:file:0").unwrap(), handle_at("agent_chat:file:1").unwrap()); + assert!(second.y > first.y + 10.0, "at 280 px the second chip goes on the next row: {first:?} {second:?}"); + let _ = std::fs::remove_dir_all(&d); +} + /// An agent's text is Markdown: a code block shows with its Copy button and a link is a button /// of its own, instead of the raw markers. #[test] diff --git a/crates/ui-egui/src/file_picker.rs b/crates/ui-egui/src/file_picker.rs index e880c07f9..3ec09ab65 100644 --- a/crates/ui-egui/src/file_picker.rs +++ b/crates/ui-egui/src/file_picker.rs @@ -46,6 +46,21 @@ impl SolveApp { self.begin_picker(future, Box::new(complete)); } + /// Ask for one file of a kind (`label`, `extensions`), with the host's dialog for it, or the + /// Open dialog where the host has none. (The agent chat asks for its files; the web build has + /// no chat.) + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + pub(crate) fn pick_file(&mut self, label: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option) + 'static) { + if self.file_picker_busy() { + return; + } + let future = match self.services.pick_file.as_ref() { + Some(pick) => Some(pick(label, extensions)), + None => self.services.pick_open.as_ref().map(|pick| pick(extensions)), + }; + self.begin_picker(future, Box::new(complete)); + } + pub(crate) fn pick_save(&mut self, name: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option) + 'static) { if self.file_picker_busy() { return; diff --git a/crates/ui-egui/src/i18n.rs b/crates/ui-egui/src/i18n.rs index 8658e8b26..f0f1e2817 100644 --- a/crates/ui-egui/src/i18n.rs +++ b/crates/ui-egui/src/i18n.rs @@ -193,6 +193,11 @@ impl Drop for LanguageScope { } } +/// This thread's language, to carry into a worker thread with [`scope`]. +pub fn language() -> &'static str { + CURRENT.with(|current| current.get().0.id) +} + pub fn tr(id: &str) -> &str { CURRENT.with(|current| current.get().text(id)) } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2b46b969b..afe53e8b5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -262,6 +262,10 @@ pub struct Services { /// the picker's first filter (`&[]`: designs, CAD files and meshes, #1801). pub pick_open: Option FilePickerFuture>>, pub pick_save: Option FilePickerFuture>>, + /// Open a dialog for one file of a kind (its name, its extensions): on macOS a dialog allows + /// only the types it names, so the agent chat's files need a dialog of their own. `None`: the + /// Open dialog. + pub pick_file: Option FilePickerFuture>>, /// Open a dialog asking for a program (a slicer); `None` when the host has no such dialog. pub pick_program: Option FilePickerFuture>>, pub show_in_folder: Option>, @@ -1275,6 +1279,8 @@ impl SolveApp { recovery_ui::tick(self); update::tick(self, ctx); if self.issue_report.draft.is_none() && self.file_picker.is_none() { + // Files dropped on the agent chat attach to its next message instead. + agent_chat::take_drops(self, ctx); for f in ctx.input(|i| i.raw.dropped_files.clone()) { let p = f.path().to_string_lossy().to_string(); if !p.is_empty() { diff --git a/crates/ui-egui/src/scenario.rs b/crates/ui-egui/src/scenario.rs index 7a3de61ae..3a83a0b96 100644 --- a/crates/ui-egui/src/scenario.rs +++ b/crates/ui-egui/src/scenario.rs @@ -217,6 +217,34 @@ impl Harness { } } + /// One frame in which the platform drops `files`, the pointer last seen at `at` (a drop + /// itself carries no position with winit 0.30). + pub fn drop_files(&mut self, at: egui::Pos2, files: Vec) { + self.drag_files(at, Vec::new(), files); + } + + /// One frame in which files are dragged over the window (`hovered`) or dropped, the pointer + /// last seen at `at`. + pub fn drag_files(&mut self, at: egui::Pos2, hovered: Vec, dropped: Vec) { + self.time += 1.0 / 60.0; + let raw = egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, self.size)), + time: Some(self.time), + focused: true, + events: vec![egui::Event::PointerMoved(at)], + hovered_files: hovered, + dropped_files: dropped, + ..Default::default() + }; + let app = &mut self.app; + let mut out = self.ctx.run_ui(raw, |ui| { + app.logic(ui.ctx()); + app.ui(ui); + }); + out.textures_delta.clear(); + self.output = out.platform_output; + } + /// Frames until the queued input is used up (and the view has settled). pub fn settle(&mut self) { for _ in 0..2000 { diff --git a/docs/agent-chat.md b/docs/agent-chat.md index 4b04ae78a..c9e39630f 100644 --- a/docs/agent-chat.md +++ b/docs/agent-chat.md @@ -53,6 +53,42 @@ when you send a message or press **Connect**. endpoint's agent also keeps it as its default model. An agent that reports no models shows no drop-down. +## Files and pictures + +A message can carry up to eight files, to show the agent what you mean: a concept picture, a +sketch, a brief, a part to fit around. + +- **+** next to **Send** has **Choose a file…** (a file dialog for the types below) and **Paste a + picture** (the picture on the clipboard). +- **Drop files on the chat.** Files dropped on the panel (or its folded strip) attach to the + next message, and the app doesn't open them too. While files are dragged over the chat, it says + so. The window gives no position while something is dragged, so the drop is the chat's when the + pointer was last seen over the chat; **+** always works. +- The files show as chips above the text box: a picture's thumbnail, the name, and × to take the + file off. A picture is read off the window thread, so a large one never stops the window: its + chip shows a spinner meanwhile, and **Send** waits for it. **Send** sends files with no text. +- **Pictures** (PNG, JPEG, GIF, WebP) go to the agent as they are when they are at most 2048 px + on the long side and 3.75 MB (a picture's base64 then stays under 5 MB, so eight still fit in a + prompt). Otherwise they are scaled down to 2048 px and re-encoded: as PNG when that fits, as + JPEG when the picture has no transparency, halved until it fits; a GIF goes as its first frame. + BMP and TIFF go as PNG. HEIC can't be read: export it as JPEG or PNG first. An agent that takes + no pictures gets none, and the message says which were left out. +- **Text files** (UTF-8, up to 256 KB: Markdown, plain text, JSON, CSV, YAML, TOML, XML, SVG…) + go in the message's text, whole as far as the message's 100,000-character cap allows: past it + a file is cut (and says how much is missing) or named only. +- **CAD files** (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) are imported into the design + at once, as components of their own. A file joins the message once its import is in (its chip + shows a spinner meanwhile, and **Send** waits), and the message tells the agent it was added; a + failed import says why and the agent is told nothing of it. Taking a part's chip off the message + leaves the part in the design (Undo removes it). +- Every file is named in the message, and listed under it in the conversation. A file that + can't be attached says why. + +Files stay with their message: one queued while the agent works keeps its files, and one sent +again after a sign-in sends them again. When several queued messages go as one prompt, their +files go with it, at most eight; the rest are named in the chat and not sent. **New chat** +clears the files waiting on the chips too. + ## Typing while the agent works The input stays enabled during a turn. **Enter** queues the message: it shows in the conversation @@ -66,9 +102,9 @@ messages go as one prompt, in order, when the turn ends normally. - **Stop** (the square next to the input, or **Esc** in the input) stops the agent and keeps the queue: the bubbles wait for **Send**. - **New chat** (the header button, **Ctrl+Shift+L** / **Cmd+Shift+L** with the chat focused) - stops what runs, clears the conversation and the queue and starts a fresh session on the same - agent, keeping the agent and its mode. It asks first when a message is running or queued. The - design's undo history is untouched. + stops what runs, clears the conversation, the queue and the files on the chips, and starts a + fresh session on the same agent, keeping the agent and its mode. It asks first when a message + is running or queued, or files are waiting. The design's undo history is untouched. ## Agents diff --git a/docs/control-protocol.md b/docs/control-protocol.md index d2729dd1d..9e8f311fa 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -156,7 +156,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox. | `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits | | `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first | | `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. `"animate": false` on a call skips it | -| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `send?`: text, `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) | +| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) | | `ui.resize` | `width`, `height` | | | `app.quit` | `force?` (quit without asking about unsaved designs) | `quitting` (the window closes), `asking` (a "save changes?" prompt is open) |