Agent chat: attach files and pictures to a message (#1744)
Thanks @onembyte.
This commit is contained in:
Generated
+30
@@ -865,6 +865,12 @@ dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.8"
|
||||
@@ -1665,6 +1671,16 @@ dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gif"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
|
||||
dependencies = [
|
||||
"color_quant",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gl_generator"
|
||||
version = "0.14.0"
|
||||
@@ -2077,6 +2093,9 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder-lite",
|
||||
"color_quant",
|
||||
"gif",
|
||||
"image-webp",
|
||||
"moxcms",
|
||||
"num-traits",
|
||||
"png",
|
||||
@@ -2085,6 +2104,16 @@ dependencies = [
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image-webp"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.2"
|
||||
@@ -4244,6 +4273,7 @@ version = "0.15.0"
|
||||
name = "solidcraft-ui-egui"
|
||||
version = "0.15.0"
|
||||
dependencies = [
|
||||
"arboard",
|
||||
"egui",
|
||||
"egui-wgpu",
|
||||
"image",
|
||||
|
||||
@@ -116,6 +116,7 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
|
||||
let parent = rfd::AsyncFileDialog::new().set_parent(cc);
|
||||
let open_parent = parent.clone();
|
||||
let program_parent = parent.clone();
|
||||
let file_parent = parent.clone();
|
||||
Services {
|
||||
pick_open: Some(Box::new(move |first: &[&str]| {
|
||||
// The file type the command reads comes first, so the picker opens on it (#1801).
|
||||
@@ -166,6 +167,21 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
|
||||
.pick_file();
|
||||
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
|
||||
})),
|
||||
// One kind of file (the agent chat's attachments): only the types it names, as given and in
|
||||
// lower and upper case (Linux dialogs match case; macOS allows just the listed types).
|
||||
pick_file: Some(Box::new(move |label: &str, exts: &[&str]| {
|
||||
let mut all: Vec<String> = Vec::new();
|
||||
for e in exts {
|
||||
for v in [e.to_string(), e.to_ascii_lowercase(), e.to_ascii_uppercase()] {
|
||||
if !all.contains(&v) {
|
||||
all.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
let all: Vec<&str> = all.iter().map(String::as_str).collect();
|
||||
let future = file_parent.clone().add_filter(label, &all).pick_file();
|
||||
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
|
||||
})),
|
||||
pick_program: Some(Box::new(move || {
|
||||
let dialog = program_parent.clone().set_title("Choose your slicer");
|
||||
let dialog = if cfg!(target_os = "windows") { dialog.add_filter("Programs", &["exe"]) } else { dialog };
|
||||
|
||||
@@ -841,12 +841,15 @@
|
||||
"ui.activate": "Activate",
|
||||
"ui.active_component": "Active component",
|
||||
"ui.agent_chat": "Agent chat",
|
||||
"ui.agent_chat_add": "Attach files",
|
||||
"ui.agent_chat_add_file": "Choose a file…",
|
||||
"ui.agent_chat_allow": "Allow",
|
||||
"ui.agent_chat_allowed": "Allowed",
|
||||
"ui.agent_chat_answered": "Answer: {option}",
|
||||
"ui.agent_chat_approval": "Allow the agent to do this?",
|
||||
"ui.agent_chat_attach_view": "Attach view",
|
||||
"ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message",
|
||||
"ui.agent_chat_attachable": "Pictures, text and CAD files",
|
||||
"ui.agent_chat_auth_agent": "Sign in with {name}",
|
||||
"ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.",
|
||||
"ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.",
|
||||
@@ -869,9 +872,30 @@
|
||||
"ui.agent_chat_dock": "Dock in the side panel",
|
||||
"ui.agent_chat_dock_left": "Dock on the left",
|
||||
"ui.agent_chat_dock_right": "Dock on the right",
|
||||
"ui.agent_chat_drop_hint": "Drop to attach to your next message",
|
||||
"ui.agent_chat_edit": "Edit",
|
||||
"ui.agent_chat_exited": "The agent stopped.",
|
||||
"ui.agent_chat_failed_start": "The agent could not start: {error}",
|
||||
"ui.agent_chat_file_cad_not_read": "{name}: a CAD file is imported into the design, not read",
|
||||
"ui.agent_chat_file_empty_picture": "{name}: an empty picture",
|
||||
"ui.agent_chat_file_error": "{name}: {error}",
|
||||
"ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first",
|
||||
"ui.agent_chat_file_importing": "Importing into the design…",
|
||||
"ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached",
|
||||
"ui.agent_chat_file_loading": "Reading the file…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}: not a file",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}: not a picture",
|
||||
"ui.agent_chat_file_not_utf8": "{name}: not UTF-8 text",
|
||||
"ui.agent_chat_file_picture_too_big": "{name}: a picture can be at most {max} MB",
|
||||
"ui.agent_chat_file_text_too_big": "{name}: a text file can be at most {max} KB",
|
||||
"ui.agent_chat_file_too_large": "{name}: too large",
|
||||
"ui.agent_chat_file_too_large_to_send": "{name}: too large to send, even scaled down",
|
||||
"ui.agent_chat_file_unreadable_picture": "{name}: not a picture this can read",
|
||||
"ui.agent_chat_files": "Files: {files}",
|
||||
"ui.agent_chat_files_left_out": "At most {max} files go with one message; left out: {files}",
|
||||
"ui.agent_chat_files_loading": "Waiting for the files to be read",
|
||||
"ui.agent_chat_files_max": "At most {max} files on one message",
|
||||
"ui.agent_chat_import_failed": "{name}: the import failed: {error}",
|
||||
"ui.agent_chat_intro": "Chat with a coding agent you already use, or a model endpoint you choose. It works on the open design through SolidCraft's tools: its changes show as it makes them and undo as one step per message. What it may do without asking is set in Preferences > AI.",
|
||||
"ui.agent_chat_key_in_env": "key set",
|
||||
"ui.agent_chat_left": "Left",
|
||||
@@ -884,11 +908,13 @@
|
||||
"ui.agent_chat_no_cli": "solidcraft-cli was not found next to the app, so the agent can't reach the design.",
|
||||
"ui.agent_chat_no_control": "The agent chat's endpoint did not start, so the agent can't reach the design.",
|
||||
"ui.agent_chat_no_images": "This agent doesn't take pictures",
|
||||
"ui.agent_chat_no_picture": "There is no picture on the clipboard",
|
||||
"ui.agent_chat_none_found": "No coding agent found. Install one of these, then press Refresh:",
|
||||
"ui.agent_chat_none_found_short": "No agent found",
|
||||
"ui.agent_chat_not_installed": "Not installed",
|
||||
"ui.agent_chat_not_signed_in": "not signed in",
|
||||
"ui.agent_chat_own_window": "Own window",
|
||||
"ui.agent_chat_paste_picture": "Paste a picture",
|
||||
"ui.agent_chat_permission": "The agent asks to run: {title}",
|
||||
"ui.agent_chat_pick": "Agent",
|
||||
"ui.agent_chat_placeholder": "Ask the agent to change the design…",
|
||||
@@ -903,6 +929,8 @@
|
||||
"ui.agent_chat_refresh": "Refresh",
|
||||
"ui.agent_chat_refused": "Not allowed for the agent: {reason}",
|
||||
"ui.agent_chat_remove": "Remove from the queue",
|
||||
"ui.agent_chat_remove_file": "Take it off this message",
|
||||
"ui.agent_chat_remove_imported": "Take it off this message; the part stays in the design (Undo removes it)",
|
||||
"ui.agent_chat_right": "Right",
|
||||
"ui.agent_chat_save": "Save",
|
||||
"ui.agent_chat_send": "Send",
|
||||
|
||||
@@ -841,12 +841,15 @@
|
||||
"ui.activate": "アクティブ化",
|
||||
"ui.active_component": "アクティブなコンポーネント",
|
||||
"ui.agent_chat": "エージェントチャット",
|
||||
"ui.agent_chat_add": "ファイルを添付",
|
||||
"ui.agent_chat_add_file": "ファイルを選択…",
|
||||
"ui.agent_chat_allow": "許可",
|
||||
"ui.agent_chat_allowed": "許可しました",
|
||||
"ui.agent_chat_answered": "回答: {option}",
|
||||
"ui.agent_chat_approval": "エージェントにこの操作を許可しますか?",
|
||||
"ui.agent_chat_attach_view": "ビューを添付",
|
||||
"ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります",
|
||||
"ui.agent_chat_attachable": "画像・テキスト・CAD ファイル",
|
||||
"ui.agent_chat_auth_agent": "{name} でサインイン",
|
||||
"ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。",
|
||||
"ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。",
|
||||
@@ -869,9 +872,30 @@
|
||||
"ui.agent_chat_dock": "サイドパネルに戻す",
|
||||
"ui.agent_chat_dock_left": "左にドッキング",
|
||||
"ui.agent_chat_dock_right": "右にドッキング",
|
||||
"ui.agent_chat_drop_hint": "ドロップして次のメッセージに添付",
|
||||
"ui.agent_chat_edit": "編集",
|
||||
"ui.agent_chat_exited": "エージェントが停止しました。",
|
||||
"ui.agent_chat_failed_start": "エージェントを起動できませんでした: {error}",
|
||||
"ui.agent_chat_file_cad_not_read": "{name}: CAD ファイルは読み込まず、デザインにインポートします",
|
||||
"ui.agent_chat_file_empty_picture": "{name}: 空の画像です",
|
||||
"ui.agent_chat_file_error": "{name}: {error}",
|
||||
"ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください",
|
||||
"ui.agent_chat_file_importing": "デザインにインポート中…",
|
||||
"ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です",
|
||||
"ui.agent_chat_file_loading": "ファイルを読み込み中…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}: ファイルではありません",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません",
|
||||
"ui.agent_chat_file_not_utf8": "{name}: UTF-8 のテキストではありません",
|
||||
"ui.agent_chat_file_picture_too_big": "{name}: 画像は {max} MB までです",
|
||||
"ui.agent_chat_file_text_too_big": "{name}: テキストファイルは {max} KB までです",
|
||||
"ui.agent_chat_file_too_large": "{name}: 大きすぎます",
|
||||
"ui.agent_chat_file_too_large_to_send": "{name}: 縮小しても大きすぎて送れません",
|
||||
"ui.agent_chat_file_unreadable_picture": "{name}: 読み込めない画像です",
|
||||
"ui.agent_chat_files": "ファイル: {files}",
|
||||
"ui.agent_chat_files_left_out": "1 つのメッセージで送れるファイルは {max} 個までです。送らなかったファイル: {files}",
|
||||
"ui.agent_chat_files_loading": "ファイルの読み込みを待っています",
|
||||
"ui.agent_chat_files_max": "1 つのメッセージに添付できるのは {max} 個までです",
|
||||
"ui.agent_chat_import_failed": "{name}: インポートに失敗しました: {error}",
|
||||
"ui.agent_chat_intro": "普段使っているコーディングエージェント、または選んだモデルのエンドポイントとチャットできます。SolidCraft のツールで開いているデザインを操作し、変更はその場で表示され、メッセージごとに1回で元に戻せます。確認なしで許可する範囲は「環境設定 > AI」で設定します。",
|
||||
"ui.agent_chat_key_in_env": "キー設定済み",
|
||||
"ui.agent_chat_left": "左",
|
||||
@@ -884,11 +908,13 @@
|
||||
"ui.agent_chat_no_cli": "アプリの隣に solidcraft-cli が見つからないため、エージェントはデザインにアクセスできません。",
|
||||
"ui.agent_chat_no_control": "エージェントチャットのエンドポイントが起動しなかったため、エージェントはデザインにアクセスできません。",
|
||||
"ui.agent_chat_no_images": "このエージェントは画像を受け付けません",
|
||||
"ui.agent_chat_no_picture": "クリップボードに画像がありません",
|
||||
"ui.agent_chat_none_found": "コーディングエージェントが見つかりません。次のいずれかをインストールしてから「再検索」を押してください:",
|
||||
"ui.agent_chat_none_found_short": "エージェントが見つかりません",
|
||||
"ui.agent_chat_not_installed": "未インストール",
|
||||
"ui.agent_chat_not_signed_in": "未サインイン",
|
||||
"ui.agent_chat_own_window": "独立したウィンドウ",
|
||||
"ui.agent_chat_paste_picture": "画像を貼り付け",
|
||||
"ui.agent_chat_permission": "エージェントが実行の許可を求めています: {title}",
|
||||
"ui.agent_chat_pick": "エージェント",
|
||||
"ui.agent_chat_placeholder": "デザインの変更をエージェントに依頼…",
|
||||
@@ -903,6 +929,8 @@
|
||||
"ui.agent_chat_refresh": "再検索",
|
||||
"ui.agent_chat_refused": "エージェントには許可されていません: {reason}",
|
||||
"ui.agent_chat_remove": "待機から削除",
|
||||
"ui.agent_chat_remove_file": "このメッセージから外す",
|
||||
"ui.agent_chat_remove_imported": "このメッセージから外します。パーツはデザインに残ります (元に戻すで削除できます)",
|
||||
"ui.agent_chat_right": "右",
|
||||
"ui.agent_chat_save": "保存",
|
||||
"ui.agent_chat_send": "送信",
|
||||
|
||||
@@ -841,12 +841,15 @@
|
||||
"ui.activate": "激活",
|
||||
"ui.active_component": "活动组件",
|
||||
"ui.agent_chat": "智能体聊天",
|
||||
"ui.agent_chat_add": "附加文件",
|
||||
"ui.agent_chat_add_file": "选择文件…",
|
||||
"ui.agent_chat_allow": "允许",
|
||||
"ui.agent_chat_allowed": "已允许",
|
||||
"ui.agent_chat_answered": "回答:{option}",
|
||||
"ui.agent_chat_approval": "允许智能体执行此操作吗?",
|
||||
"ui.agent_chat_attach_view": "附加视图",
|
||||
"ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片",
|
||||
"ui.agent_chat_attachable": "图片、文本和 CAD 文件",
|
||||
"ui.agent_chat_auth_agent": "使用 {name} 登录",
|
||||
"ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。",
|
||||
"ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。",
|
||||
@@ -869,9 +872,30 @@
|
||||
"ui.agent_chat_dock": "停靠到侧面板",
|
||||
"ui.agent_chat_dock_left": "停靠在左侧",
|
||||
"ui.agent_chat_dock_right": "停靠在右侧",
|
||||
"ui.agent_chat_drop_hint": "拖放到此处,附加到下一条消息",
|
||||
"ui.agent_chat_edit": "编辑",
|
||||
"ui.agent_chat_exited": "智能体已停止。",
|
||||
"ui.agent_chat_failed_start": "智能体无法启动:{error}",
|
||||
"ui.agent_chat_file_cad_not_read": "{name}:CAD 文件会导入设计,而不是读取",
|
||||
"ui.agent_chat_file_empty_picture": "{name}:空图片",
|
||||
"ui.agent_chat_file_error": "{name}:{error}",
|
||||
"ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG",
|
||||
"ui.agent_chat_file_importing": "正在导入设计…",
|
||||
"ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)",
|
||||
"ui.agent_chat_file_loading": "正在读取文件…",
|
||||
"ui.agent_chat_file_not_a_file": "{name}:不是文件",
|
||||
"ui.agent_chat_file_not_a_picture": "{name}:不是图片",
|
||||
"ui.agent_chat_file_not_utf8": "{name}:不是 UTF-8 文本",
|
||||
"ui.agent_chat_file_picture_too_big": "{name}:图片最大 {max} MB",
|
||||
"ui.agent_chat_file_text_too_big": "{name}:文本文件最大 {max} KB",
|
||||
"ui.agent_chat_file_too_large": "{name}:太大",
|
||||
"ui.agent_chat_file_too_large_to_send": "{name}:即使缩小也太大,无法发送",
|
||||
"ui.agent_chat_file_unreadable_picture": "{name}:无法读取的图片",
|
||||
"ui.agent_chat_files": "文件:{files}",
|
||||
"ui.agent_chat_files_left_out": "一条消息最多发送 {max} 个文件;未发送:{files}",
|
||||
"ui.agent_chat_files_loading": "正在等待文件读取完成",
|
||||
"ui.agent_chat_files_max": "一条消息最多附加 {max} 个文件",
|
||||
"ui.agent_chat_import_failed": "{name}:导入失败:{error}",
|
||||
"ui.agent_chat_intro": "与你已在使用的编程智能体或你选择的模型端点聊天。它通过 SolidCraft 的工具处理打开的设计:更改即时显示,每条消息的更改可一步撤销。无需询问即可执行的范围在“首选项 > AI”中设置。",
|
||||
"ui.agent_chat_key_in_env": "已设置密钥",
|
||||
"ui.agent_chat_left": "左侧",
|
||||
@@ -884,11 +908,13 @@
|
||||
"ui.agent_chat_no_cli": "在应用旁边未找到 solidcraft-cli,因此智能体无法访问设计。",
|
||||
"ui.agent_chat_no_control": "智能体聊天的端点未能启动,因此智能体无法访问设计。",
|
||||
"ui.agent_chat_no_images": "该智能体不接收图片",
|
||||
"ui.agent_chat_no_picture": "剪贴板上没有图片",
|
||||
"ui.agent_chat_none_found": "未找到编程智能体。请安装以下之一,然后点击刷新:",
|
||||
"ui.agent_chat_none_found_short": "未找到智能体",
|
||||
"ui.agent_chat_not_installed": "未安装",
|
||||
"ui.agent_chat_not_signed_in": "未登录",
|
||||
"ui.agent_chat_own_window": "独立窗口",
|
||||
"ui.agent_chat_paste_picture": "粘贴图片",
|
||||
"ui.agent_chat_permission": "智能体请求运行:{title}",
|
||||
"ui.agent_chat_pick": "智能体",
|
||||
"ui.agent_chat_placeholder": "让智能体修改设计…",
|
||||
@@ -903,6 +929,8 @@
|
||||
"ui.agent_chat_refresh": "刷新",
|
||||
"ui.agent_chat_refused": "不允许智能体执行:{reason}",
|
||||
"ui.agent_chat_remove": "从队列中移除",
|
||||
"ui.agent_chat_remove_file": "从这条消息中移除",
|
||||
"ui.agent_chat_remove_imported": "从这条消息中移除;零件仍保留在设计中(撤销可将其删除)",
|
||||
"ui.agent_chat_right": "右侧",
|
||||
"ui.agent_chat_save": "保存",
|
||||
"ui.agent_chat_send": "发送",
|
||||
|
||||
@@ -17,7 +17,9 @@ serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
log = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
image = { workspace = true, features = ["jpeg", "ico"] }
|
||||
# gif, webp, bmp, tiff: pictures attached to an agent chat message (GIF and WebP go as they are,
|
||||
# BMP and TIFF as PNG).
|
||||
image = { workspace = true, features = ["jpeg", "ico", "gif", "webp", "bmp", "tiff"] }
|
||||
ttf-parser = { workspace = true }
|
||||
semver = { workspace = true }
|
||||
# Updates: the downloaded package's SHA-256 (against the release's SHA256SUMS.txt), its optional
|
||||
@@ -33,6 +35,11 @@ tiny-skia = { workspace = true }
|
||||
# updated by its installer (MIT; already in the tree as a build dependency).
|
||||
winreg = "0.56"
|
||||
|
||||
# Pictures pasted into the agent chat; already in the tree, with the same features, through
|
||||
# egui-winit's clipboard.
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
arboard = { version = "3.6", default-features = false, features = ["image-data"] }
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
|
||||
# The offscreen viewport render test opens a device without a window.
|
||||
pollster = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the
|
||||
//! agent with it, text files go in as text, and CAD files are imported into the design, the agent
|
||||
//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says
|
||||
//! why.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::backend::{self, PromptPart};
|
||||
use crate::i18n::format as trf;
|
||||
|
||||
/// A refusal in the app's language: the file's name, then why.
|
||||
fn refused(key: &str, name: &str, args: &[(&str, &str)]) -> String {
|
||||
let mut all = vec![("name", name)];
|
||||
all.extend_from_slice(args);
|
||||
trf(key, &all)
|
||||
}
|
||||
|
||||
/// A refusal carrying the system's or the decoder's own words.
|
||||
fn failed(name: &str, e: impl std::fmt::Display) -> String {
|
||||
refused("ui.agent_chat_file_error", name, &[("error", &e.to_string())])
|
||||
}
|
||||
|
||||
/// Most files on one message.
|
||||
pub const MAX_FILES: usize = 8;
|
||||
/// Largest picture file read, before it is scaled down.
|
||||
const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;
|
||||
/// Largest text file sent in full.
|
||||
const MAX_TEXT_BYTES: u64 = 256 * 1024;
|
||||
/// A picture's longer side is scaled down to this before it is sent.
|
||||
pub const MAX_SIDE: u32 = 2048;
|
||||
/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,
|
||||
/// so eight of them still fit in a prompt.
|
||||
pub const MAX_SENT_BYTES: usize = 3_750_000;
|
||||
/// A picture is not halved below this to fit; past it, it is refused.
|
||||
const MIN_SIDE: u32 = 64;
|
||||
/// Largest picture decoded (pixels on a side), so a hostile header can't ask for gigabytes.
|
||||
const MAX_DECODE_SIDE: u32 = 20_000;
|
||||
/// The chip's picture.
|
||||
const THUMB: u32 = 64;
|
||||
|
||||
/// Pictures the agents read as they are; BMP and TIFF are sent as PNG.
|
||||
const PICTURE_EXTS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"];
|
||||
/// Text sent in full; SVG goes as its source, which the agents read well.
|
||||
const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv", "yaml", "yml", "toml", "xml", "log", "ini", "cfg"];
|
||||
/// CAD files and meshes, imported into the design (the file picker's list; [`kind`] asks the
|
||||
/// importers themselves).
|
||||
const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"];
|
||||
|
||||
/// Every extension that can be attached, pictures first (the file picker's list).
|
||||
pub fn extensions() -> Vec<&'static str> {
|
||||
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect()
|
||||
}
|
||||
|
||||
/// What a file is, by its extension.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Picture,
|
||||
Text,
|
||||
Cad,
|
||||
Other,
|
||||
}
|
||||
|
||||
pub fn kind(path: &Path) -> Kind {
|
||||
let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default();
|
||||
match ext.as_str() {
|
||||
e if PICTURE_EXTS.contains(&e) => Kind::Picture,
|
||||
e if TEXT_EXTS.contains(&e) => Kind::Text,
|
||||
_ => {
|
||||
let p = path.to_string_lossy();
|
||||
if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Content {
|
||||
/// The picture's bytes as sent (PNG, JPEG, GIF or WebP), and its size in pixels.
|
||||
Picture {
|
||||
mime: &'static str,
|
||||
bytes: Arc<Vec<u8>>,
|
||||
size: [u32; 2],
|
||||
},
|
||||
Text(Arc<String>),
|
||||
/// A CAD file imported into the design as its own component.
|
||||
Imported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Attachment {
|
||||
/// Its own on the message, never reused (set when it is added there; 0 until then): the
|
||||
/// chip's thumbnail is kept by it.
|
||||
pub id: u64,
|
||||
pub name: String,
|
||||
pub content: Content,
|
||||
/// A small copy of a picture for its chip.
|
||||
pub thumb: Option<Arc<egui::ColorImage>>,
|
||||
}
|
||||
|
||||
impl Attachment {
|
||||
pub fn imported(name: String) -> Self {
|
||||
Attachment { id: 0, name, content: Content::Imported, thumb: None }
|
||||
}
|
||||
pub fn is_picture(&self) -> bool {
|
||||
matches!(self.content, Content::Picture { .. })
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn name_of(path: &Path) -> String {
|
||||
path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
/// Read a picture or text file for a message. CAD files are imported by the caller, not read here.
|
||||
pub fn read(path: &Path) -> Result<Attachment, String> {
|
||||
let name = name_of(path);
|
||||
let meta = std::fs::metadata(path).map_err(|e| failed(&name, e))?;
|
||||
if !meta.is_file() {
|
||||
return Err(refused("ui.agent_chat_file_not_a_file", &name, &[]));
|
||||
}
|
||||
match kind(path) {
|
||||
Kind::Picture => {
|
||||
if meta.len() > MAX_PICTURE_BYTES {
|
||||
return Err(refused("ui.agent_chat_file_picture_too_big", &name, &[("max", &(MAX_PICTURE_BYTES / 1024 / 1024).to_string())]));
|
||||
}
|
||||
let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?;
|
||||
picture(name, &bytes)
|
||||
}
|
||||
Kind::Text => {
|
||||
if meta.len() > MAX_TEXT_BYTES {
|
||||
return Err(refused("ui.agent_chat_file_text_too_big", &name, &[("max", &(MAX_TEXT_BYTES / 1024).to_string())]));
|
||||
}
|
||||
let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?;
|
||||
let text = String::from_utf8(bytes).map_err(|_| refused("ui.agent_chat_file_not_utf8", &name, &[]))?;
|
||||
Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None })
|
||||
}
|
||||
Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])),
|
||||
Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {
|
||||
Err(refused("ui.agent_chat_file_heic", &name, &[]))
|
||||
}
|
||||
Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])),
|
||||
}
|
||||
}
|
||||
|
||||
/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read
|
||||
/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most
|
||||
/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]).
|
||||
pub fn picture(name: String, bytes: &[u8]) -> Result<Attachment, String> {
|
||||
picture_within(name, bytes, MAX_SENT_BYTES)
|
||||
}
|
||||
|
||||
fn picture_within(name: String, bytes: &[u8], limit: usize) -> Result<Attachment, String> {
|
||||
let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().map_err(|e| failed(&name, e))?;
|
||||
let format = reader.format();
|
||||
let mime = match format {
|
||||
Some(image::ImageFormat::Png) => Some("image/png"),
|
||||
Some(image::ImageFormat::Jpeg) => Some("image/jpeg"),
|
||||
Some(image::ImageFormat::Gif) => Some("image/gif"),
|
||||
Some(image::ImageFormat::WebP) => Some("image/webp"),
|
||||
Some(image::ImageFormat::Bmp | image::ImageFormat::Tiff) => None,
|
||||
_ => return Err(refused("ui.agent_chat_file_unreadable_picture", &name, &[])),
|
||||
};
|
||||
let mut limits = image::Limits::default();
|
||||
limits.max_image_width = Some(MAX_DECODE_SIDE);
|
||||
limits.max_image_height = Some(MAX_DECODE_SIDE);
|
||||
limits.max_alloc = Some(1024 * 1024 * 1024);
|
||||
reader.limits(limits);
|
||||
let img = reader.decode().map_err(|e| failed(&name, e))?;
|
||||
if img.width() == 0 || img.height() == 0 {
|
||||
return Err(refused("ui.agent_chat_file_empty_picture", &name, &[]));
|
||||
}
|
||||
if let Some(mime) = mime
|
||||
&& img.width().max(img.height()) <= MAX_SIDE
|
||||
&& bytes.len() <= limit
|
||||
{
|
||||
let thumb = thumbnail(&img);
|
||||
return Ok(Attachment {
|
||||
id: 0,
|
||||
name,
|
||||
content: Content::Picture { mime, bytes: Arc::new(bytes.to_vec()), size: [img.width(), img.height()] },
|
||||
thumb,
|
||||
});
|
||||
}
|
||||
encoded(name, img, limit)
|
||||
}
|
||||
|
||||
/// A picture copied to the clipboard (RGBA rows from the system), sent as PNG.
|
||||
pub fn from_rgba(name: String, width: usize, height: usize, rgba: &[u8]) -> Result<Attachment, String> {
|
||||
let (w, h) = (u32::try_from(width).ok(), u32::try_from(height).ok());
|
||||
let (Some(w), Some(h)) = (w, h) else { return Err(refused("ui.agent_chat_file_too_large", &name, &[])) };
|
||||
if w == 0 || h == 0 || w > MAX_DECODE_SIDE || h > MAX_DECODE_SIDE || width.checked_mul(height).and_then(|n| n.checked_mul(4)) != Some(rgba.len())
|
||||
{
|
||||
return Err(refused("ui.agent_chat_file_not_a_picture", &name, &[]));
|
||||
}
|
||||
let img = image::RgbaImage::from_raw(w, h, rgba.to_vec()).ok_or_else(|| refused("ui.agent_chat_file_not_a_picture", &name, &[]))?;
|
||||
encoded(name, image::DynamicImage::ImageRgba8(img), MAX_SENT_BYTES)
|
||||
}
|
||||
|
||||
/// Scale down to [`MAX_SIDE`] and encode within `limit` bytes: as PNG when that fits; else as JPEG
|
||||
/// when the picture has no transparency; else halved and tried again. A GIF arrives here as its
|
||||
/// first frame.
|
||||
fn encoded(name: String, img: image::DynamicImage, limit: usize) -> Result<Attachment, String> {
|
||||
let mut img = if img.width().max(img.height()) > MAX_SIDE { img.resize(MAX_SIDE, MAX_SIDE, image::imageops::FilterType::Triangle) } else { img };
|
||||
let opaque = opaque(&img);
|
||||
loop {
|
||||
let mut png = Vec::new();
|
||||
img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).map_err(|e| failed(&name, e))?;
|
||||
let mut sent = (png.len() <= limit).then_some(("image/png", png));
|
||||
if sent.is_none() && opaque {
|
||||
let mut jpeg = Vec::new();
|
||||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 85).encode_image(&img.to_rgb8()).map_err(|e| failed(&name, e))?;
|
||||
sent = (jpeg.len() <= limit).then_some(("image/jpeg", jpeg));
|
||||
}
|
||||
if let Some((mime, bytes)) = sent {
|
||||
let thumb = thumbnail(&img);
|
||||
return Ok(Attachment {
|
||||
id: 0,
|
||||
name,
|
||||
content: Content::Picture { mime, bytes: Arc::new(bytes), size: [img.width(), img.height()] },
|
||||
thumb,
|
||||
});
|
||||
}
|
||||
if img.width().max(img.height()) <= MIN_SIDE {
|
||||
return Err(refused("ui.agent_chat_file_too_large_to_send", &name, &[]));
|
||||
}
|
||||
img = img.resize((img.width() / 2).max(1), (img.height() / 2).max(1), image::imageops::FilterType::Triangle);
|
||||
}
|
||||
}
|
||||
|
||||
/// No pixel is see-through (a screenshot or a photo, even when stored with an alpha channel).
|
||||
fn opaque(img: &image::DynamicImage) -> bool {
|
||||
!img.color().has_alpha() || img.to_rgba8().pixels().all(|p| p.0[3] == 255)
|
||||
}
|
||||
|
||||
fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {
|
||||
let t = img.thumbnail(THUMB, THUMB).to_rgba8();
|
||||
let size = [usize::try_from(t.width()).ok()?, usize::try_from(t.height()).ok()?];
|
||||
Some(Arc::new(egui::ColorImage::from_rgba_unmultiplied(size, t.as_raw())))
|
||||
}
|
||||
|
||||
/// What the message says about its files, after the person's words: the text files' text, and the
|
||||
/// pictures and imported parts by name (the agent sees the pictures themselves as images).
|
||||
/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters
|
||||
/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget
|
||||
/// is named only, so the message stays within the prompt cap.
|
||||
pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String {
|
||||
let mut out = String::new();
|
||||
let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect();
|
||||
if !pictures.is_empty() {
|
||||
let list = pictures.join(", ");
|
||||
out += &if pictures_sent {
|
||||
format!("\n\nAttached pictures: {list}.")
|
||||
} else {
|
||||
format!("\n\n(Pictures attached but not sent, this agent takes none: {list}.)")
|
||||
};
|
||||
}
|
||||
let mut left = budget;
|
||||
for f in files {
|
||||
match &f.content {
|
||||
Content::Text(t) => {
|
||||
let t = t.trim_end();
|
||||
let len = t.chars().count();
|
||||
if left == 0 {
|
||||
out += &format!("\n\nAttached file `{}`: left out, the message is at its size limit.", f.name);
|
||||
} else if len <= left {
|
||||
out += &format!("\n\nAttached file `{}`:\n```\n{t}\n```", f.name);
|
||||
left -= len;
|
||||
} else {
|
||||
let kept: String = t.chars().take(left).collect();
|
||||
out += &format!("\n\nAttached file `{}` (cut: {} more characters left out):\n```\n{kept}\n```", f.name, len - left);
|
||||
left = 0;
|
||||
}
|
||||
}
|
||||
Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name),
|
||||
Content::Picture { .. } => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The pictures as image parts of the message.
|
||||
pub fn parts(files: &[Attachment]) -> Vec<PromptPart> {
|
||||
files
|
||||
.iter()
|
||||
.filter_map(|f| match &f.content {
|
||||
Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn dir(tag: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("solidcraft-attach-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
fn png(path: &Path, w: u32, h: u32) {
|
||||
image::RgbaImage::from_pixel(w, h, image::Rgba([200, 30, 30, 255])).save(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pictures_are_kept_or_scaled_down_and_text_goes_in_whole() {
|
||||
let d = dir("read");
|
||||
let small = d.join("board.png");
|
||||
png(&small, 1600, 1120);
|
||||
let a = read(&small).unwrap();
|
||||
let Content::Picture { mime, size, bytes } = &a.content else { panic!("a picture") };
|
||||
assert_eq!((*mime, *size), ("image/png", [1600, 1120]));
|
||||
assert_eq!(bytes.as_slice(), std::fs::read(&small).unwrap().as_slice(), "small enough: sent as it came");
|
||||
assert!(a.thumb.as_ref().is_some_and(|t| t.size[0] <= 64 && t.size[1] <= 64));
|
||||
let big = d.join("photo.JPG");
|
||||
image::RgbImage::from_pixel(4000, 1000, image::Rgb([10, 10, 10])).save_with_format(&big, image::ImageFormat::Jpeg).unwrap();
|
||||
let b = read(&big).unwrap();
|
||||
let Content::Picture { mime, size, .. } = &b.content else { panic!("a picture") };
|
||||
assert_eq!((*mime, *size), ("image/png", [2048, 512]), "scaled to 2048 on the long side");
|
||||
let notes = d.join("brief.md");
|
||||
std::fs::write(¬es, "# Brief\nwall 2 mm\n").unwrap();
|
||||
let c = read(¬es).unwrap();
|
||||
assert!(matches!(&c.content, Content::Text(t) if t.contains("wall 2 mm")));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Pixels that don't compress, so a small picture can be made larger than a small limit.
|
||||
fn noise(w: u32, h: u32, alpha: bool) -> image::DynamicImage {
|
||||
let mut seed = 0x2545_f491_u32;
|
||||
let mut next = move || {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
(seed & 0xff) as u8
|
||||
};
|
||||
image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(w, h, |_, _| {
|
||||
image::Rgba([next(), next(), next(), if alpha { next() } else { 255 }])
|
||||
}))
|
||||
}
|
||||
|
||||
fn encode(img: &image::DynamicImage, format: image::ImageFormat) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
img.write_to(&mut std::io::Cursor::new(&mut out), format).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
/// Review of #1744: a picture up to 2048 px went byte for byte up to the 40 MB file cap, and
|
||||
/// eight of those don't fit in a prompt. Over the limit a picture is re-encoded: PNG when that
|
||||
/// fits, JPEG when it has no transparency, halved until it fits; a GIF goes as its first frame.
|
||||
#[test]
|
||||
fn pictures_over_the_send_limit_are_reencoded_to_fit() {
|
||||
let limit = 40_000;
|
||||
let photo = encode(&noise(300, 200, false), image::ImageFormat::Png);
|
||||
assert!(photo.len() > limit, "the test picture is over the limit");
|
||||
let a = picture_within("photo.png".into(), &photo, limit).unwrap();
|
||||
let Content::Picture { mime, bytes, size } = &a.content else { panic!("a picture") };
|
||||
assert!(bytes.len() <= limit, "{} bytes", bytes.len());
|
||||
assert_eq!(*mime, "image/jpeg", "an opaque picture goes as JPEG when PNG is too big");
|
||||
assert!(size[0] <= 300 && size[1] <= 200 && size[0] * 2 == size[1] * 3, "{size:?}: smaller, same shape");
|
||||
let clear = encode(&noise(300, 200, true), image::ImageFormat::Png);
|
||||
let b = picture_within("overlay.png".into(), &clear, limit).unwrap();
|
||||
let Content::Picture { mime, bytes, size } = &b.content else { panic!("a picture") };
|
||||
assert_eq!(*mime, "image/png", "transparency stays PNG");
|
||||
assert!(bytes.len() <= limit && size[0] < 300, "{} bytes, {size:?}: halved to fit", bytes.len());
|
||||
let gif = encode(
|
||||
&image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(400, 300, image::Rgba([40, 90, 200, 255]))),
|
||||
image::ImageFormat::Gif,
|
||||
);
|
||||
let c = picture_within("loop.gif".into(), &gif, gif.len() - 1).unwrap();
|
||||
let Content::Picture { mime, size, bytes } = &c.content else { panic!("a picture") };
|
||||
assert_eq!(*mime, "image/png", "a GIF over the limit goes as its first frame, as PNG");
|
||||
assert!(bytes.len() < gif.len() && size[0] * 3 == size[1] * 4, "{size:?}: within the limit, same shape");
|
||||
assert!(a.thumb.is_some() && b.thumb.is_some() && c.thumb.is_some());
|
||||
let under = encode(&noise(60, 40, false), image::ImageFormat::Png);
|
||||
let d = picture_within("small.png".into(), &under, limit).unwrap();
|
||||
assert!(matches!(&d.content, Content::Picture { bytes, .. } if bytes.as_slice() == under.as_slice()), "under the limit: as it came");
|
||||
assert_eq!(MAX_SENT_BYTES, 3_750_000, "its base64 stays under 5 MB");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_files_are_refused_with_a_reason() {
|
||||
let d = dir("bad");
|
||||
let fake = d.join("fake.png");
|
||||
std::fs::write(&fake, b"not a picture at all").unwrap();
|
||||
assert!(read(&fake).unwrap_err().contains("fake.png"));
|
||||
let text = d.join("big.txt");
|
||||
std::fs::write(&text, vec![b'a'; 300 * 1024]).unwrap();
|
||||
assert!(read(&text).unwrap_err().contains("at most"));
|
||||
let binary = d.join("bin.json");
|
||||
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
|
||||
assert!(read(&binary).unwrap_err().contains("UTF-8"));
|
||||
let other = d.join("movie.mov");
|
||||
std::fs::write(&other, b"x").unwrap();
|
||||
assert!(read(&other).unwrap_err().contains("can be attached"));
|
||||
assert!(read(&d.join("missing.png")).is_err());
|
||||
assert_eq!(kind(Path::new("part.STEP")), Kind::Cad);
|
||||
assert_eq!(kind(Path::new("mount.stl")), Kind::Cad);
|
||||
assert_eq!(kind(Path::new("frame.3dm")), Kind::Cad);
|
||||
assert!(from_rgba("x".into(), 2, 2, &[0; 15]).is_err(), "wrong length");
|
||||
assert!(from_rgba("x".into(), 0, 2, &[]).is_err());
|
||||
assert!(from_rgba("x".into(), usize::MAX, 2, &[]).is_err());
|
||||
let ok = from_rgba("pasted.png".into(), 3, 2, &[255; 24]).unwrap();
|
||||
assert!(matches!(ok.content, Content::Picture { size: [3, 2], .. }));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: the refusals were English whatever the app's language.
|
||||
#[test]
|
||||
fn refusals_speak_the_apps_language() {
|
||||
let d = dir("lang");
|
||||
let binary = d.join("bin.json");
|
||||
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
|
||||
let other = d.join("movie.mov");
|
||||
std::fs::write(&other, b"x").unwrap();
|
||||
{
|
||||
let _ja = crate::i18n::scope("ja");
|
||||
assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません"));
|
||||
assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは"));
|
||||
}
|
||||
{
|
||||
let _zh = crate::i18n::scope("zh");
|
||||
assert!(read(&binary).unwrap_err().contains("不是 UTF-8 文本"));
|
||||
}
|
||||
assert!(read(&binary).unwrap_err().contains("not UTF-8 text"), "English again outside the scope");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Every picture format the agents read goes as it is (PNG, JPEG, GIF, WebP); BMP and TIFF are
|
||||
/// sent as PNG; SVG goes as its source text; HEIC says why not.
|
||||
#[test]
|
||||
fn each_picture_format_goes_the_way_the_agent_reads_it() {
|
||||
let d = dir("formats");
|
||||
let img = image::RgbaImage::from_pixel(30, 20, image::Rgba([20, 120, 220, 255]));
|
||||
for (file, format, mime) in [
|
||||
("a.gif", image::ImageFormat::Gif, "image/gif"),
|
||||
("a.webp", image::ImageFormat::WebP, "image/webp"),
|
||||
("a.bmp", image::ImageFormat::Bmp, "image/png"),
|
||||
("a.tiff", image::ImageFormat::Tiff, "image/png"),
|
||||
] {
|
||||
let path = d.join(file);
|
||||
image::DynamicImage::ImageRgba8(img.clone()).save_with_format(&path, format).unwrap();
|
||||
let a = read(&path).unwrap_or_else(|e| panic!("{file}: {e}"));
|
||||
let Content::Picture { mime: m, size, bytes } = &a.content else { panic!("{file}: a picture") };
|
||||
assert_eq!((*m, *size), (mime, [30, 20]), "{file}");
|
||||
if mime != "image/png" {
|
||||
assert_eq!(bytes.as_slice(), std::fs::read(&path).unwrap().as_slice(), "{file}: sent as it came");
|
||||
}
|
||||
assert!(a.thumb.is_some(), "{file}: a thumbnail");
|
||||
}
|
||||
let svg = d.join("sketch.svg");
|
||||
std::fs::write(&svg, "<svg xmlns='http://www.w3.org/2000/svg'><rect width='10' height='5'/></svg>").unwrap();
|
||||
assert!(matches!(&read(&svg).unwrap().content, Content::Text(t) if t.contains("<rect")));
|
||||
let heic = d.join("photo.HEIC");
|
||||
std::fs::write(&heic, b"....ftypheic").unwrap();
|
||||
assert!(read(&heic).unwrap_err().contains("HEIC"));
|
||||
for ext in ["png", "jpg", "jpeg", "gif", "webp", "bmp", "tiff", "svg", "md", "txt", "step", "FCStd", "3dm", "stl", "3mf", "obj", "glb"] {
|
||||
assert!(extensions().contains(&ext), "{ext} can be chosen");
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: eight 256 KB text files took a message to about 2 MB, past the prompt cap
|
||||
/// that only trimmed the typed text. The files' text now gets a budget: a file that doesn't
|
||||
/// fit is cut and says so, and a file past it is named only.
|
||||
#[test]
|
||||
fn attached_text_is_kept_within_the_prompt_budget() {
|
||||
let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None };
|
||||
let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)];
|
||||
let n = note(&files, true, 1000);
|
||||
assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}");
|
||||
assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget");
|
||||
assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}");
|
||||
assert!(n.contains("left out"), "a file past the budget is named only: {n}");
|
||||
assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_message_names_its_files_and_carries_the_pictures() {
|
||||
let pic = from_rgba("target.png".into(), 2, 2, &[9; 16]).unwrap();
|
||||
let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None };
|
||||
let part = Attachment::imported("motor.step".into());
|
||||
let files = vec![pic, text, part];
|
||||
let n = note(&files, true, usize::MAX);
|
||||
assert!(n.contains("Attached pictures: target.png."), "{n}");
|
||||
assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}");
|
||||
assert!(n.contains("`motor.step` to the design"), "{n}");
|
||||
assert!(note(&files, false, usize::MAX).contains("not sent"));
|
||||
let p = parts(&files);
|
||||
assert_eq!(p.len(), 1);
|
||||
assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo")));
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ mod acp_client;
|
||||
pub mod agents;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod api_client;
|
||||
pub mod attach;
|
||||
pub mod backend;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod discover;
|
||||
@@ -48,6 +49,8 @@ pub use panel::{show, tip as panel_tip, toggle};
|
||||
const MAX_ITEMS: usize = 2000;
|
||||
/// Longest message sent.
|
||||
const MAX_PROMPT: usize = 100_000;
|
||||
/// Room kept in a message for the files' names and the words around their text.
|
||||
const FILE_NAMES_ROOM: usize = 2_000;
|
||||
/// How long a request waits for Allow or Deny before it is declined (the agent's MCP server gives
|
||||
/// up on a request after five minutes).
|
||||
const APPROVAL_TIMEOUT_MS: f64 = 240_000.0;
|
||||
@@ -95,6 +98,8 @@ pub enum Item {
|
||||
User {
|
||||
text: String,
|
||||
image: bool,
|
||||
/// The names of the files that went with it.
|
||||
files: Vec<String>,
|
||||
},
|
||||
Agent {
|
||||
thought: bool,
|
||||
@@ -152,6 +157,8 @@ struct Turn {
|
||||
pub struct Pending {
|
||||
pub text: String,
|
||||
pub image: bool,
|
||||
/// The files attached to it (they go with it).
|
||||
pub files: Vec<attach::Attachment>,
|
||||
}
|
||||
|
||||
/// A request held for Allow or Deny.
|
||||
@@ -165,6 +172,30 @@ struct Held {
|
||||
/// it): requests arriving there go to the sender; the port it listens on.
|
||||
pub type ControlStarter = Box<dyn FnMut(Sender<ControlRequest>) -> Option<u16>>;
|
||||
|
||||
/// A file not yet on the next message: a picture being read off the window thread (so a large one
|
||||
/// doesn't stop the window), or a CAD file being imported into the design. Its chip shows
|
||||
/// meanwhile; it keeps the id it was given when attached, so the files stay in the order they
|
||||
/// were attached in.
|
||||
pub(crate) struct Loading {
|
||||
pub id: u64,
|
||||
pub name: String,
|
||||
what: Reading,
|
||||
}
|
||||
|
||||
enum Reading {
|
||||
/// A picture decoding, or a pasted one encoding: the file, or why not.
|
||||
File(Receiver<std::result::Result<attach::Attachment, String>>),
|
||||
/// A CAD file's import into the design: its answer, as a control reply.
|
||||
Import(Receiver<crate::control::ControlResponse>),
|
||||
}
|
||||
|
||||
impl Loading {
|
||||
/// A CAD file being imported (its part lands in the design whatever the message does).
|
||||
pub fn importing(&self) -> bool {
|
||||
matches!(self.what, Reading::Import(_))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct AgentChat {
|
||||
/// The panel is shown (View menu, the application bar's chat button, Ctrl+Shift+A).
|
||||
@@ -178,8 +209,8 @@ pub struct AgentChat {
|
||||
pub attach_view: bool,
|
||||
auth: Vec<acp::AuthMethod>,
|
||||
auth_status: Option<String>,
|
||||
/// A message typed before the agent was ready.
|
||||
queued: Option<(String, bool)>,
|
||||
/// A message typed before the agent was ready, with its files.
|
||||
queued: Option<(String, bool, Vec<attach::Attachment>)>,
|
||||
/// Messages typed while the agent works, in order. They go as one prompt when the turn ends
|
||||
/// (or at once with Send now); after Stop they wait for Send.
|
||||
pub pending: Vec<Pending>,
|
||||
@@ -199,8 +230,21 @@ pub struct AgentChat {
|
||||
pub confirm_new: bool,
|
||||
/// Where the panel was drawn last (for its shortcuts).
|
||||
pub panel_rect: Option<egui::Rect>,
|
||||
/// The last message sent, kept to send again after a sign-in.
|
||||
last: Option<(String, bool)>,
|
||||
/// The last message sent, kept to send again after a sign-in (its files too).
|
||||
last: Option<(String, bool, Vec<attach::Attachment>)>,
|
||||
/// Files for the next message (pictures, text files, imported parts), shown as chips.
|
||||
pub(crate) attachments: Vec<attach::Attachment>,
|
||||
/// Pictures pasted so far, to name the next one.
|
||||
pasted: usize,
|
||||
/// The chips' thumbnails, by the picture they show.
|
||||
thumbs: std::collections::HashMap<u64, egui::TextureHandle>,
|
||||
/// The last id given to a file added to a message.
|
||||
last_file_id: u64,
|
||||
/// Files still being read for the next message.
|
||||
pub(crate) loading: Vec<Loading>,
|
||||
/// Where the chat was drawn last frame (the panel, or its folded strip), for drops; `None`
|
||||
/// when it isn't drawn.
|
||||
pub(crate) drop_rect: Option<egui::Rect>,
|
||||
/// The command the running agent was started with.
|
||||
argv: Vec<String>,
|
||||
/// The agent picked for this conversation (the header's switcher); `None`: the default.
|
||||
@@ -246,6 +290,54 @@ impl Drop for AgentChat {
|
||||
}
|
||||
|
||||
impl AgentChat {
|
||||
/// Put a file on the next message with an id of its own; its name back.
|
||||
fn add_file(&mut self, file: attach::Attachment) -> String {
|
||||
let id = self.next_file_id();
|
||||
self.place(id, file)
|
||||
}
|
||||
fn next_file_id(&mut self) -> u64 {
|
||||
self.last_file_id += 1;
|
||||
self.last_file_id
|
||||
}
|
||||
/// A file goes where its id says: the order the files were attached in.
|
||||
fn place(&mut self, id: u64, mut file: attach::Attachment) -> String {
|
||||
file.id = id;
|
||||
let name = file.name.clone();
|
||||
let at = self.attachments.partition_point(|f| f.id < id);
|
||||
self.attachments.insert(at, file);
|
||||
name
|
||||
}
|
||||
/// Files on the next message, read or still being read.
|
||||
fn files(&self) -> usize {
|
||||
self.attachments.len() + self.loading.len()
|
||||
}
|
||||
/// Read a file for the next message on a worker thread, in the window's language.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn load(
|
||||
&mut self,
|
||||
name: String,
|
||||
job: impl FnOnce() -> std::result::Result<attach::Attachment, String> + Send + 'static,
|
||||
) -> std::result::Result<String, String> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let language = crate::i18n::language();
|
||||
std::thread::Builder::new()
|
||||
.name("chat-file".into())
|
||||
.spawn(move || {
|
||||
let _language = crate::i18n::scope(language);
|
||||
let _ = tx.send(job());
|
||||
})
|
||||
.map_err(|e| trf("ui.agent_chat_file_error", &[("name", &name), ("error", &e.to_string())]))?;
|
||||
let id = self.next_file_id();
|
||||
self.loading.push(Loading { id, name: name.clone(), what: Reading::File(rx) });
|
||||
Ok(name)
|
||||
}
|
||||
/// A CAD file whose import was asked for: it joins the message once the import is in.
|
||||
fn importing(&mut self, name: String, rx: Receiver<crate::control::ControlResponse>) -> String {
|
||||
let id = self.next_file_id();
|
||||
self.loading.push(Loading { id, name: name.clone(), what: Reading::Import(rx) });
|
||||
name
|
||||
}
|
||||
|
||||
pub fn state(&self) -> State {
|
||||
self.state.unwrap_or(State::Off)
|
||||
}
|
||||
@@ -573,37 +665,50 @@ const INTERRUPT_NOTE: &str = "[The user interrupted your previous step.]";
|
||||
/// Send what the user typed: now if the agent is ready, queued behind the running message if it
|
||||
/// is working, else once it is ready (starting it if needed).
|
||||
pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
|
||||
let text: String = text.chars().take(MAX_PROMPT).collect();
|
||||
if text.trim().is_empty() {
|
||||
// A message waits for its files: the words go back in the box until they are read.
|
||||
if !app.agent_chat.loading.is_empty() {
|
||||
if app.agent_chat.input.trim().is_empty() {
|
||||
app.agent_chat.input = text;
|
||||
}
|
||||
app.agent_chat.notice(tr("ui.agent_chat_files_loading"), false);
|
||||
return;
|
||||
}
|
||||
let text: String = text.chars().take(MAX_PROMPT).collect();
|
||||
// Files alone make a message.
|
||||
if text.trim().is_empty() && app.agent_chat.attachments.is_empty() {
|
||||
return;
|
||||
}
|
||||
let text = if text.trim().is_empty() { String::new() } else { text };
|
||||
let image = app.agent_chat.attach_view && app.agent_chat.images;
|
||||
if app.agent_chat.state() == State::Busy {
|
||||
let chat = &mut app.agent_chat;
|
||||
if chat.pending.len() >= MAX_PENDING {
|
||||
// The files stay on the chips for when there is room.
|
||||
chat.notice(tr("ui.agent_chat_queue_full"), false);
|
||||
} else {
|
||||
chat.pending.push(Pending { text, image });
|
||||
let files = std::mem::take(&mut chat.attachments);
|
||||
chat.pending.push(Pending { text, image, files });
|
||||
}
|
||||
return;
|
||||
}
|
||||
let files = std::mem::take(&mut app.agent_chat.attachments);
|
||||
// Messages left waiting by Stop go first, in one prompt with this one.
|
||||
if !app.agent_chat.pending.is_empty() {
|
||||
app.agent_chat.pending.push(Pending { text, image });
|
||||
app.agent_chat.pending.push(Pending { text, image, files });
|
||||
send_queue(app, ctx, false);
|
||||
return;
|
||||
}
|
||||
submit(app, ctx, text, image, false);
|
||||
submit(app, ctx, text, image, files, false);
|
||||
}
|
||||
|
||||
/// Put `text` in the conversation and send it (`interrupted`: the agent is told it cut in on its
|
||||
/// previous step).
|
||||
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, interrupted: bool) {
|
||||
app.agent_chat.push(Item::User { text: text.clone(), image });
|
||||
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
|
||||
/// cut in on its previous step).
|
||||
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
|
||||
app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() });
|
||||
match app.agent_chat.state() {
|
||||
State::Ready => start_turn_with(app, text, image, interrupted.then_some(INTERRUPT_NOTE)),
|
||||
State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)),
|
||||
s => {
|
||||
app.agent_chat.queued = Some((text, image));
|
||||
app.agent_chat.queued = Some((text, image, files));
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if !s.running() {
|
||||
connect(app, ctx);
|
||||
@@ -623,9 +728,23 @@ fn send_queue(app: &mut SolveApp, ctx: &egui::Context, interrupted: bool) {
|
||||
}
|
||||
let all = std::mem::take(&mut chat.pending);
|
||||
let image = all.iter().any(|p| p.image);
|
||||
let text = all.into_iter().map(|p| p.text).collect::<Vec<_>>().join("\n\n");
|
||||
let mut files = Vec::new();
|
||||
let mut texts = Vec::new();
|
||||
for p in all {
|
||||
files.extend(p.files);
|
||||
if !p.text.trim().is_empty() {
|
||||
texts.push(p.text);
|
||||
}
|
||||
}
|
||||
// One message carries at most MAX_FILES files; the rest are named, not sent.
|
||||
if files.len() > attach::MAX_FILES {
|
||||
let left: Vec<String> = files.split_off(attach::MAX_FILES).into_iter().map(|f| f.name).collect();
|
||||
let max = attach::MAX_FILES.to_string();
|
||||
chat.notice(trf("ui.agent_chat_files_left_out", &[("max", &max), ("files", &left.join(", "))]), true);
|
||||
}
|
||||
let text = texts.join("\n\n");
|
||||
let text: String = text.chars().take(MAX_PROMPT).collect();
|
||||
submit(app, ctx, text, image, interrupted);
|
||||
submit(app, ctx, text, image, files, interrupted);
|
||||
}
|
||||
|
||||
/// Send now: stop the running step and send the queue as the next prompt once the agent has said
|
||||
@@ -657,6 +776,8 @@ pub fn new_chat(app: &mut SolveApp) {
|
||||
chat.confirm_new = false;
|
||||
chat.items.clear();
|
||||
chat.pending.clear();
|
||||
chat.attachments.clear();
|
||||
chat.loading.clear();
|
||||
chat.pending_edit = None;
|
||||
chat.queued = None;
|
||||
chat.last = None;
|
||||
@@ -675,7 +796,7 @@ pub fn new_chat(app: &mut SolveApp) {
|
||||
|
||||
/// Whether New chat would throw something away that the user may want (so it asks first).
|
||||
pub fn new_chat_needs_confirm(app: &SolveApp) -> bool {
|
||||
app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty()
|
||||
app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty() || app.agent_chat.files() > 0
|
||||
}
|
||||
|
||||
/// Take a queued message out of the queue.
|
||||
@@ -702,16 +823,23 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
}
|
||||
}
|
||||
|
||||
fn start_turn(app: &mut SolveApp, text: String, image: bool) {
|
||||
start_turn_with(app, text, image, None);
|
||||
fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>) {
|
||||
start_turn_with(app, text, image, files, None);
|
||||
}
|
||||
|
||||
/// Start a message; `note` goes before the text for the agent, not into the conversation.
|
||||
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&str>) {
|
||||
let sent = match note {
|
||||
/// Start a message; `note` goes before the text for the agent, not into the conversation. The
|
||||
/// files are named after the text (text files in full), and pictures go as images to an agent
|
||||
/// that takes them.
|
||||
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>, note: Option<&str>) {
|
||||
let pictures = app.agent_chat.images;
|
||||
let lead = match note {
|
||||
Some(n) => format!("{n}\n\n{text}"),
|
||||
None => text.clone(),
|
||||
};
|
||||
// The whole message stays within the cap: the files' text gets what the words leave.
|
||||
let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM);
|
||||
let about_files = attach::note(&files, pictures, budget);
|
||||
let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect();
|
||||
let mut parts = vec![PromptPart::Text(sent)];
|
||||
if image {
|
||||
match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) {
|
||||
@@ -719,10 +847,15 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&
|
||||
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
|
||||
}
|
||||
}
|
||||
let label: String = text.lines().next().unwrap_or("").chars().take(48).collect();
|
||||
if pictures {
|
||||
parts.extend(attach::parts(&files));
|
||||
}
|
||||
// The undo step is named after the message, or its first file when it has no words.
|
||||
let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or("");
|
||||
let label: String = first_line.chars().take(48).collect();
|
||||
app.agent_chat.turn =
|
||||
Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label });
|
||||
app.agent_chat.last = Some((text, image));
|
||||
app.agent_chat.last = Some((text, image, files));
|
||||
app.agent_chat.state = Some(State::Busy);
|
||||
app.agent_chat.stopped = false;
|
||||
app.agent_chat.steering = false;
|
||||
@@ -732,6 +865,100 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a file to the next message. A picture or text file is read and kept (see [`attach`]); a CAD
|
||||
/// file is imported into the design at once, as its own component, and the message says so.
|
||||
pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result<String, String> {
|
||||
if app.agent_chat.files() >= attach::MAX_FILES {
|
||||
return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())]));
|
||||
}
|
||||
let p = std::path::Path::new(path);
|
||||
let file = if attach::kind(p) == attach::Kind::Cad {
|
||||
if !p.is_file() {
|
||||
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", path)]));
|
||||
}
|
||||
// Imported now, as a dropped file is; it joins the message once the import is in, so the
|
||||
// agent is told it was added only when it was. A full queue is refused at once.
|
||||
let name = attach::name_of(p);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let mut request = crate::file_load::Request::new(path, crate::file_load::Kind::Insert);
|
||||
request.reply = Some(tx);
|
||||
crate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?;
|
||||
return Ok(app.agent_chat.importing(name, rx));
|
||||
} else {
|
||||
// A picture is decoded off the window thread: a large one takes long enough to stop it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if attach::kind(p) == attach::Kind::Picture {
|
||||
// A missing file is said at once; reading and decoding wait for the worker.
|
||||
if !p.is_file() {
|
||||
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))]));
|
||||
}
|
||||
let path = p.to_path_buf();
|
||||
return app.agent_chat.load(attach::name_of(p), move || attach::read(&path));
|
||||
}
|
||||
attach::read(p)?
|
||||
};
|
||||
Ok(app.agent_chat.add_file(file))
|
||||
}
|
||||
|
||||
/// Add a picture from the clipboard (RGBA rows, as the system gives them) to the next message.
|
||||
pub fn attach_rgba(app: &mut SolveApp, width: usize, height: usize, rgba: &[u8]) -> std::result::Result<String, String> {
|
||||
if app.agent_chat.files() >= attach::MAX_FILES {
|
||||
return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())]));
|
||||
}
|
||||
let n = app.agent_chat.pasted + 1;
|
||||
let name = format!("pasted picture {n}.png");
|
||||
app.agent_chat.pasted = n;
|
||||
// Encoded off the window thread, as a decoded picture is.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let rgba = rgba.to_vec();
|
||||
let named = name.clone();
|
||||
app.agent_chat.load(name, move || attach::from_rgba(named, width, height, &rgba))
|
||||
}
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
{
|
||||
let file = attach::from_rgba(name, width, height, rgba)?;
|
||||
Ok(app.agent_chat.add_file(file))
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach the picture on the clipboard, if it holds one.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn paste_picture(app: &mut SolveApp) -> std::result::Result<String, String> {
|
||||
let image = arboard::Clipboard::new().and_then(|mut c| c.get_image()).map_err(|_| tr("ui.agent_chat_no_picture").to_string())?;
|
||||
attach_rgba(app, image.width, image.height, &image.bytes)
|
||||
}
|
||||
|
||||
/// Files dropped on the chat attach to the next message; called before the app handles drops, so
|
||||
/// those taken aren't opened too. A drop is the chat's when the pointer was last seen over it:
|
||||
/// the platform gives no position during a drag (winit 0.30), so that is the best the chat can
|
||||
/// know; the + button always works.
|
||||
pub fn take_drops(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
if !drop_is_ours(app, ctx) || !ctx.input(|i| !i.raw.dropped_files.is_empty()) {
|
||||
return;
|
||||
}
|
||||
let files = ctx.input_mut(|i| std::mem::take(&mut i.raw.dropped_files));
|
||||
attach_dropped(app, files.iter().map(|f| f.path().to_path_buf()));
|
||||
}
|
||||
|
||||
/// Whether files dropped now would be the chat's: the pointer was last seen over it.
|
||||
pub(crate) fn drop_is_ours(app: &SolveApp, ctx: &egui::Context) -> bool {
|
||||
let Some(rect) = app.agent_chat.drop_rect else { return false };
|
||||
ctx.input(|i| i.pointer.latest_pos()).is_some_and(|p| rect.contains(p))
|
||||
}
|
||||
|
||||
/// Attach dropped files; a refused one says why in the conversation.
|
||||
pub(crate) fn attach_dropped(app: &mut SolveApp, paths: impl Iterator<Item = std::path::PathBuf>) {
|
||||
for p in paths {
|
||||
if p.as_os_str().is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = attach_path(app, &p.to_string_lossy()) {
|
||||
app.agent_chat.notice(e, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop: ask the agent to stop the message it is on; requests waiting for approval are declined.
|
||||
/// Messages queued meanwhile stay queued (they wait for Send).
|
||||
pub fn cancel(app: &mut SolveApp) {
|
||||
@@ -958,6 +1185,48 @@ fn pump_tools(app: &mut SolveApp) {
|
||||
}
|
||||
|
||||
/// Read what the agent sent since the last frame, its tool requests, and the agent search.
|
||||
/// Files read off the window thread join the message as they are done, where their ids put them;
|
||||
/// a refused one says why.
|
||||
pub(crate) fn take_loaded(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
use std::sync::mpsc::TryRecvError;
|
||||
let chat = &mut app.agent_chat;
|
||||
let mut i = 0;
|
||||
while i < chat.loading.len() {
|
||||
let name = chat.loading[i].name.clone();
|
||||
let done = match &chat.loading[i].what {
|
||||
Reading::File(rx) => match rx.try_recv() {
|
||||
Ok(read) => Some(read),
|
||||
Err(TryRecvError::Empty) => None,
|
||||
Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_file_error", &[("name", &name), ("error", "stopped")]))),
|
||||
},
|
||||
Reading::Import(rx) => match rx.try_recv() {
|
||||
Ok(answer) if answer["ok"].as_bool() == Some(true) => Some(Ok(attach::Attachment::imported(name))),
|
||||
Ok(answer) => {
|
||||
let why = answer["error"].as_str().unwrap_or_default().to_string();
|
||||
Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &why)])))
|
||||
}
|
||||
Err(TryRecvError::Empty) => None,
|
||||
Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", "stopped")]))),
|
||||
},
|
||||
};
|
||||
match done {
|
||||
None => i += 1,
|
||||
Some(read) => {
|
||||
let l = chat.loading.remove(i);
|
||||
match read {
|
||||
Ok(file) => {
|
||||
chat.place(l.id, file);
|
||||
}
|
||||
Err(e) => chat.notice(e, true),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !chat.loading.is_empty() {
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(50));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if let Some(rx) = &app.agent_chat.detecting
|
||||
@@ -974,6 +1243,7 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
take_loaded(app, ctx);
|
||||
let events = app.agent_chat.backend.as_mut().map(|b| b.poll()).unwrap_or_default();
|
||||
for e in events {
|
||||
let ready = matches!(e, Event::Ready { .. });
|
||||
@@ -988,8 +1258,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
if ready {
|
||||
apply_saved_model(app);
|
||||
}
|
||||
if ready && let Some((text, image)) = app.agent_chat.queued.take() {
|
||||
start_turn(app, text, image);
|
||||
if ready && let Some((text, image, files)) = app.agent_chat.queued.take() {
|
||||
start_turn(app, text, image, files);
|
||||
}
|
||||
}
|
||||
pump_tools(app);
|
||||
@@ -1055,8 +1325,9 @@ pub fn set_mode(app: &mut SolveApp, mode: &str) {
|
||||
}
|
||||
|
||||
/// `ui.agent_chat` on the control channel (tests and screenshots): `enabled`, `command`, `open`,
|
||||
/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `send`, `attach_view`,
|
||||
/// `permission` (an option id, or `"cancel"`), `approve` (`true`/`false`, the oldest held
|
||||
/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `attach` (a file's path, for the
|
||||
/// next message), `remove_file` (index), `send`, `attach_view`, `permission` (an option id, or
|
||||
/// `"cancel"`), `approve` (`true`/`false`, the oldest held
|
||||
/// request), `cancel` (Stop), `send_now` (steer: stop, then send the queue), `remove_pending` (index), `undo` (the newest message's changes); returns the panel's state.
|
||||
pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result<Value, String> {
|
||||
if let Some(e) = p.get("features").and_then(Value::as_bool) {
|
||||
@@ -1140,6 +1411,13 @@ pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result<Val
|
||||
app.preferences.agent_chat_width = (w as f32).clamp(panel::MIN_WIDTH, panel::MAX_WIDTH);
|
||||
panel::forget_width(ctx);
|
||||
}
|
||||
if let Some(path) = p.get("attach").and_then(Value::as_str) {
|
||||
attach_path(app, path)?;
|
||||
}
|
||||
if let Some(n) = p.get("remove_file").and_then(Value::as_u64) {
|
||||
let n = usize::try_from(n).ok().filter(|n| *n < app.agent_chat.attachments.len()).ok_or("no such file on the next message")?;
|
||||
app.agent_chat.attachments.remove(n);
|
||||
}
|
||||
if let Some(a) = p.get("attach_view").and_then(Value::as_bool) {
|
||||
app.agent_chat.attach_view = a;
|
||||
}
|
||||
@@ -1200,7 +1478,7 @@ pub fn status(app: &SolveApp) -> Value {
|
||||
.items
|
||||
.iter()
|
||||
.map(|i| match i {
|
||||
Item::User { text, image } => json!({"user": text, "image": image}),
|
||||
Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}),
|
||||
Item::Agent { thought, text } => json!({"agent": text, "thought": thought}),
|
||||
Item::Tool { id, title, kind, status, output } => {
|
||||
json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output})
|
||||
@@ -1249,7 +1527,9 @@ pub fn status(app: &SolveApp) -> Value {
|
||||
"toolsPort": c.tools_port,
|
||||
"pid": pid,
|
||||
"waiting": c.waiting(),
|
||||
"pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image})).collect::<Vec<_>>(),
|
||||
"pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image, "files": p.files.iter().map(|f| f.name.clone()).collect::<Vec<_>>()})).collect::<Vec<_>>(),
|
||||
"attachments": c.attachments.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
|
||||
"loading": c.loading.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
|
||||
"stopped": c.stopped,
|
||||
"turn": c.turn.as_ref().map(|t| json!({"changes": t.changes, "requests": t.requests})),
|
||||
"detection": detection,
|
||||
|
||||
@@ -50,6 +50,8 @@ pub fn tip() -> String {
|
||||
pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
let ctx = ui.ctx().clone();
|
||||
poll(app, &ctx);
|
||||
// Set again below if the chat is drawn this frame.
|
||||
app.agent_chat.drop_rect = None;
|
||||
// With AI features off nothing of the chat is drawn, and the shortcut does nothing.
|
||||
if !shown(app) {
|
||||
return;
|
||||
@@ -85,7 +87,13 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
let left = app.preferences.agent_chat_side == "left";
|
||||
let side = |id: Id| if left { egui::Panel::left(id) } else { egui::Panel::right(id) };
|
||||
if app.preferences.agent_chat_minimized {
|
||||
side(Id::new("sc_agent_chat_strip")).exact_size(STRIP).resizable(false).frame(egui::Frame::NONE.fill(t.panel)).show(ui, |ui| strip(app, ui));
|
||||
let r = side(Id::new("sc_agent_chat_strip"))
|
||||
.exact_size(STRIP)
|
||||
.resizable(false)
|
||||
.frame(egui::Frame::NONE.fill(t.panel))
|
||||
.show(ui, |ui| strip(app, ui));
|
||||
app.agent_chat.drop_rect = Some(r.response.rect);
|
||||
show_drop_hint(app, &ctx);
|
||||
return;
|
||||
}
|
||||
let width = app.preferences.agent_chat_width.clamp(MIN_WIDTH, MAX_WIDTH);
|
||||
@@ -97,12 +105,41 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
.show(ui, |ui| body(app, ui, left));
|
||||
crate::scenario::publish_rect("agent_chat:panel", r.response.rect);
|
||||
app.agent_chat.panel_rect = Some(r.response.rect);
|
||||
app.agent_chat.drop_rect = Some(r.response.rect);
|
||||
show_drop_hint(app, &ctx);
|
||||
let w = r.response.rect.width();
|
||||
if w.is_finite() && (MIN_WIDTH..=MAX_WIDTH).contains(&w) {
|
||||
app.preferences.agent_chat_width = w;
|
||||
}
|
||||
}
|
||||
|
||||
/// While files are dragged over the window: if they would drop on the chat, say so over it.
|
||||
fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if let Some(rect) = app.agent_chat.drop_rect
|
||||
&& ctx.input(|i| !i.raw.hovered_files.is_empty())
|
||||
&& drop_is_ours(app, ctx)
|
||||
{
|
||||
drop_hint(ctx, rect);
|
||||
}
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let _ = (app, ctx);
|
||||
}
|
||||
|
||||
/// Where dragged files will go: a tinted zone over the chat with a line of text.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn drop_hint(ctx: &egui::Context, area: Rect) {
|
||||
let t = Tokens::get();
|
||||
let painter = ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, Id::new("sc_agent_chat_drop_hint")));
|
||||
let zone = area.shrink(6.0);
|
||||
painter.rect(zone, 10.0, t.accent.gamma_multiply(0.16), Stroke::new(2.0, t.accent), egui::StrokeKind::Inside);
|
||||
let font = egui::FontId::proportional(14.0);
|
||||
let text = painter.layout(tr("ui.agent_chat_drop_hint").to_string(), font, t.text, (zone.width() - 24.0).max(40.0));
|
||||
let at = zone.center() - text.size() / 2.0;
|
||||
painter.galley(at, text, t.text);
|
||||
crate::scenario::publish_rect("agent_chat:drop_hint", zone);
|
||||
}
|
||||
|
||||
/// New chat (when the chat has focus).
|
||||
pub const NEW_CHAT_SHORTCUT: egui::KeyboardShortcut =
|
||||
egui::KeyboardShortcut::new(egui::Modifiers::COMMAND.plus(egui::Modifiers::SHIFT), egui::Key::L);
|
||||
@@ -431,14 +468,16 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
auth_box(app, ui);
|
||||
}
|
||||
ui.separator();
|
||||
// The input sits at the bottom; the conversation takes the rest.
|
||||
let input_h = 92.0;
|
||||
// The input sits at the bottom (with a row per line of file chips); the conversation takes
|
||||
// the rest.
|
||||
let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0;
|
||||
let avail = (ui.available_height() - input_h).max(80.0);
|
||||
egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| {
|
||||
conversation(app, ui);
|
||||
});
|
||||
ui.separator();
|
||||
let busy = state == State::Busy;
|
||||
attachment_chips(app, ui);
|
||||
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
|
||||
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
|
||||
.desired_rows(2)
|
||||
@@ -466,9 +505,18 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
cancel(app);
|
||||
}
|
||||
ui.horizontal(|ui| {
|
||||
let send_b = ui.button(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" }));
|
||||
add_menu(app, ui);
|
||||
// A message waits for files still being read.
|
||||
let reading = !app.agent_chat.loading.is_empty();
|
||||
let send_b = ui
|
||||
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
|
||||
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
|
||||
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
|
||||
if (send_b.clicked() || enter || steer) && !app.agent_chat.input.trim().is_empty() {
|
||||
// Files alone make a message.
|
||||
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
|
||||
if reading {
|
||||
// Enter waits with the button.
|
||||
} else if (send_b.clicked() || enter || steer) && something {
|
||||
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
|
||||
send(app, ui.ctx(), text);
|
||||
if steer {
|
||||
@@ -744,14 +792,17 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
let current_doc = app.session.doc.clone();
|
||||
for (n, item) in app.agent_chat.items.iter().enumerate() {
|
||||
match item {
|
||||
Item::User { text, image } => {
|
||||
Item::User { text, image, files } => {
|
||||
ui.add_space(6.0);
|
||||
egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| {
|
||||
ui.set_width(ui.available_width());
|
||||
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
|
||||
if !text.is_empty() {
|
||||
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
|
||||
}
|
||||
if *image {
|
||||
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
|
||||
}
|
||||
files_line(ui, files.iter().map(String::as_str));
|
||||
});
|
||||
}
|
||||
Item::Agent { thought, text } => {
|
||||
@@ -900,10 +951,13 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap());
|
||||
if !p.text.is_empty() {
|
||||
ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap());
|
||||
}
|
||||
if p.image {
|
||||
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
|
||||
}
|
||||
files_line(ui, p.files.iter().map(|f| f.name.as_str()));
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -960,6 +1014,199 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
}
|
||||
}
|
||||
|
||||
/// The files that went (or will go) with a message, by name.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator<Item = &'a str>) {
|
||||
let list = names.collect::<Vec<_>>().join(", ");
|
||||
if !list.is_empty() {
|
||||
let t = Tokens::get();
|
||||
ui.add(egui::Label::new(RichText::new(trf("ui.agent_chat_files", &[("files", &list)])).color(t.text_dim).size(11.0)).wrap());
|
||||
}
|
||||
}
|
||||
|
||||
/// The widest a file chip gets; a narrower chat makes them narrower.
|
||||
const CHIP_W: f32 = 170.0;
|
||||
|
||||
/// How many rows `n` chips take in a chat `avail` wide (`gap` between them).
|
||||
fn chip_rows(n: usize, avail: f32, gap: f32) -> usize {
|
||||
let avail = avail.max(1.0);
|
||||
let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1);
|
||||
n.div_ceil(per_row)
|
||||
}
|
||||
|
||||
/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or
|
||||
/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui
|
||||
/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn attachment_chips(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
let chat = &mut app.agent_chat;
|
||||
// Thumbnails of files no longer on the message are let go.
|
||||
let keep: Vec<u64> = chat.attachments.iter().filter(|f| f.thumb.is_some()).map(|f| f.id).collect();
|
||||
chat.thumbs.retain(|k, _| keep.contains(k));
|
||||
if chat.files() == 0 {
|
||||
return;
|
||||
}
|
||||
let gap = ui.spacing().item_spacing.x;
|
||||
let avail = ui.available_width().max(1.0);
|
||||
let w = CHIP_W.min(avail);
|
||||
let per_row = (((avail + gap) / (w + gap)).floor() as usize).max(1);
|
||||
let (mut remove, mut stop) = (None, None);
|
||||
// The files read, then those still being read (each takes its place once it is done).
|
||||
let read = chat.attachments.len();
|
||||
let indices: Vec<usize> = (0..chat.files()).collect();
|
||||
for row in indices.chunks(per_row) {
|
||||
ui.horizontal(|ui| {
|
||||
for &i in row {
|
||||
if let Some(f) = chat.attachments.get(i) {
|
||||
if chip(ui, &mut chat.thumbs, f, i, w) {
|
||||
remove = Some(i);
|
||||
}
|
||||
} else if let Some(l) = chat.loading.get(i - read)
|
||||
&& loading_chip(ui, &l.name, l.importing(), i - read, w)
|
||||
{
|
||||
stop = Some(i - read);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if let Some(i) = remove
|
||||
&& i < chat.attachments.len()
|
||||
{
|
||||
chat.attachments.remove(i);
|
||||
}
|
||||
// Taken off while it is read: its reader's answer is dropped with it.
|
||||
if let Some(i) = stop
|
||||
&& i < chat.loading.len()
|
||||
{
|
||||
chat.loading.remove(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// What a chip's × says: an imported part stays in the design when it comes off the message.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(super) fn remove_tip(f: &attach::Attachment) -> &'static str {
|
||||
tr(if matches!(f.content, attach::Content::Imported) { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" })
|
||||
}
|
||||
|
||||
/// The chip of a file still being read or imported: a spinner, the name, and a × that takes it
|
||||
/// off (an import goes on: its part lands in the design).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn loading_chip(ui: &mut egui::Ui, name: &str, importing: bool, i: usize, w: f32) -> bool {
|
||||
let t = Tokens::get();
|
||||
let mut clicked = false;
|
||||
let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show(
|
||||
ui,
|
||||
|ui| {
|
||||
let inner = (w - 10.0).max(40.0);
|
||||
ui.set_width(inner);
|
||||
ui.horizontal(|ui| {
|
||||
ui.spacing_mut().item_spacing.x = 4.0;
|
||||
let doing = tr(if importing { "ui.agent_chat_file_importing" } else { "ui.agent_chat_file_loading" });
|
||||
let mark = ui.add(egui::Spinner::new().size(16.0)).on_hover_text(doing).rect.width();
|
||||
let name_w = (inner - mark - 18.0 - 8.0).max(8.0);
|
||||
ui.allocate_ui(vec2(name_w, 20.0), |ui| {
|
||||
ui.set_width(name_w);
|
||||
ui.add(egui::Label::new(RichText::new(name).size(11.5).color(t.text_dim)).truncate())
|
||||
})
|
||||
.inner
|
||||
.on_hover_text(name);
|
||||
let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click());
|
||||
let c = r.center();
|
||||
let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim });
|
||||
ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s);
|
||||
ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s);
|
||||
crate::scenario::publish_handle(&format!("agent_chat:file_loading_remove:{i}"), c);
|
||||
clicked = x.on_hover_text(tr(if importing { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" })).clicked();
|
||||
});
|
||||
},
|
||||
);
|
||||
crate::scenario::publish_handle(&format!("agent_chat:file_loading:{i}"), r.response.rect.center());
|
||||
clicked
|
||||
}
|
||||
|
||||
/// One file chip, `w` wide in all; true when its × was clicked.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap<u64, egui::TextureHandle>, f: &attach::Attachment, i: usize, w: f32) -> bool {
|
||||
let t = Tokens::get();
|
||||
let mut clicked = false;
|
||||
let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show(
|
||||
ui,
|
||||
|ui| {
|
||||
let inner = (w - 10.0).max(40.0);
|
||||
ui.set_width(inner);
|
||||
ui.horizontal(|ui| {
|
||||
ui.spacing_mut().item_spacing.x = 4.0;
|
||||
let mark = match &f.thumb {
|
||||
Some(img) => {
|
||||
let tex = thumbs
|
||||
.entry(f.id)
|
||||
.or_insert_with(|| ui.ctx().load_texture(format!("sc_attach_{}", f.id), (**img).clone(), egui::TextureOptions::LINEAR));
|
||||
let size = tex.size_vec2();
|
||||
ui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width()
|
||||
}
|
||||
None => {
|
||||
let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover());
|
||||
ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside);
|
||||
r.width()
|
||||
}
|
||||
};
|
||||
let name_w = (inner - mark - 18.0 - 8.0).max(8.0);
|
||||
ui.allocate_ui(vec2(name_w, 20.0), |ui| {
|
||||
ui.set_width(name_w);
|
||||
ui.add(egui::Label::new(RichText::new(&f.name).size(11.5)).truncate())
|
||||
})
|
||||
.inner
|
||||
.on_hover_text(&f.name);
|
||||
let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click());
|
||||
let c = r.center();
|
||||
let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim });
|
||||
ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s);
|
||||
ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s);
|
||||
crate::scenario::publish_handle(&format!("agent_chat:file_remove:{i}"), c);
|
||||
clicked = x.on_hover_text(remove_tip(f)).clicked();
|
||||
});
|
||||
},
|
||||
);
|
||||
crate::scenario::publish_rect(&format!("agent_chat:file:{i}"), r.response.rect);
|
||||
crate::scenario::publish_handle(&format!("agent_chat:file:{i}"), r.response.rect.center());
|
||||
clicked
|
||||
}
|
||||
|
||||
/// The + button: choose a file with the app's picker (pictures, text and CAD files), or paste
|
||||
/// the clipboard's picture.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn add_menu(app: &mut SolveApp, ui: &mut egui::Ui) {
|
||||
let (mut choose, mut paste) = (false, false);
|
||||
let r = ui.menu_button(RichText::new("+").strong().size(15.0), |ui| {
|
||||
let a = ui.button(tr("ui.agent_chat_add_file"));
|
||||
crate::scenario::publish_handle("agent_chat:add_file", a.rect.center());
|
||||
let b = ui.button(tr("ui.agent_chat_paste_picture"));
|
||||
crate::scenario::publish_handle("agent_chat:paste_picture", b.rect.center());
|
||||
if a.clicked() {
|
||||
choose = true;
|
||||
ui.close();
|
||||
}
|
||||
if b.clicked() {
|
||||
paste = true;
|
||||
ui.close();
|
||||
}
|
||||
});
|
||||
crate::scenario::publish_handle("agent_chat:add", r.response.rect.center());
|
||||
r.response.on_hover_text(tr("ui.agent_chat_add"));
|
||||
if choose {
|
||||
app.pick_file(tr("ui.agent_chat_attachable"), &attach::extensions(), |app, path| {
|
||||
if let Some(path) = path
|
||||
&& let Err(e) = attach_path(app, &path)
|
||||
{
|
||||
app.agent_chat.notice(e, true);
|
||||
}
|
||||
});
|
||||
}
|
||||
if paste && let Err(e) = paste_picture(app) {
|
||||
app.agent_chat.notice(e, true);
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool call as a card: its state, name and kind on one line; a click unfolds what it returned.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn tool_card(ui: &mut egui::Ui, n: usize, title: &str, kind: &str, status: Option<ToolStatus>, output: Option<&str>) {
|
||||
|
||||
@@ -26,18 +26,20 @@ fn streamed_text_joins_and_tool_updates_change_their_call() {
|
||||
#[test]
|
||||
fn a_sign_in_request_keeps_the_question_for_after_the_login() {
|
||||
let mut c = chat();
|
||||
c.last = Some(("make a cube".into(), false));
|
||||
c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())]));
|
||||
c.state = Some(State::Busy);
|
||||
assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some());
|
||||
c.on_event(Event::AuthRequired { methods: vec![] });
|
||||
assert_eq!(c.state(), State::NeedsAuth);
|
||||
assert_eq!(c.queued, Some(("make a cube".into(), false)));
|
||||
let (text, image, files) = c.queued.clone().expect("queued for after the login");
|
||||
assert_eq!((text.as_str(), image), ("make a cube", false));
|
||||
assert_eq!(files.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), ["motor.step"], "its files go again too");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plan_update_replaces_the_turns_plan() {
|
||||
let mut c = chat();
|
||||
c.push(Item::User { text: "go".into(), image: false });
|
||||
c.push(Item::User { text: "go".into(), image: false, files: Vec::new() });
|
||||
c.on_event(Event::Plan(vec![("a".into(), "pending".into())]));
|
||||
c.on_event(Event::Text { thought: false, text: "working".into() });
|
||||
c.on_event(Event::Plan(vec![("a".into(), "completed".into())]));
|
||||
@@ -66,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender<ControlRequest>) {
|
||||
let (tx, rx) = channel();
|
||||
app.agent_chat.tools_rx = Some(rx);
|
||||
app.agent_chat.state = Some(State::Busy);
|
||||
start_turn(&mut app, "two boxes".into(), false);
|
||||
start_turn(&mut app, "two boxes".into(), false, Vec::new());
|
||||
(app, tx)
|
||||
}
|
||||
|
||||
@@ -416,12 +418,17 @@ use std::sync::Mutex;
|
||||
struct Script {
|
||||
calls: Arc<Mutex<Vec<String>>>,
|
||||
events: Arc<Mutex<Vec<Event>>>,
|
||||
/// Every prompt in full (its text and pictures).
|
||||
prompts: Arc<Mutex<Vec<Vec<PromptPart>>>>,
|
||||
}
|
||||
|
||||
impl Script {
|
||||
fn calls(&self) -> Vec<String> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
fn prompts(&self) -> Vec<Vec<PromptPart>> {
|
||||
self.prompts.lock().unwrap().clone()
|
||||
}
|
||||
fn report(&self, e: Event) {
|
||||
self.events.lock().unwrap().push(e);
|
||||
}
|
||||
@@ -434,6 +441,7 @@ impl ChatBackend for Script {
|
||||
fn prompt(&mut self, parts: &[PromptPart]) {
|
||||
let text = parts.iter().find_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).unwrap_or_default();
|
||||
self.calls.lock().unwrap().push(format!("prompt: {text}"));
|
||||
self.prompts.lock().unwrap().push(parts.to_vec());
|
||||
}
|
||||
fn cancel(&mut self) {
|
||||
self.calls.lock().unwrap().push("cancel".into());
|
||||
@@ -860,6 +868,500 @@ done
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
// ---- Files attached to a message ----
|
||||
|
||||
/// A folder of files to attach: a picture and a text file.
|
||||
fn attachable(tag: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("solidcraft-chat-files-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
image::RgbaImage::from_pixel(40, 30, image::Rgba([250, 250, 250, 255])).save(d.join("target.png")).unwrap();
|
||||
std::fs::write(d.join("brief.md"), "wall 2 mm, Pi 5 inside\n").unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
fn at(d: &std::path::Path, f: &str) -> String {
|
||||
d.join(f).to_string_lossy().to_string()
|
||||
}
|
||||
|
||||
/// Wait for the files read off the window thread to join the message.
|
||||
fn settle_files(app: &mut SolveApp, ctx: &egui::Context) {
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
|
||||
while !app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline {
|
||||
take_loaded(app, ctx);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
take_loaded(app, ctx);
|
||||
}
|
||||
|
||||
/// The same with the whole app running frames.
|
||||
fn settle_chat(h: &mut crate::scenario::Harness) {
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
|
||||
while !h.app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline {
|
||||
h.frames(1);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
}
|
||||
h.frames(2);
|
||||
}
|
||||
|
||||
fn text_of(parts: &[PromptPart]) -> String {
|
||||
parts.iter().filter_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).collect()
|
||||
}
|
||||
|
||||
fn pictures_in(parts: &[PromptPart]) -> usize {
|
||||
parts.iter().filter(|p| matches!(p, PromptPart::Image { .. })).count()
|
||||
}
|
||||
|
||||
/// Files attached to a message go with it: the pictures as images, the text files in its text,
|
||||
/// and every file by name; the message lists them in the conversation.
|
||||
#[test]
|
||||
fn a_message_carries_its_files() {
|
||||
let d = attachable("carry");
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
app.agent_chat.images = true;
|
||||
control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap();
|
||||
control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap();
|
||||
settle_files(&mut app, &ctx);
|
||||
assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"]));
|
||||
assert!(control(&mut app, &ctx, &json!({"attach": at(&d, "missing.png")})).is_err());
|
||||
send(&mut app, &ctx, "make the case like this".into());
|
||||
let sent = script.prompts();
|
||||
assert_eq!(sent.len(), 1);
|
||||
let text = text_of(&sent[0]);
|
||||
assert!(text.starts_with("make the case like this"), "{text}");
|
||||
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm, Pi 5 inside"), "{text}");
|
||||
assert_eq!(sent[0].iter().filter(|p| matches!(p, PromptPart::Image { mime, .. } if mime == "image/png")).count(), 1);
|
||||
assert!(app.agent_chat.attachments.is_empty(), "the files went with the message");
|
||||
assert_eq!(status(&app)["items"][0]["files"], json!(["target.png", "brief.md"]));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: a picture was decoded on the window thread, so a large one froze the window.
|
||||
/// It is read off it now: the message waits for it (Send waits too), the files keep the order they
|
||||
/// were attached in, and a refused picture says why later, in the app's language.
|
||||
#[test]
|
||||
fn pictures_are_read_off_the_window_thread() {
|
||||
let d = attachable("worker");
|
||||
std::fs::write(d.join("fake.png"), b"not a picture at all").unwrap();
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
app.agent_chat.images = true;
|
||||
control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap();
|
||||
control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap();
|
||||
assert_eq!(status(&app)["loading"], json!(["target.png"]), "the picture is still being read");
|
||||
send(&mut app, &ctx, "like this".into());
|
||||
assert!(script.prompts().is_empty(), "Send waits for the files");
|
||||
assert_eq!(app.agent_chat.input, "like this", "and keeps the words");
|
||||
settle_files(&mut app, &ctx);
|
||||
assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"]), "in the order they were attached");
|
||||
let words = std::mem::take(&mut app.agent_chat.input);
|
||||
send(&mut app, &ctx, words);
|
||||
assert_eq!(script.prompts().len(), 1);
|
||||
{
|
||||
let _ja = crate::i18n::scope("ja");
|
||||
control(&mut app, &ctx, &json!({"attach": at(&d, "fake.png")})).unwrap();
|
||||
}
|
||||
settle_files(&mut app, &ctx);
|
||||
assert!(
|
||||
matches!(app.agent_chat.items.last(), Some(Item::Notice { text, error: true }) if text == "fake.png: 読み込めない画像です"),
|
||||
"a refused picture says why, in the language it was attached in"
|
||||
);
|
||||
assert!(app.agent_chat.attachments.is_empty());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: the prompt cap applied to the typed text only, so eight 256 KB text files
|
||||
/// took a message to about 2 MB. The whole message stays within it now, every file still named.
|
||||
#[test]
|
||||
fn a_message_with_large_text_files_stays_within_the_prompt_cap() {
|
||||
let d = attachable("cap");
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
for i in 0..attach::MAX_FILES {
|
||||
let path = d.join(format!("notes{i}.txt"));
|
||||
std::fs::write(&path, "y".repeat(250 * 1024)).unwrap();
|
||||
control(&mut app, &ctx, &json!({"attach": path.to_string_lossy()})).unwrap();
|
||||
}
|
||||
send(&mut app, &ctx, "read these".into());
|
||||
let text = text_of(&script.prompts()[0]);
|
||||
assert!(text.chars().count() <= MAX_PROMPT, "{} characters", text.chars().count());
|
||||
assert!((0..attach::MAX_FILES).all(|i| text.contains(&format!("`notes{i}.txt`"))), "every file is named");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// A message sent before the agent is ready keeps its files until it goes; an agent that takes
|
||||
/// no pictures gets none, and the message says which were left out. Files alone make a message,
|
||||
/// and its undo step is named after the first file.
|
||||
#[test]
|
||||
fn files_wait_with_their_message_and_pictures_skip_an_agent_that_takes_none() {
|
||||
let d = attachable("wait");
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
app.agent_chat.state = Some(State::Starting);
|
||||
attach_path(&mut app, &at(&d, "target.png")).unwrap();
|
||||
settle_files(&mut app, &ctx);
|
||||
send(&mut app, &ctx, " ".into());
|
||||
assert!(script.prompts().is_empty(), "not sent before the agent is ready");
|
||||
assert_eq!(status(&app)["items"][0]["user"], "", "no words, just the file");
|
||||
script.report(Event::Ready { agent: "Test".into(), images: false });
|
||||
poll(&mut app, &ctx);
|
||||
let sent = script.prompts();
|
||||
assert_eq!(sent.len(), 1, "sent once the agent is ready");
|
||||
assert_eq!(pictures_in(&sent[0]), 0, "no pictures to an agent that takes none");
|
||||
assert!(text_of(&sent[0]).contains("not sent"), "{}", text_of(&sent[0]));
|
||||
assert_eq!(app.agent_chat.turn.as_ref().map(|t| t.label.as_str()), Some("target.png"));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_most_eight_files_on_a_message() {
|
||||
let d = attachable("cap");
|
||||
let mut app = SolveApp::new(solidcraft_engine::Session::default(), crate::Services::default());
|
||||
for _ in 0..attach::MAX_FILES {
|
||||
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
|
||||
}
|
||||
assert!(attach_path(&mut app, &at(&d, "brief.md")).unwrap_err().contains('8'));
|
||||
assert!(attach_rgba(&mut app, 2, 2, &[9; 16]).is_err());
|
||||
assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES);
|
||||
let ctx = egui::Context::default();
|
||||
assert!(control(&mut app, &ctx, &json!({"remove_file": 8})).is_err(), "no ninth file to take off");
|
||||
control(&mut app, &ctx, &json!({"remove_file": 0})).unwrap();
|
||||
assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES - 1);
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// While the agent works, a message queues with its files: the queue shows them, Stop keeps
|
||||
/// them, and they go with the queue.
|
||||
#[test]
|
||||
fn a_queued_message_keeps_its_files_through_stop_and_sends_them() {
|
||||
let d = attachable("queue");
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
app.agent_chat.images = true;
|
||||
send(&mut app, &ctx, "make a box".into());
|
||||
attach_path(&mut app, &at(&d, "target.png")).unwrap();
|
||||
settle_files(&mut app, &ctx);
|
||||
send(&mut app, &ctx, "like this".into());
|
||||
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
|
||||
send(&mut app, &ctx, String::new());
|
||||
assert!(app.agent_chat.attachments.is_empty(), "the files moved to the queue");
|
||||
assert_eq!(status(&app)["pending"][0]["files"], json!(["target.png"]));
|
||||
assert_eq!(status(&app)["pending"][1]["files"], json!(["brief.md"]));
|
||||
cancel(&mut app);
|
||||
ended(&script, "cancelled");
|
||||
poll(&mut app, &ctx);
|
||||
assert_eq!(app.agent_chat.pending.len(), 2, "Stop keeps the queue and its files");
|
||||
send_now(&mut app, &ctx);
|
||||
let sent = script.prompts();
|
||||
let last = sent.last().expect("the queue went");
|
||||
let text = text_of(last);
|
||||
assert!(text.starts_with("like this") && !text.contains("\n\n\n\n"), "a message of files alone adds no empty paragraph: {text:?}");
|
||||
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm"), "{text}");
|
||||
assert_eq!(pictures_in(last), 1);
|
||||
assert_eq!(status(&app)["items"].as_array().and_then(|i| i.last()).map(|i| i["files"].clone()), Some(json!(["target.png", "brief.md"])));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Queued messages go as one prompt with at most eight files; the rest are named, not sent.
|
||||
#[test]
|
||||
fn the_queue_sends_at_most_eight_files_and_names_the_rest() {
|
||||
let d = attachable("queue-cap");
|
||||
let (mut app, script, ctx) = app_with_script();
|
||||
send(&mut app, &ctx, "first".into());
|
||||
for n in 0..3 {
|
||||
for _ in 0..4 {
|
||||
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
|
||||
}
|
||||
send(&mut app, &ctx, format!("batch {n}"));
|
||||
}
|
||||
ended(&script, "");
|
||||
poll(&mut app, &ctx);
|
||||
let sent = script.prompts();
|
||||
assert_eq!(sent.len(), 2);
|
||||
assert_eq!(text_of(&sent[1]).matches("Attached file `brief.md`").count(), attach::MAX_FILES);
|
||||
let told = app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, .. } if text.contains("brief.md") && text.contains('8')));
|
||||
assert!(told, "the files left out are named");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// New chat clears the files waiting on the chips too, and asks first when there are some.
|
||||
#[test]
|
||||
fn new_chat_clears_the_files_waiting_and_asks_first() {
|
||||
let d = attachable("new");
|
||||
let (mut app, _script, _ctx) = app_with_script();
|
||||
assert!(!new_chat_needs_confirm(&app));
|
||||
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
|
||||
assert!(new_chat_needs_confirm(&app), "files would be lost");
|
||||
new_chat(&mut app);
|
||||
assert!(app.agent_chat.attachments.is_empty());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// A harness with the chat open at its narrowest and nothing to look for.
|
||||
fn narrow_chat() -> crate::scenario::Harness {
|
||||
let mut h = crate::scenario::Harness::new();
|
||||
h.app.preferences.agent_chat_width = panel::MIN_WIDTH;
|
||||
h.app.agent_chat.open = true;
|
||||
h.app.agent_chat.detection = Some(Default::default());
|
||||
h
|
||||
}
|
||||
|
||||
/// Every chip and the text box sit inside the panel.
|
||||
fn assert_inside_the_panel(chips: usize) {
|
||||
use crate::scenario::rect_of;
|
||||
let panel = rect_of("agent_chat:panel").expect("the panel");
|
||||
let mut rects: Vec<(String, egui::Rect)> =
|
||||
(0..chips).map(|i| (format!("file:{i}"), rect_of(&format!("agent_chat:file:{i}")).expect("a chip"))).collect();
|
||||
rects.push(("input".into(), rect_of("agent_chat:input").expect("the text box")));
|
||||
for (name, r) in rects {
|
||||
assert!(r.left() >= panel.left() && r.right() <= panel.right(), "{name} {r:?} in {panel:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Files for the next message show as chips above the text box (a picture's thumbnail, the name,
|
||||
/// a × that takes it off), inside the chat, and Send sends files with no text.
|
||||
#[test]
|
||||
fn attached_files_show_as_chips_that_can_be_taken_off_and_sent_alone() {
|
||||
use crate::scenario::handle_at;
|
||||
let d = attachable("chips");
|
||||
let mut h = narrow_chat();
|
||||
let script = Script::default();
|
||||
h.app.agent_chat.backend = Some(Box::new(script.clone()));
|
||||
h.app.agent_chat.state = Some(State::Ready);
|
||||
h.app.agent_chat.images = true;
|
||||
h.frames(2);
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "brief.md")}));
|
||||
settle_chat(&mut h);
|
||||
assert!(handle_at("agent_chat:file:0").is_some() && handle_at("agent_chat:file:1").is_some(), "a chip per file");
|
||||
assert_inside_the_panel(2);
|
||||
let x = handle_at("agent_chat:file_remove:0").expect("a × on the chip");
|
||||
h.call("ui.click", json!({"x": x.x, "y": x.y}));
|
||||
h.frames(2);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["brief.md"]));
|
||||
let send = handle_at("agent_chat:send").expect("Send");
|
||||
h.call("ui.click", json!({"x": send.x, "y": send.y}));
|
||||
h.frames(2);
|
||||
assert_eq!(script.prompts().len(), 1, "files alone make a message");
|
||||
assert!(h.app.agent_chat.attachments.is_empty() && handle_at("agent_chat:file:0").is_none());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: chips' thumbnails were cached by the picture's address, so a new picture at a
|
||||
/// freed address could show an old one. Each file has an id of its own, never reused, and the
|
||||
/// cache follows the files on the message.
|
||||
#[test]
|
||||
fn each_file_has_its_own_id_and_thumbnail() {
|
||||
let d = attachable("ids");
|
||||
image::RgbaImage::from_pixel(20, 20, image::Rgba([10, 200, 10, 255])).save(d.join("green.png")).unwrap();
|
||||
let mut h = narrow_chat();
|
||||
h.frames(2);
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "green.png")}));
|
||||
settle_chat(&mut h);
|
||||
let first: Vec<u64> = h.app.agent_chat.attachments.iter().map(|f| f.id).collect();
|
||||
assert!(first.len() == 2 && first[0] != first[1], "{first:?}");
|
||||
let mut cached: Vec<u64> = h.app.agent_chat.thumbs.keys().copied().collect();
|
||||
cached.sort_unstable();
|
||||
assert_eq!(cached, first, "a thumbnail per file, by its id");
|
||||
h.call("ui.agent_chat", json!({"remove_file": 0}));
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
|
||||
settle_chat(&mut h);
|
||||
let now: Vec<u64> = h.app.agent_chat.attachments.iter().map(|f| f.id).collect();
|
||||
assert!(!now.contains(&first[0]), "a removed file's id is not used again: {now:?}");
|
||||
let mut cached: Vec<u64> = h.app.agent_chat.thumbs.keys().copied().collect();
|
||||
cached.sort_unstable();
|
||||
let mut want = now.clone();
|
||||
want.sort_unstable();
|
||||
assert_eq!(cached, want, "the cache follows the files on the message");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// The + button: "Choose a file…" asks the app's file picker for the types that can be attached
|
||||
/// (not the design types Open shows) and attaches what it answers; "Paste a picture" attaches
|
||||
/// the clipboard's picture (here given as RGBA, as the system does).
|
||||
#[test]
|
||||
fn the_plus_button_attaches_a_chosen_file_or_a_pasted_picture() {
|
||||
use crate::scenario::handle_at;
|
||||
let d = attachable("plus");
|
||||
let mut h = narrow_chat();
|
||||
let chosen = at(&d, "brief.md");
|
||||
let asked = std::rc::Rc::new(std::cell::RefCell::new(Vec::<String>::new()));
|
||||
let seen = asked.clone();
|
||||
h.app.services.pick_file = Some(Box::new(move |_label: &str, exts: &[&str]| {
|
||||
*seen.borrow_mut() = exts.iter().map(|e| e.to_string()).collect();
|
||||
let p = chosen.clone();
|
||||
Box::pin(async move { Some(p) })
|
||||
}));
|
||||
h.frames(2);
|
||||
let plus = handle_at("agent_chat:add").expect("a + button");
|
||||
h.call("ui.click", json!({"x": plus.x, "y": plus.y}));
|
||||
h.frames(2);
|
||||
let choose = handle_at("agent_chat:add_file").expect("Choose a file… in its menu");
|
||||
assert!(handle_at("agent_chat:paste_picture").is_some(), "Paste a picture in its menu");
|
||||
h.call("ui.click", json!({"x": choose.x, "y": choose.y}));
|
||||
h.frames(4);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["brief.md"]));
|
||||
let exts = asked.borrow().clone();
|
||||
for ext in ["png", "jpg", "gif", "webp", "svg", "md", "step", "stl"] {
|
||||
assert!(exts.iter().any(|e| e == ext), "{ext} in {exts:?}");
|
||||
}
|
||||
assert!(!exts.iter().any(|e| e == "solidcraft"));
|
||||
attach_rgba(&mut h.app, 4, 3, &[200; 48]).unwrap();
|
||||
attach_rgba(&mut h.app, 4, 3, &[200; 47]).unwrap();
|
||||
settle_chat(&mut h);
|
||||
assert!(
|
||||
h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("pasted picture 2.png:"))),
|
||||
"not a picture: it says so once it has been read"
|
||||
);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["brief.md", "pasted picture 1.png"]));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DroppedPath(std::path::PathBuf);
|
||||
|
||||
impl egui::DroppedFile for DroppedPath {
|
||||
fn path(&self) -> &std::path::Path {
|
||||
&self.0
|
||||
}
|
||||
fn bytes(&self) -> Result<Vec<u8>, String> {
|
||||
std::fs::read(&self.0).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn drop_files(h: &mut crate::scenario::Harness, at: egui::Pos2, files: &[std::path::PathBuf]) {
|
||||
h.drop_files(at, files.iter().map(|p| Arc::new(DroppedPath(p.clone())) as egui::DroppedFileHandle).collect());
|
||||
}
|
||||
|
||||
/// A tetrahedron, as a small ASCII STL.
|
||||
const TETRA_STL: &str = "solid t
|
||||
facet normal 0 0 -1
|
||||
outer loop
|
||||
vertex 0 0 0
|
||||
vertex 0 10 0
|
||||
vertex 10 0 0
|
||||
endloop
|
||||
endfacet
|
||||
facet normal 0 -1 0
|
||||
outer loop
|
||||
vertex 0 0 0
|
||||
vertex 10 0 0
|
||||
vertex 0 0 10
|
||||
endloop
|
||||
endfacet
|
||||
facet normal -1 0 0
|
||||
outer loop
|
||||
vertex 0 0 0
|
||||
vertex 0 0 10
|
||||
vertex 0 10 0
|
||||
endloop
|
||||
endfacet
|
||||
facet normal 1 1 1
|
||||
outer loop
|
||||
vertex 10 0 0
|
||||
vertex 0 10 0
|
||||
vertex 0 0 10
|
||||
endloop
|
||||
endfacet
|
||||
endsolid t
|
||||
";
|
||||
|
||||
/// Review of #1744: a CAD file's import wasn't waited on, so a failed or queued-out import still
|
||||
/// told the agent "I added x to the design". The file joins the message once its import is in;
|
||||
/// a failed import says why and the agent is told nothing of it.
|
||||
#[test]
|
||||
fn a_cad_file_joins_the_message_once_its_import_is_in() {
|
||||
let d = attachable("cad");
|
||||
std::fs::write(d.join("block.stl"), TETRA_STL).unwrap();
|
||||
std::fs::write(d.join("broken.stl"), [0u8, 1, 2]).unwrap();
|
||||
let mut h = narrow_chat();
|
||||
let script = Script::default();
|
||||
h.app.agent_chat.backend = Some(Box::new(script.clone()));
|
||||
h.app.agent_chat.state = Some(State::Ready);
|
||||
h.frames(2);
|
||||
attach_path(&mut h.app, &at(&d, "block.stl")).unwrap();
|
||||
assert_eq!(status(&h.app)["loading"], json!(["block.stl"]), "importing, not yet on the message");
|
||||
assert_eq!(status(&h.app)["attachments"], json!([]));
|
||||
settle_chat(&mut h);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "in, and on the message");
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "broken.stl")}));
|
||||
settle_chat(&mut h);
|
||||
assert!(
|
||||
h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("broken.stl:"))),
|
||||
"a failed import says why"
|
||||
);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "and is not on the message");
|
||||
h.call("ui.agent_chat", json!({"send": "fit around this"}));
|
||||
h.frames(2);
|
||||
let text = text_of(&script.prompts()[0]);
|
||||
assert!(text.contains("`block.stl` to the design") && !text.contains("broken.stl"), "{text}");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Review of #1744: taking an imported part's chip off leaves the part in the design, and its ×
|
||||
/// says so; any other file's × just takes it off.
|
||||
#[test]
|
||||
fn an_imported_parts_chip_says_the_part_stays() {
|
||||
assert_eq!(panel::remove_tip(&attach::Attachment::imported("motor.step".into())), tr("ui.agent_chat_remove_imported"));
|
||||
assert!(tr("ui.agent_chat_remove_imported").contains("stays in the design"));
|
||||
let picture = attach::from_rgba("a.png".into(), 2, 2, &[9; 16]).unwrap();
|
||||
assert_eq!(panel::remove_tip(&picture), tr("ui.agent_chat_remove_file"));
|
||||
}
|
||||
|
||||
/// Files dropped on the chat attach to the next message, and the app doesn't open them as well
|
||||
/// (while they are dragged over it, the chat says so); dropped on the 3D view they are not the
|
||||
/// chat's (the app handles them as before), nor when the chat is closed.
|
||||
#[test]
|
||||
fn files_dropped_on_the_chat_attach_and_the_app_does_not_open_them() {
|
||||
use crate::scenario::rect_of;
|
||||
let d = attachable("drop");
|
||||
let mut h = narrow_chat();
|
||||
h.call("ui.resize", json!({"width": 1600, "height": 1000}));
|
||||
h.frames(3);
|
||||
let panel = rect_of("agent_chat:panel").expect("docked");
|
||||
// While files are dragged over the chat, it says they will attach; over the 3D view, not.
|
||||
let hovered = || vec![egui::HoveredFile { path: Some(d.join("target.png")), mime: String::new() }];
|
||||
h.drag_files(panel.center(), hovered(), Vec::new());
|
||||
assert!(rect_of("agent_chat:drop_hint").is_some_and(|r| panel.contains(r.center())), "the hint over the chat");
|
||||
h.drag_files(egui::pos2(300.0, 500.0), hovered(), Vec::new());
|
||||
assert!(rect_of("agent_chat:drop_hint").is_none(), "no hint over the 3D view");
|
||||
drop_files(&mut h, panel.center(), &[d.join("target.png"), d.join("brief.md")]);
|
||||
settle_chat(&mut h);
|
||||
assert_eq!(status(&h.app)["attachments"], json!(["target.png", "brief.md"]));
|
||||
assert!(!h.app.file_load.busy(), "the app doesn't open them as well");
|
||||
drop_files(&mut h, egui::pos2(300.0, 500.0), &[d.join("brief.md")]);
|
||||
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(2), "over the 3D view: not the chat's");
|
||||
// Folded to its strip, a drop on the strip is the chat's too.
|
||||
h.call("ui.agent_chat", json!({"minimized": true}));
|
||||
h.frames(3);
|
||||
let strip = h.app.agent_chat.drop_rect.expect("the strip takes drops");
|
||||
assert!(strip.width() < 60.0, "{strip:?}");
|
||||
drop_files(&mut h, strip.center(), &[d.join("brief.md")]);
|
||||
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3));
|
||||
// Closed: nothing is the chat's, wherever the pointer was.
|
||||
h.app.agent_chat.open = false;
|
||||
h.frames(2);
|
||||
assert!(h.app.agent_chat.drop_rect.is_none());
|
||||
drop_files(&mut h, strip.center(), &[d.join("brief.md")]);
|
||||
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// Many files and a long message in a narrow chat: the chips wrap onto more rows and the text box
|
||||
/// keeps to the chat (a row of chips wider than the chat would widen everything under it).
|
||||
#[test]
|
||||
fn chips_and_a_long_message_wrap_inside_a_narrow_chat() {
|
||||
use crate::scenario::handle_at;
|
||||
let d = attachable("wrap");
|
||||
let mut h = narrow_chat();
|
||||
for _ in 0..6 {
|
||||
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
|
||||
}
|
||||
h.app.agent_chat.input = "a separate design next to the robot body, the main version, with these measures for the second one".into();
|
||||
settle_chat(&mut h);
|
||||
assert_inside_the_panel(6);
|
||||
let (first, second) = (handle_at("agent_chat:file:0").unwrap(), handle_at("agent_chat:file:1").unwrap());
|
||||
assert!(second.y > first.y + 10.0, "at 280 px the second chip goes on the next row: {first:?} {second:?}");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
/// An agent's text is Markdown: a code block shows with its Copy button and a link is a button
|
||||
/// of its own, instead of the raw markers.
|
||||
#[test]
|
||||
|
||||
@@ -46,6 +46,21 @@ impl SolveApp {
|
||||
self.begin_picker(future, Box::new(complete));
|
||||
}
|
||||
|
||||
/// Ask for one file of a kind (`label`, `extensions`), with the host's dialog for it, or the
|
||||
/// Open dialog where the host has none. (The agent chat asks for its files; the web build has
|
||||
/// no chat.)
|
||||
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
|
||||
pub(crate) fn pick_file(&mut self, label: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option<String>) + 'static) {
|
||||
if self.file_picker_busy() {
|
||||
return;
|
||||
}
|
||||
let future = match self.services.pick_file.as_ref() {
|
||||
Some(pick) => Some(pick(label, extensions)),
|
||||
None => self.services.pick_open.as_ref().map(|pick| pick(extensions)),
|
||||
};
|
||||
self.begin_picker(future, Box::new(complete));
|
||||
}
|
||||
|
||||
pub(crate) fn pick_save(&mut self, name: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option<String>) + 'static) {
|
||||
if self.file_picker_busy() {
|
||||
return;
|
||||
|
||||
@@ -193,6 +193,11 @@ impl Drop for LanguageScope {
|
||||
}
|
||||
}
|
||||
|
||||
/// This thread's language, to carry into a worker thread with [`scope`].
|
||||
pub fn language() -> &'static str {
|
||||
CURRENT.with(|current| current.get().0.id)
|
||||
}
|
||||
|
||||
pub fn tr(id: &str) -> &str {
|
||||
CURRENT.with(|current| current.get().text(id))
|
||||
}
|
||||
|
||||
@@ -262,6 +262,10 @@ pub struct Services {
|
||||
/// the picker's first filter (`&[]`: designs, CAD files and meshes, #1801).
|
||||
pub pick_open: Option<Box<dyn Fn(&[&str]) -> FilePickerFuture>>,
|
||||
pub pick_save: Option<Box<dyn Fn(&str, &[&str]) -> FilePickerFuture>>,
|
||||
/// Open a dialog for one file of a kind (its name, its extensions): on macOS a dialog allows
|
||||
/// only the types it names, so the agent chat's files need a dialog of their own. `None`: the
|
||||
/// Open dialog.
|
||||
pub pick_file: Option<Box<dyn Fn(&str, &[&str]) -> FilePickerFuture>>,
|
||||
/// Open a dialog asking for a program (a slicer); `None` when the host has no such dialog.
|
||||
pub pick_program: Option<Box<dyn Fn() -> FilePickerFuture>>,
|
||||
pub show_in_folder: Option<Box<dyn Fn(&str)>>,
|
||||
@@ -1275,6 +1279,8 @@ impl SolveApp {
|
||||
recovery_ui::tick(self);
|
||||
update::tick(self, ctx);
|
||||
if self.issue_report.draft.is_none() && self.file_picker.is_none() {
|
||||
// Files dropped on the agent chat attach to its next message instead.
|
||||
agent_chat::take_drops(self, ctx);
|
||||
for f in ctx.input(|i| i.raw.dropped_files.clone()) {
|
||||
let p = f.path().to_string_lossy().to_string();
|
||||
if !p.is_empty() {
|
||||
|
||||
@@ -217,6 +217,34 @@ impl Harness {
|
||||
}
|
||||
}
|
||||
|
||||
/// One frame in which the platform drops `files`, the pointer last seen at `at` (a drop
|
||||
/// itself carries no position with winit 0.30).
|
||||
pub fn drop_files(&mut self, at: egui::Pos2, files: Vec<egui::DroppedFileHandle>) {
|
||||
self.drag_files(at, Vec::new(), files);
|
||||
}
|
||||
|
||||
/// One frame in which files are dragged over the window (`hovered`) or dropped, the pointer
|
||||
/// last seen at `at`.
|
||||
pub fn drag_files(&mut self, at: egui::Pos2, hovered: Vec<egui::HoveredFile>, dropped: Vec<egui::DroppedFileHandle>) {
|
||||
self.time += 1.0 / 60.0;
|
||||
let raw = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, self.size)),
|
||||
time: Some(self.time),
|
||||
focused: true,
|
||||
events: vec![egui::Event::PointerMoved(at)],
|
||||
hovered_files: hovered,
|
||||
dropped_files: dropped,
|
||||
..Default::default()
|
||||
};
|
||||
let app = &mut self.app;
|
||||
let mut out = self.ctx.run_ui(raw, |ui| {
|
||||
app.logic(ui.ctx());
|
||||
app.ui(ui);
|
||||
});
|
||||
out.textures_delta.clear();
|
||||
self.output = out.platform_output;
|
||||
}
|
||||
|
||||
/// Frames until the queued input is used up (and the view has settled).
|
||||
pub fn settle(&mut self) {
|
||||
for _ in 0..2000 {
|
||||
|
||||
+39
-3
@@ -53,6 +53,42 @@ when you send a message or press **Connect**.
|
||||
endpoint's agent also keeps it as its default model. An agent that reports no models shows no
|
||||
drop-down.
|
||||
|
||||
## Files and pictures
|
||||
|
||||
A message can carry up to eight files, to show the agent what you mean: a concept picture, a
|
||||
sketch, a brief, a part to fit around.
|
||||
|
||||
- **+** next to **Send** has **Choose a file…** (a file dialog for the types below) and **Paste a
|
||||
picture** (the picture on the clipboard).
|
||||
- **Drop files on the chat.** Files dropped on the panel (or its folded strip) attach to the
|
||||
next message, and the app doesn't open them too. While files are dragged over the chat, it says
|
||||
so. The window gives no position while something is dragged, so the drop is the chat's when the
|
||||
pointer was last seen over the chat; **+** always works.
|
||||
- The files show as chips above the text box: a picture's thumbnail, the name, and × to take the
|
||||
file off. A picture is read off the window thread, so a large one never stops the window: its
|
||||
chip shows a spinner meanwhile, and **Send** waits for it. **Send** sends files with no text.
|
||||
- **Pictures** (PNG, JPEG, GIF, WebP) go to the agent as they are when they are at most 2048 px
|
||||
on the long side and 3.75 MB (a picture's base64 then stays under 5 MB, so eight still fit in a
|
||||
prompt). Otherwise they are scaled down to 2048 px and re-encoded: as PNG when that fits, as
|
||||
JPEG when the picture has no transparency, halved until it fits; a GIF goes as its first frame.
|
||||
BMP and TIFF go as PNG. HEIC can't be read: export it as JPEG or PNG first. An agent that takes
|
||||
no pictures gets none, and the message says which were left out.
|
||||
- **Text files** (UTF-8, up to 256 KB: Markdown, plain text, JSON, CSV, YAML, TOML, XML, SVG…)
|
||||
go in the message's text, whole as far as the message's 100,000-character cap allows: past it
|
||||
a file is cut (and says how much is missing) or named only.
|
||||
- **CAD files** (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) are imported into the design
|
||||
at once, as components of their own. A file joins the message once its import is in (its chip
|
||||
shows a spinner meanwhile, and **Send** waits), and the message tells the agent it was added; a
|
||||
failed import says why and the agent is told nothing of it. Taking a part's chip off the message
|
||||
leaves the part in the design (Undo removes it).
|
||||
- Every file is named in the message, and listed under it in the conversation. A file that
|
||||
can't be attached says why.
|
||||
|
||||
Files stay with their message: one queued while the agent works keeps its files, and one sent
|
||||
again after a sign-in sends them again. When several queued messages go as one prompt, their
|
||||
files go with it, at most eight; the rest are named in the chat and not sent. **New chat**
|
||||
clears the files waiting on the chips too.
|
||||
|
||||
## Typing while the agent works
|
||||
|
||||
The input stays enabled during a turn. **Enter** queues the message: it shows in the conversation
|
||||
@@ -66,9 +102,9 @@ messages go as one prompt, in order, when the turn ends normally.
|
||||
- **Stop** (the square next to the input, or **Esc** in the input) stops the agent and keeps
|
||||
the queue: the bubbles wait for **Send**.
|
||||
- **New chat** (the header button, **Ctrl+Shift+L** / **Cmd+Shift+L** with the chat focused)
|
||||
stops what runs, clears the conversation and the queue and starts a fresh session on the same
|
||||
agent, keeping the agent and its mode. It asks first when a message is running or queued. The
|
||||
design's undo history is untouched.
|
||||
stops what runs, clears the conversation, the queue and the files on the chips, and starts a
|
||||
fresh session on the same agent, keeping the agent and its mode. It asks first when a message
|
||||
is running or queued, or files are waiting. The design's undo history is untouched.
|
||||
|
||||
## Agents
|
||||
|
||||
|
||||
@@ -156,7 +156,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox.
|
||||
| `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits |
|
||||
| `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first |
|
||||
| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. `"animate": false` on a call skips it |
|
||||
| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `send?`: text, `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) |
|
||||
| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) |
|
||||
| `ui.resize` | `width`, `height` | |
|
||||
| `app.quit` | `force?` (quit without asking about unsaved designs) | `quitting` (the window closes), `asking` (a "save changes?" prompt is open) |
|
||||
|
||||
|
||||
Reference in New Issue
Block a user