Agent chat: attach files and pictures to a message (#1744)

Thanks @onembyte.
This commit is contained in:
franco michetti
2026-10-18 09:18:29 -03:00
committed by GitHub
parent 4ebaca97e7
commit a4edea9ad2
16 changed files with 1797 additions and 48 deletions
Generated
+30
View File
@@ -865,6 +865,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "combine"
version = "4.6.8"
@@ -1665,6 +1671,16 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "gif"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
dependencies = [
"color_quant",
"weezl",
]
[[package]]
name = "gl_generator"
version = "0.14.0"
@@ -2077,6 +2093,9 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
dependencies = [
"bytemuck",
"byteorder-lite",
"color_quant",
"gif",
"image-webp",
"moxcms",
"num-traits",
"png",
@@ -2085,6 +2104,16 @@ dependencies = [
"zune-jpeg",
]
[[package]]
name = "image-webp"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
dependencies = [
"byteorder-lite",
"quick-error",
]
[[package]]
name = "indexmap"
version = "2.14.2"
@@ -4244,6 +4273,7 @@ version = "0.15.0"
name = "solidcraft-ui-egui"
version = "0.15.0"
dependencies = [
"arboard",
"egui",
"egui-wgpu",
"image",
+16
View File
@@ -116,6 +116,7 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
let parent = rfd::AsyncFileDialog::new().set_parent(cc);
let open_parent = parent.clone();
let program_parent = parent.clone();
let file_parent = parent.clone();
Services {
pick_open: Some(Box::new(move |first: &[&str]| {
// The file type the command reads comes first, so the picker opens on it (#1801).
@@ -166,6 +167,21 @@ fn services(cc: &eframe::CreationContext<'_>) -> Services {
.pick_file();
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
})),
// One kind of file (the agent chat's attachments): only the types it names, as given and in
// lower and upper case (Linux dialogs match case; macOS allows just the listed types).
pick_file: Some(Box::new(move |label: &str, exts: &[&str]| {
let mut all: Vec<String> = Vec::new();
for e in exts {
for v in [e.to_string(), e.to_ascii_lowercase(), e.to_ascii_uppercase()] {
if !all.contains(&v) {
all.push(v);
}
}
}
let all: Vec<&str> = all.iter().map(String::as_str).collect();
let future = file_parent.clone().add_filter(label, &all).pick_file();
Box::pin(async move { future.await.map(|p| p.path().to_string_lossy().into_owned()) })
})),
pick_program: Some(Box::new(move || {
let dialog = program_parent.clone().set_title("Choose your slicer");
let dialog = if cfg!(target_os = "windows") { dialog.add_filter("Programs", &["exe"]) } else { dialog };
+28
View File
@@ -841,12 +841,15 @@
"ui.activate": "Activate",
"ui.active_component": "Active component",
"ui.agent_chat": "Agent chat",
"ui.agent_chat_add": "Attach files",
"ui.agent_chat_add_file": "Choose a file…",
"ui.agent_chat_allow": "Allow",
"ui.agent_chat_allowed": "Allowed",
"ui.agent_chat_answered": "Answer: {option}",
"ui.agent_chat_approval": "Allow the agent to do this?",
"ui.agent_chat_attach_view": "Attach view",
"ui.agent_chat_attach_view_tip": "Send a picture of the current view with the message",
"ui.agent_chat_attachable": "Pictures, text and CAD files",
"ui.agent_chat_auth_agent": "Sign in with {name}",
"ui.agent_chat_auth_env": "Set {vars} in your environment and restart SolidCraft.",
"ui.agent_chat_auth_needed": "The agent asks you to sign in. You sign in with the agent itself; SolidCraft never sees your password or keys.",
@@ -869,9 +872,30 @@
"ui.agent_chat_dock": "Dock in the side panel",
"ui.agent_chat_dock_left": "Dock on the left",
"ui.agent_chat_dock_right": "Dock on the right",
"ui.agent_chat_drop_hint": "Drop to attach to your next message",
"ui.agent_chat_edit": "Edit",
"ui.agent_chat_exited": "The agent stopped.",
"ui.agent_chat_failed_start": "The agent could not start: {error}",
"ui.agent_chat_file_cad_not_read": "{name}: a CAD file is imported into the design, not read",
"ui.agent_chat_file_empty_picture": "{name}: an empty picture",
"ui.agent_chat_file_error": "{name}: {error}",
"ui.agent_chat_file_heic": "{name}: HEIC pictures can't be read here; export it as JPEG or PNG first",
"ui.agent_chat_file_importing": "Importing into the design…",
"ui.agent_chat_file_kinds": "{name}: pictures (PNG, JPEG, GIF, WebP, BMP, TIFF), text files (SVG too) and CAD files (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) can be attached",
"ui.agent_chat_file_loading": "Reading the file…",
"ui.agent_chat_file_not_a_file": "{name}: not a file",
"ui.agent_chat_file_not_a_picture": "{name}: not a picture",
"ui.agent_chat_file_not_utf8": "{name}: not UTF-8 text",
"ui.agent_chat_file_picture_too_big": "{name}: a picture can be at most {max} MB",
"ui.agent_chat_file_text_too_big": "{name}: a text file can be at most {max} KB",
"ui.agent_chat_file_too_large": "{name}: too large",
"ui.agent_chat_file_too_large_to_send": "{name}: too large to send, even scaled down",
"ui.agent_chat_file_unreadable_picture": "{name}: not a picture this can read",
"ui.agent_chat_files": "Files: {files}",
"ui.agent_chat_files_left_out": "At most {max} files go with one message; left out: {files}",
"ui.agent_chat_files_loading": "Waiting for the files to be read",
"ui.agent_chat_files_max": "At most {max} files on one message",
"ui.agent_chat_import_failed": "{name}: the import failed: {error}",
"ui.agent_chat_intro": "Chat with a coding agent you already use, or a model endpoint you choose. It works on the open design through SolidCraft's tools: its changes show as it makes them and undo as one step per message. What it may do without asking is set in Preferences > AI.",
"ui.agent_chat_key_in_env": "key set",
"ui.agent_chat_left": "Left",
@@ -884,11 +908,13 @@
"ui.agent_chat_no_cli": "solidcraft-cli was not found next to the app, so the agent can't reach the design.",
"ui.agent_chat_no_control": "The agent chat's endpoint did not start, so the agent can't reach the design.",
"ui.agent_chat_no_images": "This agent doesn't take pictures",
"ui.agent_chat_no_picture": "There is no picture on the clipboard",
"ui.agent_chat_none_found": "No coding agent found. Install one of these, then press Refresh:",
"ui.agent_chat_none_found_short": "No agent found",
"ui.agent_chat_not_installed": "Not installed",
"ui.agent_chat_not_signed_in": "not signed in",
"ui.agent_chat_own_window": "Own window",
"ui.agent_chat_paste_picture": "Paste a picture",
"ui.agent_chat_permission": "The agent asks to run: {title}",
"ui.agent_chat_pick": "Agent",
"ui.agent_chat_placeholder": "Ask the agent to change the design…",
@@ -903,6 +929,8 @@
"ui.agent_chat_refresh": "Refresh",
"ui.agent_chat_refused": "Not allowed for the agent: {reason}",
"ui.agent_chat_remove": "Remove from the queue",
"ui.agent_chat_remove_file": "Take it off this message",
"ui.agent_chat_remove_imported": "Take it off this message; the part stays in the design (Undo removes it)",
"ui.agent_chat_right": "Right",
"ui.agent_chat_save": "Save",
"ui.agent_chat_send": "Send",
+28
View File
@@ -841,12 +841,15 @@
"ui.activate": "アクティブ化",
"ui.active_component": "アクティブなコンポーネント",
"ui.agent_chat": "エージェントチャット",
"ui.agent_chat_add": "ファイルを添付",
"ui.agent_chat_add_file": "ファイルを選択…",
"ui.agent_chat_allow": "許可",
"ui.agent_chat_allowed": "許可しました",
"ui.agent_chat_answered": "回答: {option}",
"ui.agent_chat_approval": "エージェントにこの操作を許可しますか?",
"ui.agent_chat_attach_view": "ビューを添付",
"ui.agent_chat_attach_view_tip": "現在のビューの画像をメッセージと一緒に送ります",
"ui.agent_chat_attachable": "画像・テキスト・CAD ファイル",
"ui.agent_chat_auth_agent": "{name} でサインイン",
"ui.agent_chat_auth_env": "環境変数 {vars} を設定して SolidCraft を再起動してください。",
"ui.agent_chat_auth_needed": "エージェントがサインインを求めています。サインインはエージェント自身で行います。SolidCraft がパスワードやキーを見ることはありません。",
@@ -869,9 +872,30 @@
"ui.agent_chat_dock": "サイドパネルに戻す",
"ui.agent_chat_dock_left": "左にドッキング",
"ui.agent_chat_dock_right": "右にドッキング",
"ui.agent_chat_drop_hint": "ドロップして次のメッセージに添付",
"ui.agent_chat_edit": "編集",
"ui.agent_chat_exited": "エージェントが停止しました。",
"ui.agent_chat_failed_start": "エージェントを起動できませんでした: {error}",
"ui.agent_chat_file_cad_not_read": "{name}: CAD ファイルは読み込まず、デザインにインポートします",
"ui.agent_chat_file_empty_picture": "{name}: 空の画像です",
"ui.agent_chat_file_error": "{name}: {error}",
"ui.agent_chat_file_heic": "{name}: HEIC 画像はここでは読めません。先に JPEG か PNG に書き出してください",
"ui.agent_chat_file_importing": "デザインにインポート中…",
"ui.agent_chat_file_kinds": "{name}: 添付できるのは画像 (PNG、JPEG、GIF、WebP、BMP、TIFF)、テキストファイル (SVG も)、CAD ファイル (STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF) です",
"ui.agent_chat_file_loading": "ファイルを読み込み中…",
"ui.agent_chat_file_not_a_file": "{name}: ファイルではありません",
"ui.agent_chat_file_not_a_picture": "{name}: 画像ではありません",
"ui.agent_chat_file_not_utf8": "{name}: UTF-8 のテキストではありません",
"ui.agent_chat_file_picture_too_big": "{name}: 画像は {max} MB までです",
"ui.agent_chat_file_text_too_big": "{name}: テキストファイルは {max} KB までです",
"ui.agent_chat_file_too_large": "{name}: 大きすぎます",
"ui.agent_chat_file_too_large_to_send": "{name}: 縮小しても大きすぎて送れません",
"ui.agent_chat_file_unreadable_picture": "{name}: 読み込めない画像です",
"ui.agent_chat_files": "ファイル: {files}",
"ui.agent_chat_files_left_out": "1 つのメッセージで送れるファイルは {max} 個までです。送らなかったファイル: {files}",
"ui.agent_chat_files_loading": "ファイルの読み込みを待っています",
"ui.agent_chat_files_max": "1 つのメッセージに添付できるのは {max} 個までです",
"ui.agent_chat_import_failed": "{name}: インポートに失敗しました: {error}",
"ui.agent_chat_intro": "普段使っているコーディングエージェント、または選んだモデルのエンドポイントとチャットできます。SolidCraft のツールで開いているデザインを操作し、変更はその場で表示され、メッセージごとに1回で元に戻せます。確認なしで許可する範囲は「環境設定 > AI」で設定します。",
"ui.agent_chat_key_in_env": "キー設定済み",
"ui.agent_chat_left": "左",
@@ -884,11 +908,13 @@
"ui.agent_chat_no_cli": "アプリの隣に solidcraft-cli が見つからないため、エージェントはデザインにアクセスできません。",
"ui.agent_chat_no_control": "エージェントチャットのエンドポイントが起動しなかったため、エージェントはデザインにアクセスできません。",
"ui.agent_chat_no_images": "このエージェントは画像を受け付けません",
"ui.agent_chat_no_picture": "クリップボードに画像がありません",
"ui.agent_chat_none_found": "コーディングエージェントが見つかりません。次のいずれかをインストールしてから「再検索」を押してください:",
"ui.agent_chat_none_found_short": "エージェントが見つかりません",
"ui.agent_chat_not_installed": "未インストール",
"ui.agent_chat_not_signed_in": "未サインイン",
"ui.agent_chat_own_window": "独立したウィンドウ",
"ui.agent_chat_paste_picture": "画像を貼り付け",
"ui.agent_chat_permission": "エージェントが実行の許可を求めています: {title}",
"ui.agent_chat_pick": "エージェント",
"ui.agent_chat_placeholder": "デザインの変更をエージェントに依頼…",
@@ -903,6 +929,8 @@
"ui.agent_chat_refresh": "再検索",
"ui.agent_chat_refused": "エージェントには許可されていません: {reason}",
"ui.agent_chat_remove": "待機から削除",
"ui.agent_chat_remove_file": "このメッセージから外す",
"ui.agent_chat_remove_imported": "このメッセージから外します。パーツはデザインに残ります (元に戻すで削除できます)",
"ui.agent_chat_right": "右",
"ui.agent_chat_save": "保存",
"ui.agent_chat_send": "送信",
+28
View File
@@ -841,12 +841,15 @@
"ui.activate": "激活",
"ui.active_component": "活动组件",
"ui.agent_chat": "智能体聊天",
"ui.agent_chat_add": "附加文件",
"ui.agent_chat_add_file": "选择文件…",
"ui.agent_chat_allow": "允许",
"ui.agent_chat_allowed": "已允许",
"ui.agent_chat_answered": "回答:{option}",
"ui.agent_chat_approval": "允许智能体执行此操作吗?",
"ui.agent_chat_attach_view": "附加视图",
"ui.agent_chat_attach_view_tip": "随消息发送当前视图的图片",
"ui.agent_chat_attachable": "图片、文本和 CAD 文件",
"ui.agent_chat_auth_agent": "使用 {name} 登录",
"ui.agent_chat_auth_env": "请在环境中设置 {vars},然后重启 SolidCraft。",
"ui.agent_chat_auth_needed": "智能体要求你登录。登录在智能体本身完成;SolidCraft 从不接触你的密码或密钥。",
@@ -869,9 +872,30 @@
"ui.agent_chat_dock": "停靠到侧面板",
"ui.agent_chat_dock_left": "停靠在左侧",
"ui.agent_chat_dock_right": "停靠在右侧",
"ui.agent_chat_drop_hint": "拖放到此处,附加到下一条消息",
"ui.agent_chat_edit": "编辑",
"ui.agent_chat_exited": "智能体已停止。",
"ui.agent_chat_failed_start": "智能体无法启动:{error}",
"ui.agent_chat_file_cad_not_read": "{name}:CAD 文件会导入设计,而不是读取",
"ui.agent_chat_file_empty_picture": "{name}:空图片",
"ui.agent_chat_file_error": "{name}:{error}",
"ui.agent_chat_file_heic": "{name}:这里无法读取 HEIC 图片;请先导出为 JPEG 或 PNG",
"ui.agent_chat_file_importing": "正在导入设计…",
"ui.agent_chat_file_kinds": "{name}:可以附加图片(PNG、JPEG、GIF、WebP、BMP、TIFF)、文本文件(包括 SVG)和 CAD 文件(STEP、IGES、FreeCAD、Rhino、STL、3MF、OBJ、glTF)",
"ui.agent_chat_file_loading": "正在读取文件…",
"ui.agent_chat_file_not_a_file": "{name}:不是文件",
"ui.agent_chat_file_not_a_picture": "{name}:不是图片",
"ui.agent_chat_file_not_utf8": "{name}:不是 UTF-8 文本",
"ui.agent_chat_file_picture_too_big": "{name}:图片最大 {max} MB",
"ui.agent_chat_file_text_too_big": "{name}:文本文件最大 {max} KB",
"ui.agent_chat_file_too_large": "{name}:太大",
"ui.agent_chat_file_too_large_to_send": "{name}:即使缩小也太大,无法发送",
"ui.agent_chat_file_unreadable_picture": "{name}:无法读取的图片",
"ui.agent_chat_files": "文件:{files}",
"ui.agent_chat_files_left_out": "一条消息最多发送 {max} 个文件;未发送:{files}",
"ui.agent_chat_files_loading": "正在等待文件读取完成",
"ui.agent_chat_files_max": "一条消息最多附加 {max} 个文件",
"ui.agent_chat_import_failed": "{name}:导入失败:{error}",
"ui.agent_chat_intro": "与你已在使用的编程智能体或你选择的模型端点聊天。它通过 SolidCraft 的工具处理打开的设计:更改即时显示,每条消息的更改可一步撤销。无需询问即可执行的范围在“首选项 > AI”中设置。",
"ui.agent_chat_key_in_env": "已设置密钥",
"ui.agent_chat_left": "左侧",
@@ -884,11 +908,13 @@
"ui.agent_chat_no_cli": "在应用旁边未找到 solidcraft-cli,因此智能体无法访问设计。",
"ui.agent_chat_no_control": "智能体聊天的端点未能启动,因此智能体无法访问设计。",
"ui.agent_chat_no_images": "该智能体不接收图片",
"ui.agent_chat_no_picture": "剪贴板上没有图片",
"ui.agent_chat_none_found": "未找到编程智能体。请安装以下之一,然后点击刷新:",
"ui.agent_chat_none_found_short": "未找到智能体",
"ui.agent_chat_not_installed": "未安装",
"ui.agent_chat_not_signed_in": "未登录",
"ui.agent_chat_own_window": "独立窗口",
"ui.agent_chat_paste_picture": "粘贴图片",
"ui.agent_chat_permission": "智能体请求运行:{title}",
"ui.agent_chat_pick": "智能体",
"ui.agent_chat_placeholder": "让智能体修改设计…",
@@ -903,6 +929,8 @@
"ui.agent_chat_refresh": "刷新",
"ui.agent_chat_refused": "不允许智能体执行:{reason}",
"ui.agent_chat_remove": "从队列中移除",
"ui.agent_chat_remove_file": "从这条消息中移除",
"ui.agent_chat_remove_imported": "从这条消息中移除;零件仍保留在设计中(撤销可将其删除)",
"ui.agent_chat_right": "右侧",
"ui.agent_chat_save": "保存",
"ui.agent_chat_send": "发送",
+8 -1
View File
@@ -17,7 +17,9 @@ serde = { workspace = true }
serde_json = { workspace = true }
log = { workspace = true }
web-time = { workspace = true }
image = { workspace = true, features = ["jpeg", "ico"] }
# gif, webp, bmp, tiff: pictures attached to an agent chat message (GIF and WebP go as they are,
# BMP and TIFF as PNG).
image = { workspace = true, features = ["jpeg", "ico", "gif", "webp", "bmp", "tiff"] }
ttf-parser = { workspace = true }
semver = { workspace = true }
# Updates: the downloaded package's SHA-256 (against the release's SHA256SUMS.txt), its optional
@@ -33,6 +35,11 @@ tiny-skia = { workspace = true }
# updated by its installer (MIT; already in the tree as a build dependency).
winreg = "0.56"
# Pictures pasted into the agent chat; already in the tree, with the same features, through
# egui-winit's clipboard.
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
arboard = { version = "3.6", default-features = false, features = ["image-data"] }
[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
# The offscreen viewport render test opens a device without a window.
pollster = { workspace = true }
+493
View File
@@ -0,0 +1,493 @@
//! Files a person adds to a message (the + button, a drop on the chat, a paste): pictures go to the
//! agent with it, text files go in as text, and CAD files are imported into the design, the agent
//! being told what arrived. Each file is checked and bounded before it is kept; a refused one says
//! why.
use std::path::Path;
use std::sync::Arc;
use super::backend::{self, PromptPart};
use crate::i18n::format as trf;
/// A refusal in the app's language: the file's name, then why.
fn refused(key: &str, name: &str, args: &[(&str, &str)]) -> String {
let mut all = vec![("name", name)];
all.extend_from_slice(args);
trf(key, &all)
}
/// A refusal carrying the system's or the decoder's own words.
fn failed(name: &str, e: impl std::fmt::Display) -> String {
refused("ui.agent_chat_file_error", name, &[("error", &e.to_string())])
}
/// Most files on one message.
pub const MAX_FILES: usize = 8;
/// Largest picture file read, before it is scaled down.
const MAX_PICTURE_BYTES: u64 = 40 * 1024 * 1024;
/// Largest text file sent in full.
const MAX_TEXT_BYTES: u64 = 256 * 1024;
/// A picture's longer side is scaled down to this before it is sent.
pub const MAX_SIDE: u32 = 2048;
/// Largest picture sent, as it came or re-encoded: its base64 (a third larger) stays under 5 MB,
/// so eight of them still fit in a prompt.
pub const MAX_SENT_BYTES: usize = 3_750_000;
/// A picture is not halved below this to fit; past it, it is refused.
const MIN_SIDE: u32 = 64;
/// Largest picture decoded (pixels on a side), so a hostile header can't ask for gigabytes.
const MAX_DECODE_SIDE: u32 = 20_000;
/// The chip's picture.
const THUMB: u32 = 64;
/// Pictures the agents read as they are; BMP and TIFF are sent as PNG.
const PICTURE_EXTS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff"];
/// Text sent in full; SVG goes as its source, which the agents read well.
const TEXT_EXTS: &[&str] = &["svg", "txt", "md", "markdown", "json", "csv", "tsv", "yaml", "yml", "toml", "xml", "log", "ini", "cfg"];
/// CAD files and meshes, imported into the design (the file picker's list; [`kind`] asks the
/// importers themselves).
const CAD_EXTS: &[&str] = &["step", "stp", "igs", "iges", "FCStd", "3dm", "stl", "3mf", "obj", "glb", "gltf"];
/// Every extension that can be attached, pictures first (the file picker's list).
pub fn extensions() -> Vec<&'static str> {
PICTURE_EXTS.iter().chain(TEXT_EXTS).chain(CAD_EXTS).copied().collect()
}
/// What a file is, by its extension.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Kind {
Picture,
Text,
Cad,
Other,
}
pub fn kind(path: &Path) -> Kind {
let ext = path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).unwrap_or_default();
match ext.as_str() {
e if PICTURE_EXTS.contains(&e) => Kind::Picture,
e if TEXT_EXTS.contains(&e) => Kind::Text,
_ => {
let p = path.to_string_lossy();
if solidcraft_engine::io::is_step_path(&p) || solidcraft_engine::io::is_mesh_path(&p) { Kind::Cad } else { Kind::Other }
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum Content {
/// The picture's bytes as sent (PNG, JPEG, GIF or WebP), and its size in pixels.
Picture {
mime: &'static str,
bytes: Arc<Vec<u8>>,
size: [u32; 2],
},
Text(Arc<String>),
/// A CAD file imported into the design as its own component.
Imported,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Attachment {
/// Its own on the message, never reused (set when it is added there; 0 until then): the
/// chip's thumbnail is kept by it.
pub id: u64,
pub name: String,
pub content: Content,
/// A small copy of a picture for its chip.
pub thumb: Option<Arc<egui::ColorImage>>,
}
impl Attachment {
pub fn imported(name: String) -> Self {
Attachment { id: 0, name, content: Content::Imported, thumb: None }
}
pub fn is_picture(&self) -> bool {
matches!(self.content, Content::Picture { .. })
}
}
pub(crate) fn name_of(path: &Path) -> String {
path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string_lossy().to_string())
}
/// Read a picture or text file for a message. CAD files are imported by the caller, not read here.
pub fn read(path: &Path) -> Result<Attachment, String> {
let name = name_of(path);
let meta = std::fs::metadata(path).map_err(|e| failed(&name, e))?;
if !meta.is_file() {
return Err(refused("ui.agent_chat_file_not_a_file", &name, &[]));
}
match kind(path) {
Kind::Picture => {
if meta.len() > MAX_PICTURE_BYTES {
return Err(refused("ui.agent_chat_file_picture_too_big", &name, &[("max", &(MAX_PICTURE_BYTES / 1024 / 1024).to_string())]));
}
let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?;
picture(name, &bytes)
}
Kind::Text => {
if meta.len() > MAX_TEXT_BYTES {
return Err(refused("ui.agent_chat_file_text_too_big", &name, &[("max", &(MAX_TEXT_BYTES / 1024).to_string())]));
}
let bytes = std::fs::read(path).map_err(|e| failed(&name, e))?;
let text = String::from_utf8(bytes).map_err(|_| refused("ui.agent_chat_file_not_utf8", &name, &[]))?;
Ok(Attachment { id: 0, name, content: Content::Text(Arc::new(text)), thumb: None })
}
Kind::Cad => Err(refused("ui.agent_chat_file_cad_not_read", &name, &[])),
Kind::Other if matches!(path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("heic" | "heif")) => {
Err(refused("ui.agent_chat_file_heic", &name, &[]))
}
Kind::Other => Err(refused("ui.agent_chat_file_kinds", &name, &[])),
}
}
/// A picture from a file's bytes: decoded within limits, and sent as it came when the agents read
/// that format (PNG, JPEG, GIF, WebP), it is at most [`MAX_SIDE`] on a side and at most
/// [`MAX_SENT_BYTES`]; otherwise re-encoded (see [`encoded`]).
pub fn picture(name: String, bytes: &[u8]) -> Result<Attachment, String> {
picture_within(name, bytes, MAX_SENT_BYTES)
}
fn picture_within(name: String, bytes: &[u8], limit: usize) -> Result<Attachment, String> {
let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().map_err(|e| failed(&name, e))?;
let format = reader.format();
let mime = match format {
Some(image::ImageFormat::Png) => Some("image/png"),
Some(image::ImageFormat::Jpeg) => Some("image/jpeg"),
Some(image::ImageFormat::Gif) => Some("image/gif"),
Some(image::ImageFormat::WebP) => Some("image/webp"),
Some(image::ImageFormat::Bmp | image::ImageFormat::Tiff) => None,
_ => return Err(refused("ui.agent_chat_file_unreadable_picture", &name, &[])),
};
let mut limits = image::Limits::default();
limits.max_image_width = Some(MAX_DECODE_SIDE);
limits.max_image_height = Some(MAX_DECODE_SIDE);
limits.max_alloc = Some(1024 * 1024 * 1024);
reader.limits(limits);
let img = reader.decode().map_err(|e| failed(&name, e))?;
if img.width() == 0 || img.height() == 0 {
return Err(refused("ui.agent_chat_file_empty_picture", &name, &[]));
}
if let Some(mime) = mime
&& img.width().max(img.height()) <= MAX_SIDE
&& bytes.len() <= limit
{
let thumb = thumbnail(&img);
return Ok(Attachment {
id: 0,
name,
content: Content::Picture { mime, bytes: Arc::new(bytes.to_vec()), size: [img.width(), img.height()] },
thumb,
});
}
encoded(name, img, limit)
}
/// A picture copied to the clipboard (RGBA rows from the system), sent as PNG.
pub fn from_rgba(name: String, width: usize, height: usize, rgba: &[u8]) -> Result<Attachment, String> {
let (w, h) = (u32::try_from(width).ok(), u32::try_from(height).ok());
let (Some(w), Some(h)) = (w, h) else { return Err(refused("ui.agent_chat_file_too_large", &name, &[])) };
if w == 0 || h == 0 || w > MAX_DECODE_SIDE || h > MAX_DECODE_SIDE || width.checked_mul(height).and_then(|n| n.checked_mul(4)) != Some(rgba.len())
{
return Err(refused("ui.agent_chat_file_not_a_picture", &name, &[]));
}
let img = image::RgbaImage::from_raw(w, h, rgba.to_vec()).ok_or_else(|| refused("ui.agent_chat_file_not_a_picture", &name, &[]))?;
encoded(name, image::DynamicImage::ImageRgba8(img), MAX_SENT_BYTES)
}
/// Scale down to [`MAX_SIDE`] and encode within `limit` bytes: as PNG when that fits; else as JPEG
/// when the picture has no transparency; else halved and tried again. A GIF arrives here as its
/// first frame.
fn encoded(name: String, img: image::DynamicImage, limit: usize) -> Result<Attachment, String> {
let mut img = if img.width().max(img.height()) > MAX_SIDE { img.resize(MAX_SIDE, MAX_SIDE, image::imageops::FilterType::Triangle) } else { img };
let opaque = opaque(&img);
loop {
let mut png = Vec::new();
img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).map_err(|e| failed(&name, e))?;
let mut sent = (png.len() <= limit).then_some(("image/png", png));
if sent.is_none() && opaque {
let mut jpeg = Vec::new();
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 85).encode_image(&img.to_rgb8()).map_err(|e| failed(&name, e))?;
sent = (jpeg.len() <= limit).then_some(("image/jpeg", jpeg));
}
if let Some((mime, bytes)) = sent {
let thumb = thumbnail(&img);
return Ok(Attachment {
id: 0,
name,
content: Content::Picture { mime, bytes: Arc::new(bytes), size: [img.width(), img.height()] },
thumb,
});
}
if img.width().max(img.height()) <= MIN_SIDE {
return Err(refused("ui.agent_chat_file_too_large_to_send", &name, &[]));
}
img = img.resize((img.width() / 2).max(1), (img.height() / 2).max(1), image::imageops::FilterType::Triangle);
}
}
/// No pixel is see-through (a screenshot or a photo, even when stored with an alpha channel).
fn opaque(img: &image::DynamicImage) -> bool {
!img.color().has_alpha() || img.to_rgba8().pixels().all(|p| p.0[3] == 255)
}
fn thumbnail(img: &image::DynamicImage) -> Option<Arc<egui::ColorImage>> {
let t = img.thumbnail(THUMB, THUMB).to_rgba8();
let size = [usize::try_from(t.width()).ok()?, usize::try_from(t.height()).ok()?];
Some(Arc::new(egui::ColorImage::from_rgba_unmultiplied(size, t.as_raw())))
}
/// What the message says about its files, after the person's words: the text files' text, and the
/// pictures and imported parts by name (the agent sees the pictures themselves as images).
/// `pictures_sent`: whether this agent takes pictures. The text files share `budget` characters
/// in order: one that doesn't fit is cut and says how much is missing, and one past the budget
/// is named only, so the message stays within the prompt cap.
pub fn note(files: &[Attachment], pictures_sent: bool, budget: usize) -> String {
let mut out = String::new();
let pictures: Vec<&str> = files.iter().filter(|f| f.is_picture()).map(|f| f.name.as_str()).collect();
if !pictures.is_empty() {
let list = pictures.join(", ");
out += &if pictures_sent {
format!("\n\nAttached pictures: {list}.")
} else {
format!("\n\n(Pictures attached but not sent, this agent takes none: {list}.)")
};
}
let mut left = budget;
for f in files {
match &f.content {
Content::Text(t) => {
let t = t.trim_end();
let len = t.chars().count();
if left == 0 {
out += &format!("\n\nAttached file `{}`: left out, the message is at its size limit.", f.name);
} else if len <= left {
out += &format!("\n\nAttached file `{}`:\n```\n{t}\n```", f.name);
left -= len;
} else {
let kept: String = t.chars().take(left).collect();
out += &format!("\n\nAttached file `{}` (cut: {} more characters left out):\n```\n{kept}\n```", f.name, len - left);
left = 0;
}
}
Content::Imported => out += &format!("\n\n(I added `{}` to the design as a component of its own.)", f.name),
Content::Picture { .. } => {}
}
}
out
}
/// The pictures as image parts of the message.
pub fn parts(files: &[Attachment]) -> Vec<PromptPart> {
files
.iter()
.filter_map(|f| match &f.content {
Content::Picture { mime, bytes, .. } => Some(PromptPart::Image { mime: (*mime).to_string(), base64: backend::base64(bytes) }),
_ => None,
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn dir(tag: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("solidcraft-attach-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn png(path: &Path, w: u32, h: u32) {
image::RgbaImage::from_pixel(w, h, image::Rgba([200, 30, 30, 255])).save(path).unwrap();
}
#[test]
fn pictures_are_kept_or_scaled_down_and_text_goes_in_whole() {
let d = dir("read");
let small = d.join("board.png");
png(&small, 1600, 1120);
let a = read(&small).unwrap();
let Content::Picture { mime, size, bytes } = &a.content else { panic!("a picture") };
assert_eq!((*mime, *size), ("image/png", [1600, 1120]));
assert_eq!(bytes.as_slice(), std::fs::read(&small).unwrap().as_slice(), "small enough: sent as it came");
assert!(a.thumb.as_ref().is_some_and(|t| t.size[0] <= 64 && t.size[1] <= 64));
let big = d.join("photo.JPG");
image::RgbImage::from_pixel(4000, 1000, image::Rgb([10, 10, 10])).save_with_format(&big, image::ImageFormat::Jpeg).unwrap();
let b = read(&big).unwrap();
let Content::Picture { mime, size, .. } = &b.content else { panic!("a picture") };
assert_eq!((*mime, *size), ("image/png", [2048, 512]), "scaled to 2048 on the long side");
let notes = d.join("brief.md");
std::fs::write(&notes, "# Brief\nwall 2 mm\n").unwrap();
let c = read(&notes).unwrap();
assert!(matches!(&c.content, Content::Text(t) if t.contains("wall 2 mm")));
let _ = std::fs::remove_dir_all(&d);
}
/// Pixels that don't compress, so a small picture can be made larger than a small limit.
fn noise(w: u32, h: u32, alpha: bool) -> image::DynamicImage {
let mut seed = 0x2545_f491_u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
(seed & 0xff) as u8
};
image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(w, h, |_, _| {
image::Rgba([next(), next(), next(), if alpha { next() } else { 255 }])
}))
}
fn encode(img: &image::DynamicImage, format: image::ImageFormat) -> Vec<u8> {
let mut out = Vec::new();
img.write_to(&mut std::io::Cursor::new(&mut out), format).unwrap();
out
}
/// Review of #1744: a picture up to 2048 px went byte for byte up to the 40 MB file cap, and
/// eight of those don't fit in a prompt. Over the limit a picture is re-encoded: PNG when that
/// fits, JPEG when it has no transparency, halved until it fits; a GIF goes as its first frame.
#[test]
fn pictures_over_the_send_limit_are_reencoded_to_fit() {
let limit = 40_000;
let photo = encode(&noise(300, 200, false), image::ImageFormat::Png);
assert!(photo.len() > limit, "the test picture is over the limit");
let a = picture_within("photo.png".into(), &photo, limit).unwrap();
let Content::Picture { mime, bytes, size } = &a.content else { panic!("a picture") };
assert!(bytes.len() <= limit, "{} bytes", bytes.len());
assert_eq!(*mime, "image/jpeg", "an opaque picture goes as JPEG when PNG is too big");
assert!(size[0] <= 300 && size[1] <= 200 && size[0] * 2 == size[1] * 3, "{size:?}: smaller, same shape");
let clear = encode(&noise(300, 200, true), image::ImageFormat::Png);
let b = picture_within("overlay.png".into(), &clear, limit).unwrap();
let Content::Picture { mime, bytes, size } = &b.content else { panic!("a picture") };
assert_eq!(*mime, "image/png", "transparency stays PNG");
assert!(bytes.len() <= limit && size[0] < 300, "{} bytes, {size:?}: halved to fit", bytes.len());
let gif = encode(
&image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(400, 300, image::Rgba([40, 90, 200, 255]))),
image::ImageFormat::Gif,
);
let c = picture_within("loop.gif".into(), &gif, gif.len() - 1).unwrap();
let Content::Picture { mime, size, bytes } = &c.content else { panic!("a picture") };
assert_eq!(*mime, "image/png", "a GIF over the limit goes as its first frame, as PNG");
assert!(bytes.len() < gif.len() && size[0] * 3 == size[1] * 4, "{size:?}: within the limit, same shape");
assert!(a.thumb.is_some() && b.thumb.is_some() && c.thumb.is_some());
let under = encode(&noise(60, 40, false), image::ImageFormat::Png);
let d = picture_within("small.png".into(), &under, limit).unwrap();
assert!(matches!(&d.content, Content::Picture { bytes, .. } if bytes.as_slice() == under.as_slice()), "under the limit: as it came");
assert_eq!(MAX_SENT_BYTES, 3_750_000, "its base64 stays under 5 MB");
}
#[test]
fn bad_files_are_refused_with_a_reason() {
let d = dir("bad");
let fake = d.join("fake.png");
std::fs::write(&fake, b"not a picture at all").unwrap();
assert!(read(&fake).unwrap_err().contains("fake.png"));
let text = d.join("big.txt");
std::fs::write(&text, vec![b'a'; 300 * 1024]).unwrap();
assert!(read(&text).unwrap_err().contains("at most"));
let binary = d.join("bin.json");
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
assert!(read(&binary).unwrap_err().contains("UTF-8"));
let other = d.join("movie.mov");
std::fs::write(&other, b"x").unwrap();
assert!(read(&other).unwrap_err().contains("can be attached"));
assert!(read(&d.join("missing.png")).is_err());
assert_eq!(kind(Path::new("part.STEP")), Kind::Cad);
assert_eq!(kind(Path::new("mount.stl")), Kind::Cad);
assert_eq!(kind(Path::new("frame.3dm")), Kind::Cad);
assert!(from_rgba("x".into(), 2, 2, &[0; 15]).is_err(), "wrong length");
assert!(from_rgba("x".into(), 0, 2, &[]).is_err());
assert!(from_rgba("x".into(), usize::MAX, 2, &[]).is_err());
let ok = from_rgba("pasted.png".into(), 3, 2, &[255; 24]).unwrap();
assert!(matches!(ok.content, Content::Picture { size: [3, 2], .. }));
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: the refusals were English whatever the app's language.
#[test]
fn refusals_speak_the_apps_language() {
let d = dir("lang");
let binary = d.join("bin.json");
std::fs::write(&binary, [0xff, 0xfe, 0x00, 0x80]).unwrap();
let other = d.join("movie.mov");
std::fs::write(&other, b"x").unwrap();
{
let _ja = crate::i18n::scope("ja");
assert!(read(&binary).unwrap_err().contains("UTF-8 のテキストではありません"));
assert!(read(&other).unwrap_err().starts_with("movie.mov: 添付できるのは"));
}
{
let _zh = crate::i18n::scope("zh");
assert!(read(&binary).unwrap_err().contains("不是 UTF-8 文本"));
}
assert!(read(&binary).unwrap_err().contains("not UTF-8 text"), "English again outside the scope");
let _ = std::fs::remove_dir_all(&d);
}
/// Every picture format the agents read goes as it is (PNG, JPEG, GIF, WebP); BMP and TIFF are
/// sent as PNG; SVG goes as its source text; HEIC says why not.
#[test]
fn each_picture_format_goes_the_way_the_agent_reads_it() {
let d = dir("formats");
let img = image::RgbaImage::from_pixel(30, 20, image::Rgba([20, 120, 220, 255]));
for (file, format, mime) in [
("a.gif", image::ImageFormat::Gif, "image/gif"),
("a.webp", image::ImageFormat::WebP, "image/webp"),
("a.bmp", image::ImageFormat::Bmp, "image/png"),
("a.tiff", image::ImageFormat::Tiff, "image/png"),
] {
let path = d.join(file);
image::DynamicImage::ImageRgba8(img.clone()).save_with_format(&path, format).unwrap();
let a = read(&path).unwrap_or_else(|e| panic!("{file}: {e}"));
let Content::Picture { mime: m, size, bytes } = &a.content else { panic!("{file}: a picture") };
assert_eq!((*m, *size), (mime, [30, 20]), "{file}");
if mime != "image/png" {
assert_eq!(bytes.as_slice(), std::fs::read(&path).unwrap().as_slice(), "{file}: sent as it came");
}
assert!(a.thumb.is_some(), "{file}: a thumbnail");
}
let svg = d.join("sketch.svg");
std::fs::write(&svg, "<svg xmlns='http://www.w3.org/2000/svg'><rect width='10' height='5'/></svg>").unwrap();
assert!(matches!(&read(&svg).unwrap().content, Content::Text(t) if t.contains("<rect")));
let heic = d.join("photo.HEIC");
std::fs::write(&heic, b"....ftypheic").unwrap();
assert!(read(&heic).unwrap_err().contains("HEIC"));
for ext in ["png", "jpg", "jpeg", "gif", "webp", "bmp", "tiff", "svg", "md", "txt", "step", "FCStd", "3dm", "stl", "3mf", "obj", "glb"] {
assert!(extensions().contains(&ext), "{ext} can be chosen");
}
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: eight 256 KB text files took a message to about 2 MB, past the prompt cap
/// that only trimmed the typed text. The files' text now gets a budget: a file that doesn't
/// fit is cut and says so, and a file past it is named only.
#[test]
fn attached_text_is_kept_within_the_prompt_budget() {
let file = |n: &str, len: usize| Attachment { id: 0, name: n.into(), content: Content::Text(Arc::new("x".repeat(len))), thumb: None };
let files = vec![file("a.md", 600), file("b.md", 600), file("c.md", 600)];
let n = note(&files, true, 1000);
assert!(n.contains("`a.md`") && n.contains("`b.md`") && n.contains("`c.md`"), "every file is named: {n}");
assert!(n.matches('x').count() <= 1000, "the files' text stays within the budget");
assert!(n.contains("200 more characters"), "a cut file says how much is missing: {n}");
assert!(n.contains("left out"), "a file past the budget is named only: {n}");
assert_eq!(note(&files, true, usize::MAX).matches('x').count(), 1800, "with room, every file goes whole");
}
#[test]
fn the_message_names_its_files_and_carries_the_pictures() {
let pic = from_rgba("target.png".into(), 2, 2, &[9; 16]).unwrap();
let text = Attachment { id: 0, name: "brief.md".into(), content: Content::Text(Arc::new("wall 2 mm\n".into())), thumb: None };
let part = Attachment::imported("motor.step".into());
let files = vec![pic, text, part];
let n = note(&files, true, usize::MAX);
assert!(n.contains("Attached pictures: target.png."), "{n}");
assert!(n.contains("Attached file `brief.md`:\n```\nwall 2 mm\n```"), "{n}");
assert!(n.contains("`motor.step` to the design"), "{n}");
assert!(note(&files, false, usize::MAX).contains("not sent"));
let p = parts(&files);
assert_eq!(p.len(), 1);
assert!(matches!(&p[0], PromptPart::Image { mime, base64 } if mime == "image/png" && base64.starts_with("iVBORw0KGgo")));
}
}
+311 -31
View File
@@ -22,6 +22,7 @@ mod acp_client;
pub mod agents;
#[cfg(not(target_arch = "wasm32"))]
mod api_client;
pub mod attach;
pub mod backend;
#[cfg(not(target_arch = "wasm32"))]
pub mod discover;
@@ -48,6 +49,8 @@ pub use panel::{show, tip as panel_tip, toggle};
const MAX_ITEMS: usize = 2000;
/// Longest message sent.
const MAX_PROMPT: usize = 100_000;
/// Room kept in a message for the files' names and the words around their text.
const FILE_NAMES_ROOM: usize = 2_000;
/// How long a request waits for Allow or Deny before it is declined (the agent's MCP server gives
/// up on a request after five minutes).
const APPROVAL_TIMEOUT_MS: f64 = 240_000.0;
@@ -95,6 +98,8 @@ pub enum Item {
User {
text: String,
image: bool,
/// The names of the files that went with it.
files: Vec<String>,
},
Agent {
thought: bool,
@@ -152,6 +157,8 @@ struct Turn {
pub struct Pending {
pub text: String,
pub image: bool,
/// The files attached to it (they go with it).
pub files: Vec<attach::Attachment>,
}
/// A request held for Allow or Deny.
@@ -165,6 +172,30 @@ struct Held {
/// it): requests arriving there go to the sender; the port it listens on.
pub type ControlStarter = Box<dyn FnMut(Sender<ControlRequest>) -> Option<u16>>;
/// A file not yet on the next message: a picture being read off the window thread (so a large one
/// doesn't stop the window), or a CAD file being imported into the design. Its chip shows
/// meanwhile; it keeps the id it was given when attached, so the files stay in the order they
/// were attached in.
pub(crate) struct Loading {
pub id: u64,
pub name: String,
what: Reading,
}
enum Reading {
/// A picture decoding, or a pasted one encoding: the file, or why not.
File(Receiver<std::result::Result<attach::Attachment, String>>),
/// A CAD file's import into the design: its answer, as a control reply.
Import(Receiver<crate::control::ControlResponse>),
}
impl Loading {
/// A CAD file being imported (its part lands in the design whatever the message does).
pub fn importing(&self) -> bool {
matches!(self.what, Reading::Import(_))
}
}
#[derive(Default)]
pub struct AgentChat {
/// The panel is shown (View menu, the application bar's chat button, Ctrl+Shift+A).
@@ -178,8 +209,8 @@ pub struct AgentChat {
pub attach_view: bool,
auth: Vec<acp::AuthMethod>,
auth_status: Option<String>,
/// A message typed before the agent was ready.
queued: Option<(String, bool)>,
/// A message typed before the agent was ready, with its files.
queued: Option<(String, bool, Vec<attach::Attachment>)>,
/// Messages typed while the agent works, in order. They go as one prompt when the turn ends
/// (or at once with Send now); after Stop they wait for Send.
pub pending: Vec<Pending>,
@@ -199,8 +230,21 @@ pub struct AgentChat {
pub confirm_new: bool,
/// Where the panel was drawn last (for its shortcuts).
pub panel_rect: Option<egui::Rect>,
/// The last message sent, kept to send again after a sign-in.
last: Option<(String, bool)>,
/// The last message sent, kept to send again after a sign-in (its files too).
last: Option<(String, bool, Vec<attach::Attachment>)>,
/// Files for the next message (pictures, text files, imported parts), shown as chips.
pub(crate) attachments: Vec<attach::Attachment>,
/// Pictures pasted so far, to name the next one.
pasted: usize,
/// The chips' thumbnails, by the picture they show.
thumbs: std::collections::HashMap<u64, egui::TextureHandle>,
/// The last id given to a file added to a message.
last_file_id: u64,
/// Files still being read for the next message.
pub(crate) loading: Vec<Loading>,
/// Where the chat was drawn last frame (the panel, or its folded strip), for drops; `None`
/// when it isn't drawn.
pub(crate) drop_rect: Option<egui::Rect>,
/// The command the running agent was started with.
argv: Vec<String>,
/// The agent picked for this conversation (the header's switcher); `None`: the default.
@@ -246,6 +290,54 @@ impl Drop for AgentChat {
}
impl AgentChat {
/// Put a file on the next message with an id of its own; its name back.
fn add_file(&mut self, file: attach::Attachment) -> String {
let id = self.next_file_id();
self.place(id, file)
}
fn next_file_id(&mut self) -> u64 {
self.last_file_id += 1;
self.last_file_id
}
/// A file goes where its id says: the order the files were attached in.
fn place(&mut self, id: u64, mut file: attach::Attachment) -> String {
file.id = id;
let name = file.name.clone();
let at = self.attachments.partition_point(|f| f.id < id);
self.attachments.insert(at, file);
name
}
/// Files on the next message, read or still being read.
fn files(&self) -> usize {
self.attachments.len() + self.loading.len()
}
/// Read a file for the next message on a worker thread, in the window's language.
#[cfg(not(target_arch = "wasm32"))]
fn load(
&mut self,
name: String,
job: impl FnOnce() -> std::result::Result<attach::Attachment, String> + Send + 'static,
) -> std::result::Result<String, String> {
let (tx, rx) = std::sync::mpsc::channel();
let language = crate::i18n::language();
std::thread::Builder::new()
.name("chat-file".into())
.spawn(move || {
let _language = crate::i18n::scope(language);
let _ = tx.send(job());
})
.map_err(|e| trf("ui.agent_chat_file_error", &[("name", &name), ("error", &e.to_string())]))?;
let id = self.next_file_id();
self.loading.push(Loading { id, name: name.clone(), what: Reading::File(rx) });
Ok(name)
}
/// A CAD file whose import was asked for: it joins the message once the import is in.
fn importing(&mut self, name: String, rx: Receiver<crate::control::ControlResponse>) -> String {
let id = self.next_file_id();
self.loading.push(Loading { id, name: name.clone(), what: Reading::Import(rx) });
name
}
pub fn state(&self) -> State {
self.state.unwrap_or(State::Off)
}
@@ -573,37 +665,50 @@ const INTERRUPT_NOTE: &str = "[The user interrupted your previous step.]";
/// Send what the user typed: now if the agent is ready, queued behind the running message if it
/// is working, else once it is ready (starting it if needed).
pub fn send(app: &mut SolveApp, ctx: &egui::Context, text: String) {
let text: String = text.chars().take(MAX_PROMPT).collect();
if text.trim().is_empty() {
// A message waits for its files: the words go back in the box until they are read.
if !app.agent_chat.loading.is_empty() {
if app.agent_chat.input.trim().is_empty() {
app.agent_chat.input = text;
}
app.agent_chat.notice(tr("ui.agent_chat_files_loading"), false);
return;
}
let text: String = text.chars().take(MAX_PROMPT).collect();
// Files alone make a message.
if text.trim().is_empty() && app.agent_chat.attachments.is_empty() {
return;
}
let text = if text.trim().is_empty() { String::new() } else { text };
let image = app.agent_chat.attach_view && app.agent_chat.images;
if app.agent_chat.state() == State::Busy {
let chat = &mut app.agent_chat;
if chat.pending.len() >= MAX_PENDING {
// The files stay on the chips for when there is room.
chat.notice(tr("ui.agent_chat_queue_full"), false);
} else {
chat.pending.push(Pending { text, image });
let files = std::mem::take(&mut chat.attachments);
chat.pending.push(Pending { text, image, files });
}
return;
}
let files = std::mem::take(&mut app.agent_chat.attachments);
// Messages left waiting by Stop go first, in one prompt with this one.
if !app.agent_chat.pending.is_empty() {
app.agent_chat.pending.push(Pending { text, image });
app.agent_chat.pending.push(Pending { text, image, files });
send_queue(app, ctx, false);
return;
}
submit(app, ctx, text, image, false);
submit(app, ctx, text, image, files, false);
}
/// Put `text` in the conversation and send it (`interrupted`: the agent is told it cut in on its
/// previous step).
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, interrupted: bool) {
app.agent_chat.push(Item::User { text: text.clone(), image });
/// Put `text` in the conversation and send it with its files (`interrupted`: the agent is told it
/// cut in on its previous step).
fn submit(app: &mut SolveApp, ctx: &egui::Context, text: String, image: bool, files: Vec<attach::Attachment>, interrupted: bool) {
app.agent_chat.push(Item::User { text: text.clone(), image, files: files.iter().map(|f| f.name.clone()).collect() });
match app.agent_chat.state() {
State::Ready => start_turn_with(app, text, image, interrupted.then_some(INTERRUPT_NOTE)),
State::Ready => start_turn_with(app, text, image, files, interrupted.then_some(INTERRUPT_NOTE)),
s => {
app.agent_chat.queued = Some((text, image));
app.agent_chat.queued = Some((text, image, files));
#[cfg(not(target_arch = "wasm32"))]
if !s.running() {
connect(app, ctx);
@@ -623,9 +728,23 @@ fn send_queue(app: &mut SolveApp, ctx: &egui::Context, interrupted: bool) {
}
let all = std::mem::take(&mut chat.pending);
let image = all.iter().any(|p| p.image);
let text = all.into_iter().map(|p| p.text).collect::<Vec<_>>().join("\n\n");
let mut files = Vec::new();
let mut texts = Vec::new();
for p in all {
files.extend(p.files);
if !p.text.trim().is_empty() {
texts.push(p.text);
}
}
// One message carries at most MAX_FILES files; the rest are named, not sent.
if files.len() > attach::MAX_FILES {
let left: Vec<String> = files.split_off(attach::MAX_FILES).into_iter().map(|f| f.name).collect();
let max = attach::MAX_FILES.to_string();
chat.notice(trf("ui.agent_chat_files_left_out", &[("max", &max), ("files", &left.join(", "))]), true);
}
let text = texts.join("\n\n");
let text: String = text.chars().take(MAX_PROMPT).collect();
submit(app, ctx, text, image, interrupted);
submit(app, ctx, text, image, files, interrupted);
}
/// Send now: stop the running step and send the queue as the next prompt once the agent has said
@@ -657,6 +776,8 @@ pub fn new_chat(app: &mut SolveApp) {
chat.confirm_new = false;
chat.items.clear();
chat.pending.clear();
chat.attachments.clear();
chat.loading.clear();
chat.pending_edit = None;
chat.queued = None;
chat.last = None;
@@ -675,7 +796,7 @@ pub fn new_chat(app: &mut SolveApp) {
/// Whether New chat would throw something away that the user may want (so it asks first).
pub fn new_chat_needs_confirm(app: &SolveApp) -> bool {
app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty()
app.agent_chat.state() == State::Busy || !app.agent_chat.pending.is_empty() || app.agent_chat.files() > 0
}
/// Take a queued message out of the queue.
@@ -702,16 +823,23 @@ fn after_turn(app: &mut SolveApp, ctx: &egui::Context) {
}
}
fn start_turn(app: &mut SolveApp, text: String, image: bool) {
start_turn_with(app, text, image, None);
fn start_turn(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>) {
start_turn_with(app, text, image, files, None);
}
/// Start a message; `note` goes before the text for the agent, not into the conversation.
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&str>) {
let sent = match note {
/// Start a message; `note` goes before the text for the agent, not into the conversation. The
/// files are named after the text (text files in full), and pictures go as images to an agent
/// that takes them.
fn start_turn_with(app: &mut SolveApp, text: String, image: bool, files: Vec<attach::Attachment>, note: Option<&str>) {
let pictures = app.agent_chat.images;
let lead = match note {
Some(n) => format!("{n}\n\n{text}"),
None => text.clone(),
};
// The whole message stays within the cap: the files' text gets what the words leave.
let budget = MAX_PROMPT.saturating_sub(lead.chars().count() + FILE_NAMES_ROOM);
let about_files = attach::note(&files, pictures, budget);
let sent: String = format!("{lead}{about_files}").chars().take(MAX_PROMPT).collect();
let mut parts = vec![PromptPart::Text(sent)];
if image {
match crate::control::view_png(app, VIEW_IMAGE.0, VIEW_IMAGE.1) {
@@ -719,10 +847,15 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&
Err(e) => log::warn!("the view's picture is left out of the prompt: {e}"),
}
}
let label: String = text.lines().next().unwrap_or("").chars().take(48).collect();
if pictures {
parts.extend(attach::parts(&files));
}
// The undo step is named after the message, or its first file when it has no words.
let first_line = text.lines().find(|l| !l.trim().is_empty()).or_else(|| files.first().map(|f| f.name.as_str())).unwrap_or("");
let label: String = first_line.chars().take(48).collect();
app.agent_chat.turn =
Some(Turn { start_doc: app.session.doc.clone(), start_undo: app.session.undo.len(), changes: 0, requests: 0, limit_told: false, label });
app.agent_chat.last = Some((text, image));
app.agent_chat.last = Some((text, image, files));
app.agent_chat.state = Some(State::Busy);
app.agent_chat.stopped = false;
app.agent_chat.steering = false;
@@ -732,6 +865,100 @@ fn start_turn_with(app: &mut SolveApp, text: String, image: bool, note: Option<&
}
}
/// Add a file to the next message. A picture or text file is read and kept (see [`attach`]); a CAD
/// file is imported into the design at once, as its own component, and the message says so.
pub fn attach_path(app: &mut SolveApp, path: &str) -> std::result::Result<String, String> {
if app.agent_chat.files() >= attach::MAX_FILES {
return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())]));
}
let p = std::path::Path::new(path);
let file = if attach::kind(p) == attach::Kind::Cad {
if !p.is_file() {
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", path)]));
}
// Imported now, as a dropped file is; it joins the message once the import is in, so the
// agent is told it was added only when it was. A full queue is refused at once.
let name = attach::name_of(p);
let (tx, rx) = std::sync::mpsc::channel();
let mut request = crate::file_load::Request::new(path, crate::file_load::Kind::Insert);
request.reply = Some(tx);
crate::file_load::begin_request(app, request).map_err(|e| trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &e)]))?;
return Ok(app.agent_chat.importing(name, rx));
} else {
// A picture is decoded off the window thread: a large one takes long enough to stop it.
#[cfg(not(target_arch = "wasm32"))]
if attach::kind(p) == attach::Kind::Picture {
// A missing file is said at once; reading and decoding wait for the worker.
if !p.is_file() {
return Err(trf("ui.agent_chat_file_not_a_file", &[("name", &attach::name_of(p))]));
}
let path = p.to_path_buf();
return app.agent_chat.load(attach::name_of(p), move || attach::read(&path));
}
attach::read(p)?
};
Ok(app.agent_chat.add_file(file))
}
/// Add a picture from the clipboard (RGBA rows, as the system gives them) to the next message.
pub fn attach_rgba(app: &mut SolveApp, width: usize, height: usize, rgba: &[u8]) -> std::result::Result<String, String> {
if app.agent_chat.files() >= attach::MAX_FILES {
return Err(trf("ui.agent_chat_files_max", &[("max", &attach::MAX_FILES.to_string())]));
}
let n = app.agent_chat.pasted + 1;
let name = format!("pasted picture {n}.png");
app.agent_chat.pasted = n;
// Encoded off the window thread, as a decoded picture is.
#[cfg(not(target_arch = "wasm32"))]
{
let rgba = rgba.to_vec();
let named = name.clone();
app.agent_chat.load(name, move || attach::from_rgba(named, width, height, &rgba))
}
#[cfg(target_arch = "wasm32")]
{
let file = attach::from_rgba(name, width, height, rgba)?;
Ok(app.agent_chat.add_file(file))
}
}
/// Attach the picture on the clipboard, if it holds one.
#[cfg(not(target_arch = "wasm32"))]
pub fn paste_picture(app: &mut SolveApp) -> std::result::Result<String, String> {
let image = arboard::Clipboard::new().and_then(|mut c| c.get_image()).map_err(|_| tr("ui.agent_chat_no_picture").to_string())?;
attach_rgba(app, image.width, image.height, &image.bytes)
}
/// Files dropped on the chat attach to the next message; called before the app handles drops, so
/// those taken aren't opened too. A drop is the chat's when the pointer was last seen over it:
/// the platform gives no position during a drag (winit 0.30), so that is the best the chat can
/// know; the + button always works.
pub fn take_drops(app: &mut SolveApp, ctx: &egui::Context) {
if !drop_is_ours(app, ctx) || !ctx.input(|i| !i.raw.dropped_files.is_empty()) {
return;
}
let files = ctx.input_mut(|i| std::mem::take(&mut i.raw.dropped_files));
attach_dropped(app, files.iter().map(|f| f.path().to_path_buf()));
}
/// Whether files dropped now would be the chat's: the pointer was last seen over it.
pub(crate) fn drop_is_ours(app: &SolveApp, ctx: &egui::Context) -> bool {
let Some(rect) = app.agent_chat.drop_rect else { return false };
ctx.input(|i| i.pointer.latest_pos()).is_some_and(|p| rect.contains(p))
}
/// Attach dropped files; a refused one says why in the conversation.
pub(crate) fn attach_dropped(app: &mut SolveApp, paths: impl Iterator<Item = std::path::PathBuf>) {
for p in paths {
if p.as_os_str().is_empty() {
continue;
}
if let Err(e) = attach_path(app, &p.to_string_lossy()) {
app.agent_chat.notice(e, true);
}
}
}
/// Stop: ask the agent to stop the message it is on; requests waiting for approval are declined.
/// Messages queued meanwhile stay queued (they wait for Send).
pub fn cancel(app: &mut SolveApp) {
@@ -958,6 +1185,48 @@ fn pump_tools(app: &mut SolveApp) {
}
/// Read what the agent sent since the last frame, its tool requests, and the agent search.
/// Files read off the window thread join the message as they are done, where their ids put them;
/// a refused one says why.
pub(crate) fn take_loaded(app: &mut SolveApp, ctx: &egui::Context) {
use std::sync::mpsc::TryRecvError;
let chat = &mut app.agent_chat;
let mut i = 0;
while i < chat.loading.len() {
let name = chat.loading[i].name.clone();
let done = match &chat.loading[i].what {
Reading::File(rx) => match rx.try_recv() {
Ok(read) => Some(read),
Err(TryRecvError::Empty) => None,
Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_file_error", &[("name", &name), ("error", "stopped")]))),
},
Reading::Import(rx) => match rx.try_recv() {
Ok(answer) if answer["ok"].as_bool() == Some(true) => Some(Ok(attach::Attachment::imported(name))),
Ok(answer) => {
let why = answer["error"].as_str().unwrap_or_default().to_string();
Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", &why)])))
}
Err(TryRecvError::Empty) => None,
Err(TryRecvError::Disconnected) => Some(Err(trf("ui.agent_chat_import_failed", &[("name", &name), ("error", "stopped")]))),
},
};
match done {
None => i += 1,
Some(read) => {
let l = chat.loading.remove(i);
match read {
Ok(file) => {
chat.place(l.id, file);
}
Err(e) => chat.notice(e, true),
}
}
}
}
if !chat.loading.is_empty() {
ctx.request_repaint_after(std::time::Duration::from_millis(50));
}
}
pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
#[cfg(not(target_arch = "wasm32"))]
if let Some(rx) = &app.agent_chat.detecting
@@ -974,6 +1243,7 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
}
return;
}
take_loaded(app, ctx);
let events = app.agent_chat.backend.as_mut().map(|b| b.poll()).unwrap_or_default();
for e in events {
let ready = matches!(e, Event::Ready { .. });
@@ -988,8 +1258,8 @@ pub fn poll(app: &mut SolveApp, ctx: &egui::Context) {
if ready {
apply_saved_model(app);
}
if ready && let Some((text, image)) = app.agent_chat.queued.take() {
start_turn(app, text, image);
if ready && let Some((text, image, files)) = app.agent_chat.queued.take() {
start_turn(app, text, image, files);
}
}
pump_tools(app);
@@ -1055,8 +1325,9 @@ pub fn set_mode(app: &mut SolveApp, mode: &str) {
}
/// `ui.agent_chat` on the control channel (tests and screenshots): `enabled`, `command`, `open`,
/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `send`, `attach_view`,
/// `permission` (an option id, or `"cancel"`), `approve` (`true`/`false`, the oldest held
/// `side`, `minimized`, `width`, `refresh`, `connect`, `stop`, `attach` (a file's path, for the
/// next message), `remove_file` (index), `send`, `attach_view`, `permission` (an option id, or
/// `"cancel"`), `approve` (`true`/`false`, the oldest held
/// request), `cancel` (Stop), `send_now` (steer: stop, then send the queue), `remove_pending` (index), `undo` (the newest message's changes); returns the panel's state.
pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result<Value, String> {
if let Some(e) = p.get("features").and_then(Value::as_bool) {
@@ -1140,6 +1411,13 @@ pub fn control(app: &mut SolveApp, ctx: &egui::Context, p: &Value) -> Result<Val
app.preferences.agent_chat_width = (w as f32).clamp(panel::MIN_WIDTH, panel::MAX_WIDTH);
panel::forget_width(ctx);
}
if let Some(path) = p.get("attach").and_then(Value::as_str) {
attach_path(app, path)?;
}
if let Some(n) = p.get("remove_file").and_then(Value::as_u64) {
let n = usize::try_from(n).ok().filter(|n| *n < app.agent_chat.attachments.len()).ok_or("no such file on the next message")?;
app.agent_chat.attachments.remove(n);
}
if let Some(a) = p.get("attach_view").and_then(Value::as_bool) {
app.agent_chat.attach_view = a;
}
@@ -1200,7 +1478,7 @@ pub fn status(app: &SolveApp) -> Value {
.items
.iter()
.map(|i| match i {
Item::User { text, image } => json!({"user": text, "image": image}),
Item::User { text, image, files } => json!({"user": text, "image": image, "files": files}),
Item::Agent { thought, text } => json!({"agent": text, "thought": thought}),
Item::Tool { id, title, kind, status, output } => {
json!({"tool": id, "title": title, "kind": kind, "status": status.map(|s| format!("{s:?}")), "output": output})
@@ -1249,7 +1527,9 @@ pub fn status(app: &SolveApp) -> Value {
"toolsPort": c.tools_port,
"pid": pid,
"waiting": c.waiting(),
"pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image})).collect::<Vec<_>>(),
"pending": c.pending.iter().map(|p| json!({"text": p.text, "image": p.image, "files": p.files.iter().map(|f| f.name.clone()).collect::<Vec<_>>()})).collect::<Vec<_>>(),
"attachments": c.attachments.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
"loading": c.loading.iter().map(|f| f.name.clone()).collect::<Vec<_>>(),
"stopped": c.stopped,
"turn": c.turn.as_ref().map(|t| json!({"changes": t.changes, "requests": t.requests})),
"detection": detection,
+255 -8
View File
@@ -50,6 +50,8 @@ pub fn tip() -> String {
pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
let ctx = ui.ctx().clone();
poll(app, &ctx);
// Set again below if the chat is drawn this frame.
app.agent_chat.drop_rect = None;
// With AI features off nothing of the chat is drawn, and the shortcut does nothing.
if !shown(app) {
return;
@@ -85,7 +87,13 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
let left = app.preferences.agent_chat_side == "left";
let side = |id: Id| if left { egui::Panel::left(id) } else { egui::Panel::right(id) };
if app.preferences.agent_chat_minimized {
side(Id::new("sc_agent_chat_strip")).exact_size(STRIP).resizable(false).frame(egui::Frame::NONE.fill(t.panel)).show(ui, |ui| strip(app, ui));
let r = side(Id::new("sc_agent_chat_strip"))
.exact_size(STRIP)
.resizable(false)
.frame(egui::Frame::NONE.fill(t.panel))
.show(ui, |ui| strip(app, ui));
app.agent_chat.drop_rect = Some(r.response.rect);
show_drop_hint(app, &ctx);
return;
}
let width = app.preferences.agent_chat_width.clamp(MIN_WIDTH, MAX_WIDTH);
@@ -97,12 +105,41 @@ pub fn show(app: &mut SolveApp, ui: &mut egui::Ui) {
.show(ui, |ui| body(app, ui, left));
crate::scenario::publish_rect("agent_chat:panel", r.response.rect);
app.agent_chat.panel_rect = Some(r.response.rect);
app.agent_chat.drop_rect = Some(r.response.rect);
show_drop_hint(app, &ctx);
let w = r.response.rect.width();
if w.is_finite() && (MIN_WIDTH..=MAX_WIDTH).contains(&w) {
app.preferences.agent_chat_width = w;
}
}
/// While files are dragged over the window: if they would drop on the chat, say so over it.
fn show_drop_hint(app: &SolveApp, ctx: &egui::Context) {
#[cfg(not(target_arch = "wasm32"))]
if let Some(rect) = app.agent_chat.drop_rect
&& ctx.input(|i| !i.raw.hovered_files.is_empty())
&& drop_is_ours(app, ctx)
{
drop_hint(ctx, rect);
}
#[cfg(target_arch = "wasm32")]
let _ = (app, ctx);
}
/// Where dragged files will go: a tinted zone over the chat with a line of text.
#[cfg(not(target_arch = "wasm32"))]
fn drop_hint(ctx: &egui::Context, area: Rect) {
let t = Tokens::get();
let painter = ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, Id::new("sc_agent_chat_drop_hint")));
let zone = area.shrink(6.0);
painter.rect(zone, 10.0, t.accent.gamma_multiply(0.16), Stroke::new(2.0, t.accent), egui::StrokeKind::Inside);
let font = egui::FontId::proportional(14.0);
let text = painter.layout(tr("ui.agent_chat_drop_hint").to_string(), font, t.text, (zone.width() - 24.0).max(40.0));
let at = zone.center() - text.size() / 2.0;
painter.galley(at, text, t.text);
crate::scenario::publish_rect("agent_chat:drop_hint", zone);
}
/// New chat (when the chat has focus).
pub const NEW_CHAT_SHORTCUT: egui::KeyboardShortcut =
egui::KeyboardShortcut::new(egui::Modifiers::COMMAND.plus(egui::Modifiers::SHIFT), egui::Key::L);
@@ -431,14 +468,16 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
auth_box(app, ui);
}
ui.separator();
// The input sits at the bottom; the conversation takes the rest.
let input_h = 92.0;
// The input sits at the bottom (with a row per line of file chips); the conversation takes
// the rest.
let input_h = 92.0 + chip_rows(app.agent_chat.files(), ui.available_width(), ui.spacing().item_spacing.x) as f32 * 32.0;
let avail = (ui.available_height() - input_h).max(80.0);
egui::ScrollArea::vertical().id_salt("sc_agent_chat_scroll").max_height(avail).auto_shrink([false, false]).stick_to_bottom(true).show(ui, |ui| {
conversation(app, ui);
});
ui.separator();
let busy = state == State::Busy;
attachment_chips(app, ui);
let edit = egui::TextEdit::multiline(&mut app.agent_chat.input)
.hint_text(tr(if busy { "ui.agent_chat_placeholder_busy" } else { "ui.agent_chat_placeholder" }))
.desired_rows(2)
@@ -466,9 +505,18 @@ fn desktop_body(app: &mut SolveApp, ui: &mut egui::Ui) {
cancel(app);
}
ui.horizontal(|ui| {
let send_b = ui.button(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" }));
add_menu(app, ui);
// A message waits for files still being read.
let reading = !app.agent_chat.loading.is_empty();
let send_b = ui
.add_enabled(!reading, egui::Button::new(tr(if busy { "ui.agent_chat_queue" } else { "ui.agent_chat_send" })))
.on_disabled_hover_text(tr("ui.agent_chat_files_loading"));
crate::scenario::publish_handle("agent_chat:send", send_b.rect.center());
if (send_b.clicked() || enter || steer) && !app.agent_chat.input.trim().is_empty() {
// Files alone make a message.
let something = !app.agent_chat.input.trim().is_empty() || !app.agent_chat.attachments.is_empty();
if reading {
// Enter waits with the button.
} else if (send_b.clicked() || enter || steer) && something {
let text = std::mem::take(&mut app.agent_chat.input).trim_end_matches('\n').to_string();
send(app, ui.ctx(), text);
if steer {
@@ -744,14 +792,17 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
let current_doc = app.session.doc.clone();
for (n, item) in app.agent_chat.items.iter().enumerate() {
match item {
Item::User { text, image } => {
Item::User { text, image, files } => {
ui.add_space(6.0);
egui::Frame::NONE.fill(t.accent_soft).corner_radius(6.0).inner_margin(egui::Margin::same(6)).show(ui, |ui| {
ui.set_width(ui.available_width());
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
if !text.is_empty() {
ui.add(egui::Label::new(RichText::new(text).color(t.text)).selectable(true).wrap());
}
if *image {
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
}
files_line(ui, files.iter().map(String::as_str));
});
}
Item::Agent { thought, text } => {
@@ -900,10 +951,13 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
});
}
_ => {
ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap());
if !p.text.is_empty() {
ui.add(egui::Label::new(RichText::new(&p.text).color(t.text)).selectable(true).wrap());
}
if p.image {
ui.label(RichText::new(tr("ui.agent_chat_view_attached")).color(t.text_dim).size(11.0));
}
files_line(ui, p.files.iter().map(|f| f.name.as_str()));
}
}
});
@@ -960,6 +1014,199 @@ fn conversation(app: &mut SolveApp, ui: &mut egui::Ui) {
}
}
/// The files that went (or will go) with a message, by name.
#[cfg(not(target_arch = "wasm32"))]
fn files_line<'a>(ui: &mut egui::Ui, names: impl Iterator<Item = &'a str>) {
let list = names.collect::<Vec<_>>().join(", ");
if !list.is_empty() {
let t = Tokens::get();
ui.add(egui::Label::new(RichText::new(trf("ui.agent_chat_files", &[("files", &list)])).color(t.text_dim).size(11.0)).wrap());
}
}
/// The widest a file chip gets; a narrower chat makes them narrower.
const CHIP_W: f32 = 170.0;
/// How many rows `n` chips take in a chat `avail` wide (`gap` between them).
fn chip_rows(n: usize, avail: f32, gap: f32) -> usize {
let avail = avail.max(1.0);
let per_row = (((avail + gap) / (CHIP_W.min(avail) + gap)).floor() as usize).max(1);
n.div_ceil(per_row)
}
/// The files for the next message, as chips in rows that fit the chat: a picture's thumbnail (or
/// a file mark), the name cut to fit, and a × that takes the file off. Laid out by hand: egui
/// wraps plain widgets, not framed groups, and one row wider than the chat widens all below it.
#[cfg(not(target_arch = "wasm32"))]
fn attachment_chips(app: &mut SolveApp, ui: &mut egui::Ui) {
let chat = &mut app.agent_chat;
// Thumbnails of files no longer on the message are let go.
let keep: Vec<u64> = chat.attachments.iter().filter(|f| f.thumb.is_some()).map(|f| f.id).collect();
chat.thumbs.retain(|k, _| keep.contains(k));
if chat.files() == 0 {
return;
}
let gap = ui.spacing().item_spacing.x;
let avail = ui.available_width().max(1.0);
let w = CHIP_W.min(avail);
let per_row = (((avail + gap) / (w + gap)).floor() as usize).max(1);
let (mut remove, mut stop) = (None, None);
// The files read, then those still being read (each takes its place once it is done).
let read = chat.attachments.len();
let indices: Vec<usize> = (0..chat.files()).collect();
for row in indices.chunks(per_row) {
ui.horizontal(|ui| {
for &i in row {
if let Some(f) = chat.attachments.get(i) {
if chip(ui, &mut chat.thumbs, f, i, w) {
remove = Some(i);
}
} else if let Some(l) = chat.loading.get(i - read)
&& loading_chip(ui, &l.name, l.importing(), i - read, w)
{
stop = Some(i - read);
}
}
});
}
if let Some(i) = remove
&& i < chat.attachments.len()
{
chat.attachments.remove(i);
}
// Taken off while it is read: its reader's answer is dropped with it.
if let Some(i) = stop
&& i < chat.loading.len()
{
chat.loading.remove(i);
}
}
/// What a chip's × says: an imported part stays in the design when it comes off the message.
#[cfg(not(target_arch = "wasm32"))]
pub(super) fn remove_tip(f: &attach::Attachment) -> &'static str {
tr(if matches!(f.content, attach::Content::Imported) { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" })
}
/// The chip of a file still being read or imported: a spinner, the name, and a × that takes it
/// off (an import goes on: its part lands in the design).
#[cfg(not(target_arch = "wasm32"))]
fn loading_chip(ui: &mut egui::Ui, name: &str, importing: bool, i: usize, w: f32) -> bool {
let t = Tokens::get();
let mut clicked = false;
let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show(
ui,
|ui| {
let inner = (w - 10.0).max(40.0);
ui.set_width(inner);
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 4.0;
let doing = tr(if importing { "ui.agent_chat_file_importing" } else { "ui.agent_chat_file_loading" });
let mark = ui.add(egui::Spinner::new().size(16.0)).on_hover_text(doing).rect.width();
let name_w = (inner - mark - 18.0 - 8.0).max(8.0);
ui.allocate_ui(vec2(name_w, 20.0), |ui| {
ui.set_width(name_w);
ui.add(egui::Label::new(RichText::new(name).size(11.5).color(t.text_dim)).truncate())
})
.inner
.on_hover_text(name);
let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click());
let c = r.center();
let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim });
ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s);
ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s);
crate::scenario::publish_handle(&format!("agent_chat:file_loading_remove:{i}"), c);
clicked = x.on_hover_text(tr(if importing { "ui.agent_chat_remove_imported" } else { "ui.agent_chat_remove_file" })).clicked();
});
},
);
crate::scenario::publish_handle(&format!("agent_chat:file_loading:{i}"), r.response.rect.center());
clicked
}
/// One file chip, `w` wide in all; true when its × was clicked.
#[cfg(not(target_arch = "wasm32"))]
fn chip(ui: &mut egui::Ui, thumbs: &mut std::collections::HashMap<u64, egui::TextureHandle>, f: &attach::Attachment, i: usize, w: f32) -> bool {
let t = Tokens::get();
let mut clicked = false;
let r = egui::Frame::NONE.fill(t.toolbar).stroke(Stroke::new(1.0, t.border)).corner_radius(6.0).inner_margin(egui::Margin::symmetric(4, 3)).show(
ui,
|ui| {
let inner = (w - 10.0).max(40.0);
ui.set_width(inner);
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 4.0;
let mark = match &f.thumb {
Some(img) => {
let tex = thumbs
.entry(f.id)
.or_insert_with(|| ui.ctx().load_texture(format!("sc_attach_{}", f.id), (**img).clone(), egui::TextureOptions::LINEAR));
let size = tex.size_vec2();
ui.image((tex.id(), size * (22.0 / size.x.max(size.y).max(1.0)))).rect.width()
}
None => {
let (r, _) = ui.allocate_exact_size(vec2(16.0, 20.0), Sense::hover());
ui.painter().rect_stroke(r.shrink2(vec2(2.0, 3.0)), 2.0, Stroke::new(1.2, t.text_dim), egui::StrokeKind::Inside);
r.width()
}
};
let name_w = (inner - mark - 18.0 - 8.0).max(8.0);
ui.allocate_ui(vec2(name_w, 20.0), |ui| {
ui.set_width(name_w);
ui.add(egui::Label::new(RichText::new(&f.name).size(11.5)).truncate())
})
.inner
.on_hover_text(&f.name);
let (r, x) = ui.allocate_exact_size(vec2(18.0, 18.0), Sense::click());
let c = r.center();
let s = Stroke::new(1.3, if x.hovered() { t.text } else { t.text_dim });
ui.painter().line_segment([c + vec2(-3.5, -3.5), c + vec2(3.5, 3.5)], s);
ui.painter().line_segment([c + vec2(-3.5, 3.5), c + vec2(3.5, -3.5)], s);
crate::scenario::publish_handle(&format!("agent_chat:file_remove:{i}"), c);
clicked = x.on_hover_text(remove_tip(f)).clicked();
});
},
);
crate::scenario::publish_rect(&format!("agent_chat:file:{i}"), r.response.rect);
crate::scenario::publish_handle(&format!("agent_chat:file:{i}"), r.response.rect.center());
clicked
}
/// The + button: choose a file with the app's picker (pictures, text and CAD files), or paste
/// the clipboard's picture.
#[cfg(not(target_arch = "wasm32"))]
fn add_menu(app: &mut SolveApp, ui: &mut egui::Ui) {
let (mut choose, mut paste) = (false, false);
let r = ui.menu_button(RichText::new("+").strong().size(15.0), |ui| {
let a = ui.button(tr("ui.agent_chat_add_file"));
crate::scenario::publish_handle("agent_chat:add_file", a.rect.center());
let b = ui.button(tr("ui.agent_chat_paste_picture"));
crate::scenario::publish_handle("agent_chat:paste_picture", b.rect.center());
if a.clicked() {
choose = true;
ui.close();
}
if b.clicked() {
paste = true;
ui.close();
}
});
crate::scenario::publish_handle("agent_chat:add", r.response.rect.center());
r.response.on_hover_text(tr("ui.agent_chat_add"));
if choose {
app.pick_file(tr("ui.agent_chat_attachable"), &attach::extensions(), |app, path| {
if let Some(path) = path
&& let Err(e) = attach_path(app, &path)
{
app.agent_chat.notice(e, true);
}
});
}
if paste && let Err(e) = paste_picture(app) {
app.agent_chat.notice(e, true);
}
}
/// A tool call as a card: its state, name and kind on one line; a click unfolds what it returned.
#[cfg(not(target_arch = "wasm32"))]
fn tool_card(ui: &mut egui::Ui, n: usize, title: &str, kind: &str, status: Option<ToolStatus>, output: Option<&str>) {
+506 -4
View File
@@ -26,18 +26,20 @@ fn streamed_text_joins_and_tool_updates_change_their_call() {
#[test]
fn a_sign_in_request_keeps_the_question_for_after_the_login() {
let mut c = chat();
c.last = Some(("make a cube".into(), false));
c.last = Some(("make a cube".into(), false, vec![attach::Attachment::imported("motor.step".into())]));
c.state = Some(State::Busy);
assert!(c.on_event(Event::TurnEnded { reason: "auth".into() }).is_some());
c.on_event(Event::AuthRequired { methods: vec![] });
assert_eq!(c.state(), State::NeedsAuth);
assert_eq!(c.queued, Some(("make a cube".into(), false)));
let (text, image, files) = c.queued.clone().expect("queued for after the login");
assert_eq!((text.as_str(), image), ("make a cube", false));
assert_eq!(files.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), ["motor.step"], "its files go again too");
}
#[test]
fn a_plan_update_replaces_the_turns_plan() {
let mut c = chat();
c.push(Item::User { text: "go".into(), image: false });
c.push(Item::User { text: "go".into(), image: false, files: Vec::new() });
c.on_event(Event::Plan(vec![("a".into(), "pending".into())]));
c.on_event(Event::Text { thought: false, text: "working".into() });
c.on_event(Event::Plan(vec![("a".into(), "completed".into())]));
@@ -66,7 +68,7 @@ fn app_in_a_turn() -> (SolveApp, Sender<ControlRequest>) {
let (tx, rx) = channel();
app.agent_chat.tools_rx = Some(rx);
app.agent_chat.state = Some(State::Busy);
start_turn(&mut app, "two boxes".into(), false);
start_turn(&mut app, "two boxes".into(), false, Vec::new());
(app, tx)
}
@@ -416,12 +418,17 @@ use std::sync::Mutex;
struct Script {
calls: Arc<Mutex<Vec<String>>>,
events: Arc<Mutex<Vec<Event>>>,
/// Every prompt in full (its text and pictures).
prompts: Arc<Mutex<Vec<Vec<PromptPart>>>>,
}
impl Script {
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
fn prompts(&self) -> Vec<Vec<PromptPart>> {
self.prompts.lock().unwrap().clone()
}
fn report(&self, e: Event) {
self.events.lock().unwrap().push(e);
}
@@ -434,6 +441,7 @@ impl ChatBackend for Script {
fn prompt(&mut self, parts: &[PromptPart]) {
let text = parts.iter().find_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).unwrap_or_default();
self.calls.lock().unwrap().push(format!("prompt: {text}"));
self.prompts.lock().unwrap().push(parts.to_vec());
}
fn cancel(&mut self) {
self.calls.lock().unwrap().push("cancel".into());
@@ -860,6 +868,500 @@ done
let _ = std::fs::remove_dir_all(&dir);
}
// ---- Files attached to a message ----
/// A folder of files to attach: a picture and a text file.
fn attachable(tag: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("solidcraft-chat-files-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
image::RgbaImage::from_pixel(40, 30, image::Rgba([250, 250, 250, 255])).save(d.join("target.png")).unwrap();
std::fs::write(d.join("brief.md"), "wall 2 mm, Pi 5 inside\n").unwrap();
d
}
fn at(d: &std::path::Path, f: &str) -> String {
d.join(f).to_string_lossy().to_string()
}
/// Wait for the files read off the window thread to join the message.
fn settle_files(app: &mut SolveApp, ctx: &egui::Context) {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
while !app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline {
take_loaded(app, ctx);
std::thread::sleep(std::time::Duration::from_millis(5));
}
take_loaded(app, ctx);
}
/// The same with the whole app running frames.
fn settle_chat(h: &mut crate::scenario::Harness) {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
while !h.app.agent_chat.loading.is_empty() && std::time::Instant::now() < deadline {
h.frames(1);
std::thread::sleep(std::time::Duration::from_millis(5));
}
h.frames(2);
}
fn text_of(parts: &[PromptPart]) -> String {
parts.iter().filter_map(|p| if let PromptPart::Text(t) = p { Some(t.clone()) } else { None }).collect()
}
fn pictures_in(parts: &[PromptPart]) -> usize {
parts.iter().filter(|p| matches!(p, PromptPart::Image { .. })).count()
}
/// Files attached to a message go with it: the pictures as images, the text files in its text,
/// and every file by name; the message lists them in the conversation.
#[test]
fn a_message_carries_its_files() {
let d = attachable("carry");
let (mut app, script, ctx) = app_with_script();
app.agent_chat.images = true;
control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap();
control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap();
settle_files(&mut app, &ctx);
assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"]));
assert!(control(&mut app, &ctx, &json!({"attach": at(&d, "missing.png")})).is_err());
send(&mut app, &ctx, "make the case like this".into());
let sent = script.prompts();
assert_eq!(sent.len(), 1);
let text = text_of(&sent[0]);
assert!(text.starts_with("make the case like this"), "{text}");
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm, Pi 5 inside"), "{text}");
assert_eq!(sent[0].iter().filter(|p| matches!(p, PromptPart::Image { mime, .. } if mime == "image/png")).count(), 1);
assert!(app.agent_chat.attachments.is_empty(), "the files went with the message");
assert_eq!(status(&app)["items"][0]["files"], json!(["target.png", "brief.md"]));
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: a picture was decoded on the window thread, so a large one froze the window.
/// It is read off it now: the message waits for it (Send waits too), the files keep the order they
/// were attached in, and a refused picture says why later, in the app's language.
#[test]
fn pictures_are_read_off_the_window_thread() {
let d = attachable("worker");
std::fs::write(d.join("fake.png"), b"not a picture at all").unwrap();
let (mut app, script, ctx) = app_with_script();
app.agent_chat.images = true;
control(&mut app, &ctx, &json!({"attach": at(&d, "target.png")})).unwrap();
control(&mut app, &ctx, &json!({"attach": at(&d, "brief.md")})).unwrap();
assert_eq!(status(&app)["loading"], json!(["target.png"]), "the picture is still being read");
send(&mut app, &ctx, "like this".into());
assert!(script.prompts().is_empty(), "Send waits for the files");
assert_eq!(app.agent_chat.input, "like this", "and keeps the words");
settle_files(&mut app, &ctx);
assert_eq!(status(&app)["attachments"], json!(["target.png", "brief.md"]), "in the order they were attached");
let words = std::mem::take(&mut app.agent_chat.input);
send(&mut app, &ctx, words);
assert_eq!(script.prompts().len(), 1);
{
let _ja = crate::i18n::scope("ja");
control(&mut app, &ctx, &json!({"attach": at(&d, "fake.png")})).unwrap();
}
settle_files(&mut app, &ctx);
assert!(
matches!(app.agent_chat.items.last(), Some(Item::Notice { text, error: true }) if text == "fake.png: 読み込めない画像です"),
"a refused picture says why, in the language it was attached in"
);
assert!(app.agent_chat.attachments.is_empty());
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: the prompt cap applied to the typed text only, so eight 256 KB text files
/// took a message to about 2 MB. The whole message stays within it now, every file still named.
#[test]
fn a_message_with_large_text_files_stays_within_the_prompt_cap() {
let d = attachable("cap");
let (mut app, script, ctx) = app_with_script();
for i in 0..attach::MAX_FILES {
let path = d.join(format!("notes{i}.txt"));
std::fs::write(&path, "y".repeat(250 * 1024)).unwrap();
control(&mut app, &ctx, &json!({"attach": path.to_string_lossy()})).unwrap();
}
send(&mut app, &ctx, "read these".into());
let text = text_of(&script.prompts()[0]);
assert!(text.chars().count() <= MAX_PROMPT, "{} characters", text.chars().count());
assert!((0..attach::MAX_FILES).all(|i| text.contains(&format!("`notes{i}.txt`"))), "every file is named");
let _ = std::fs::remove_dir_all(&d);
}
/// A message sent before the agent is ready keeps its files until it goes; an agent that takes
/// no pictures gets none, and the message says which were left out. Files alone make a message,
/// and its undo step is named after the first file.
#[test]
fn files_wait_with_their_message_and_pictures_skip_an_agent_that_takes_none() {
let d = attachable("wait");
let (mut app, script, ctx) = app_with_script();
app.agent_chat.state = Some(State::Starting);
attach_path(&mut app, &at(&d, "target.png")).unwrap();
settle_files(&mut app, &ctx);
send(&mut app, &ctx, " ".into());
assert!(script.prompts().is_empty(), "not sent before the agent is ready");
assert_eq!(status(&app)["items"][0]["user"], "", "no words, just the file");
script.report(Event::Ready { agent: "Test".into(), images: false });
poll(&mut app, &ctx);
let sent = script.prompts();
assert_eq!(sent.len(), 1, "sent once the agent is ready");
assert_eq!(pictures_in(&sent[0]), 0, "no pictures to an agent that takes none");
assert!(text_of(&sent[0]).contains("not sent"), "{}", text_of(&sent[0]));
assert_eq!(app.agent_chat.turn.as_ref().map(|t| t.label.as_str()), Some("target.png"));
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn at_most_eight_files_on_a_message() {
let d = attachable("cap");
let mut app = SolveApp::new(solidcraft_engine::Session::default(), crate::Services::default());
for _ in 0..attach::MAX_FILES {
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
}
assert!(attach_path(&mut app, &at(&d, "brief.md")).unwrap_err().contains('8'));
assert!(attach_rgba(&mut app, 2, 2, &[9; 16]).is_err());
assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES);
let ctx = egui::Context::default();
assert!(control(&mut app, &ctx, &json!({"remove_file": 8})).is_err(), "no ninth file to take off");
control(&mut app, &ctx, &json!({"remove_file": 0})).unwrap();
assert_eq!(app.agent_chat.attachments.len(), attach::MAX_FILES - 1);
let _ = std::fs::remove_dir_all(&d);
}
/// While the agent works, a message queues with its files: the queue shows them, Stop keeps
/// them, and they go with the queue.
#[test]
fn a_queued_message_keeps_its_files_through_stop_and_sends_them() {
let d = attachable("queue");
let (mut app, script, ctx) = app_with_script();
app.agent_chat.images = true;
send(&mut app, &ctx, "make a box".into());
attach_path(&mut app, &at(&d, "target.png")).unwrap();
settle_files(&mut app, &ctx);
send(&mut app, &ctx, "like this".into());
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
send(&mut app, &ctx, String::new());
assert!(app.agent_chat.attachments.is_empty(), "the files moved to the queue");
assert_eq!(status(&app)["pending"][0]["files"], json!(["target.png"]));
assert_eq!(status(&app)["pending"][1]["files"], json!(["brief.md"]));
cancel(&mut app);
ended(&script, "cancelled");
poll(&mut app, &ctx);
assert_eq!(app.agent_chat.pending.len(), 2, "Stop keeps the queue and its files");
send_now(&mut app, &ctx);
let sent = script.prompts();
let last = sent.last().expect("the queue went");
let text = text_of(last);
assert!(text.starts_with("like this") && !text.contains("\n\n\n\n"), "a message of files alone adds no empty paragraph: {text:?}");
assert!(text.contains("Attached pictures: target.png.") && text.contains("wall 2 mm"), "{text}");
assert_eq!(pictures_in(last), 1);
assert_eq!(status(&app)["items"].as_array().and_then(|i| i.last()).map(|i| i["files"].clone()), Some(json!(["target.png", "brief.md"])));
let _ = std::fs::remove_dir_all(&d);
}
/// Queued messages go as one prompt with at most eight files; the rest are named, not sent.
#[test]
fn the_queue_sends_at_most_eight_files_and_names_the_rest() {
let d = attachable("queue-cap");
let (mut app, script, ctx) = app_with_script();
send(&mut app, &ctx, "first".into());
for n in 0..3 {
for _ in 0..4 {
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
}
send(&mut app, &ctx, format!("batch {n}"));
}
ended(&script, "");
poll(&mut app, &ctx);
let sent = script.prompts();
assert_eq!(sent.len(), 2);
assert_eq!(text_of(&sent[1]).matches("Attached file `brief.md`").count(), attach::MAX_FILES);
let told = app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, .. } if text.contains("brief.md") && text.contains('8')));
assert!(told, "the files left out are named");
let _ = std::fs::remove_dir_all(&d);
}
/// New chat clears the files waiting on the chips too, and asks first when there are some.
#[test]
fn new_chat_clears_the_files_waiting_and_asks_first() {
let d = attachable("new");
let (mut app, _script, _ctx) = app_with_script();
assert!(!new_chat_needs_confirm(&app));
attach_path(&mut app, &at(&d, "brief.md")).unwrap();
assert!(new_chat_needs_confirm(&app), "files would be lost");
new_chat(&mut app);
assert!(app.agent_chat.attachments.is_empty());
let _ = std::fs::remove_dir_all(&d);
}
/// A harness with the chat open at its narrowest and nothing to look for.
fn narrow_chat() -> crate::scenario::Harness {
let mut h = crate::scenario::Harness::new();
h.app.preferences.agent_chat_width = panel::MIN_WIDTH;
h.app.agent_chat.open = true;
h.app.agent_chat.detection = Some(Default::default());
h
}
/// Every chip and the text box sit inside the panel.
fn assert_inside_the_panel(chips: usize) {
use crate::scenario::rect_of;
let panel = rect_of("agent_chat:panel").expect("the panel");
let mut rects: Vec<(String, egui::Rect)> =
(0..chips).map(|i| (format!("file:{i}"), rect_of(&format!("agent_chat:file:{i}")).expect("a chip"))).collect();
rects.push(("input".into(), rect_of("agent_chat:input").expect("the text box")));
for (name, r) in rects {
assert!(r.left() >= panel.left() && r.right() <= panel.right(), "{name} {r:?} in {panel:?}");
}
}
/// Files for the next message show as chips above the text box (a picture's thumbnail, the name,
/// a × that takes it off), inside the chat, and Send sends files with no text.
#[test]
fn attached_files_show_as_chips_that_can_be_taken_off_and_sent_alone() {
use crate::scenario::handle_at;
let d = attachable("chips");
let mut h = narrow_chat();
let script = Script::default();
h.app.agent_chat.backend = Some(Box::new(script.clone()));
h.app.agent_chat.state = Some(State::Ready);
h.app.agent_chat.images = true;
h.frames(2);
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
h.call("ui.agent_chat", json!({"attach": at(&d, "brief.md")}));
settle_chat(&mut h);
assert!(handle_at("agent_chat:file:0").is_some() && handle_at("agent_chat:file:1").is_some(), "a chip per file");
assert_inside_the_panel(2);
let x = handle_at("agent_chat:file_remove:0").expect("a × on the chip");
h.call("ui.click", json!({"x": x.x, "y": x.y}));
h.frames(2);
assert_eq!(status(&h.app)["attachments"], json!(["brief.md"]));
let send = handle_at("agent_chat:send").expect("Send");
h.call("ui.click", json!({"x": send.x, "y": send.y}));
h.frames(2);
assert_eq!(script.prompts().len(), 1, "files alone make a message");
assert!(h.app.agent_chat.attachments.is_empty() && handle_at("agent_chat:file:0").is_none());
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: chips' thumbnails were cached by the picture's address, so a new picture at a
/// freed address could show an old one. Each file has an id of its own, never reused, and the
/// cache follows the files on the message.
#[test]
fn each_file_has_its_own_id_and_thumbnail() {
let d = attachable("ids");
image::RgbaImage::from_pixel(20, 20, image::Rgba([10, 200, 10, 255])).save(d.join("green.png")).unwrap();
let mut h = narrow_chat();
h.frames(2);
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
h.call("ui.agent_chat", json!({"attach": at(&d, "green.png")}));
settle_chat(&mut h);
let first: Vec<u64> = h.app.agent_chat.attachments.iter().map(|f| f.id).collect();
assert!(first.len() == 2 && first[0] != first[1], "{first:?}");
let mut cached: Vec<u64> = h.app.agent_chat.thumbs.keys().copied().collect();
cached.sort_unstable();
assert_eq!(cached, first, "a thumbnail per file, by its id");
h.call("ui.agent_chat", json!({"remove_file": 0}));
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
settle_chat(&mut h);
let now: Vec<u64> = h.app.agent_chat.attachments.iter().map(|f| f.id).collect();
assert!(!now.contains(&first[0]), "a removed file's id is not used again: {now:?}");
let mut cached: Vec<u64> = h.app.agent_chat.thumbs.keys().copied().collect();
cached.sort_unstable();
let mut want = now.clone();
want.sort_unstable();
assert_eq!(cached, want, "the cache follows the files on the message");
let _ = std::fs::remove_dir_all(&d);
}
/// The + button: "Choose a file…" asks the app's file picker for the types that can be attached
/// (not the design types Open shows) and attaches what it answers; "Paste a picture" attaches
/// the clipboard's picture (here given as RGBA, as the system does).
#[test]
fn the_plus_button_attaches_a_chosen_file_or_a_pasted_picture() {
use crate::scenario::handle_at;
let d = attachable("plus");
let mut h = narrow_chat();
let chosen = at(&d, "brief.md");
let asked = std::rc::Rc::new(std::cell::RefCell::new(Vec::<String>::new()));
let seen = asked.clone();
h.app.services.pick_file = Some(Box::new(move |_label: &str, exts: &[&str]| {
*seen.borrow_mut() = exts.iter().map(|e| e.to_string()).collect();
let p = chosen.clone();
Box::pin(async move { Some(p) })
}));
h.frames(2);
let plus = handle_at("agent_chat:add").expect("a + button");
h.call("ui.click", json!({"x": plus.x, "y": plus.y}));
h.frames(2);
let choose = handle_at("agent_chat:add_file").expect("Choose a file… in its menu");
assert!(handle_at("agent_chat:paste_picture").is_some(), "Paste a picture in its menu");
h.call("ui.click", json!({"x": choose.x, "y": choose.y}));
h.frames(4);
assert_eq!(status(&h.app)["attachments"], json!(["brief.md"]));
let exts = asked.borrow().clone();
for ext in ["png", "jpg", "gif", "webp", "svg", "md", "step", "stl"] {
assert!(exts.iter().any(|e| e == ext), "{ext} in {exts:?}");
}
assert!(!exts.iter().any(|e| e == "solidcraft"));
attach_rgba(&mut h.app, 4, 3, &[200; 48]).unwrap();
attach_rgba(&mut h.app, 4, 3, &[200; 47]).unwrap();
settle_chat(&mut h);
assert!(
h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("pasted picture 2.png:"))),
"not a picture: it says so once it has been read"
);
assert_eq!(status(&h.app)["attachments"], json!(["brief.md", "pasted picture 1.png"]));
let _ = std::fs::remove_dir_all(&d);
}
#[derive(Debug)]
struct DroppedPath(std::path::PathBuf);
impl egui::DroppedFile for DroppedPath {
fn path(&self) -> &std::path::Path {
&self.0
}
fn bytes(&self) -> Result<Vec<u8>, String> {
std::fs::read(&self.0).map_err(|e| e.to_string())
}
}
fn drop_files(h: &mut crate::scenario::Harness, at: egui::Pos2, files: &[std::path::PathBuf]) {
h.drop_files(at, files.iter().map(|p| Arc::new(DroppedPath(p.clone())) as egui::DroppedFileHandle).collect());
}
/// A tetrahedron, as a small ASCII STL.
const TETRA_STL: &str = "solid t
facet normal 0 0 -1
outer loop
vertex 0 0 0
vertex 0 10 0
vertex 10 0 0
endloop
endfacet
facet normal 0 -1 0
outer loop
vertex 0 0 0
vertex 10 0 0
vertex 0 0 10
endloop
endfacet
facet normal -1 0 0
outer loop
vertex 0 0 0
vertex 0 0 10
vertex 0 10 0
endloop
endfacet
facet normal 1 1 1
outer loop
vertex 10 0 0
vertex 0 10 0
vertex 0 0 10
endloop
endfacet
endsolid t
";
/// Review of #1744: a CAD file's import wasn't waited on, so a failed or queued-out import still
/// told the agent "I added x to the design". The file joins the message once its import is in;
/// a failed import says why and the agent is told nothing of it.
#[test]
fn a_cad_file_joins_the_message_once_its_import_is_in() {
let d = attachable("cad");
std::fs::write(d.join("block.stl"), TETRA_STL).unwrap();
std::fs::write(d.join("broken.stl"), [0u8, 1, 2]).unwrap();
let mut h = narrow_chat();
let script = Script::default();
h.app.agent_chat.backend = Some(Box::new(script.clone()));
h.app.agent_chat.state = Some(State::Ready);
h.frames(2);
attach_path(&mut h.app, &at(&d, "block.stl")).unwrap();
assert_eq!(status(&h.app)["loading"], json!(["block.stl"]), "importing, not yet on the message");
assert_eq!(status(&h.app)["attachments"], json!([]));
settle_chat(&mut h);
assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "in, and on the message");
h.call("ui.agent_chat", json!({"attach": at(&d, "broken.stl")}));
settle_chat(&mut h);
assert!(
h.app.agent_chat.items.iter().any(|i| matches!(i, Item::Notice { text, error: true } if text.starts_with("broken.stl:"))),
"a failed import says why"
);
assert_eq!(status(&h.app)["attachments"], json!(["block.stl"]), "and is not on the message");
h.call("ui.agent_chat", json!({"send": "fit around this"}));
h.frames(2);
let text = text_of(&script.prompts()[0]);
assert!(text.contains("`block.stl` to the design") && !text.contains("broken.stl"), "{text}");
let _ = std::fs::remove_dir_all(&d);
}
/// Review of #1744: taking an imported part's chip off leaves the part in the design, and its ×
/// says so; any other file's × just takes it off.
#[test]
fn an_imported_parts_chip_says_the_part_stays() {
assert_eq!(panel::remove_tip(&attach::Attachment::imported("motor.step".into())), tr("ui.agent_chat_remove_imported"));
assert!(tr("ui.agent_chat_remove_imported").contains("stays in the design"));
let picture = attach::from_rgba("a.png".into(), 2, 2, &[9; 16]).unwrap();
assert_eq!(panel::remove_tip(&picture), tr("ui.agent_chat_remove_file"));
}
/// Files dropped on the chat attach to the next message, and the app doesn't open them as well
/// (while they are dragged over it, the chat says so); dropped on the 3D view they are not the
/// chat's (the app handles them as before), nor when the chat is closed.
#[test]
fn files_dropped_on_the_chat_attach_and_the_app_does_not_open_them() {
use crate::scenario::rect_of;
let d = attachable("drop");
let mut h = narrow_chat();
h.call("ui.resize", json!({"width": 1600, "height": 1000}));
h.frames(3);
let panel = rect_of("agent_chat:panel").expect("docked");
// While files are dragged over the chat, it says they will attach; over the 3D view, not.
let hovered = || vec![egui::HoveredFile { path: Some(d.join("target.png")), mime: String::new() }];
h.drag_files(panel.center(), hovered(), Vec::new());
assert!(rect_of("agent_chat:drop_hint").is_some_and(|r| panel.contains(r.center())), "the hint over the chat");
h.drag_files(egui::pos2(300.0, 500.0), hovered(), Vec::new());
assert!(rect_of("agent_chat:drop_hint").is_none(), "no hint over the 3D view");
drop_files(&mut h, panel.center(), &[d.join("target.png"), d.join("brief.md")]);
settle_chat(&mut h);
assert_eq!(status(&h.app)["attachments"], json!(["target.png", "brief.md"]));
assert!(!h.app.file_load.busy(), "the app doesn't open them as well");
drop_files(&mut h, egui::pos2(300.0, 500.0), &[d.join("brief.md")]);
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(2), "over the 3D view: not the chat's");
// Folded to its strip, a drop on the strip is the chat's too.
h.call("ui.agent_chat", json!({"minimized": true}));
h.frames(3);
let strip = h.app.agent_chat.drop_rect.expect("the strip takes drops");
assert!(strip.width() < 60.0, "{strip:?}");
drop_files(&mut h, strip.center(), &[d.join("brief.md")]);
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3));
// Closed: nothing is the chat's, wherever the pointer was.
h.app.agent_chat.open = false;
h.frames(2);
assert!(h.app.agent_chat.drop_rect.is_none());
drop_files(&mut h, strip.center(), &[d.join("brief.md")]);
assert_eq!(status(&h.app)["attachments"].as_array().map(Vec::len), Some(3));
let _ = std::fs::remove_dir_all(&d);
}
/// Many files and a long message in a narrow chat: the chips wrap onto more rows and the text box
/// keeps to the chat (a row of chips wider than the chat would widen everything under it).
#[test]
fn chips_and_a_long_message_wrap_inside_a_narrow_chat() {
use crate::scenario::handle_at;
let d = attachable("wrap");
let mut h = narrow_chat();
for _ in 0..6 {
h.call("ui.agent_chat", json!({"attach": at(&d, "target.png")}));
}
h.app.agent_chat.input = "a separate design next to the robot body, the main version, with these measures for the second one".into();
settle_chat(&mut h);
assert_inside_the_panel(6);
let (first, second) = (handle_at("agent_chat:file:0").unwrap(), handle_at("agent_chat:file:1").unwrap());
assert!(second.y > first.y + 10.0, "at 280 px the second chip goes on the next row: {first:?} {second:?}");
let _ = std::fs::remove_dir_all(&d);
}
/// An agent's text is Markdown: a code block shows with its Copy button and a link is a button
/// of its own, instead of the raw markers.
#[test]
+15
View File
@@ -46,6 +46,21 @@ impl SolveApp {
self.begin_picker(future, Box::new(complete));
}
/// Ask for one file of a kind (`label`, `extensions`), with the host's dialog for it, or the
/// Open dialog where the host has none. (The agent chat asks for its files; the web build has
/// no chat.)
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
pub(crate) fn pick_file(&mut self, label: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option<String>) + 'static) {
if self.file_picker_busy() {
return;
}
let future = match self.services.pick_file.as_ref() {
Some(pick) => Some(pick(label, extensions)),
None => self.services.pick_open.as_ref().map(|pick| pick(extensions)),
};
self.begin_picker(future, Box::new(complete));
}
pub(crate) fn pick_save(&mut self, name: &str, extensions: &[&str], complete: impl FnOnce(&mut SolveApp, Option<String>) + 'static) {
if self.file_picker_busy() {
return;
+5
View File
@@ -193,6 +193,11 @@ impl Drop for LanguageScope {
}
}
/// This thread's language, to carry into a worker thread with [`scope`].
pub fn language() -> &'static str {
CURRENT.with(|current| current.get().0.id)
}
pub fn tr(id: &str) -> &str {
CURRENT.with(|current| current.get().text(id))
}
+6
View File
@@ -262,6 +262,10 @@ pub struct Services {
/// the picker's first filter (`&[]`: designs, CAD files and meshes, #1801).
pub pick_open: Option<Box<dyn Fn(&[&str]) -> FilePickerFuture>>,
pub pick_save: Option<Box<dyn Fn(&str, &[&str]) -> FilePickerFuture>>,
/// Open a dialog for one file of a kind (its name, its extensions): on macOS a dialog allows
/// only the types it names, so the agent chat's files need a dialog of their own. `None`: the
/// Open dialog.
pub pick_file: Option<Box<dyn Fn(&str, &[&str]) -> FilePickerFuture>>,
/// Open a dialog asking for a program (a slicer); `None` when the host has no such dialog.
pub pick_program: Option<Box<dyn Fn() -> FilePickerFuture>>,
pub show_in_folder: Option<Box<dyn Fn(&str)>>,
@@ -1275,6 +1279,8 @@ impl SolveApp {
recovery_ui::tick(self);
update::tick(self, ctx);
if self.issue_report.draft.is_none() && self.file_picker.is_none() {
// Files dropped on the agent chat attach to its next message instead.
agent_chat::take_drops(self, ctx);
for f in ctx.input(|i| i.raw.dropped_files.clone()) {
let p = f.path().to_string_lossy().to_string();
if !p.is_empty() {
+28
View File
@@ -217,6 +217,34 @@ impl Harness {
}
}
/// One frame in which the platform drops `files`, the pointer last seen at `at` (a drop
/// itself carries no position with winit 0.30).
pub fn drop_files(&mut self, at: egui::Pos2, files: Vec<egui::DroppedFileHandle>) {
self.drag_files(at, Vec::new(), files);
}
/// One frame in which files are dragged over the window (`hovered`) or dropped, the pointer
/// last seen at `at`.
pub fn drag_files(&mut self, at: egui::Pos2, hovered: Vec<egui::HoveredFile>, dropped: Vec<egui::DroppedFileHandle>) {
self.time += 1.0 / 60.0;
let raw = egui::RawInput {
screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, self.size)),
time: Some(self.time),
focused: true,
events: vec![egui::Event::PointerMoved(at)],
hovered_files: hovered,
dropped_files: dropped,
..Default::default()
};
let app = &mut self.app;
let mut out = self.ctx.run_ui(raw, |ui| {
app.logic(ui.ctx());
app.ui(ui);
});
out.textures_delta.clear();
self.output = out.platform_output;
}
/// Frames until the queued input is used up (and the view has settled).
pub fn settle(&mut self) {
for _ in 0..2000 {
+39 -3
View File
@@ -53,6 +53,42 @@ when you send a message or press **Connect**.
endpoint's agent also keeps it as its default model. An agent that reports no models shows no
drop-down.
## Files and pictures
A message can carry up to eight files, to show the agent what you mean: a concept picture, a
sketch, a brief, a part to fit around.
- **+** next to **Send** has **Choose a file…** (a file dialog for the types below) and **Paste a
picture** (the picture on the clipboard).
- **Drop files on the chat.** Files dropped on the panel (or its folded strip) attach to the
next message, and the app doesn't open them too. While files are dragged over the chat, it says
so. The window gives no position while something is dragged, so the drop is the chat's when the
pointer was last seen over the chat; **+** always works.
- The files show as chips above the text box: a picture's thumbnail, the name, and × to take the
file off. A picture is read off the window thread, so a large one never stops the window: its
chip shows a spinner meanwhile, and **Send** waits for it. **Send** sends files with no text.
- **Pictures** (PNG, JPEG, GIF, WebP) go to the agent as they are when they are at most 2048 px
on the long side and 3.75 MB (a picture's base64 then stays under 5 MB, so eight still fit in a
prompt). Otherwise they are scaled down to 2048 px and re-encoded: as PNG when that fits, as
JPEG when the picture has no transparency, halved until it fits; a GIF goes as its first frame.
BMP and TIFF go as PNG. HEIC can't be read: export it as JPEG or PNG first. An agent that takes
no pictures gets none, and the message says which were left out.
- **Text files** (UTF-8, up to 256 KB: Markdown, plain text, JSON, CSV, YAML, TOML, XML, SVG…)
go in the message's text, whole as far as the message's 100,000-character cap allows: past it
a file is cut (and says how much is missing) or named only.
- **CAD files** (STEP, IGES, FreeCAD, Rhino, STL, 3MF, OBJ, glTF) are imported into the design
at once, as components of their own. A file joins the message once its import is in (its chip
shows a spinner meanwhile, and **Send** waits), and the message tells the agent it was added; a
failed import says why and the agent is told nothing of it. Taking a part's chip off the message
leaves the part in the design (Undo removes it).
- Every file is named in the message, and listed under it in the conversation. A file that
can't be attached says why.
Files stay with their message: one queued while the agent works keeps its files, and one sent
again after a sign-in sends them again. When several queued messages go as one prompt, their
files go with it, at most eight; the rest are named in the chat and not sent. **New chat**
clears the files waiting on the chips too.
## Typing while the agent works
The input stays enabled during a turn. **Enter** queues the message: it shows in the conversation
@@ -66,9 +102,9 @@ messages go as one prompt, in order, when the turn ends normally.
- **Stop** (the square next to the input, or **Esc** in the input) stops the agent and keeps
the queue: the bubbles wait for **Send**.
- **New chat** (the header button, **Ctrl+Shift+L** / **Cmd+Shift+L** with the chat focused)
stops what runs, clears the conversation and the queue and starts a fresh session on the same
agent, keeping the agent and its mode. It asks first when a message is running or queued. The
design's undo history is untouched.
stops what runs, clears the conversation, the queue and the files on the chips, and starts a
fresh session on the same agent, keeping the agent and its mode. It asks first when a message
is running or queued, or files are waiting. The design's undo history is untouched.
## Agents
+1 -1
View File
@@ -156,7 +156,7 @@ labels; `prefs:measure_axis_distances` identifies the checkbox.
| `ui.capture` | `action?`: `capture`, `revert`, `cancel` | answers the Capture Position question (asked when a command starts or the design is saved while components were moved without capturing); without `action`, what waits |
| `ui.window` | `action`: minimize, maximize, restore, toggle, close | what the title bar's caption buttons do (on Windows and Linux the application bar is the title bar); close asks about unsaved designs first |
| `ui.agent_cursor` | `show` (bool), `speed`: a number from 1 to 5 (times the normal speed) or `"instant"` (no animation: the command runs at once and the pointer jumps to where it acted), `follow_camera` (bool); all optional | the settings. While the cursor is shown, `engine.execute` first glides to the command's toolbar button and to each pick, then runs and replies; later requests wait their turn. `"animate": false` on a call skips it |
| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `send?`: text, `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) |
| `ui.agent_chat` | `features?`, `toolbar_button?`, `permissions?` (`ask`, `edits`, `all`), `add_agent?` ({`name`, `kind`: `acp` or `api`, `command` or `base_url` + `model`}; no keys), `default_agent?`, `agent?` (the conversation's agent, by id), `mode?` (the agent's own mode), `model?` (one of its models), `open?`, `side?` (`left`, `right`), `minimized?`, `popped?` (in its own window), `width?`, `picker?`, `refresh?`, `connect?`, `stop?`, `attach?`: a file's path (a picture, a text file or a CAD file, for the next message; CAD files are imported at once), `remove_file?` (index of a file on the next message), `send?`: text (files alone make a message too), `attach_view?`, `permission?`: an option id or `"cancel"` (the newest question of the agent's own), `approve?`: true, false or `"always"` (the oldest request held for Allow/Deny), `cancel?` (Stop: the queue stays), `send_now?` (stop the running step, then send the queued messages), `remove_pending?` (index), `new_chat?` (clear the chat, new session), `undo?` (the last message's changes) | the chat's state: `features`, `agents` (found and added), `currentAgent`, `state` (off, starting, ready, busy, auth, exited), `popped`, `window` (its place and size), `agent`, `modes`, `mode`, `models`, `model`, `authMethods`, `toolsPort`, `pid`, `waiting`, `turn`, `pending` (queued messages, with their `files`), `attachments` (the files on the next message, by name), `loading` (pictures still being read for it, by name), `stopped`, `detection` and the conversation (`items`). With `features` off only the settings apply. Desktop only; see [Agent chat](agent-chat.md) |
| `ui.resize` | `width`, `height` | |
| `app.quit` | `force?` (quit without asking about unsaved designs) | `quitting` (the window closes), `asking` (a "save changes?" prompt is open) |