Files
vibemake/tests/cad/assistant-browser.mjs
T
2026-11-05 17:50:35 +08:00

408 lines
25 KiB
JavaScript

import assert from 'node:assert/strict';
import { mkdir, writeFile } from 'node:fs/promises';
import { resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { chromium } from 'playwright';
const base = (process.env.VM_CAD_BASE_URL || 'http://localhost:8094').replace(/\/$/, '');
const provider = process.env.VM_CAD_MODEL_URL || 'http://127.0.0.1:8092/__assistant/v1';
const auth = { token: 'cad-browser-test-token', user: { userCode: 'cad-browser-test-user', nickname: 'CAD Browser Test' } };
const headers = { Authorization: `Bearer ${auth.token}`, 'Content-Type': 'application/json' };
const request = (page, method, params = {}) => page.evaluate(([method, params]) => window.solidcraft.request(method, params), [method, params]);
const chat = (page, params = {}) => request(page, 'ui.agent_chat', params);
const inspect = (page) => page.evaluate(() => window.solidcraft.inspect({ measure: true }));
async function until(page, predicate, message, timeoutMs = 30000) {
for (let i = 0; i < Math.ceil(timeoutMs / 100); i++) { const state = await chat(page); if (predicate(state)) return state; await page.waitForTimeout(100); }
throw new Error(message + ': ' + JSON.stringify(await chat(page)).slice(0, 2000));
}
async function ready(page) {
await page.waitForFunction(() => window.vibemakeCad && window.solidcraft, null, { timeout: 180000 });
let stable = 0;
for (let i = 0; i < 900; i++) {
const state = await page.evaluate(() => window.vibemakeCad.getState());
const hostReady = await page.evaluate(() => document.querySelector('#cad-main')?.hidden && !document.querySelector('#cad-viewport')?.inert);
stable = state.binding && !state.locked && state.assistant?.enabled && hostReady ? stable + 1 : 0;
if (stable >= 3) return;
await page.waitForTimeout(100);
}
throw new Error('Host and assistant did not become interactive');
}
async function clickHandle(page, name) {
for (let i = 0; i < 40; i++) {
const handles = await request(page, 'ui.handles', { prefix: name });
const handle = handles.find((handle) => handle.name === name);
if (handle) {
const canvas = await page.locator('canvas').boundingBox();
assert.ok(canvas, 'editor canvas must be visible');
await page.mouse.click(canvas.x + handle.x, canvas.y + handle.y);
await page.waitForTimeout(150); return;
}
await page.waitForTimeout(100);
}
throw new Error('Missing UI handle: ' + name);
}
async function openAttachmentMenu(page) {
for (let attempt = 0; attempt < 5; attempt++) {
const handles = await request(page, 'ui.handles', { prefix: 'agent_chat:add_file' });
if (handles.some((handle) => handle.name === 'agent_chat:add_file')) return;
await clickHandle(page, 'agent_chat:add');
await page.waitForTimeout(200);
}
throw new Error('Attachment menu did not open');
}
async function openAssistantSettings(page, language) {
await page.locator('[data-engine-menu="preferences"]').click();
// The host menu is delivered on the next engine frame; select the page only
// after that frame, otherwise its General-page request can win the race.
await page.waitForTimeout(350);
await clickHandle(page, 'prefs:page:Assistant');
await page.waitForFunction(async () => {
const handles = await window.solidcraft.request('ui.handles', { prefix: 'prefs:ai_features' });
return handles.some((handle) => handle.name === 'prefs:ai_features');
});
await request(page, 'ui.prefs', { language, button: 'apply' });
await page.waitForFunction(async (language) => (await window.vibemakeCad.getState()).locale === language, language);
await page.waitForTimeout(250);
}
async function configure(page, { id = 'browser-api', kind = 'api', model = 'mock-tools', url = provider, timeout = 30 } = {}) {
await chat(page, { add_agent: { id, name: id, kind, base_url: url, model, request_timeout_s: timeout }, agent: id, open: true });
}
async function setKey(page, id, key, remember) {
await chat(page, { edit_dialog: id });
await page.waitForTimeout(350);
await clickHandle(page, 'ai_add:key');
await page.keyboard.press('ControlOrMeta+A');
await page.keyboard.type(key, { delay: 15 });
await page.waitForTimeout(150);
if (process.env.VM_CAD_ARTIFACT_DIR) await page.screenshot({ path: resolve(process.env.VM_CAD_ARTIFACT_DIR, 'assistant-key-setup.png') });
if (remember) await clickHandle(page, 'ai_add:remember');
await clickHandle(page, 'ai_add:add');
await page.waitForTimeout(200);
}
async function reconnect(page, id) {
await chat(page, { agent: id, connect: true });
await until(page, (state) => state.state === 'ready', 'model did not connect');
}
async function turn(page, text, approve = true) {
await chat(page, { send: text, attach_view: false, attach_selection: false });
const state = await until(page, (state) => state.waiting > 0 || state.state !== 'busy', 'tool never reached approval');
if (state.waiting) await chat(page, { approve });
return until(page, (state) => state.state !== 'busy' && state.waiting === 0, 'turn did not end');
}
async function remembered(page) {
return page.evaluate(async () => {
const req = indexedDB.open('vm.cad.agent.credentials.v1', 1);
const db = await new Promise((resolve, reject) => { req.onsuccess = () => resolve(req.result); req.onerror = () => reject(req.error); });
const tx = db.transaction('credentials', 'readonly'), get = tx.objectStore('credentials').get('cad-browser-test-user');
const value = await new Promise((resolve, reject) => { get.onsuccess = () => resolve(get.result); get.onerror = () => reject(get.error); });
db.close(); return JSON.parse(value || '{}');
});
}
async function saveAndReload(page) {
await page.locator('[data-save]').click();
await page.waitForFunction(() => document.querySelector('.cad-save-state')?.dataset.state === 'saved', null, { timeout: 60000 });
await page.reload(); await ready(page);
}
export async function runAssistantBrowserChecks(browser, artifactDir) {
const checks = [], errors = [], context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
await context.addInitScript((value) => { if (!localStorage.getItem('vm.auth')) localStorage.setItem('vm.auth', JSON.stringify(value)); }, auth);
const page = await context.newPage(); page.on('pageerror', (error) => errors.push(String(error)));
page.on('dialog', (dialog) => dialog.accept());
let url;
const screenshot = async (name) => { if (artifactDir) { await mkdir(artifactDir, { recursive: true }); await page.screenshot({ path: resolve(artifactDir, name) }); } };
try {
const created = await (await fetch(`${base}/api/cad/workspace/projects`, { method: 'POST', headers, body: JSON.stringify({ name: 'Assistant browser regression' }) })).json();
assert.equal(created.code, 'OK');
url = `${base}/cad/editor/?project=${created.data.cadProjectCode}`;
await page.goto(url); await ready(page);
const readyInfo = await page.evaluate(() => window.vibemakeCad.ready());
assert.equal(readyInfo.capabilities.assistant, true);
const utilities = await page.locator('.cad-editor-utility button').evaluateAll((buttons) => buttons.map((button) => button.dataset.engineMenu));
assert.deepEqual(utilities, ['preferences', 'help']);
assert.equal((await chat(page)).permissions, 'ask');
await until(page, (state) => state.open, 'assistant did not open');
await screenshot('assistant-zh.png');
checks.push('original-panel-and-ask-default');
await page.route('http://localhost:11434/v1/models', (route) => route.fulfill({ json: { data: [] } }));
await page.route('https://openrouter.ai/api/v1/models', (route) => route.fulfill({ json: { data: [] } }));
await chat(page, { add_dialog: 'api' });
await page.waitForTimeout(350);
// All original connection kinds stay visible. Unsupported native kinds cannot change the form.
await clickHandle(page, 'ai_add:ui.ai_add_installed');
let dialogHandles = await request(page, 'ui.handles', { prefix: 'ai_add:' });
assert.ok(dialogHandles.some((handle) => handle.name === 'ai_add:protocol:openai'));
assert.ok(dialogHandles.some((handle) => handle.name === 'ai_add:ui.ai_add_custom'));
await clickHandle(page, 'ai_add:preset');
await clickHandle(page, 'ai_add:preset:OpenRouter');
await screenshot('assistant-original-presets-zh.png');
await chat(page, { add_dialog: false });
checks.push('original-connection-kind-layout-and-service-presets');
await context.grantPermissions(['clipboard-read', 'clipboard-write']);
await page.evaluate(async () => {
const canvas = document.createElement('canvas'); canvas.width = 12; canvas.height = 12;
canvas.getContext('2d').fillRect(0, 0, 12, 12);
const blob = await new Promise((resolve) => canvas.toBlob(resolve, 'image/png'));
await navigator.clipboard.write([new ClipboardItem({ 'image/png': blob })]);
});
await openAttachmentMenu(page);
await screenshot('assistant-attachment-menu-zh.png');
await clickHandle(page, 'agent_chat:paste_picture');
await until(page, (state) => state.attachments.length === 1, 'clipboard picture did not attach');
await screenshot('assistant-clipboard-picture-zh.png');
await clickHandle(page, 'agent_chat:file_remove:0');
await openAttachmentMenu(page);
const [fileChoice] = await Promise.all([page.waitForEvent('filechooser'), clickHandle(page, 'agent_chat:add_file')]);
await fileChoice.setFiles({ name: 'design-notes.svg', mimeType: 'image/svg+xml', buffer: Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"><title>CAD design notes</title></svg>') });
await until(page, (state) => state.attachments.includes('design-notes.svg'), 'original SVG text attachment did not load');
await clickHandle(page, 'agent_chat:file_remove:0');
const panel = await request(page, 'ui.handles', { prefix: 'agent_chat:add' });
const addTarget = panel.find((handle) => handle.name === 'agent_chat:add');
const canvas = await page.locator('canvas').boundingBox();
await page.mouse.move(canvas.x + addTarget.x, canvas.y + addTarget.y - 60);
await page.waitForTimeout(200);
await page.evaluate(() => {
const data = new DataTransfer();
data.items.add(new File(['First design note'], 'dropped-first.md', { type: 'text/plain' }));
data.items.add(new File(['Second design note'], 'dropped-second.txt', { type: 'text/plain' }));
document.querySelector('canvas').dispatchEvent(new DragEvent('drop', { bubbles: true, cancelable: true, dataTransfer: data }));
});
await until(page, (state) => state.attachments.includes('dropped-first.md') && state.attachments.includes('dropped-second.txt'), 'both dropped text files must attach');
await screenshot('assistant-dropped-files-zh.png');
await clickHandle(page, 'agent_chat:file_remove:0');
await clickHandle(page, 'agent_chat:file_remove:0');
checks.push('original-attachment-menu-clipboard-picture-and-text-file-picker');
checks.push('browser-drops-attach-multiple-files-without-opening-another-design');
await configure(page); await setKey(page, 'browser-api', 'synthetic-memory-key', false);
assert.deepEqual(await remembered(page), {});
await reconnect(page, 'browser-api');
const connected = await chat(page);
assert.deepEqual(connected.models.map((model) => model.id).sort(), ['chat-only', 'mock-tools']);
assert.equal(connected.models.find((model) => model.id === 'mock-tools').name, 'Deterministic CAD test model');
assert.equal(connected.modelsNote, null);
checks.push('connection-loads-original-model-picker-and-provider-display-names');
const before = await inspect(page);
await chat(page, { send: 'vm-test:create', attach_view: false, attach_selection: false });
await until(page, (state) => state.waiting > 0, 'missing approval');
assert.equal((await inspect(page)).bodies.length, before.bodies.length);
await screenshot('assistant-approval-zh.png');
await chat(page, { approve: true });
await until(page, (state) => state.state === 'ready' && !state.waiting, 'approved tool did not finish');
assert.equal((await inspect(page)).bodies.length, 1);
assert.equal((await page.evaluate(() => window.vibemakeCad.getState())).dirty, true);
await screenshot('assistant-result-zh.png');
const providerCalls = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.ok(providerCalls.some((call) => call.authorization === 'Bearer synthetic-memory-key'));
assert.ok(providerCalls.every((call) => call.cookie === '' && call.authorization !== `Bearer ${auth.token}`));
assert.ok(!JSON.stringify(await page.evaluate(() => window.vibemakeCad.getState())).includes('synthetic-memory-key'));
assert.ok(!(await page.evaluate(() => localStorage.getItem('vm.cad.engine.prefs') || '')).includes('browser-api'));
checks.push('browser-provider-request-no-site-credentials-and-approval-before-mutation');
await chat(page, { new_chat: true });
await page.waitForTimeout(200);
assert.deepEqual((await chat(page)).items, [], 'new conversation must not begin with a stale cancelled notice');
checks.push('new-conversation-starts-with-a-clean-transcript');
await turn(page, 'vm-test:repeat-id first');
await turn(page, 'vm-test:repeat-id second');
assert.equal((await chat(page)).items.filter((item) => item.tool).length, 2, 'reused provider IDs must retain both tool cards');
await screenshot('assistant-tool-history.png');
checks.push('reused-provider-tool-ids-retain-separate-transcript-cards');
for (const scenario of ['tool-error', 'unknown-tool', 'invalid-arguments']) {
await chat(page, { new_chat: true });
const state = await turn(page, `vm-test:${scenario}`);
assert.ok(JSON.stringify(state.items).includes('CAD tool result received'), `${scenario}: model must receive the error and finish its answer`);
assert.ok(state.items.some((item) => item.tool && item.status === 'Failed'), `${scenario}: failed tool stays visible`);
}
checks.push('failed-unknown-and-malformed-tools-return-feedback-to-model');
await saveAndReload(page);
assert.equal((await inspect(page)).bodies.length, 1);
assert.equal((await chat(page)).items.length, 0);
// Ordinary configuration remains, but memory-only credentials do not.
await reconnect(page, 'browser-api');
await turn(page, 'vm-test:read');
assert.ok(JSON.stringify((await chat(page)).items).includes('AIBox'), 'read tool result must describe the current design');
const afterReload = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.equal(afterReload.at(-1).authorization, '');
checks.push('ai-native-model-saves-reopens-and-chat-memory-clears');
await chat(page, { new_chat: true });
const count = (await inspect(page)).bodies.length;
await turn(page, 'vm-test:create', false);
assert.equal((await inspect(page)).bodies.length, count);
checks.push('denial-prevents-model-changes');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:create', attach_view: false, attach_selection: false });
await until(page, (state) => state.waiting > 0, 'missing stale approval');
await page.evaluate(() => window.solidcraft.execute('solid.box', { length: 3, width: 4, height: 5, body_name: 'Human' }));
await until(page, (state) => !state.waiting && !state.turn, 'stale approval was not cancelled');
assert.equal((await inspect(page)).bodies.length, count + 1);
checks.push('manual-edits-invalidate-ai-approval');
await configure(page, { id: 'anthropic-api', kind: 'anthropic' });
await setKey(page, 'anthropic-api', 'synthetic-remembered-key', true);
for (let i = 0; i < 40 && !(await remembered(page))['anthropic-api']; i++) await page.waitForTimeout(100);
assert.equal((await remembered(page))['anthropic-api'], 'synthetic-remembered-key');
await reconnect(page, 'anthropic-api');
await chat(page, { permissions: 'edits' });
await turn(page, 'vm-test:read');
checks.push('anthropic-compatible-browser-protocol-and-opt-in-idb-key');
await chat(page, { new_chat: true });
await turn(page, 'vm-test:tool-error');
const failedCalls = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.ok(failedCalls.some((call) => call.path.endsWith('/messages') && call.body.messages?.at(-1)?.content?.some?.((part) => part.type === 'tool_result' && part.is_error === true)));
checks.push('anthropic-failed-tools-carry-is-error-and-continue-conversation');
await chat(page, { new_chat: true });
await turn(page, 'vm-test:batch');
assert.equal((await inspect(page)).bodies.length, count + 3);
await chat(page, { undo: true });
assert.equal((await inspect(page)).bodies.length, count + 1, 'one AI undo must remove the complete batch');
checks.push('multi-step-ai-batch-retains-original-undo-group');
await openAssistantSettings(page, 'zh');
await screenshot('assistant-settings-zh.png');
await request(page, 'ui.prefs', { button: 'cancel' });
await saveAndReload(page);
await reconnect(page, 'anthropic-api');
const rememberedCalls = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.equal(rememberedCalls.at(-1).key, 'synthetic-remembered-key');
checks.push('opt-in-remembered-key-restores-after-refresh');
// Changing a service URL must not use its old key for listing, testing or a new connection.
await chat(page, { edit_dialog: 'anthropic-api' });
await page.waitForTimeout(350);
await clickHandle(page, 'ai_add:url');
await page.keyboard.press('ControlOrMeta+A');
await page.keyboard.type(provider.replace('/v1', '/other/v1'), { delay: 10 });
await page.waitForTimeout(150);
await clickHandle(page, 'ai_add:add');
assert.equal((await chat(page)).state, 'off', 'editing the active endpoint must release its old connection and key');
await reconnect(page, 'anthropic-api');
const changedCalls = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.ok(changedCalls.some((call) => call.path.includes('/other/')), 'edited endpoint must actually receive a request');
assert.ok(changedCalls.filter((call) => call.path.includes('/other/')).every((call) => !call.key && !call.authorization));
for (let i = 0; i < 50 && (await remembered(page))['anthropic-api']; i++) await page.waitForTimeout(100);
assert.equal((await remembered(page))['anthropic-api'], undefined);
checks.push('changed-service-does-not-inherit-key-and-deletes-persisted-copy');
await page.evaluate(() => window.vibemakeCad.setLocale('en'));
await page.evaluate(() => window.vibemakeCad.ready());
await openAssistantSettings(page, 'en');
await screenshot('assistant-settings-en.png');
await request(page, 'ui.prefs', { button: 'cancel' });
await chat(page, { open: true });
await screenshot('assistant-en.png');
await configure(page, { id: 'chat-only', model: 'chat-only' }); await reconnect(page, 'chat-only');
await turn(page, 'vm-test:create');
assert.equal((await inspect(page)).bodies.length, count + 1);
const only = await chat(page);
assert.ok(only.modelWarning?.includes('Chat only') || JSON.stringify(only).includes('Chat only'));
checks.push('chat-only-model-never-advertises-cad-editing');
for (const model of ['unauthorized', 'rate-limit', 'bad-json', 'redirect']) {
await configure(page, { id: model, model });
await chat(page, { connect: true });
const failed = await until(page, (state) => !['starting', 'busy'].includes(state.state), model + ' did not fail');
assert.notEqual(failed.state, 'ready');
assert.equal(await page.evaluate(() => JSON.parse(localStorage.getItem('vm.auth')).token), auth.token);
}
checks.push('provider-auth-rate-limit-malformed-and-redirect-errors-do-not-log-out');
await configure(page, { id: 'no-cors', url: provider.replace('/v1', '/no-cors/v1') });
await chat(page, { connect: true });
const cors = await until(page, (state) => !['starting', 'busy'].includes(state.state), 'CORS rejection did not settle');
assert.notEqual(cors.state, 'ready');
assert.ok(JSON.stringify(cors.items).includes('Cannot reach'));
checks.push('browser-cors-rejection-reports-network-context');
await reconnect(page, 'browser-api');
await chat(page, { permissions: 'all', send: 'vm-test:forbidden' });
await until(page, (state) => state.state === 'ready' && !state.turn, 'platform restriction did not settle');
assert.equal((await inspect(page)).bodies.length, count + 1);
checks.push('all-mode-preserves-host-project-binding');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:slow', attach_view: false, attach_selection: false });
await page.waitForTimeout(100);
await chat(page, { cancel: true });
await page.waitForTimeout(1800);
assert.equal((await inspect(page)).bodies.length, count + 1);
checks.push('stop-cancels-in-flight-request-and-preserves-completed-model');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:timeout', attach_view: false, attach_selection: false });
const timedOut = await until(page, (state) => state.state !== 'busy' && !state.turn, 'request timeout did not abort the response', 45000);
assert.ok(JSON.stringify(timedOut.items).includes('timed out'));
assert.equal((await inspect(page)).bodies.length, count + 1);
checks.push('configured-timeout-aborts-response-without-model-changes');
await context.setOffline(true);
try {
await chat(page, { connect: true });
const offline = await until(page, (state) => !['starting', 'busy'].includes(state.state), 'offline request did not settle');
assert.notEqual(offline.state, 'ready');
assert.ok(JSON.stringify(offline.items).includes('Cannot reach'));
assert.equal(await page.evaluate(() => JSON.parse(localStorage.getItem('vm.auth')).token), auth.token);
} finally { await context.setOffline(false); }
checks.push('offline-model-request-does-not-change-site-authentication');
await configure(page, { id: 'storage-unavailable' });
await page.evaluate(() => {
window.__cadTestIdbOpen = IDBFactory.prototype.open;
IDBFactory.prototype.open = function(name, ...args) {
if (name === 'vm.cad.agent.credentials.v1') throw new DOMException('Test storage denied', 'SecurityError');
return window.__cadTestIdbOpen.call(this, name, ...args);
};
});
try {
await setKey(page, 'storage-unavailable', 'synthetic-unremembered-key', true);
const failedStorage = await until(page, (state) => JSON.stringify(state.items).includes('could not be remembered'), 'storage failure notice is missing');
assert.ok(failedStorage.items.length);
await reconnect(page, 'storage-unavailable');
const calls = await (await fetch(provider.replace(/\/v1$/, '/requests'))).json();
assert.equal(calls.at(-1).authorization, 'Bearer synthetic-unremembered-key', 'failed persistence retains the page-memory key');
} finally {
await page.evaluate(() => { IDBFactory.prototype.open = window.__cadTestIdbOpen; delete window.__cadTestIdbOpen; });
}
assert.equal((await remembered(page))['storage-unavailable'], undefined);
checks.push('credential-storage-failure-is-explicit-and-memory-key-remains-usable');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:slow', attach_view: false, attach_selection: false });
await page.waitForTimeout(100);
const bound = await page.evaluate(() => window.vibemakeCad.getState());
await page.evaluate(([binding, project]) => window.vibemakeCad.setAgentContext({ userId: 'other-test-account', projectId: project, binding, sessionId: 'other-session' }), [bound.binding, created.data.cadProjectCode]);
await page.waitForTimeout(1800);
assert.equal((await inspect(page)).bodies.length, count + 1);
assert.equal((await chat(page)).permissions, 'ask');
assert.equal((await chat(page)).items.length, 0);
const other = await chat(page);
assert.ok(!JSON.stringify(other).includes('synthetic-remembered-key'));
checks.push('late-responses-and-credentials-isolated-after-context-change');
assert.deepEqual(errors, []);
return { checks, errors, url };
} catch (error) {
await screenshot('assistant-failure.png').catch(() => {});
if (artifactDir) await writeFile(resolve(artifactDir, 'assistant-failure.json'), JSON.stringify({ checks, errors, error: String(error), state: await chat(page).catch(() => null) }, null, 2));
throw error;
} finally { await context.close(); }
}
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
const options = { headless: true, args: ['--enable-unsafe-webgpu', '--use-angle=swiftshader', '--enable-webgl'] };
if (process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE) options.executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE;
const browser = await chromium.launch(options), artifacts = process.env.VM_CAD_ARTIFACT_DIR;
try {
const result = await runAssistantBrowserChecks(browser, artifacts);
if (artifacts) await writeFile(resolve(artifacts, 'assistant-browser-results.json'), JSON.stringify(result, null, 2));
console.log(JSON.stringify(result));
} finally { await browser.close(); }
}