615 lines
42 KiB
JavaScript
615 lines
42 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { writeFile } from 'node:fs/promises';
|
|
import { pathToFileURL } from 'node:url';
|
|
import { chromium } from 'playwright';
|
|
|
|
const base = (process.env.VM_CAD_BASE_URL || 'http://localhost:8091').replace(/\/$/, '');
|
|
const auth = { token: 'cad-browser-test-token', user: { userCode: 'cad-browser-test-user', nickname: 'CAD Browser Test' } };
|
|
const headers = { Authorization: `Bearer ${auth.token}`, 'Content-Type': 'application/json' };
|
|
const chat = (page, params = {}) => page.evaluate((params) => window.solidcraft.request('ui.agent_chat', params), params);
|
|
const inspect = (page) => page.evaluate(() => window.solidcraft.inspect({ measure: true }));
|
|
const agentCalls = async () => (await fetch(`${base}/api/cad/ai/requests`, { headers })).json();
|
|
|
|
async function clickHandle(page, name) {
|
|
for (let i = 0; i < 40; i++) {
|
|
const handle = (await page.evaluate((prefix) => window.solidcraft.request('ui.handles', { prefix }), name)).find((h) => h.name === name);
|
|
if (handle) {
|
|
const canvas = await page.locator('canvas').boundingBox();
|
|
await page.mouse.click(canvas.x + handle.x, canvas.y + handle.y);
|
|
return;
|
|
}
|
|
await page.waitForTimeout(100);
|
|
}
|
|
throw new Error(`Missing CAD handle: ${name}`);
|
|
}
|
|
|
|
async function until(page, predicate, message, timeoutMs = 30000) {
|
|
for (let i = 0; i < timeoutMs / 100; i++) {
|
|
// Draft recovery can replace the engine after the host's first binding.
|
|
// Poll read-only state through that transition; never retry a mutation here.
|
|
const state = await page.evaluate(async () => {
|
|
if (!window.solidcraft) return null;
|
|
try { return await window.solidcraft.request('ui.agent_chat', {}); }
|
|
catch (error) {
|
|
if (String(error).includes('HOST_INTERACTION_LOCKED')) return null;
|
|
throw error;
|
|
}
|
|
});
|
|
if (!state) { await page.waitForTimeout(100); continue; }
|
|
if (predicate(state)) return state;
|
|
await page.waitForTimeout(100);
|
|
}
|
|
throw new Error(`${message}: ${JSON.stringify(await chat(page)).slice(0, 2000)}`);
|
|
}
|
|
|
|
async function run(browser) {
|
|
const checks = [], errors = [];
|
|
const project = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: 'Platform Agent browser test', description: '' }),
|
|
})).json();
|
|
assert.equal(project.code, 'OK');
|
|
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
|
|
await context.addInitScript((value) => localStorage.setItem('vm.auth', JSON.stringify(value)), auth);
|
|
const page = await context.newPage();
|
|
page.on('pageerror', (error) => errors.push(String(error)));
|
|
try {
|
|
await page.goto(`${base}/cad/editor/?project=${project.data.cadProjectCode}`);
|
|
await page.waitForFunction(() => window.vibemakeCad && window.solidcraft, null, { timeout: 180000 });
|
|
await page.waitForFunction(async () => {
|
|
const host = await window.vibemakeCad.getState();
|
|
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await page.waitForFunction(async () => {
|
|
const state = await window.vibemakeCad.getState();
|
|
return state.assistant?.fullHeight && state.assistant?.open && state.assistant?.connection === 'ready'
|
|
&& !state.locked && !document.querySelector('#cad-viewport').inert && document.body.classList.contains('cad-full-height');
|
|
});
|
|
// Binding, draft recovery and agent capabilities settle over separate engine frames.
|
|
await page.waitForTimeout(1500);
|
|
async function assertSplitLayout() {
|
|
await page.waitForFunction(async () => {
|
|
const state = await window.vibemakeCad.getState();
|
|
const canvas = document.querySelector('#cad-viewport').getBoundingClientRect();
|
|
const bar = document.querySelector('body > header.nav').getBoundingClientRect();
|
|
return !state.locked && !document.querySelector('#cad-viewport').inert && canvas.y === 0 && canvas.height === innerHeight
|
|
&& Math.abs(bar.width - innerWidth * (1 - state.assistant.panelFraction)) < 2;
|
|
});
|
|
const host = await page.evaluate(() => window.vibemakeCad.getState());
|
|
const canvas = await page.locator('#cad-viewport').boundingBox();
|
|
const bar = await page.locator('body > header.nav').boundingBox();
|
|
const size = page.viewportSize();
|
|
assert.equal(canvas.y, 0);
|
|
assert.equal(canvas.height, size.height);
|
|
assert.equal(bar.y, 0);
|
|
assert.equal(bar.height, 42);
|
|
assert.ok(Math.abs(bar.width - size.width * (1 - host.assistant.panelFraction)) < 2, 'HTML menu ends at the native divider');
|
|
const handles = await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'agent_chat:' }));
|
|
const newChat = handles.find((handle) => handle.name === 'agent_chat:newchat');
|
|
assert.ok(newChat.x > bar.width && Math.abs(newChat.y - bar.height / 2) < 2, 'assistant actions share the menu baseline');
|
|
for (const action of ['close', 'minimize', 'pop_out', 'grip']) {
|
|
assert.ok(!handles.some((handle) => handle.name === `agent_chat:${action}`), `no ${action} action in the permanent column`);
|
|
}
|
|
const input = handles.find((handle) => handle.name === 'agent_chat:input');
|
|
assert.ok(input.x > bar.width && input.y < size.height && input.y > size.height / 2, 'composer stays inside the right column');
|
|
const ui = await page.evaluate(() => window.solidcraft.request('ui.inspect', {}));
|
|
assert.ok(ui.viewport[0] + ui.viewport[2] <= bar.width + 2, 'CAD viewport clears the assistant');
|
|
const chrome = await page.locator('header .nav-logo, header .cad-appbar-left, header .nav-actions').evaluateAll((nodes) => nodes.map((node) => {
|
|
const rect = node.getBoundingClientRect(); return { left: rect.left, right: rect.right };
|
|
}));
|
|
for (let i = 1; i < chrome.length; i++) assert.ok(chrome[i - 1].right <= chrome[i].left + 1, 'menu controls never overlap');
|
|
assert.ok(chrome.at(-1).right <= bar.width + 1, 'menu actions stay in the left column');
|
|
}
|
|
await assertSplitLayout();
|
|
checks.push('assistant-opens-automatically-in-full-height-split-with-aligned-title-bar');
|
|
await until(page, (state) => state.open && state.state === 'ready' && state.agent === 'Mock CAD Agent', 'platform agent did not become ready', 60000);
|
|
await chat(page, { open: false, minimized: true, popped: true, side: 'left' });
|
|
await page.waitForTimeout(250);
|
|
assert.equal((await chat(page)).open, true);
|
|
await assertSplitLayout();
|
|
checks.push('assistant-cannot-be-hidden-folded-popped-out-or-moved-left');
|
|
await chat(page, { minimized: false, popped: false, side: 'right' });
|
|
const beforeResize = (await page.evaluate(() => window.vibemakeCad.getState())).assistant.panelFraction;
|
|
const divider = (await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'agent_chat:resize' })))[0];
|
|
await page.mouse.move(divider.x, divider.y);
|
|
await page.mouse.down();
|
|
await page.mouse.move(divider.x - 80, divider.y, { steps: 12 });
|
|
await page.mouse.up();
|
|
await page.waitForTimeout(500);
|
|
assert.ok((await page.evaluate(() => window.vibemakeCad.getState())).assistant.panelFraction > beforeResize + 0.02, 'divider resizes the actual column');
|
|
await assertSplitLayout();
|
|
await chat(page, { width: 380 });
|
|
await page.waitForTimeout(350);
|
|
for (const width of [1024, 821]) {
|
|
await page.setViewportSize({ width, height: 720 });
|
|
await page.waitForTimeout(500);
|
|
await assertSplitLayout();
|
|
}
|
|
await page.setViewportSize({ width: 1440, height: 1000 });
|
|
await page.waitForTimeout(350);
|
|
await assertSplitLayout();
|
|
checks.push('divider-resizes-both-columns-and-menus-fit-at-821-and-1024-pixels');
|
|
|
|
const beforeCalls = await (await fetch(`${base}/api/cad/ai/requests`, { headers })).json();
|
|
assert.equal(beforeCalls.at(-1).authorization, `Bearer ${auth.token}`);
|
|
assert.ok(beforeCalls.at(-1).path === '/cad/ai/capabilities');
|
|
assert.equal(await page.evaluate(() => window.__VIBEMAKE_CAD__.agentChat instanceof Function), true);
|
|
assert.equal(await page.locator('input[type=password], [data-engine-menu="assistant"] + input').count(), 0);
|
|
checks.push('capabilities-use-platform-auth-and-fixed-agent-has-no-key-input');
|
|
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-empty.png') });
|
|
await page.locator('[data-engine-menu="preferences"]').click();
|
|
await page.waitForFunction(async () => (await window.solidcraft.request('ui.handles', { prefix: 'prefs:page:' })).some((h) => h.name === 'prefs:page:General'));
|
|
const pages = await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'prefs:page:' }));
|
|
assert.ok(!pages.some((h) => h.name === 'prefs:page:Assistant'));
|
|
await page.evaluate(() => window.solidcraft.request('ui.prefs', { button: 'cancel' }));
|
|
checks.push('platform-preferences-hide-local-assistant-configuration');
|
|
|
|
// Real browser IME events: the hidden text bridge must retain its native caret after
|
|
// each commit, while the visible egui editor still supports insertion in the middle.
|
|
await page.bringToFront();
|
|
await page.waitForFunction(() => document.hasFocus());
|
|
await clickHandle(page, 'agent_chat:input');
|
|
await page.keyboard.type('vm-test:read ', { delay: 35 });
|
|
const cdp = await context.newCDPSession(page);
|
|
for (const [pinyin, text] of [['nihao', '你好'], ['shijie', '世界']]) {
|
|
await cdp.send('Input.imeSetComposition', { text: pinyin, selectionStart: pinyin.length, selectionEnd: pinyin.length });
|
|
await page.waitForTimeout(100);
|
|
await cdp.send('Input.insertText', { text });
|
|
await page.waitForTimeout(200);
|
|
}
|
|
await page.keyboard.press('ArrowLeft');
|
|
await page.keyboard.press('ArrowLeft');
|
|
await page.keyboard.type('X');
|
|
await page.keyboard.press('End');
|
|
await page.keyboard.press('Shift+Enter');
|
|
await page.keyboard.type('ok', { delay: 50 });
|
|
// Wait for actual engine frames; software rendering can exceed a fixed delay.
|
|
const flushInput = () => page.evaluate(async () => {
|
|
for (let frame = 0; frame < 2; frame++) await window.solidcraft.request('ui.handles', { prefix: 'agent_chat:input' });
|
|
});
|
|
await flushInput();
|
|
await page.waitForFunction(() => {
|
|
const input = document.activeElement;
|
|
return input instanceof HTMLInputElement && input.value.endsWith('ok')
|
|
&& input.selectionStart === input.value.length && input.selectionEnd === input.value.length;
|
|
});
|
|
// Release only after the engine consumes key-down; software rendering can
|
|
// take longer than a fixed key-press duration when several editors are open.
|
|
await page.keyboard.down('Backspace');
|
|
await flushInput();
|
|
await page.keyboard.up('Backspace');
|
|
await page.waitForFunction(() => document.activeElement instanceof HTMLInputElement && document.activeElement.value === '');
|
|
await flushInput();
|
|
await page.keyboard.press('Enter');
|
|
await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'typed prompt did not finish');
|
|
assert.equal((await chat(page)).items.find((item) => item.user).user, 'vm-test:read 你好X世界\no');
|
|
checks.push('chinese-ime-commits-preserve-order-and-middle-editing-newlines-and-backspace');
|
|
if (process.env.VM_CAD_AGENT_BROWSER_SUITE === 'input') {
|
|
assert.deepEqual(errors, []);
|
|
return { checks, errors, url: page.url() };
|
|
}
|
|
await chat(page, { new_chat: true });
|
|
|
|
const empty = await inspect(page);
|
|
assert.equal(empty.bodies.length, 0);
|
|
const initialRevision = await page.evaluate(async () => (await window.vibemakeCad.getState()).revision);
|
|
await chat(page, { send: 'vm-test:create', attach_view: false, attach_selection: false });
|
|
let calls = [];
|
|
for (let i = 0; i < 300; i++) {
|
|
calls = await agentCalls();
|
|
if (calls.some((call) => call.path === '/cad/ai/chat' && call.body?.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:create')))) break;
|
|
await page.waitForTimeout(100);
|
|
}
|
|
const firstChat = calls.find((call) => call.path === '/cad/ai/chat' && call.body?.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:create')));
|
|
assert.ok(firstChat, `chat request was not recorded: ${JSON.stringify(calls)}`);
|
|
assert.equal(firstChat.body.cadProjectCode, project.data.cadProjectCode);
|
|
assert.equal(typeof firstChat.body.context.documentRevision, 'string');
|
|
const firstRuntime = JSON.parse(firstChat.body.history[0].content.find((part) => part.text)?.text.split('[CAD Web runtime context]\n')[1]).cadWebRuntime;
|
|
assert.equal(firstRuntime.backendLimits.imagesPerRequest, 2);
|
|
assert.equal(firstRuntime.remainingImages, 2);
|
|
assert.ok(!Object.hasOwn(firstRuntime.backendLimits, 'maxModelCallsPerTurn'));
|
|
assert.match(firstRuntime.rules, /Do not request screenshots when remainingImages is zero/);
|
|
assert.match(firstChat.body.context.documentRevision, /^[a-f0-9]{64}$/);
|
|
assert.equal(firstChat.body.context.documentRevision, createHash('sha256').update(String(initialRevision)).digest('hex'));
|
|
assert.ok(firstChat.body.documentSessionCode);
|
|
assert.ok(firstChat.body.turnCode);
|
|
await page.waitForTimeout(100);
|
|
assert.equal((await inspect(page)).bodies.length, 0, 'tool_call display must not execute before completed');
|
|
await until(page, (state) => !state.turn && state.state === 'ready', 'completed model turn did not finish', 60000);
|
|
assert.equal((await inspect(page)).bodies.length, 1);
|
|
assert.equal((await inspect(page)).bodies[0].name, 'PlatformAIBox');
|
|
checks.push('fragmented-sse-and-completed-output-authorize-tool-execution');
|
|
|
|
await chat(page, { new_chat: true });
|
|
await chat(page, { send: 'vm-test:truncated', attach_view: false, attach_selection: false });
|
|
await until(page, (state) => !state.turn && state.state === 'ready', 'truncated stream did not stop', 30000);
|
|
assert.equal((await inspect(page)).bodies.length, 1, 'EOF without completed must never execute an announced tool');
|
|
checks.push('eof-without-completed-does-not-run-announced-tool');
|
|
|
|
await chat(page, { new_chat: true });
|
|
await chat(page, { send: 'vm-test:screenshot', attach_view: false, attach_selection: false });
|
|
await until(page, (state) => !state.turn && state.state === 'ready', 'screenshot tool turn did not finish', 60000);
|
|
const finalCalls = await agentCalls();
|
|
const firstTurn = finalCalls.filter((call) => call.path === '/cad/ai/chat' && call.body.turnCode === firstChat.body.turnCode);
|
|
assert.equal(firstTurn.length, 2);
|
|
assert.notEqual(firstTurn[0].body.context.documentRevision, firstTurn[1].body.context.documentRevision);
|
|
const screenshotCall = finalCalls.find((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === project.data.cadProjectCode && call.body.history.some((item) => item.type === 'toolResult' && item.content?.some((part) => part.type === 'image')));
|
|
assert.ok(screenshotCall, 'model continuation must receive the CAD-only screenshot image');
|
|
const picture = screenshotCall.body.history.flatMap((item) => item.content || []).find((part) => part.type === 'image');
|
|
assert.equal(picture.mimeType, 'image/png');
|
|
assert.ok(picture.data.startsWith('iVBORw0KGgo'));
|
|
checks.push('screenshot-tool-returns-model-render-as-an-image-block');
|
|
|
|
await chat(page, { new_chat: true });
|
|
const beforeCameraShot = await inspect(page);
|
|
await chat(page, { send: 'vm-test:screenshot-camera', attach_view: false, attach_selection: false });
|
|
const cameraState = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'custom camera screenshot did not finish', 60000);
|
|
assert.equal(cameraState.items.filter((i) => i.status === 'Failed').length, 0, JSON.stringify(cameraState.items));
|
|
assert.equal(cameraState.items.filter((i) => i.status === 'Completed').length, 1, JSON.stringify(cameraState.items));
|
|
const cameraCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:screenshot-camera')));
|
|
assert.equal(cameraCalls.length, 2, 'custom camera screenshot must continue the same model turn');
|
|
assert.equal(cameraCalls[0].body.turnCode, cameraCalls[1].body.turnCode);
|
|
const cameraResult = cameraCalls[1].body.history.at(-1);
|
|
assert.equal(cameraResult.type, 'toolResult');
|
|
assert.equal(cameraResult.isError, false);
|
|
const cameraImage = cameraResult.content.find((part) => part.type === 'image');
|
|
assert.equal(cameraImage?.mimeType, 'image/png');
|
|
const cameraPng = Buffer.from(cameraImage.data, 'base64');
|
|
assert.equal(cameraPng.subarray(0, 8).toString('hex'), '89504e470d0a1a0a');
|
|
assert.equal(cameraPng.readUInt32BE(16), 320);
|
|
assert.equal(cameraPng.readUInt32BE(20), 200);
|
|
// ui.render uses the stored camera; ui.camera reports the viewport preference's FOV.
|
|
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
|
|
assert.deepEqual(camera.target, [1, 1.5, 2]);
|
|
for (const [key, value] of Object.entries({ yaw: 1, pitch: 0.2, distance: 12, fov: 0.5, roll: 0.3 })) {
|
|
assert.ok(Math.abs(camera[key] - value) < 1e-9, `screenshot camera ${key}: ${camera[key]}`);
|
|
}
|
|
assert.deepEqual((await inspect(page)).bodies, beforeCameraShot.bodies, 'taking a screenshot must preserve the model');
|
|
checks.push('custom-camera-screenshot-applies-camera-and-returns-png-to-the-same-model-turn');
|
|
if (process.env.VM_CAD_SCREENSHOT) {
|
|
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-camera.png') });
|
|
await writeFile(process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-camera-render.png'), cameraPng);
|
|
}
|
|
|
|
const savedCamera = { target: [1, 1.5, 2], yaw: 0.4, pitch: 0.7, distance: 20, fov: 0 };
|
|
await page.evaluate((camera) => window.solidcraft.request('engine.execute', {
|
|
command: 'view.save', params: { name: 'AgentSavedView', camera },
|
|
}), savedCamera);
|
|
for (const view of ['named', 'isometric']) {
|
|
await chat(page, { new_chat: true });
|
|
await chat(page, { send: `vm-test:screenshot-${view}`, attach_view: false, attach_selection: false });
|
|
const state = await until(page, (s) => !s.turn && s.items.some((i) => i.user), `${view} screenshot did not finish`, 60000);
|
|
assert.equal(state.items.filter((i) => i.status === 'Completed').length, 1, JSON.stringify(state.items));
|
|
const requests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes(`vm-test:screenshot-${view}`)));
|
|
assert.equal(requests.length, 2, 'named and alias views must return their screenshot to the model');
|
|
assert.ok(requests[1].body.history.at(-1).content.some((part) => part.type === 'image'));
|
|
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
|
|
if (view === 'named') {
|
|
for (const [key, value] of Object.entries(savedCamera)) assert.deepEqual(camera[key], value);
|
|
} else {
|
|
assert.ok(Math.abs(camera.yaw + Math.PI / 4) < 1e-9);
|
|
assert.ok(Math.abs(camera.pitch - 0.6154797086703874) < 1e-9);
|
|
}
|
|
checks.push(`${view}-screenshot-view-matches-the-native-engine-and-returns-an-image`);
|
|
}
|
|
|
|
await chat(page, { new_chat: true });
|
|
await chat(page, { send: 'vm-test:screenshot-recover', attach_view: false, attach_selection: false });
|
|
const recoveredShot = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'invalid screenshot view did not recover', 60000);
|
|
assert.equal(recoveredShot.items.filter((i) => i.status === 'Failed').length, 1, JSON.stringify(recoveredShot.items));
|
|
assert.equal(recoveredShot.items.filter((i) => i.status === 'Completed').length, 2, JSON.stringify(recoveredShot.items));
|
|
const shotRecoveryCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:screenshot-recover')));
|
|
assert.equal(shotRecoveryCalls.length, 4);
|
|
assert.equal(new Set(shotRecoveryCalls.map((call) => call.body.turnCode)).size, 1);
|
|
const viewFailure = shotRecoveryCalls[1].body.history.at(-1);
|
|
assert.equal(viewFailure.isError, true);
|
|
assert.match(viewFailure.content[0].text, /no named view `MissingAgentView`/);
|
|
assert.ok(viewFailure.content.some((part) => part.text?.includes('Omit view and camera')));
|
|
assert.ok(shotRecoveryCalls[3].body.history.at(-1).content.some((part) => part.type === 'image'));
|
|
checks.push('invalid-screenshot-view-is-returned-to-the-model-inspected-and-corrected-in-the-same-turn');
|
|
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-view-recovery.png') });
|
|
|
|
for (const scenario of ['recover', 'partial']) {
|
|
await chat(page, { new_chat: true });
|
|
const before = (await inspect(page)).bodies.length;
|
|
await chat(page, { send: `vm-test:${scenario}`, attach_view: false, attach_selection: false });
|
|
const state = await until(page, (s) => !s.turn && s.items.some((i) => i.user), `${scenario} did not recover`, 60000);
|
|
assert.equal(state.items.filter((i) => i.status === 'Failed').length, 1, JSON.stringify(state.items));
|
|
assert.equal(state.items.filter((i) => i.status === 'Completed').length, 2, JSON.stringify(state.items));
|
|
assert.ok(!state.items.some((i) => i.notice?.includes('模型已变化')));
|
|
assert.equal((await inspect(page)).bodies.length, before + (scenario === 'partial' ? 2 : 1));
|
|
const recorded = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes(`vm-test:${scenario}`)));
|
|
assert.equal(recorded.length, 4, JSON.stringify(recorded.map((c) => c.body.history)));
|
|
assert.equal(new Set(recorded.map((c) => c.body.turnCode)).size, 1);
|
|
const failed = recorded[1].body.history.at(-1);
|
|
assert.equal(failed.isError, true);
|
|
const detail = JSON.parse(failed.content[0].text);
|
|
assert.match(detail.failed.error, /`m` is a unit name/);
|
|
assert.deepEqual(detail.applied, scenario === 'partial' ? [0] : []);
|
|
const feedback = JSON.parse(failed.content.at(-1).text);
|
|
assert.ok(!Object.hasOwn(feedback, 'remainingChanges'));
|
|
assert.ok(!Object.hasOwn(feedback, 'remainingModelCalls'));
|
|
assert.match(feedback.recovery, /Analyse the tool error/);
|
|
checks.push(`${scenario}-tool-failure-is-analysed-inspected-and-corrected-without-replaying-applied-steps`);
|
|
}
|
|
|
|
await chat(page, { new_chat: true });
|
|
const beforeLongTurn = (await inspect(page)).bodies.length;
|
|
await chat(page, { send: 'vm-test:long-turn', attach_view: false, attach_selection: false });
|
|
const longTurn = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'long turn did not finish', 120000);
|
|
assert.equal(longTurn.items.filter((i) => i.status === 'Failed').length, 0, JSON.stringify(longTurn.items));
|
|
assert.equal(longTurn.items.filter((i) => i.status === 'Completed').length, 15, JSON.stringify(longTurn.items));
|
|
assert.ok(JSON.stringify(longTurn.items).includes('Completed 30 CAD changes over 16 model calls'));
|
|
const afterLongTurn = await inspect(page);
|
|
assert.equal(afterLongTurn.bodies.length, beforeLongTurn + 30);
|
|
assert.equal(afterLongTurn.bodies.filter((body) => body.name.startsWith('LongTurnBox_')).length, 30);
|
|
const longCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === project.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:long-turn')));
|
|
assert.equal(longCalls.length, 16);
|
|
assert.equal(new Set(longCalls.map((call) => call.body.turnCode)).size, 1);
|
|
assert.equal(new Set(longCalls.map((call) => call.body.documentSessionCode)).size, 1);
|
|
assert.equal(longCalls.at(-1).body.history.filter((item) => item.type === 'toolResult').length, 15);
|
|
checks.push('one-turn-completes-16-model-calls-and-30-changes-despite-capabilities-12-and-24');
|
|
const arrivals = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === project.data.cadProjectCode).map((call) => call.receivedAt);
|
|
for (const end of arrivals) assert.ok(arrivals.filter((time) => time > end - 60_000 && time <= end).length <= 30,
|
|
'tool continuations must stay under 30 network requests in every rolling minute');
|
|
checks.push('all-model-and-tool-continuation-requests-stay-under-30-per-rolling-minute');
|
|
assert.deepEqual(errors, []);
|
|
if (process.env.VM_CAD_SCREENSHOT) {
|
|
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT });
|
|
}
|
|
|
|
// A real populated model produces an oversized inspect result.
|
|
await context.route('**/api/cad/ai/capabilities', async (route) => {
|
|
const response = await route.fetch(), body = await response.json();
|
|
body.data.limits.toolResultTextBytes = 1024;
|
|
await route.fulfill({ response, json: body });
|
|
});
|
|
const textBudgetProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: 'Tool result budget', description: '' }),
|
|
})).json();
|
|
assert.equal(textBudgetProject.code, 'OK');
|
|
await page.goto(`${base}/cad/editor/?project=${textBudgetProject.data.cadProjectCode}`);
|
|
await page.waitForFunction(async () => {
|
|
if (!window.vibemakeCad || !window.solidcraft) return false;
|
|
const state = await window.vibemakeCad.getState();
|
|
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await until(page, (state) => state.open && state.state === 'ready', 'text budget agent did not become ready', 60000);
|
|
for (let index = 0; index < 4; index++) await page.evaluate((index) => window.solidcraft.execute('solid.box', {
|
|
length: 2, width: 3, height: 4, corner: [index * 4, 0, 0], body_name: 'LongTurnBox_' + index,
|
|
}), index);
|
|
const beforeTextBudget = await inspect(page);
|
|
assert.ok(Buffer.byteLength(JSON.stringify(beforeTextBudget)) > 1024, 'real inspect result must exceed the advertised test quota');
|
|
await chat(page, { send: 'vm-test:tool-text-budget', attach_view: false, attach_selection: false });
|
|
const textBudget = await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'text budget recovery did not finish', 60000);
|
|
const textRequests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === textBudgetProject.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:tool-text-budget')));
|
|
assert.equal(textRequests.length, 3, JSON.stringify(textBudget));
|
|
const textResults = textRequests.at(-1).body.history.filter((item) => item.type === 'toolResult');
|
|
assert.equal(textResults[0].isError, true);
|
|
const withheld = JSON.parse(textResults[0].content[0].text);
|
|
assert.equal(withheld.error, 'tool_result_too_large');
|
|
assert.equal(withheld.toolResultTextBytes, 1024);
|
|
assert.ok(withheld.textBytes > 1024);
|
|
assert.match(withheld.recovery, /No partial JSON was sent/);
|
|
assert.match(withheld.recovery, /never replay modifications/);
|
|
assert.equal(textResults[1].isError, false, JSON.stringify(textResults[1]));
|
|
assert.deepEqual((await inspect(page)).bodies, beforeTextBudget.bodies);
|
|
for (const result of textResults) assert.ok(result.content.reduce((total, part) => total + Buffer.byteLength(part.text || ''), 0) <= 1024);
|
|
checks.push('oversized-tool-result-is-an-explicit-delivery-error-and-recovers-with-read-only-summary-without-partial-json');
|
|
await context.unroute('**/api/cad/ai/capabilities');
|
|
|
|
await context.route('**/api/cad/ai/capabilities', async (route) => {
|
|
const response = await route.fetch(), body = await response.json();
|
|
body.data.limits.textBytes = 8192;
|
|
await route.fulfill({ response, json: body });
|
|
});
|
|
const textProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: 'UTF-8 request budget', description: '' }),
|
|
})).json();
|
|
assert.equal(textProject.code, 'OK');
|
|
await page.goto(`${base}/cad/editor/?project=${textProject.data.cadProjectCode}`);
|
|
await page.waitForFunction(async () => {
|
|
if (!window.vibemakeCad || !window.solidcraft) return false;
|
|
const state = await window.vibemakeCad.getState();
|
|
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await until(page, (state) => state.open && state.state === 'ready', 'request budget agent did not become ready', 60000);
|
|
// The user text alone fits (8100 UTF-8 bytes); the mandatory runtime context
|
|
// pushes the combined request beyond 8192, exercising aggregate accounting.
|
|
await chat(page, { send: '限'.repeat(2700), attach_view: false, attach_selection: false });
|
|
const overText = await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'oversized request did not stop');
|
|
assert.match(JSON.stringify(overText), /textBytes=\d+\/8192/);
|
|
assert.equal((await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === textProject.data.cadProjectCode).length, 0, 'oversized UTF-8 request never reaches the backend');
|
|
await context.unroute('**/api/cad/ai/capabilities');
|
|
checks.push('oversized-current-turn-stops-before-network-with-the-exact-backend-byte-quota');
|
|
|
|
for (const scenario of ['images', 'higher', 'unlimited', 'compressed']) {
|
|
await context.route('**/api/cad/ai/capabilities', async (route) => {
|
|
const response = await route.fetch(), body = await response.json();
|
|
if (scenario === 'higher') Object.assign(body.data.limits, {
|
|
imagesPerRequest: 5, historyItems: 1024, imageBytes: 2097152, imageMaxEdge: 4096,
|
|
requestBytes: 8388608, textBytes: 524288, toolResultTextBytes: 65536, requestTimeoutSeconds: 180,
|
|
});
|
|
if (scenario === 'unlimited') body.data.limits = { imagesPerRequest: 0, historyItems: 0 };
|
|
if (scenario === 'compressed') Object.assign(body.data.limits, { imagesPerRequest: 3, imageBytes: 128 });
|
|
await route.fulfill({ response, json: body });
|
|
});
|
|
const budgetProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: `Backend budget ${scenario}`, description: '' }),
|
|
})).json();
|
|
assert.equal(budgetProject.code, 'OK');
|
|
await page.goto(`${base}/cad/editor/?project=${budgetProject.data.cadProjectCode}`);
|
|
await page.waitForFunction(async () => {
|
|
if (!window.vibemakeCad || !window.solidcraft) return false;
|
|
const state = await window.vibemakeCad.getState();
|
|
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await until(page, (state) => state.open && state.state === 'ready', 'budget agent did not become ready', 60000);
|
|
await chat(page, { send: `vm-test:budget-${scenario}`, attach_view: false, attach_selection: false });
|
|
const finished = await until(page, (state) => !state.turn && state.items.some((item) => item.user), `${scenario} budget did not finish`, 60000);
|
|
const requests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === budgetProject.data.cadProjectCode);
|
|
assert.equal(requests.length, 5, JSON.stringify(finished));
|
|
assert.equal(new Set(requests.map((call) => call.body.turnCode)).size, 1);
|
|
const notes = requests.map((call) => {
|
|
const text = call.body.history[0].content.find((part) => part.type === 'text').text;
|
|
assert.equal(text.split('[CAD Web runtime context]').length, 2, 'runtime context must be replaced, never accumulated');
|
|
return JSON.parse(text.split('[CAD Web runtime context]\n')[1]).cadWebRuntime;
|
|
});
|
|
const results = requests.at(-1).body.history.filter((item) => item.type === 'toolResult');
|
|
if (scenario === 'images') {
|
|
assert.equal(finished.items.filter((item) => item.status === 'Failed').length, 1);
|
|
assert.deepEqual(notes.map((note) => note.remainingImages), [2, 1, 0, 0, 0]);
|
|
assert.equal(results[2].isError, true);
|
|
assert.match(results[2].content[0].text, /Screenshot not executed: backend imagesPerRequest=2/);
|
|
assert.ok(!results[2].content.some((part) => part.type === 'image'));
|
|
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
|
|
assert.ok(Math.abs(camera.pitch) < 1e-9, 'blocked top screenshot must preserve the previous front camera');
|
|
checks.push('backend-image-quota-is-explained-to-model-and-blocked-before-camera-or-render-with-recovery');
|
|
} else {
|
|
assert.equal(finished.items.filter((item) => item.status === 'Failed').length, 0, JSON.stringify(finished));
|
|
assert.equal(results.flatMap((item) => item.content).filter((part) => part.type === 'image').length, 3);
|
|
if (scenario === 'compressed') {
|
|
for (const picture of results.flatMap((item) => item.content).filter((part) => part.type === 'image')) {
|
|
const png = Buffer.from(picture.data, 'base64');
|
|
assert.ok(png.length <= 128, 'actual image bytes must fit the advertised quota');
|
|
assert.ok(png.readUInt32BE(16) < 160 && png.readUInt32BE(20) < 100, 'oversized PNG is scaled down before sending');
|
|
}
|
|
checks.push('screenshots-are-automatically-scaled-and-reencoded-to-the-advertised-image-byte-quota');
|
|
} else if (scenario === 'higher') {
|
|
assert.deepEqual(notes.map((note) => note.remainingImages), [5, 4, 3, 2, 2]);
|
|
assert.equal(notes[0].backendLimits.historyItems, 1024);
|
|
assert.equal(notes[0].backendLimits.requestTimeoutSeconds, 180);
|
|
checks.push('larger-backend-quotas-are-used-without-the-old-two-image-and-256-item-clamps');
|
|
} else {
|
|
assert.deepEqual(notes.map((note) => note.remainingImages), [null, null, null, null, null]);
|
|
assert.deepEqual(notes[0].backendLimits, {});
|
|
checks.push('missing-or-zero-backend-quotas-do-not-acquire-frontend-image-history-or-timeout-limits');
|
|
}
|
|
}
|
|
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, `-budget-${scenario}.png`) });
|
|
if (scenario === 'images') {
|
|
// No new chat: a new message frees old screenshots by dropping whole
|
|
// completed rounds, including their call/result pairs.
|
|
await chat(page, { send: 'vm-test:screenshot after-budget', attach_view: false, attach_selection: false });
|
|
const resumed = await until(page, (state) => !state.turn && state.items.some((item) => item.user?.includes('after-budget')),
|
|
'new message did not recover image budget', 60000);
|
|
const resumedRequests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === budgetProject.data.cadProjectCode
|
|
&& call.body.history[0]?.content?.some((part) => part.text?.includes('after-budget')));
|
|
assert.equal(resumedRequests.length, 2, JSON.stringify(resumed));
|
|
assert.equal(resumedRequests[0].body.history.length, 1, 'only the new complete user round is retained');
|
|
assert.equal(resumedRequests[1].body.history.at(-1).isError, false);
|
|
assert.equal(resumedRequests[1].body.history.at(-1).content.filter((part) => part.type === 'image').length, 1);
|
|
checks.push('new-user-message-reclaims-old-image-budget-by-trimming-only-complete-rounds');
|
|
}
|
|
await context.unroute('**/api/cad/ai/capabilities');
|
|
}
|
|
|
|
// A long turn waits at the request rate boundary and can still be stopped.
|
|
await context.route('**/api/cad/ai/capabilities', async (route) => {
|
|
const response = await route.fetch();
|
|
const body = await response.json();
|
|
body.data.limits.requestsPerMinute = 1;
|
|
body.data.limits.requestTimeoutSeconds = 3;
|
|
await route.fulfill({ response, json: body });
|
|
});
|
|
const pacedProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: 'Paced Agent browser test', description: '' }),
|
|
})).json();
|
|
assert.equal(pacedProject.code, 'OK');
|
|
await page.goto(`${base}/cad/editor/?project=${pacedProject.data.cadProjectCode}`);
|
|
await page.waitForFunction(async () => {
|
|
if (!window.vibemakeCad || !window.solidcraft) return false;
|
|
const host = await window.vibemakeCad.getState();
|
|
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await page.waitForFunction(() => window.solidcraft && window.vibemakeCad, null, { timeout: 60000 });
|
|
await until(page, (s) => s.open && s.state === 'ready', 'paced agent did not become ready', 60000);
|
|
// Isolate this scenario's quota from the previous long turn.
|
|
await page.evaluate(() => {
|
|
for (const key of Object.keys(localStorage)) if (key.startsWith('vm.cad.ai-rate:')) localStorage.removeItem(key);
|
|
});
|
|
await chat(page, { send: 'vm-test:read paced-cancel', attach_view: false, attach_selection: false });
|
|
await until(page, (s) => s.turn && s.items.some((i) => i.status === 'Completed'), 'first paced tool did not execute');
|
|
await page.waitForTimeout(3500);
|
|
const waiting = await chat(page);
|
|
assert.ok(waiting.turn, 'reaching the rate limit must wait instead of ending the turn');
|
|
const pacedCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === pacedProject.data.cadProjectCode
|
|
&& call.body.history.some((item) => item.content?.some((part) => part.text?.includes('paced-cancel'))));
|
|
assert.equal(pacedCalls.length, 1, 'no continuation may be sent above the rate limit');
|
|
await chat(page, { cancel: true });
|
|
await until(page, (s) => !s.turn, 'Stop did not cancel rate-limit waiting');
|
|
await page.waitForTimeout(500);
|
|
assert.equal((await agentCalls()).filter((call) => call.body?.turnCode === pacedCalls[0].body.turnCode).length, 1);
|
|
checks.push('rate-limit-waits-without-ending-the-turn-and-stop-cancels-the-wait');
|
|
checks.push('queued-time-is-excluded-from-the-model-request-timeout');
|
|
|
|
const otherProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
|
|
method: 'POST', headers, body: JSON.stringify({ name: 'Shared account quota test', description: '' }),
|
|
})).json();
|
|
assert.equal(otherProject.code, 'OK');
|
|
const otherPage = await context.newPage();
|
|
otherPage.on('pageerror', (error) => errors.push(String(error)));
|
|
try {
|
|
await otherPage.goto(`${base}/cad/editor/?project=${otherProject.data.cadProjectCode}`);
|
|
await otherPage.waitForFunction(async () => {
|
|
if (!window.vibemakeCad || !window.solidcraft) return false;
|
|
const host = await window.vibemakeCad.getState();
|
|
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
|
|
}, null, { timeout: 180000 });
|
|
await until(otherPage, (s) => s.open && s.state === 'ready', 'second tab agent did not become ready', 60000);
|
|
await chat(otherPage, { send: 'vm-test:read shared-account-quota', attach_view: false, attach_selection: false });
|
|
await until(otherPage, (s) => s.turn, 'second tab did not queue its request');
|
|
await otherPage.waitForTimeout(500);
|
|
assert.ok((await chat(otherPage)).turn);
|
|
assert.equal((await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
|
|
&& call.body.cadProjectCode === otherProject.data.cadProjectCode).length, 0,
|
|
'a new document/tab must retain the account quota');
|
|
await chat(otherPage, { cancel: true });
|
|
await until(otherPage, (s) => !s.turn, 'second tab queued request did not cancel');
|
|
checks.push('new-documents-and-browser-tabs-share-the-account-request-quota');
|
|
} finally {
|
|
await otherPage.close();
|
|
}
|
|
assert.deepEqual(errors, []);
|
|
return { checks, errors, url: page.url() };
|
|
} catch (error) {
|
|
if (process.env.VM_CAD_SCREENSHOT) {
|
|
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-failure.png') }).catch(() => {});
|
|
await writeFile(process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-failure.json'), JSON.stringify({
|
|
checks, error: String(error), host: await page.evaluate(() => window.vibemakeCad?.getState()).catch(() => null),
|
|
inert: await page.locator('#cad-viewport').evaluate((element) => element.inert).catch(() => null),
|
|
}, null, 2));
|
|
}
|
|
throw error;
|
|
} finally {
|
|
await context.close();
|
|
}
|
|
}
|
|
|
|
export { run as runPlatformAgentBrowserChecks };
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
const options = { headless: true, args: ['--enable-unsafe-webgpu', '--use-angle=swiftshader', '--enable-webgl'] };
|
|
if (process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE) options.executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE;
|
|
const browser = await chromium.launch(options);
|
|
try { console.log(JSON.stringify(await run(browser))); }
|
|
finally { await browser.close(); }
|
|
}
|