Files
vibemake/tests/cad/platform-agent-browser.mjs
T
2026-11-05 17:50:35 +08:00

595 lines
40 KiB
JavaScript

import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { writeFile } from 'node:fs/promises';
import { pathToFileURL } from 'node:url';
import { chromium } from 'playwright';
const base = (process.env.VM_CAD_BASE_URL || 'http://localhost:8091').replace(/\/$/, '');
const auth = { token: 'cad-browser-test-token', user: { userCode: 'cad-browser-test-user', nickname: 'CAD Browser Test' } };
const headers = { Authorization: `Bearer ${auth.token}`, 'Content-Type': 'application/json' };
const chat = (page, params = {}) => page.evaluate((params) => window.solidcraft.request('ui.agent_chat', params), params);
const inspect = (page) => page.evaluate(() => window.solidcraft.inspect({ measure: true }));
const agentCalls = async () => (await fetch(`${base}/api/cad/ai/requests`, { headers })).json();
async function clickHandle(page, name) {
for (let i = 0; i < 40; i++) {
const handle = (await page.evaluate((prefix) => window.solidcraft.request('ui.handles', { prefix }), name)).find((h) => h.name === name);
if (handle) {
const canvas = await page.locator('canvas').boundingBox();
await page.mouse.click(canvas.x + handle.x, canvas.y + handle.y);
return;
}
await page.waitForTimeout(100);
}
throw new Error(`Missing CAD handle: ${name}`);
}
async function until(page, predicate, message, timeoutMs = 30000) {
for (let i = 0; i < timeoutMs / 100; i++) {
// Draft recovery can replace the engine after the host's first binding.
// Poll read-only state through that transition; never retry a mutation here.
const state = await page.evaluate(async () => {
if (!window.solidcraft) return null;
try { return await window.solidcraft.request('ui.agent_chat', {}); }
catch (error) {
if (String(error).includes('HOST_INTERACTION_LOCKED')) return null;
throw error;
}
});
if (!state) { await page.waitForTimeout(100); continue; }
if (predicate(state)) return state;
await page.waitForTimeout(100);
}
throw new Error(`${message}: ${JSON.stringify(await chat(page)).slice(0, 2000)}`);
}
async function run(browser) {
const checks = [], errors = [];
const project = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: 'Platform Agent browser test', description: '' }),
})).json();
assert.equal(project.code, 'OK');
const context = await browser.newContext({ viewport: { width: 1440, height: 1000 } });
await context.addInitScript((value) => localStorage.setItem('vm.auth', JSON.stringify(value)), auth);
const page = await context.newPage();
page.on('pageerror', (error) => errors.push(String(error)));
try {
await page.goto(`${base}/cad/editor/?project=${project.data.cadProjectCode}`);
await page.waitForFunction(() => window.vibemakeCad && window.solidcraft, null, { timeout: 180000 });
await page.waitForFunction(async () => {
const host = await window.vibemakeCad.getState();
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await page.waitForFunction(async () => {
const state = await window.vibemakeCad.getState();
return state.assistant?.fullHeight && state.assistant?.open && state.assistant?.connection === 'ready'
&& !state.locked && !document.querySelector('#cad-viewport').inert && document.body.classList.contains('cad-full-height');
});
// Binding, draft recovery and agent capabilities settle over separate engine frames.
await page.waitForTimeout(1500);
async function assertSplitLayout() {
await page.waitForFunction(async () => {
const state = await window.vibemakeCad.getState();
const canvas = document.querySelector('#cad-viewport').getBoundingClientRect();
const bar = document.querySelector('body > header.nav').getBoundingClientRect();
return !state.locked && !document.querySelector('#cad-viewport').inert && canvas.y === 0 && canvas.height === innerHeight
&& Math.abs(bar.width - innerWidth * (1 - state.assistant.panelFraction)) < 2;
});
const host = await page.evaluate(() => window.vibemakeCad.getState());
const canvas = await page.locator('#cad-viewport').boundingBox();
const bar = await page.locator('body > header.nav').boundingBox();
const size = page.viewportSize();
assert.equal(canvas.y, 0);
assert.equal(canvas.height, size.height);
assert.equal(bar.y, 0);
assert.equal(bar.height, 42);
assert.ok(Math.abs(bar.width - size.width * (1 - host.assistant.panelFraction)) < 2, 'HTML menu ends at the native divider');
const handles = await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'agent_chat:' }));
const newChat = handles.find((handle) => handle.name === 'agent_chat:newchat');
assert.ok(newChat.x > bar.width && Math.abs(newChat.y - bar.height / 2) < 2, 'assistant actions share the menu baseline');
for (const action of ['close', 'minimize', 'pop_out', 'grip']) {
assert.ok(!handles.some((handle) => handle.name === `agent_chat:${action}`), `no ${action} action in the permanent column`);
}
const input = handles.find((handle) => handle.name === 'agent_chat:input');
assert.ok(input.x > bar.width && input.y < size.height && input.y > size.height / 2, 'composer stays inside the right column');
const ui = await page.evaluate(() => window.solidcraft.request('ui.inspect', {}));
assert.ok(ui.viewport[0] + ui.viewport[2] <= bar.width + 2, 'CAD viewport clears the assistant');
const chrome = await page.locator('header .nav-logo, header .cad-appbar-left, header .nav-actions').evaluateAll((nodes) => nodes.map((node) => {
const rect = node.getBoundingClientRect(); return { left: rect.left, right: rect.right };
}));
for (let i = 1; i < chrome.length; i++) assert.ok(chrome[i - 1].right <= chrome[i].left + 1, 'menu controls never overlap');
assert.ok(chrome.at(-1).right <= bar.width + 1, 'menu actions stay in the left column');
}
await assertSplitLayout();
checks.push('assistant-opens-automatically-in-full-height-split-with-aligned-title-bar');
await until(page, (state) => state.open && state.state === 'ready' && state.agent === 'Mock CAD Agent', 'platform agent did not become ready', 60000);
await chat(page, { open: false, minimized: true, popped: true, side: 'left' });
await page.waitForTimeout(250);
assert.equal((await chat(page)).open, true);
await assertSplitLayout();
checks.push('assistant-cannot-be-hidden-folded-popped-out-or-moved-left');
await chat(page, { minimized: false, popped: false, side: 'right' });
const beforeResize = (await page.evaluate(() => window.vibemakeCad.getState())).assistant.panelFraction;
const divider = (await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'agent_chat:resize' })))[0];
await page.mouse.move(divider.x, divider.y);
await page.mouse.down();
await page.mouse.move(divider.x - 80, divider.y, { steps: 12 });
await page.mouse.up();
await page.waitForTimeout(500);
assert.ok((await page.evaluate(() => window.vibemakeCad.getState())).assistant.panelFraction > beforeResize + 0.02, 'divider resizes the actual column');
await assertSplitLayout();
await chat(page, { width: 380 });
await page.waitForTimeout(350);
for (const width of [1024, 821]) {
await page.setViewportSize({ width, height: 720 });
await page.waitForTimeout(500);
await assertSplitLayout();
}
await page.setViewportSize({ width: 1440, height: 1000 });
await page.waitForTimeout(350);
await assertSplitLayout();
checks.push('divider-resizes-both-columns-and-menus-fit-at-821-and-1024-pixels');
const beforeCalls = await (await fetch(`${base}/api/cad/ai/requests`, { headers })).json();
assert.equal(beforeCalls.at(-1).authorization, `Bearer ${auth.token}`);
assert.ok(beforeCalls.at(-1).path === '/cad/ai/capabilities');
assert.equal(await page.evaluate(() => window.__VIBEMAKE_CAD__.agentChat instanceof Function), true);
assert.equal(await page.locator('input[type=password], [data-engine-menu="assistant"] + input').count(), 0);
checks.push('capabilities-use-platform-auth-and-fixed-agent-has-no-key-input');
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-empty.png') });
await page.locator('[data-engine-menu="preferences"]').click();
await page.waitForFunction(async () => (await window.solidcraft.request('ui.handles', { prefix: 'prefs:page:' })).some((h) => h.name === 'prefs:page:General'));
const pages = await page.evaluate(() => window.solidcraft.request('ui.handles', { prefix: 'prefs:page:' }));
assert.ok(!pages.some((h) => h.name === 'prefs:page:Assistant'));
await page.evaluate(() => window.solidcraft.request('ui.prefs', { button: 'cancel' }));
checks.push('platform-preferences-hide-local-assistant-configuration');
// Real browser IME events: the hidden text bridge must retain its native caret after
// each commit, while the visible egui editor still supports insertion in the middle.
await clickHandle(page, 'agent_chat:input');
await page.keyboard.type('vm-test:read ', { delay: 35 });
const cdp = await context.newCDPSession(page);
for (const [pinyin, text] of [['nihao', '你好'], ['shijie', '世界']]) {
await cdp.send('Input.imeSetComposition', { text: pinyin, selectionStart: pinyin.length, selectionEnd: pinyin.length });
await page.waitForTimeout(100);
await cdp.send('Input.insertText', { text });
await page.waitForTimeout(200);
}
await page.keyboard.press('ArrowLeft');
await page.keyboard.press('ArrowLeft');
await page.keyboard.type('X');
await page.keyboard.press('End');
await page.keyboard.press('Shift+Enter');
await page.keyboard.type('ok', { delay: 50 });
// Let egui consume the browser text event before sending the next key.
await page.waitForTimeout(150);
await page.keyboard.press('Backspace');
await page.waitForTimeout(150);
await page.keyboard.press('Enter');
await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'typed prompt did not finish');
assert.equal((await chat(page)).items.find((item) => item.user).user, 'vm-test:read 你好X世界\no');
checks.push('chinese-ime-commits-preserve-order-and-middle-editing-newlines-and-backspace');
await chat(page, { new_chat: true });
const empty = await inspect(page);
assert.equal(empty.bodies.length, 0);
const initialRevision = await page.evaluate(async () => (await window.vibemakeCad.getState()).revision);
await chat(page, { send: 'vm-test:create', attach_view: false, attach_selection: false });
let calls = [];
for (let i = 0; i < 300; i++) {
calls = await agentCalls();
if (calls.some((call) => call.path === '/cad/ai/chat' && call.body?.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:create')))) break;
await page.waitForTimeout(100);
}
const firstChat = calls.find((call) => call.path === '/cad/ai/chat' && call.body?.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:create')));
assert.ok(firstChat, `chat request was not recorded: ${JSON.stringify(calls)}`);
assert.equal(firstChat.body.cadProjectCode, project.data.cadProjectCode);
assert.equal(typeof firstChat.body.context.documentRevision, 'string');
const firstRuntime = JSON.parse(firstChat.body.history[0].content.find((part) => part.text)?.text.split('[CAD Web runtime context]\n')[1]).cadWebRuntime;
assert.equal(firstRuntime.backendLimits.imagesPerRequest, 2);
assert.equal(firstRuntime.remainingImages, 2);
assert.ok(!Object.hasOwn(firstRuntime.backendLimits, 'maxModelCallsPerTurn'));
assert.match(firstRuntime.rules, /Do not request screenshots when remainingImages is zero/);
assert.match(firstChat.body.context.documentRevision, /^[a-f0-9]{64}$/);
assert.equal(firstChat.body.context.documentRevision, createHash('sha256').update(String(initialRevision)).digest('hex'));
assert.ok(firstChat.body.documentSessionCode);
assert.ok(firstChat.body.turnCode);
await page.waitForTimeout(100);
assert.equal((await inspect(page)).bodies.length, 0, 'tool_call display must not execute before completed');
await until(page, (state) => !state.turn && state.state === 'ready', 'completed model turn did not finish', 60000);
assert.equal((await inspect(page)).bodies.length, 1);
assert.equal((await inspect(page)).bodies[0].name, 'PlatformAIBox');
checks.push('fragmented-sse-and-completed-output-authorize-tool-execution');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:truncated', attach_view: false, attach_selection: false });
await until(page, (state) => !state.turn && state.state === 'ready', 'truncated stream did not stop', 30000);
assert.equal((await inspect(page)).bodies.length, 1, 'EOF without completed must never execute an announced tool');
checks.push('eof-without-completed-does-not-run-announced-tool');
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:screenshot', attach_view: false, attach_selection: false });
await until(page, (state) => !state.turn && state.state === 'ready', 'screenshot tool turn did not finish', 60000);
const finalCalls = await agentCalls();
const firstTurn = finalCalls.filter((call) => call.path === '/cad/ai/chat' && call.body.turnCode === firstChat.body.turnCode);
assert.equal(firstTurn.length, 2);
assert.notEqual(firstTurn[0].body.context.documentRevision, firstTurn[1].body.context.documentRevision);
const screenshotCall = finalCalls.find((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === project.data.cadProjectCode && call.body.history.some((item) => item.type === 'toolResult' && item.content?.some((part) => part.type === 'image')));
assert.ok(screenshotCall, 'model continuation must receive the CAD-only screenshot image');
const picture = screenshotCall.body.history.flatMap((item) => item.content || []).find((part) => part.type === 'image');
assert.equal(picture.mimeType, 'image/png');
assert.ok(picture.data.startsWith('iVBORw0KGgo'));
checks.push('screenshot-tool-returns-model-render-as-an-image-block');
await chat(page, { new_chat: true });
const beforeCameraShot = await inspect(page);
await chat(page, { send: 'vm-test:screenshot-camera', attach_view: false, attach_selection: false });
const cameraState = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'custom camera screenshot did not finish', 60000);
assert.equal(cameraState.items.filter((i) => i.status === 'Failed').length, 0, JSON.stringify(cameraState.items));
assert.equal(cameraState.items.filter((i) => i.status === 'Completed').length, 1, JSON.stringify(cameraState.items));
const cameraCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:screenshot-camera')));
assert.equal(cameraCalls.length, 2, 'custom camera screenshot must continue the same model turn');
assert.equal(cameraCalls[0].body.turnCode, cameraCalls[1].body.turnCode);
const cameraResult = cameraCalls[1].body.history.at(-1);
assert.equal(cameraResult.type, 'toolResult');
assert.equal(cameraResult.isError, false);
const cameraImage = cameraResult.content.find((part) => part.type === 'image');
assert.equal(cameraImage?.mimeType, 'image/png');
const cameraPng = Buffer.from(cameraImage.data, 'base64');
assert.equal(cameraPng.subarray(0, 8).toString('hex'), '89504e470d0a1a0a');
assert.equal(cameraPng.readUInt32BE(16), 320);
assert.equal(cameraPng.readUInt32BE(20), 200);
// ui.render uses the stored camera; ui.camera reports the viewport preference's FOV.
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
assert.deepEqual(camera.target, [1, 1.5, 2]);
for (const [key, value] of Object.entries({ yaw: 1, pitch: 0.2, distance: 12, fov: 0.5, roll: 0.3 })) {
assert.ok(Math.abs(camera[key] - value) < 1e-9, `screenshot camera ${key}: ${camera[key]}`);
}
assert.deepEqual((await inspect(page)).bodies, beforeCameraShot.bodies, 'taking a screenshot must preserve the model');
checks.push('custom-camera-screenshot-applies-camera-and-returns-png-to-the-same-model-turn');
if (process.env.VM_CAD_SCREENSHOT) {
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-camera.png') });
await writeFile(process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-camera-render.png'), cameraPng);
}
const savedCamera = { target: [1, 1.5, 2], yaw: 0.4, pitch: 0.7, distance: 20, fov: 0 };
await page.evaluate((camera) => window.solidcraft.request('engine.execute', {
command: 'view.save', params: { name: 'AgentSavedView', camera },
}), savedCamera);
for (const view of ['named', 'isometric']) {
await chat(page, { new_chat: true });
await chat(page, { send: `vm-test:screenshot-${view}`, attach_view: false, attach_selection: false });
const state = await until(page, (s) => !s.turn && s.items.some((i) => i.user), `${view} screenshot did not finish`, 60000);
assert.equal(state.items.filter((i) => i.status === 'Completed').length, 1, JSON.stringify(state.items));
const requests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes(`vm-test:screenshot-${view}`)));
assert.equal(requests.length, 2, 'named and alias views must return their screenshot to the model');
assert.ok(requests[1].body.history.at(-1).content.some((part) => part.type === 'image'));
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
if (view === 'named') {
for (const [key, value] of Object.entries(savedCamera)) assert.deepEqual(camera[key], value);
} else {
assert.ok(Math.abs(camera.yaw + Math.PI / 4) < 1e-9);
assert.ok(Math.abs(camera.pitch - 0.6154797086703874) < 1e-9);
}
checks.push(`${view}-screenshot-view-matches-the-native-engine-and-returns-an-image`);
}
await chat(page, { new_chat: true });
await chat(page, { send: 'vm-test:screenshot-recover', attach_view: false, attach_selection: false });
const recoveredShot = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'invalid screenshot view did not recover', 60000);
assert.equal(recoveredShot.items.filter((i) => i.status === 'Failed').length, 1, JSON.stringify(recoveredShot.items));
assert.equal(recoveredShot.items.filter((i) => i.status === 'Completed').length, 2, JSON.stringify(recoveredShot.items));
const shotRecoveryCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:screenshot-recover')));
assert.equal(shotRecoveryCalls.length, 4);
assert.equal(new Set(shotRecoveryCalls.map((call) => call.body.turnCode)).size, 1);
const viewFailure = shotRecoveryCalls[1].body.history.at(-1);
assert.equal(viewFailure.isError, true);
assert.match(viewFailure.content[0].text, /no named view `MissingAgentView`/);
assert.ok(viewFailure.content.some((part) => part.text?.includes('Omit view and camera')));
assert.ok(shotRecoveryCalls[3].body.history.at(-1).content.some((part) => part.type === 'image'));
checks.push('invalid-screenshot-view-is-returned-to-the-model-inspected-and-corrected-in-the-same-turn');
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-view-recovery.png') });
for (const scenario of ['recover', 'partial']) {
await chat(page, { new_chat: true });
const before = (await inspect(page)).bodies.length;
await chat(page, { send: `vm-test:${scenario}`, attach_view: false, attach_selection: false });
const state = await until(page, (s) => !s.turn && s.items.some((i) => i.user), `${scenario} did not recover`, 60000);
assert.equal(state.items.filter((i) => i.status === 'Failed').length, 1, JSON.stringify(state.items));
assert.equal(state.items.filter((i) => i.status === 'Completed').length, 2, JSON.stringify(state.items));
assert.ok(!state.items.some((i) => i.notice?.includes('模型已变化')));
assert.equal((await inspect(page)).bodies.length, before + (scenario === 'partial' ? 2 : 1));
const recorded = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes(`vm-test:${scenario}`)));
assert.equal(recorded.length, 4, JSON.stringify(recorded.map((c) => c.body.history)));
assert.equal(new Set(recorded.map((c) => c.body.turnCode)).size, 1);
const failed = recorded[1].body.history.at(-1);
assert.equal(failed.isError, true);
const detail = JSON.parse(failed.content[0].text);
assert.match(detail.failed.error, /`m` is a unit name/);
assert.deepEqual(detail.applied, scenario === 'partial' ? [0] : []);
const feedback = JSON.parse(failed.content.at(-1).text);
assert.ok(!Object.hasOwn(feedback, 'remainingChanges'));
assert.ok(!Object.hasOwn(feedback, 'remainingModelCalls'));
assert.match(feedback.recovery, /Analyse the tool error/);
checks.push(`${scenario}-tool-failure-is-analysed-inspected-and-corrected-without-replaying-applied-steps`);
}
await chat(page, { new_chat: true });
const beforeLongTurn = (await inspect(page)).bodies.length;
await chat(page, { send: 'vm-test:long-turn', attach_view: false, attach_selection: false });
const longTurn = await until(page, (s) => !s.turn && s.items.some((i) => i.user), 'long turn did not finish', 120000);
assert.equal(longTurn.items.filter((i) => i.status === 'Failed').length, 0, JSON.stringify(longTurn.items));
assert.equal(longTurn.items.filter((i) => i.status === 'Completed').length, 15, JSON.stringify(longTurn.items));
assert.ok(JSON.stringify(longTurn.items).includes('Completed 30 CAD changes over 16 model calls'));
const afterLongTurn = await inspect(page);
assert.equal(afterLongTurn.bodies.length, beforeLongTurn + 30);
assert.equal(afterLongTurn.bodies.filter((body) => body.name.startsWith('LongTurnBox_')).length, 30);
const longCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === project.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:long-turn')));
assert.equal(longCalls.length, 16);
assert.equal(new Set(longCalls.map((call) => call.body.turnCode)).size, 1);
assert.equal(new Set(longCalls.map((call) => call.body.documentSessionCode)).size, 1);
assert.equal(longCalls.at(-1).body.history.filter((item) => item.type === 'toolResult').length, 15);
checks.push('one-turn-completes-16-model-calls-and-30-changes-despite-capabilities-12-and-24');
const arrivals = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === project.data.cadProjectCode).map((call) => call.receivedAt);
for (const end of arrivals) assert.ok(arrivals.filter((time) => time > end - 60_000 && time <= end).length <= 30,
'tool continuations must stay under 30 network requests in every rolling minute');
checks.push('all-model-and-tool-continuation-requests-stay-under-30-per-rolling-minute');
assert.deepEqual(errors, []);
if (process.env.VM_CAD_SCREENSHOT) {
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT });
}
// A real populated model produces an oversized inspect result.
await context.route('**/api/cad/ai/capabilities', async (route) => {
const response = await route.fetch(), body = await response.json();
body.data.limits.toolResultTextBytes = 1024;
await route.fulfill({ response, json: body });
});
const textBudgetProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: 'Tool result budget', description: '' }),
})).json();
assert.equal(textBudgetProject.code, 'OK');
await page.goto(`${base}/cad/editor/?project=${textBudgetProject.data.cadProjectCode}`);
await page.waitForFunction(async () => {
if (!window.vibemakeCad || !window.solidcraft) return false;
const state = await window.vibemakeCad.getState();
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await until(page, (state) => state.open && state.state === 'ready', 'text budget agent did not become ready', 60000);
for (let index = 0; index < 4; index++) await page.evaluate((index) => window.solidcraft.execute('solid.box', {
length: 2, width: 3, height: 4, corner: [index * 4, 0, 0], body_name: 'LongTurnBox_' + index,
}), index);
const beforeTextBudget = await inspect(page);
assert.ok(Buffer.byteLength(JSON.stringify(beforeTextBudget)) > 1024, 'real inspect result must exceed the advertised test quota');
await chat(page, { send: 'vm-test:tool-text-budget', attach_view: false, attach_selection: false });
const textBudget = await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'text budget recovery did not finish', 60000);
const textRequests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === textBudgetProject.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('vm-test:tool-text-budget')));
assert.equal(textRequests.length, 3, JSON.stringify(textBudget));
const textResults = textRequests.at(-1).body.history.filter((item) => item.type === 'toolResult');
assert.equal(textResults[0].isError, true);
const withheld = JSON.parse(textResults[0].content[0].text);
assert.equal(withheld.error, 'tool_result_too_large');
assert.equal(withheld.toolResultTextBytes, 1024);
assert.ok(withheld.textBytes > 1024);
assert.match(withheld.recovery, /No partial JSON was sent/);
assert.match(withheld.recovery, /never replay modifications/);
assert.equal(textResults[1].isError, false, JSON.stringify(textResults[1]));
assert.deepEqual((await inspect(page)).bodies, beforeTextBudget.bodies);
for (const result of textResults) assert.ok(result.content.reduce((total, part) => total + Buffer.byteLength(part.text || ''), 0) <= 1024);
checks.push('oversized-tool-result-is-an-explicit-delivery-error-and-recovers-with-read-only-summary-without-partial-json');
await context.unroute('**/api/cad/ai/capabilities');
await context.route('**/api/cad/ai/capabilities', async (route) => {
const response = await route.fetch(), body = await response.json();
body.data.limits.textBytes = 8192;
await route.fulfill({ response, json: body });
});
const textProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: 'UTF-8 request budget', description: '' }),
})).json();
assert.equal(textProject.code, 'OK');
await page.goto(`${base}/cad/editor/?project=${textProject.data.cadProjectCode}`);
await page.waitForFunction(async () => {
if (!window.vibemakeCad || !window.solidcraft) return false;
const state = await window.vibemakeCad.getState();
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await until(page, (state) => state.open && state.state === 'ready', 'request budget agent did not become ready', 60000);
// The user text alone fits (8100 UTF-8 bytes); the mandatory runtime context
// pushes the combined request beyond 8192, exercising aggregate accounting.
await chat(page, { send: '限'.repeat(2700), attach_view: false, attach_selection: false });
const overText = await until(page, (state) => !state.turn && state.items.some((item) => item.user), 'oversized request did not stop');
assert.match(JSON.stringify(overText), /textBytes=\d+\/8192/);
assert.equal((await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === textProject.data.cadProjectCode).length, 0, 'oversized UTF-8 request never reaches the backend');
await context.unroute('**/api/cad/ai/capabilities');
checks.push('oversized-current-turn-stops-before-network-with-the-exact-backend-byte-quota');
for (const scenario of ['images', 'higher', 'unlimited', 'compressed']) {
await context.route('**/api/cad/ai/capabilities', async (route) => {
const response = await route.fetch(), body = await response.json();
if (scenario === 'higher') Object.assign(body.data.limits, {
imagesPerRequest: 5, historyItems: 1024, imageBytes: 2097152, imageMaxEdge: 4096,
requestBytes: 8388608, textBytes: 524288, toolResultTextBytes: 65536, requestTimeoutSeconds: 180,
});
if (scenario === 'unlimited') body.data.limits = { imagesPerRequest: 0, historyItems: 0 };
if (scenario === 'compressed') Object.assign(body.data.limits, { imagesPerRequest: 3, imageBytes: 128 });
await route.fulfill({ response, json: body });
});
const budgetProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: `Backend budget ${scenario}`, description: '' }),
})).json();
assert.equal(budgetProject.code, 'OK');
await page.goto(`${base}/cad/editor/?project=${budgetProject.data.cadProjectCode}`);
await page.waitForFunction(async () => {
if (!window.vibemakeCad || !window.solidcraft) return false;
const state = await window.vibemakeCad.getState();
return state.binding && !state.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await until(page, (state) => state.open && state.state === 'ready', 'budget agent did not become ready', 60000);
await chat(page, { send: `vm-test:budget-${scenario}`, attach_view: false, attach_selection: false });
const finished = await until(page, (state) => !state.turn && state.items.some((item) => item.user), `${scenario} budget did not finish`, 60000);
const requests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat' && call.body.cadProjectCode === budgetProject.data.cadProjectCode);
assert.equal(requests.length, 5, JSON.stringify(finished));
assert.equal(new Set(requests.map((call) => call.body.turnCode)).size, 1);
const notes = requests.map((call) => {
const text = call.body.history[0].content.find((part) => part.type === 'text').text;
assert.equal(text.split('[CAD Web runtime context]').length, 2, 'runtime context must be replaced, never accumulated');
return JSON.parse(text.split('[CAD Web runtime context]\n')[1]).cadWebRuntime;
});
const results = requests.at(-1).body.history.filter((item) => item.type === 'toolResult');
if (scenario === 'images') {
assert.equal(finished.items.filter((item) => item.status === 'Failed').length, 1);
assert.deepEqual(notes.map((note) => note.remainingImages), [2, 1, 0, 0, 0]);
assert.equal(results[2].isError, true);
assert.match(results[2].content[0].text, /Screenshot not executed: backend imagesPerRequest=2/);
assert.ok(!results[2].content.some((part) => part.type === 'image'));
const camera = await page.evaluate(async () => (await window.solidcraft.request('ui.inspect', {})).camera);
assert.ok(Math.abs(camera.pitch) < 1e-9, 'blocked top screenshot must preserve the previous front camera');
checks.push('backend-image-quota-is-explained-to-model-and-blocked-before-camera-or-render-with-recovery');
} else {
assert.equal(finished.items.filter((item) => item.status === 'Failed').length, 0, JSON.stringify(finished));
assert.equal(results.flatMap((item) => item.content).filter((part) => part.type === 'image').length, 3);
if (scenario === 'compressed') {
for (const picture of results.flatMap((item) => item.content).filter((part) => part.type === 'image')) {
const png = Buffer.from(picture.data, 'base64');
assert.ok(png.length <= 128, 'actual image bytes must fit the advertised quota');
assert.ok(png.readUInt32BE(16) < 160 && png.readUInt32BE(20) < 100, 'oversized PNG is scaled down before sending');
}
checks.push('screenshots-are-automatically-scaled-and-reencoded-to-the-advertised-image-byte-quota');
} else if (scenario === 'higher') {
assert.deepEqual(notes.map((note) => note.remainingImages), [5, 4, 3, 2, 2]);
assert.equal(notes[0].backendLimits.historyItems, 1024);
assert.equal(notes[0].backendLimits.requestTimeoutSeconds, 180);
checks.push('larger-backend-quotas-are-used-without-the-old-two-image-and-256-item-clamps');
} else {
assert.deepEqual(notes.map((note) => note.remainingImages), [null, null, null, null, null]);
assert.deepEqual(notes[0].backendLimits, {});
checks.push('missing-or-zero-backend-quotas-do-not-acquire-frontend-image-history-or-timeout-limits');
}
}
if (process.env.VM_CAD_SCREENSHOT) await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, `-budget-${scenario}.png`) });
if (scenario === 'images') {
// No new chat: a new message frees old screenshots by dropping whole
// completed rounds, including their call/result pairs.
await chat(page, { send: 'vm-test:screenshot after-budget', attach_view: false, attach_selection: false });
const resumed = await until(page, (state) => !state.turn && state.items.some((item) => item.user?.includes('after-budget')),
'new message did not recover image budget', 60000);
const resumedRequests = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === budgetProject.data.cadProjectCode
&& call.body.history[0]?.content?.some((part) => part.text?.includes('after-budget')));
assert.equal(resumedRequests.length, 2, JSON.stringify(resumed));
assert.equal(resumedRequests[0].body.history.length, 1, 'only the new complete user round is retained');
assert.equal(resumedRequests[1].body.history.at(-1).isError, false);
assert.equal(resumedRequests[1].body.history.at(-1).content.filter((part) => part.type === 'image').length, 1);
checks.push('new-user-message-reclaims-old-image-budget-by-trimming-only-complete-rounds');
}
await context.unroute('**/api/cad/ai/capabilities');
}
// A long turn waits at the request rate boundary and can still be stopped.
await context.route('**/api/cad/ai/capabilities', async (route) => {
const response = await route.fetch();
const body = await response.json();
body.data.limits.requestsPerMinute = 1;
body.data.limits.requestTimeoutSeconds = 3;
await route.fulfill({ response, json: body });
});
const pacedProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: 'Paced Agent browser test', description: '' }),
})).json();
assert.equal(pacedProject.code, 'OK');
await page.goto(`${base}/cad/editor/?project=${pacedProject.data.cadProjectCode}`);
await page.waitForFunction(async () => {
if (!window.vibemakeCad || !window.solidcraft) return false;
const host = await window.vibemakeCad.getState();
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await page.waitForFunction(() => window.solidcraft && window.vibemakeCad, null, { timeout: 60000 });
await until(page, (s) => s.open && s.state === 'ready', 'paced agent did not become ready', 60000);
// Isolate this scenario's quota from the previous long turn.
await page.evaluate(() => {
for (const key of Object.keys(localStorage)) if (key.startsWith('vm.cad.ai-rate:')) localStorage.removeItem(key);
});
await chat(page, { send: 'vm-test:read paced-cancel', attach_view: false, attach_selection: false });
await until(page, (s) => s.turn && s.items.some((i) => i.status === 'Completed'), 'first paced tool did not execute');
await page.waitForTimeout(3500);
const waiting = await chat(page);
assert.ok(waiting.turn, 'reaching the rate limit must wait instead of ending the turn');
const pacedCalls = (await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.history.some((item) => item.content?.some((part) => part.text?.includes('paced-cancel'))));
assert.equal(pacedCalls.length, 1, 'no continuation may be sent above the rate limit');
await chat(page, { cancel: true });
await until(page, (s) => !s.turn, 'Stop did not cancel rate-limit waiting');
await page.waitForTimeout(500);
assert.equal((await agentCalls()).filter((call) => call.body?.turnCode === pacedCalls[0].body.turnCode).length, 1);
checks.push('rate-limit-waits-without-ending-the-turn-and-stop-cancels-the-wait');
checks.push('queued-time-is-excluded-from-the-model-request-timeout');
const otherProject = await (await fetch(`${base}/api/cad/workspace/projects`, {
method: 'POST', headers, body: JSON.stringify({ name: 'Shared account quota test', description: '' }),
})).json();
assert.equal(otherProject.code, 'OK');
const otherPage = await context.newPage();
otherPage.on('pageerror', (error) => errors.push(String(error)));
try {
await otherPage.goto(`${base}/cad/editor/?project=${otherProject.data.cadProjectCode}`);
await otherPage.waitForFunction(async () => {
if (!window.vibemakeCad || !window.solidcraft) return false;
const host = await window.vibemakeCad.getState();
return host.binding && !host.locked && document.querySelector('#cad-main')?.hidden;
}, null, { timeout: 180000 });
await until(otherPage, (s) => s.open && s.state === 'ready', 'second tab agent did not become ready', 60000);
await chat(otherPage, { send: 'vm-test:read shared-account-quota', attach_view: false, attach_selection: false });
await until(otherPage, (s) => s.turn, 'second tab did not queue its request');
await otherPage.waitForTimeout(500);
assert.ok((await chat(otherPage)).turn);
assert.equal((await agentCalls()).filter((call) => call.path === '/cad/ai/chat'
&& call.body.cadProjectCode === otherProject.data.cadProjectCode).length, 0,
'a new document/tab must retain the account quota');
await chat(otherPage, { cancel: true });
await until(otherPage, (s) => !s.turn, 'second tab queued request did not cancel');
checks.push('new-documents-and-browser-tabs-share-the-account-request-quota');
} finally {
await otherPage.close();
}
assert.deepEqual(errors, []);
return { checks, errors, url: page.url() };
} catch (error) {
if (process.env.VM_CAD_SCREENSHOT) {
await page.screenshot({ path: process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-failure.png') }).catch(() => {});
await writeFile(process.env.VM_CAD_SCREENSHOT.replace(/\.png$/, '-failure.json'), JSON.stringify({
checks, error: String(error), host: await page.evaluate(() => window.vibemakeCad?.getState()).catch(() => null),
inert: await page.locator('#cad-viewport').evaluate((element) => element.inert).catch(() => null),
}, null, 2));
}
throw error;
} finally {
await context.close();
}
}
export { run as runPlatformAgentBrowserChecks };
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const options = { headless: true, args: ['--enable-unsafe-webgpu', '--use-angle=swiftshader', '--enable-webgl'] };
if (process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE) options.executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE;
const browser = await chromium.launch(options);
try { console.log(JSON.stringify(await run(browser))); }
finally { await browser.close(); }
}