116 lines
3.7 KiB
JavaScript
116 lines
3.7 KiB
JavaScript
// Explicit, opt-in live evaluation. Uses the local server, never reads the provider key.
|
|
import { sseData } from "../shared/sse.mjs";
|
|
import assert from "node:assert/strict";
|
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
const cases = [
|
|
{
|
|
id: "bird",
|
|
messages: [{ role: "user", text: "我想要一个仿生鸟类的机器" }],
|
|
check(r) {
|
|
for (const term of ["鸟类", "飞行", "双足"])
|
|
assert.ok(
|
|
r.request.requirements.some((x) => x.includes(term)),
|
|
term,
|
|
);
|
|
assert.equal(r.request.requiredParts.length, 11);
|
|
assert.ok(r.analysis.unmodeled.length);
|
|
assert.equal(r.request.robotRequirements.massKg, undefined);
|
|
},
|
|
},
|
|
{
|
|
id: "penguin",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "我想要企鹅机器人,不要飞行,不要左臂,整机质量不超过五百克",
|
|
},
|
|
],
|
|
check(r) {
|
|
assert.ok(r.request.requirements.some((x) => /不.*飞行/.test(x)));
|
|
assert.ok(!r.request.requiredParts.includes("ARM_L"));
|
|
assert.equal(r.request.robotRequirements.massKg, 0.5);
|
|
assert.ok(
|
|
!r.analysis.keywords.some((k) => k.term === "飞行" && !k.negated),
|
|
);
|
|
},
|
|
},
|
|
{
|
|
id: "correction",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "我要双足跑步机器人,整体二十自由度,整机质量不超过10kg",
|
|
},
|
|
{ role: "assistant", text: "整机20自由度,质量上限10kg。" },
|
|
{ role: "user", text: "整机质量改成5kg以内,两只手臂都用POM" },
|
|
],
|
|
check(r) {
|
|
assert.equal(r.request.robotRequirements.massKg, 5);
|
|
assert.equal(r.request.robotRequirements.degreesOfFreedom, 20);
|
|
for (const category of ["ARM_L", "ARM_R"])
|
|
assert.equal(
|
|
r.request.partRequirements.find((p) => p.category === category)
|
|
?.material,
|
|
"POM",
|
|
);
|
|
},
|
|
},
|
|
{
|
|
id: "range",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "我要双足机器人,整机质量5kg到10kg,右臂扭矩必须大于20N·m",
|
|
},
|
|
],
|
|
check(r) {
|
|
assert.equal(r.analysis.canSubmit, false);
|
|
assert.ok(r.analysis.unmodeled.length >= 2);
|
|
assert.equal(r.request.robotRequirements.massKg, undefined);
|
|
assert.equal(
|
|
r.request.partRequirements.find((p) => p.category === "ARM_R")
|
|
?.torqueNm,
|
|
undefined,
|
|
);
|
|
},
|
|
},
|
|
];
|
|
const results = [];
|
|
for (let i = 0; i < cases.length; i += 2)
|
|
await Promise.all(
|
|
cases.slice(i, i + 2).map(async (item) => {
|
|
const response = await fetch("http://127.0.0.1:8093/api/ai/responses", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ messages: item.messages }),
|
|
signal: AbortSignal.timeout(185000),
|
|
});
|
|
if (!response.ok)
|
|
throw Error(
|
|
`${item.id}: HTTP ${response.status} ${await response.text()}`,
|
|
);
|
|
let result;
|
|
for await (const raw of sseData(response.body)) {
|
|
if (raw === "[DONE]") continue;
|
|
const event = JSON.parse(raw);
|
|
if (event.type === "error") throw Error(`${item.id}: ${event.message}`);
|
|
if (event.type === "analysis.completed") result = event.result;
|
|
}
|
|
if (!result) throw Error(`${item.id}: missing result`);
|
|
results.push({ id: item.id, result });
|
|
try {
|
|
item.check(result);
|
|
console.log(item.id, "PASS");
|
|
} catch (error) {
|
|
console.log(item.id, "FAIL", error.message);
|
|
process.exitCode = 1;
|
|
}
|
|
}),
|
|
);
|
|
await mkdir("docs/analysis-examples", { recursive: true });
|
|
for (const item of results)
|
|
await writeFile(
|
|
`docs/analysis-examples/${item.id}.json`,
|
|
JSON.stringify(item.result, null, 2) + "\n",
|
|
);
|