优化agent
This commit is contained in:
@@ -6,18 +6,21 @@ import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "backend"))
|
||||
|
||||
from app.models.contracts import ChatMessage, MessagePart # noqa: E402
|
||||
from app.services.agent_service import AgentService # noqa: E402
|
||||
from app.services.engine_service import load_engine # noqa: E402
|
||||
from app.services.flange_sleeve_template import TEMPLATE_ID # noqa: E402
|
||||
from app.services.library import CdslLibrary # noqa: E402
|
||||
from app.services.part_skills import PartSkillLibrary # noqa: E402
|
||||
from app.services.storage import WorkspaceStore # noqa: E402
|
||||
from app.settings import ProviderConfig, ProviderModel, Settings # noqa: E402
|
||||
from tests.test_design_intent import mounting_plate_cdsl, mounting_plate_intent # noqa: E402
|
||||
from tests.test_design_intent import mounting_plate_cdsl # noqa: E402
|
||||
|
||||
|
||||
BACKEND = ROOT / "backend"
|
||||
@@ -40,85 +43,274 @@ class DesignIntentFlowTests(unittest.TestCase):
|
||||
providers=(provider,),
|
||||
)
|
||||
|
||||
def test_design_intent_is_required_before_library_or_cdsl(self) -> None:
|
||||
def test_design_brief_is_required_before_library_or_cdsl(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), PartSkillLibrary(PART_SKILL_ROOT))
|
||||
state = {"phase": "WAITING_FOR_INTENT", "design_intent_id": ""}
|
||||
searched, _ = asyncio.run(agent._run_tool("search_cdsl_library", {"query": "mounting plate"}, "", "mounting plate", [], intent_state=state))
|
||||
generated, _ = asyncio.run(agent._run_tool("generate_cdsl_model", {"design_intent_id": "intent_aaaaaaaaaaaa", "cdsl": {}, "summary": "x", "assumptions": []}, "", "mounting plate", [], intent_state=state))
|
||||
state = {"phase": "WAITING_FOR_PLAN", "design_brief": ""}
|
||||
searched, _ = asyncio.run(agent._run_tool("search_cdsl_library", {"query": "mounting plate"}, "", "mounting plate", [], planning_state=state))
|
||||
generated, _ = asyncio.run(agent._run_tool("generate_cdsl_model", {"cdsl": {}, "summary": "x", "assumptions": []}, "", "mounting plate", [], planning_state=state))
|
||||
|
||||
self.assertEqual(searched["code"], "DESIGN_INTENT_REQUIRED")
|
||||
self.assertEqual(generated["code"], "DESIGN_INTENT_REQUIRED")
|
||||
self.assertEqual(searched["code"], "DESIGN_BRIEF_REQUIRED")
|
||||
self.assertEqual(generated["code"], "DESIGN_BRIEF_REQUIRED")
|
||||
self.assertEqual(list(settings.task_root.glob("cad_*")), [])
|
||||
|
||||
def test_ready_plan_persists_before_build_and_links_success_revision(self) -> None:
|
||||
def test_generation_normalizes_legacy_llm_cdsl_before_building(self) -> None:
|
||||
legacy_cdsl = {
|
||||
"schema": "cad.cdsl.llm.v1",
|
||||
"part_id": "legacy-flange-base",
|
||||
"geometry": {"sketches": [{
|
||||
"id": "base_sketch",
|
||||
"plane": "XY",
|
||||
"offset_mm": 12,
|
||||
"profile": {"type": "circle", "radius_mm": 20},
|
||||
}]},
|
||||
"features": [{
|
||||
"id": "base_add",
|
||||
"atomic_id": "extrude_add_blind",
|
||||
"sketch": "base_sketch",
|
||||
"params": {"distance_mm": 8},
|
||||
}],
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), PartSkillLibrary(PART_SKILL_ROOT))
|
||||
state = {"phase": "PLAN_RECORDED", "design_brief": "Create a flange base."}
|
||||
captured_build: dict[str, object] = {}
|
||||
|
||||
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
||||
captured_build.update(kwargs)
|
||||
return {"task_id": "cad_aaaaaaaaaaaa", "revision_id": "rev_001"}
|
||||
|
||||
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
||||
result, _ = asyncio.run(agent._run_tool(
|
||||
"generate_cdsl_model",
|
||||
{"cdsl": legacy_cdsl, "summary": "flange base", "assumptions": []},
|
||||
"", "Create a flange base", [], planning_state=state,
|
||||
))
|
||||
|
||||
built_cdsl = captured_build["cdsl"]
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(built_cdsl["features"][0]["sketch_id"], "base_sketch")
|
||||
self.assertEqual(built_cdsl["features"][0]["depends_on"], [])
|
||||
self.assertNotIn("sketch", built_cdsl["features"][0])
|
||||
self.assertIn("workplane", built_cdsl["geometry"]["sketches"][0])
|
||||
self.assertEqual(len(result["normalization_repairs"]), 3)
|
||||
|
||||
def test_flange_sleeve_tool_builds_cdsl_from_a_compact_semantic_plan(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), PartSkillLibrary(PART_SKILL_ROOT))
|
||||
state = {"phase": "PLAN_RECORDED", "design_brief": "Create a flange sleeve."}
|
||||
captured_build: dict[str, object] = {}
|
||||
|
||||
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
||||
captured_build.update(kwargs)
|
||||
return {"task_id": "cad_aaaaaaaaaaaa", "revision_id": "rev_001"}
|
||||
|
||||
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
||||
result, _ = asyncio.run(agent._run_tool(
|
||||
"generate_flange_sleeve_model",
|
||||
{
|
||||
"plan": {
|
||||
"template": TEMPLATE_ID,
|
||||
"name": "Image flange sleeve",
|
||||
"flange_width_mm": 140,
|
||||
"flange_height_mm": 120,
|
||||
"mount_hole_u_mm": 48,
|
||||
"mount_hole_v_mm": 38,
|
||||
},
|
||||
"summary": "flange sleeve",
|
||||
"assumptions": ["Dimensions are approximate."],
|
||||
},
|
||||
"", "Create an image flange sleeve", [], planning_state=state,
|
||||
))
|
||||
|
||||
built_cdsl = captured_build["cdsl"]
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(result["template_id"], TEMPLATE_ID)
|
||||
self.assertEqual(captured_build["operation"]["template_id"], TEMPLATE_ID)
|
||||
self.assertEqual(built_cdsl["meta"]["name"], "Image flange sleeve")
|
||||
self.assertEqual(built_cdsl["geometry"]["sketches"][0]["workplane"]["normal"], [1.0, 0.0, 0.0])
|
||||
self.assertEqual(built_cdsl["features"][-1]["depends_on"], ["mount_hole_cuts"])
|
||||
self.assertNotIn("cdsl", result["template_plan"])
|
||||
|
||||
def test_successful_generation_ends_the_agent_tool_loop(self) -> None:
|
||||
class CaptureAgent(AgentService):
|
||||
def __init__(self, *args: object, **kwargs: object) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.responses = [
|
||||
{
|
||||
"choices": [{"message": {
|
||||
"role": "assistant", "content": "", "tool_calls": [{
|
||||
"id": "brief", "type": "function", "function": {
|
||||
"name": "describe_design_intent",
|
||||
"arguments": json.dumps({"plan": "Create a flange sleeve.", "assumptions": []}),
|
||||
},
|
||||
}],
|
||||
}}],
|
||||
},
|
||||
{
|
||||
"choices": [{"message": {
|
||||
"role": "assistant", "content": "", "tool_calls": [{
|
||||
"id": "template", "type": "function", "function": {
|
||||
"name": "generate_flange_sleeve_model",
|
||||
"arguments": json.dumps({
|
||||
"plan": {"template": TEMPLATE_ID},
|
||||
"summary": "flange sleeve",
|
||||
"assumptions": [],
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}}],
|
||||
},
|
||||
]
|
||||
|
||||
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
||||
return self.responses.pop(0)
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
agent = CaptureAgent(settings, store, CdslLibrary(settings), PartSkillLibrary(PART_SKILL_ROOT))
|
||||
|
||||
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
||||
return {
|
||||
"task_id": "cad_aaaaaaaaaaaa", "revision_id": "rev_001",
|
||||
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
||||
"step_path": "revisions/rev_001/model.step",
|
||||
"glb_path": "revisions/rev_001/model.glb",
|
||||
"report_path": "revisions/rev_001/rebuild-report.json",
|
||||
"summary": str(kwargs["summary"]), "reference_ids": [], "engine": "cdsl_only",
|
||||
}
|
||||
|
||||
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
||||
async def consume() -> None:
|
||||
message = ChatMessage(id="user_1", role="user", parts=[MessagePart(type="text", text="Create a flange sleeve")])
|
||||
async for _ in agent.stream([message], None, None):
|
||||
pass
|
||||
|
||||
asyncio.run(consume())
|
||||
|
||||
saved = store.read_conversation(next(item.name for item in settings.conversation_root.iterdir()))
|
||||
parts = saved["messages"][-1]["parts"]
|
||||
self.assertEqual(agent.responses, [])
|
||||
self.assertTrue(any(part["type"] == "data-cad-result" for part in parts))
|
||||
self.assertFalse(any(part["type"] == "data-cad-error" for part in parts))
|
||||
|
||||
def test_text_brief_is_returned_to_model_and_cdsl_is_the_only_contract(self) -> None:
|
||||
class CaptureAgent(AgentService):
|
||||
def __init__(self, *args: object, **kwargs: object) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.seen_messages: list[list[dict[str, object]]] = []
|
||||
self.responses = [
|
||||
{
|
||||
"choices": [{"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{
|
||||
"id": "brief",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "describe_design_intent",
|
||||
"arguments": json.dumps({
|
||||
"plan": "Create a rectangular mounting plate, then cut four mounting holes and a center slot.",
|
||||
"assumptions": ["Use millimetres."],
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}}],
|
||||
},
|
||||
{
|
||||
"choices": [{"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{
|
||||
"id": "generate",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "generate_cdsl_model",
|
||||
"arguments": json.dumps({
|
||||
"cdsl": mounting_plate_cdsl(),
|
||||
"summary": "mounting plate",
|
||||
"assumptions": ["Use millimetres."],
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}}],
|
||||
},
|
||||
{"choices": [{"message": {"role": "assistant", "content": "已生成。", "tool_calls": []}}]},
|
||||
]
|
||||
|
||||
async def _complete(self, messages: list[dict[str, object]], *args: object, **kwargs: object) -> dict[str, object]:
|
||||
self.seen_messages.append([dict(message) for message in messages])
|
||||
return self.responses.pop(0)
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
skills = PartSkillLibrary(PART_SKILL_ROOT)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), skills)
|
||||
request = "Create a mounting plate with four holes and a center slot"
|
||||
state = {"phase": "WAITING_FOR_INTENT", "design_intent_id": ""}
|
||||
planned, _ = asyncio.run(agent._run_tool(
|
||||
"propose_design_intent",
|
||||
{"intent": mounting_plate_intent(), "summary": "plate plan", "assumptions": []},
|
||||
"", request, [], part_skill_selection=skills.select(request), intent_state=state,
|
||||
))
|
||||
agent = CaptureAgent(settings, store, CdslLibrary(settings), skills)
|
||||
captured_build: dict[str, object] = {}
|
||||
|
||||
self.assertTrue(planned["ok"])
|
||||
task = store.read_task(planned["task_id"])
|
||||
self.assertEqual(task["current_revision"], "")
|
||||
planning_path = store.artifact_path(planned["task_id"], planned["design_intent_path"])
|
||||
self.assertTrue(planning_path.is_file())
|
||||
saved = json.loads(planning_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(saved["part_skill_ids"], skills.select(request)["skill_ids"])
|
||||
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
||||
captured_build.update(kwargs)
|
||||
return {
|
||||
"task_id": "cad_aaaaaaaaaaaa",
|
||||
"revision_id": "rev_001",
|
||||
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
||||
"step_path": "revisions/rev_001/model.step",
|
||||
"glb_path": "revisions/rev_001/model.glb",
|
||||
"report_path": "revisions/rev_001/rebuild-report.json",
|
||||
"summary": str(kwargs["summary"]),
|
||||
"reference_ids": list(kwargs["reference_ids"]),
|
||||
"engine": "cdsl_only",
|
||||
}
|
||||
|
||||
searched, _ = asyncio.run(agent._run_tool("search_cdsl_library", {"query": "plate"}, planned["task_id"], request, [], intent_state=state))
|
||||
self.assertTrue(searched["ok"])
|
||||
result, built = asyncio.run(agent._run_tool(
|
||||
"generate_cdsl_model",
|
||||
{"design_intent_id": planned["design_intent_id"], "cdsl": mounting_plate_cdsl(), "summary": "plate", "assumptions": []},
|
||||
planned["task_id"], request, [], part_skill_selection=skills.select(request), intent_state=state,
|
||||
engine = load_engine(settings)
|
||||
with patch.object(engine, "validate_intent_cdsl", side_effect=AssertionError("legacy intent validation must not run")) as legacy_validation, patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
||||
async def consume() -> None:
|
||||
message = ChatMessage(id="user_1", role="user", parts=[MessagePart(type="text", text="Create a mounting plate")])
|
||||
async for _ in agent.stream([message], None, None):
|
||||
pass
|
||||
|
||||
asyncio.run(consume())
|
||||
|
||||
brief_result = json.loads(str(agent.seen_messages[1][-1]["content"]))
|
||||
self.assertEqual(brief_result["plan"], "Create a rectangular mounting plate, then cut four mounting holes and a center slot.")
|
||||
self.assertNotIn("design_intent_id", brief_result)
|
||||
self.assertNotIn("design_intent_id", captured_build)
|
||||
self.assertNotIn("design_intent", captured_build)
|
||||
self.assertEqual(captured_build["parent_revision_id"], "")
|
||||
legacy_validation.assert_not_called()
|
||||
|
||||
def test_revision_parent_is_taken_from_the_current_successful_cdsl_revision(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), PartSkillLibrary(PART_SKILL_ROOT))
|
||||
state = {"phase": "WAITING_FOR_PLAN", "design_brief": ""}
|
||||
asyncio.run(agent._run_tool(
|
||||
"describe_design_intent",
|
||||
{"plan": "Increase the plate thickness and preserve the existing hole layout.", "assumptions": []},
|
||||
"cad_aaaaaaaaaaaa", "Revise the plate", [], planning_state=state,
|
||||
))
|
||||
store.read_task = lambda _task_id: {"current_revision": "rev_007"} # type: ignore[method-assign]
|
||||
captured_build: dict[str, object] = {}
|
||||
|
||||
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
||||
captured_build.update(kwargs)
|
||||
return {"task_id": "cad_aaaaaaaaaaaa", "revision_id": "rev_008"}
|
||||
|
||||
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
||||
result, _ = asyncio.run(agent._run_tool(
|
||||
"generate_cdsl_model",
|
||||
{"cdsl": mounting_plate_cdsl(), "summary": "revised plate", "assumptions": []},
|
||||
"cad_aaaaaaaaaaaa", "Revise the plate", [], planning_state=state,
|
||||
))
|
||||
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertIsNotNone(built)
|
||||
revision = store.read_task(planned["task_id"])["revisions"][-1]
|
||||
self.assertEqual(revision["design_intent_id"], planned["design_intent_id"])
|
||||
self.assertEqual(revision["design_intent_path"], planned["design_intent_path"])
|
||||
audit = json.loads(store.artifact_path(planned["task_id"], revision["part_skills_path"]).read_text(encoding="utf-8"))
|
||||
self.assertEqual(audit["design_intent_id"], planned["design_intent_id"])
|
||||
self.assertEqual(audit["design_intent_assumptions"], [])
|
||||
|
||||
def test_blocked_plan_is_persisted_without_a_revision_and_new_plan_supersedes_it(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
settings = self.settings(Path(directory))
|
||||
store = WorkspaceStore(settings)
|
||||
skills = PartSkillLibrary(PART_SKILL_ROOT)
|
||||
agent = AgentService(settings, store, CdslLibrary(settings), skills)
|
||||
request = "Create a mounting plate"
|
||||
state = {"phase": "WAITING_FOR_INTENT", "design_intent_id": ""}
|
||||
blocked_intent = mounting_plate_intent()
|
||||
blocked_intent["open_questions"] = [{"id": "thickness", "question": "Thickness?", "blocking": True}]
|
||||
blocked_intent["status"] = "needs_clarification"
|
||||
blocked, _ = asyncio.run(agent._run_tool(
|
||||
"propose_design_intent",
|
||||
{"intent": blocked_intent, "summary": "blocked", "assumptions": []},
|
||||
"", request, [], part_skill_selection=skills.select(request), intent_state=state,
|
||||
))
|
||||
self.assertEqual(blocked["code"], "DESIGN_INTENT_BLOCKED")
|
||||
task = store.read_task(blocked["task_id"])
|
||||
self.assertEqual(task["revisions"], [])
|
||||
self.assertEqual(task["design_intents"][0]["status"], "pending")
|
||||
|
||||
ready, _ = asyncio.run(agent._run_tool(
|
||||
"propose_design_intent",
|
||||
{"intent": mounting_plate_intent(), "summary": "ready", "assumptions": []},
|
||||
blocked["task_id"], request, [], part_skill_selection=skills.select(request), intent_state=state,
|
||||
))
|
||||
self.assertTrue(ready["ok"])
|
||||
task = store.read_task(blocked["task_id"])
|
||||
self.assertEqual(task["design_intents"][0]["status"], "superseded")
|
||||
self.assertEqual(task["design_intents"][-1]["status"], "accepted")
|
||||
self.assertEqual(captured_build["parent_revision_id"], "rev_007")
|
||||
|
||||
Reference in New Issue
Block a user