899 lines
47 KiB
Python
899 lines
47 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from app.models.contracts import ChatMessage, MessagePart
|
|
from app.services.agent_service import AgentService, CDSL_TOOL_SCHEMA, RepeatedToolArgumentsError, StrictToolSchemaError, TOOL_SCHEMAS, ToolArgumentsError, engine_capability_manifest, get_repair_step_key, normalize_image_analysis, parse_tool_arguments, response_language_instruction, system_prompt, tools_for_model, user_visible_error_message
|
|
from app.services.engine_service import load_engine
|
|
from app.services.quality import QUALITY_RULE_TYPES
|
|
from app.services.library import CdslLibrary
|
|
from app.services.storage import WorkspaceStore
|
|
from app.settings import ProviderConfig, ProviderModel, Settings, get_settings
|
|
|
|
|
|
class ToolChoiceCompatibilityTests(unittest.TestCase):
|
|
def test_retries_without_tool_choice_when_thinking_mode_rejects_it(self) -> None:
|
|
class FakeResponse:
|
|
def __init__(self, status_code: int, text: str, body: dict[str, object]) -> None:
|
|
self.status_code = status_code
|
|
self.text = text
|
|
self._body = body
|
|
|
|
def json(self) -> dict[str, object]:
|
|
return self._body
|
|
|
|
class FakeClient:
|
|
def __init__(self) -> None:
|
|
self.requests: list[dict[str, object]] = []
|
|
self.responses = [
|
|
FakeResponse(400, '{"error":{"message":"Thinking mode does not support this tool_choice"}}', {}),
|
|
FakeResponse(200, "", {"choices": [{"message": {"role": "assistant", "content": "ok"}}]}),
|
|
]
|
|
|
|
async def __aenter__(self) -> "FakeClient":
|
|
return self
|
|
|
|
async def __aexit__(self, *args: object) -> None:
|
|
return None
|
|
|
|
async def post(self, _url: str, *, headers: dict[str, str], json: dict[str, object]) -> FakeResponse:
|
|
self.requests.append(dict(json))
|
|
return self.responses.pop(0)
|
|
|
|
agent = object.__new__(AgentService)
|
|
agent.settings = SimpleNamespace(llm_timeout_s=1)
|
|
client = FakeClient()
|
|
provider = ProviderConfig("deepseek", "DeepSeek", "https://example.invalid/v1", "test-key", (ProviderModel("deepseek-v4-flash-vision-exp", vision=True),))
|
|
model = provider.models[0]
|
|
|
|
with patch("app.services.agent_service.httpx.AsyncClient", return_value=client):
|
|
response = asyncio.run(agent._complete([], [], provider, model, "analyze_image_reference"))
|
|
|
|
self.assertEqual(response["choices"][0]["message"]["content"], "ok")
|
|
self.assertEqual(client.requests[0]["tool_choice"], {"type": "function", "function": {"name": "analyze_image_reference"}})
|
|
self.assertNotIn("tool_choice", client.requests[1])
|
|
|
|
|
|
class ParseToolArgumentsTests(unittest.TestCase):
|
|
def test_accepts_one_json_object(self) -> None:
|
|
payload = parse_tool_arguments(' {"summary":"water cup","cdsl":{"parts":[]}} ')
|
|
|
|
self.assertEqual(payload["summary"], "water cup")
|
|
self.assertEqual(payload["cdsl"], {"parts": []})
|
|
|
|
def test_rejects_concatenated_json_objects(self) -> None:
|
|
with self.assertRaisesRegex(ToolArgumentsError, "trailing content"):
|
|
parse_tool_arguments('{"summary":"water cup"}{"cdsl":{}}')
|
|
|
|
def test_rejects_markdown_or_prose_after_json(self) -> None:
|
|
with self.assertRaisesRegex(ToolArgumentsError, "trailing content"):
|
|
parse_tool_arguments('{"summary":"water cup"}\n```')
|
|
|
|
def test_rejects_non_object_json(self) -> None:
|
|
with self.assertRaisesRegex(ToolArgumentsError, "JSON object"):
|
|
parse_tool_arguments('["not", "tool arguments"]')
|
|
|
|
|
|
class RepairStepKeyTests(unittest.TestCase):
|
|
def test_generation_and_patch_share_the_same_feature_step_budget(self) -> None:
|
|
state = {
|
|
"phase": "CDSL_REPAIR",
|
|
"feature_plan": {
|
|
"plan_id": "plan_1",
|
|
"nodes": [
|
|
{"id": "boss", "status": "ready"},
|
|
{"id": "hole", "status": "waiting_for_selection"},
|
|
],
|
|
},
|
|
}
|
|
|
|
self.assertEqual(
|
|
get_repair_step_key(state, "generate_cdsl_model"),
|
|
get_repair_step_key(state, "patch_cdsl_model"),
|
|
)
|
|
|
|
def test_completed_nodes_do_not_change_the_active_step_key(self) -> None:
|
|
state = {
|
|
"phase": "CDSL_REPAIR",
|
|
"feature_plan": {
|
|
"plan_id": "plan_1",
|
|
"nodes": [
|
|
{"id": "base", "status": "completed"},
|
|
{"id": "boss", "status": "ready"},
|
|
],
|
|
},
|
|
}
|
|
self.assertEqual(get_repair_step_key(state, "generate_cdsl_model"), "plan:plan_1:boss")
|
|
|
|
def test_recovers_only_the_known_premature_cdsl_wrapper_close(self) -> None:
|
|
payload = parse_tool_arguments(
|
|
'{"cdsl":{"schema":"cad.cdsl.llm.v1"}}, "summary":"fixed envelope"}',
|
|
recover_cdsl_wrapper=True,
|
|
)
|
|
|
|
self.assertEqual(payload["summary"], "fixed envelope")
|
|
self.assertEqual(payload["cdsl"], {"schema": "cad.cdsl.llm.v1"})
|
|
|
|
def test_recovers_trailing_cdsl_metadata_after_a_complete_envelope(self) -> None:
|
|
payload = parse_tool_arguments(
|
|
'{"cdsl":{"schema":"cad.cdsl.llm.v1"},"summary":"fixed envelope",'
|
|
'"assumptions":["metric"]}, "summary":"repeated envelope",'
|
|
'"assumptions":["metric"]}',
|
|
recover_cdsl_wrapper=True,
|
|
)
|
|
|
|
self.assertEqual(payload, {
|
|
"cdsl": {"schema": "cad.cdsl.llm.v1"},
|
|
"summary": "fixed envelope",
|
|
"assumptions": ["metric"],
|
|
})
|
|
|
|
def test_rejects_a_second_cdsl_payload_after_a_complete_envelope(self) -> None:
|
|
with self.assertRaisesRegex(ToolArgumentsError, "trailing content"):
|
|
parse_tool_arguments(
|
|
'{"cdsl":{"schema":"cad.cdsl.llm.v1"},"summary":"original",'
|
|
'"assumptions":[]}, "cdsl":{"schema":"different"}}',
|
|
recover_cdsl_wrapper=True,
|
|
)
|
|
|
|
def test_does_not_recover_arbitrary_trailing_tool_content(self) -> None:
|
|
with self.assertRaisesRegex(ToolArgumentsError, "trailing content"):
|
|
parse_tool_arguments(
|
|
'{"cdsl":{"schema":"cad.cdsl.llm.v1"}} prose',
|
|
recover_cdsl_wrapper=True,
|
|
)
|
|
|
|
def test_identifies_chinese_output_requirement(self) -> None:
|
|
self.assertIn("Chinese", response_language_instruction("生成一个水杯"))
|
|
|
|
def test_generate_tool_requires_a_non_empty_cdsl_structure(self) -> None:
|
|
generate_tool = next(tool for tool in TOOL_SCHEMAS if tool["function"]["name"] == "generate_cdsl_model")
|
|
cdsl = generate_tool["function"]["parameters"]["properties"]["cdsl"]
|
|
|
|
self.assertEqual(set(cdsl["required"]), {"schema", "features", "geometry"})
|
|
self.assertEqual(cdsl["properties"]["features"]["minItems"], 1)
|
|
self.assertEqual(cdsl["properties"]["geometry"]["properties"]["sketches"]["minItems"], 1)
|
|
self.assertIn("extrude_add_blind", cdsl["$defs"]["feature_atomic_ids"]["enum"])
|
|
self.assertNotIn("extrude", cdsl["$defs"]["feature_atomic_ids"]["enum"])
|
|
self.assertEqual(cdsl, CDSL_TOOL_SCHEMA)
|
|
|
|
def test_generation_schema_exposes_only_runtime_atomic_ids(self) -> None:
|
|
settings = get_settings()
|
|
engine = load_engine(settings)
|
|
self.assertEqual(
|
|
set(CDSL_TOOL_SCHEMA["$defs"]["feature_atomic_ids"]["enum"]),
|
|
set(engine.SUPPORTED_ATOMIC_IDS),
|
|
)
|
|
|
|
def test_verification_schema_exposes_only_implemented_rule_types(self) -> None:
|
|
generate_tool = next(tool for tool in TOOL_SCHEMAS if tool["function"]["name"] == "generate_cdsl_model")
|
|
verification = generate_tool["function"]["parameters"]["properties"]["verification"]
|
|
rule_type = verification["properties"]["rules"]["items"]["properties"]["type"]
|
|
self.assertEqual(set(rule_type["enum"]), set(QUALITY_RULE_TYPES))
|
|
|
|
def test_verification_schema_requires_feature_for_feature_scoped_rules(self) -> None:
|
|
generate_tool = next(tool for tool in TOOL_SCHEMAS if tool["function"]["name"] == "generate_cdsl_model")
|
|
rule = generate_tool["function"]["parameters"]["properties"]["verification"]["properties"]["rules"]["items"]
|
|
self.assertTrue(any("feature" in branch.get("then", {}).get("required", []) for branch in rule["allOf"]))
|
|
self.assertIn("overall_width", rule["properties"]["type"]["enum"])
|
|
self.assertIn("overall_height", rule["properties"]["type"]["enum"])
|
|
|
|
def test_strict_tool_schema_covers_generation_arguments(self) -> None:
|
|
tools = tools_for_model(ProviderModel("strict-model", strict_tool_schema=True))
|
|
strict_tools = [tool["function"]["name"] for tool in tools if tool["function"].get("strict")]
|
|
|
|
self.assertEqual(strict_tools, ["generate_cdsl_model", "patch_cdsl_model"])
|
|
generate_tool = next(tool for tool in tools if tool["function"]["name"] == "generate_cdsl_model")
|
|
self.assertEqual(generate_tool["function"]["parameters"]["properties"]["summary"], {"type": "string", "minLength": 1})
|
|
self.assertEqual(generate_tool["function"]["parameters"]["properties"]["cdsl"], CDSL_TOOL_SCHEMA)
|
|
self.assertIn("verification", generate_tool["function"]["parameters"]["properties"])
|
|
patch_tool = next(tool for tool in tools if tool["function"]["name"] == "patch_cdsl_model")
|
|
self.assertIn("base_revision_id", patch_tool["function"]["parameters"]["required"])
|
|
self.assertFalse(generate_tool["function"]["parameters"]["additionalProperties"])
|
|
|
|
def test_default_model_does_not_receive_strict_tool_schema(self) -> None:
|
|
tools = tools_for_model(ProviderModel("default-model"))
|
|
|
|
self.assertFalse(any(tool["function"].get("strict") for tool in tools))
|
|
|
|
def test_direct_cdsl_tools_exclude_spec_and_template_generators(self) -> None:
|
|
names = [tool["function"]["name"] for tool in tools_for_model(ProviderModel("default-model"))]
|
|
self.assertIn("generate_cdsl_model", names)
|
|
self.assertIn("patch_cdsl_model", names)
|
|
self.assertNotIn("create_generation_spec", names)
|
|
self.assertNotIn("author_cdsl_from_generation_spec", names)
|
|
self.assertNotIn("patch_generation_spec", names)
|
|
self.assertNotIn("generate_flange_sleeve_model", names)
|
|
|
|
def test_recorded_image_analysis_is_not_exposed_as_a_tool(self) -> None:
|
|
tools = tools_for_model(ProviderModel("vision-model", vision=True), include_image_analysis=False)
|
|
|
|
self.assertNotIn("analyze_image_reference", [tool["function"]["name"] for tool in tools])
|
|
|
|
def test_image_analysis_allows_no_dimension_candidates(self) -> None:
|
|
analysis_tool = next(tool for tool in TOOL_SCHEMAS if tool["function"]["name"] == "analyze_image_reference")
|
|
self.assertNotIn("dimension_candidates", analysis_tool["function"]["parameters"]["required"])
|
|
|
|
result = normalize_image_analysis({
|
|
"part_type": "压铸外壳",
|
|
"visible_features": ["圆角矩形外轮廓"],
|
|
"uncertain_features": [],
|
|
})
|
|
self.assertEqual(result["dimension_candidates"], [])
|
|
|
|
def test_strict_schema_rejection_is_localized_for_chinese_requests(self) -> None:
|
|
message = user_visible_error_message(
|
|
StrictToolSchemaError("provider rejected strict schema"),
|
|
"生成一个法兰",
|
|
)
|
|
|
|
self.assertIn("不支持严格 CDSL 工具 schema", message)
|
|
self.assertNotIn("provider rejected", message)
|
|
|
|
def test_repeated_invalid_cdsl_tool_arguments_are_localized_for_chinese_requests(self) -> None:
|
|
message = user_visible_error_message(
|
|
RepeatedToolArgumentsError("arguments are not valid JSON"),
|
|
"生成一个法兰",
|
|
)
|
|
|
|
self.assertIn("连续两次未返回完整的 CDSL 工具 JSON", message)
|
|
self.assertIn("函数调用兼容性", message)
|
|
|
|
|
|
class ToolArgumentsRetryTests(unittest.TestCase):
|
|
def test_repeated_invalid_cdsl_arguments_stop_before_the_safety_limit(self) -> None:
|
|
class InvalidCdslAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.responses = [
|
|
{
|
|
"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "invalid_cdsl_1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": '{"cdsl":',
|
|
},
|
|
}],
|
|
}}],
|
|
},
|
|
{
|
|
"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "invalid_cdsl_2",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": '{"cdsl":',
|
|
},
|
|
}],
|
|
}}],
|
|
},
|
|
]
|
|
|
|
self.responses[0]["id"] = "chatcmpl_invalid_1"
|
|
self.responses[0]["model"] = "test-model"
|
|
self.responses[0]["usage"] = {"completion_tokens": 4096}
|
|
self.responses[0]["choices"][0]["finish_reason"] = "length"
|
|
self.responses[1]["id"] = "chatcmpl_invalid_2"
|
|
self.responses[1]["model"] = "test-model"
|
|
self.responses[1]["usage"] = {"completion_tokens": 4096}
|
|
self.responses[1]["choices"][0]["finish_reason"] = "length"
|
|
|
|
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
|
return self.responses.pop(0)
|
|
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
settings = Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
agent = InvalidCdslAgent(settings, WorkspaceStore(settings), CdslLibrary(settings))
|
|
message = ChatMessage(id="user_1", role="user", parts=[MessagePart(type="text", text="生成一个法兰")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], None, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
|
|
errors = [str(event.get("message", "")) for event in events if event.get("stage") == "agent"]
|
|
self.assertEqual(agent.responses, [])
|
|
self.assertTrue(any("连续两次未返回完整的 CDSL 工具 JSON" in error for error in errors))
|
|
self.assertFalse(any("safety limit" in error for error in errors))
|
|
diagnostics = sorted(settings.conversation_root.glob("conv_*/diagnostics/tool_call_*.json"))
|
|
self.assertEqual(len(diagnostics), 2)
|
|
records = [json.loads(path.read_text(encoding="utf-8")) for path in diagnostics]
|
|
self.assertEqual([record["arguments"] for record in records], ['{"cdsl":', '{"cdsl":'])
|
|
self.assertTrue(all(record["parse_error"] == "arguments are not valid JSON" for record in records))
|
|
self.assertTrue(all(record["finish_reason"] == "length" for record in records))
|
|
self.assertTrue(all(record["json_error"]["character"] == 8 for record in records))
|
|
|
|
def test_invalid_arguments_are_returned_to_the_model_for_retry(self) -> None:
|
|
class RetryAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.responses = [
|
|
{
|
|
"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "I will search for a water cup reference.",
|
|
"tool_calls": [{
|
|
"id": "bad_call",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "search_cdsl_library",
|
|
"arguments": '{"query":"water cup"}{"limit":3}',
|
|
},
|
|
}],
|
|
}}],
|
|
},
|
|
{"choices": [{"message": {"role": "assistant", "content": "已修正工具参数。", "tool_calls": []}}]},
|
|
]
|
|
|
|
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
|
return self.responses.pop(0)
|
|
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
settings = Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
store = WorkspaceStore(settings)
|
|
agent = RetryAgent(settings, store, CdslLibrary(settings))
|
|
message = ChatMessage(id="user_1", role="user", parts=[MessagePart(type="text", text="生成水杯")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], None, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
|
|
self.assertEqual(agent.responses, [])
|
|
self.assertTrue(any(event.get("status") == "error" for event in events))
|
|
self.assertFalse(any(event.get("stage") == "agent" for event in events))
|
|
self.assertFalse(any("I will search" in str(event.get("text", "")) for event in events))
|
|
self.assertTrue(any("已修正工具参数" in str(event.get("text", "")) for event in events))
|
|
self.assertEqual(list(settings.task_root.glob("cad_*")), [])
|
|
|
|
def test_persists_every_cdsl_attempt_and_validation_failure(self) -> None:
|
|
class InvalidCdslAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
plan_call = {
|
|
"id": "design_brief",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "describe_design_intent",
|
|
"arguments": json.dumps({"plan": "建立一个法兰。", "assumptions": []}),
|
|
},
|
|
}
|
|
invalid_call = {
|
|
"id": "invalid_cdsl",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({"cdsl": {}, "summary": "无效法兰", "assumptions": []}),
|
|
},
|
|
}
|
|
self.responses = [
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [plan_call]}}]},
|
|
*[
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [invalid_call]}}]}
|
|
for _ in range(7)
|
|
],
|
|
]
|
|
|
|
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
|
return self.responses.pop(0)
|
|
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
settings = Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
store = WorkspaceStore(settings)
|
|
agent = InvalidCdslAgent(settings, store, CdslLibrary(settings))
|
|
conversation_id = "conv_000000000004"
|
|
message = ChatMessage(id="user_invalid_cdsl", role="user", parts=[MessagePart(type="text", text="生成一个法兰")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], conversation_id, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
|
|
diagnostics = settings.conversation_root / conversation_id / "diagnostics"
|
|
attempts = sorted(diagnostics.glob("cdsl_attempt_*.json"))
|
|
failures = sorted(diagnostics.glob("cdsl_validation_*.json"))
|
|
self.assertEqual(len(attempts), 5)
|
|
self.assertEqual(len(failures), 5)
|
|
self.assertTrue(all(json.loads(path.read_text(encoding="utf-8")) == {} for path in attempts))
|
|
|
|
records = sorted(
|
|
(json.loads(path.read_text(encoding="utf-8")) for path in failures),
|
|
key=lambda record: int(record["iteration"]),
|
|
)
|
|
self.assertEqual([record["iteration"] for record in records], list(range(2, 7)))
|
|
self.assertTrue(all(record["kind"] == "cdsl_validation_failure" for record in records))
|
|
self.assertTrue(all(record["validation_error_type"] == "ValueError" for record in records))
|
|
self.assertTrue(all(record["validation_error"] for record in records))
|
|
self.assertEqual(
|
|
{Path(record["cdsl_attempt_path"]).name for record in records},
|
|
{path.name for path in attempts},
|
|
)
|
|
self.assertTrue(any("每次 CDSL 校验失败的诊断已保存到" in str(event.get("message", "")) for event in events))
|
|
self.assertEqual(len(agent.responses), 2)
|
|
self.assertEqual(list(settings.task_root.glob("cad_*")), [])
|
|
|
|
|
|
class ImageReferenceIntakeTests(unittest.TestCase):
|
|
@staticmethod
|
|
def _settings(temporary_root: Path, *, vision: bool = True) -> Settings:
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
provider = ProviderConfig(
|
|
"test",
|
|
"Test",
|
|
"https://example.invalid/v1",
|
|
"test-key",
|
|
(ProviderModel("vision-model", vision=vision),),
|
|
)
|
|
return Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="vision-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
|
|
@staticmethod
|
|
def _add_image_attachment(store: WorkspaceStore, conversation_id: str) -> str:
|
|
store.ensure_conversation(conversation_id)
|
|
relative_path, _ = store.write_conversation_upload(conversation_id, "flange.png", b"image-bytes")
|
|
store.add_conversation_attachment(conversation_id, {
|
|
"id": "upload_flange",
|
|
"conversation_id": conversation_id,
|
|
"name": "flange.png",
|
|
"kind": "image",
|
|
"path": relative_path,
|
|
"mime": "image/png",
|
|
})
|
|
return "upload_flange"
|
|
|
|
@staticmethod
|
|
def _analysis_arguments() -> dict[str, object]:
|
|
return {
|
|
"part_type": "四孔法兰套筒",
|
|
"visible_features": ["中空圆筒", "四孔法兰", "螺栓孔"],
|
|
"uncertain_features": ["法兰背面可能有沉孔"],
|
|
"dimension_candidates": [
|
|
{"id": "bore_diameter", "label": "中心孔直径", "reason": "图片没有标注内径"},
|
|
{"id": "bolt_circle", "label": "螺栓孔中心距", "reason": "透视图无法确定孔距"},
|
|
],
|
|
}
|
|
|
|
def test_image_request_keeps_structured_analysis_when_model_asks_a_question(self) -> None:
|
|
class ImageIntakeAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.required_tools: list[str | None] = []
|
|
self.responses = [
|
|
{"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "image_analysis",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "analyze_image_reference",
|
|
"arguments": json.dumps(ImageReferenceIntakeTests._analysis_arguments()),
|
|
},
|
|
}],
|
|
}}]},
|
|
{"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "中心孔直径会显著影响零件用途，请确认这个尺寸。",
|
|
"tool_calls": [],
|
|
}}]},
|
|
]
|
|
|
|
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
|
self.required_tools.append(kwargs.get("required_tool_name") if "required_tool_name" in kwargs else args[4] if len(args) > 4 else None)
|
|
return self.responses.pop(0)
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
settings = self._settings(temporary_root)
|
|
store = WorkspaceStore(settings)
|
|
conversation_id = "conv_000000000001"
|
|
self._add_image_attachment(store, conversation_id)
|
|
agent = ImageIntakeAgent(settings, store, CdslLibrary(settings))
|
|
message = ChatMessage(id="user_image", role="user", parts=[MessagePart(type="text", text="生成图片中的模型")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], conversation_id, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
conversation = store.read_conversation(conversation_id)
|
|
assistant_parts = conversation["messages"][-1]["parts"]
|
|
|
|
self.assertEqual(agent.required_tools, ["analyze_image_reference", None])
|
|
self.assertEqual(agent.responses, [])
|
|
self.assertTrue(any(event.get("partType") == "四孔法兰套筒" for event in events))
|
|
self.assertTrue(any("中心孔直径" in str(event.get("text", "")) for event in events))
|
|
self.assertEqual([part["type"] for part in assistant_parts], ["data-cad-image-analysis", "text"])
|
|
self.assertEqual(assistant_parts[0]["data"]["attachmentIds"], ["upload_flange"])
|
|
self.assertEqual(list(settings.task_root.glob("cad_*")), [])
|
|
|
|
def test_model_can_continue_to_generation_after_initial_analysis(self) -> None:
|
|
class EstimateAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.tool_sets: list[list[str]] = []
|
|
self.tool_calls: list[str] = []
|
|
self.responses = [
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "image_analysis", "type": "function", "function": {
|
|
"name": "analyze_image_reference",
|
|
"arguments": json.dumps(ImageReferenceIntakeTests._analysis_arguments()),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "design_brief", "type": "function", "function": {
|
|
"name": "describe_design_intent",
|
|
"arguments": json.dumps({
|
|
"plan": "按图片比例建立法兰套筒。",
|
|
"assumptions": ["所有未标注尺寸按图片比例估算，单位为 mm。"],
|
|
}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "build_cdsl", "type": "function", "function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({"cdsl": {}, "summary": "估算尺寸的法兰套筒", "assumptions": ["尺寸按比例估算"]}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "已按图片比例估算尺寸并生成模型。", "tool_calls": []}}]},
|
|
]
|
|
|
|
async def _complete(self, messages: list[dict[str, object]], tools: list[dict[str, object]], *args: object, **kwargs: object) -> dict[str, object]:
|
|
self.tool_sets.append([str(tool["function"]["name"]) for tool in tools])
|
|
return self.responses.pop(0)
|
|
|
|
async def _run_tool(self, name: str, arguments: dict[str, object], *args: object, **kwargs: object) -> tuple[dict[str, object], dict[str, object] | None]:
|
|
self.tool_calls.append(name)
|
|
if name == "generate_cdsl_model":
|
|
return {"ok": True, "summary": "估算尺寸的法兰套筒"}, None
|
|
return await super()._run_tool(name, arguments, *args, **kwargs)
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
settings = self._settings(Path(temporary_directory))
|
|
store = WorkspaceStore(settings)
|
|
conversation_id = "conv_000000000002"
|
|
self._add_image_attachment(store, conversation_id)
|
|
agent = EstimateAgent(settings, store, CdslLibrary(settings))
|
|
message = ChatMessage(id="user_estimate", role="user", parts=[MessagePart(type="text", text="根据图片直接推进建模，比例上的不确定性按合理工程判断处理。")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], conversation_id, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
assistant_parts = store.read_conversation(conversation_id)["messages"][-1]["parts"]
|
|
|
|
self.assertEqual(agent.tool_calls, ["analyze_image_reference", "describe_design_intent", "generate_cdsl_model"])
|
|
self.assertIn("analyze_image_reference", agent.tool_sets[0])
|
|
self.assertTrue(all("analyze_image_reference" not in tool_set for tool_set in agent.tool_sets[1:]))
|
|
self.assertEqual([part["type"] for part in assistant_parts], ["data-cad-image-analysis", "text"])
|
|
self.assertFalse(any("请补充以下尺寸" in str(event.get("text", "")) for event in events))
|
|
|
|
def test_recorded_analysis_reuses_context_without_reanalyzing(self) -> None:
|
|
class RecordedEstimateAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.tool_sets: list[list[str]] = []
|
|
self.tool_calls: list[str] = []
|
|
self.responses = [
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "design_brief", "type": "function", "function": {
|
|
"name": "describe_design_intent",
|
|
"arguments": json.dumps({"plan": "按既有识别结果建立法兰套筒。", "assumptions": ["尺寸按图片比例估算"]}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "build_cdsl", "type": "function", "function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({"cdsl": {}, "summary": "估算尺寸的法兰套筒", "assumptions": ["尺寸按比例估算"]}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "已按已有识别结果继续生成模型。", "tool_calls": []}}]},
|
|
]
|
|
|
|
async def _complete(self, messages: list[dict[str, object]], tools: list[dict[str, object]], *args: object, **kwargs: object) -> dict[str, object]:
|
|
self.tool_sets.append([str(tool["function"]["name"]) for tool in tools])
|
|
return self.responses.pop(0)
|
|
|
|
async def _run_tool(self, name: str, arguments: dict[str, object], *args: object, **kwargs: object) -> tuple[dict[str, object], dict[str, object] | None]:
|
|
self.tool_calls.append(name)
|
|
if name == "generate_cdsl_model":
|
|
return {"ok": True, "summary": "估算尺寸的法兰套筒"}, None
|
|
return await super()._run_tool(name, arguments, *args, **kwargs)
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
settings = self._settings(Path(temporary_directory))
|
|
store = WorkspaceStore(settings)
|
|
conversation_id = "conv_000000000003"
|
|
attachment_id = self._add_image_attachment(store, conversation_id)
|
|
analysis = self._analysis_arguments()
|
|
store.append_conversation_message(conversation_id, {
|
|
"id": "assistant_previous_analysis",
|
|
"role": "assistant",
|
|
"parts": [{"type": "data-cad-image-analysis", "data": {
|
|
"attachmentIds": [attachment_id],
|
|
"partType": analysis["part_type"],
|
|
"visibleFeatures": analysis["visible_features"],
|
|
"uncertainFeatures": analysis["uncertain_features"],
|
|
"dimensionCandidates": analysis["dimension_candidates"],
|
|
}}],
|
|
})
|
|
agent = RecordedEstimateAgent(settings, store, CdslLibrary(settings))
|
|
message = ChatMessage(id="user_estimate_again", role="user", parts=[MessagePart(type="text", text="请继续，未标注处按你的工程判断处理。")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], conversation_id, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
conversation = store.read_conversation(conversation_id)
|
|
analysis_parts = [
|
|
part
|
|
for item in conversation["messages"]
|
|
for part in item["parts"]
|
|
if part["type"] == "data-cad-image-analysis"
|
|
]
|
|
|
|
self.assertEqual(agent.tool_calls, ["describe_design_intent", "generate_cdsl_model"])
|
|
self.assertTrue(all("analyze_image_reference" not in tool_set for tool_set in agent.tool_sets))
|
|
self.assertEqual(len(analysis_parts), 1)
|
|
self.assertFalse(any("请补充以下尺寸" in str(event.get("text", "")) for event in events))
|
|
|
|
def test_direct_cdsl_generation_is_rejected_without_creating_a_task(self) -> None:
|
|
class RetryAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.seen_messages: list[list[dict[str, object]]] = []
|
|
self.required_tools: list[str | None] = []
|
|
self.responses = [
|
|
{
|
|
"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "incomplete_cdsl",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({
|
|
"cdsl": {"schema": "cad.cdsl.llm.v1"},
|
|
"summary": "incomplete",
|
|
}),
|
|
},
|
|
}],
|
|
}}],
|
|
},
|
|
{
|
|
"choices": [{"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{
|
|
"id": "corrected_cdsl",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({
|
|
"cdsl": {
|
|
"schema": "cad.cdsl.llm.v1",
|
|
"features": [{"id": "f01", "atomic_id": "extrude_add_blind", "depends_on": [], "params": {}, "sketch_id": "s01"}],
|
|
"geometry": {"sketches": [{"id": "s01", "workplane": {}, "profile": {"type": "circle"}}]},
|
|
},
|
|
"summary": "complete",
|
|
}),
|
|
},
|
|
}],
|
|
}}],
|
|
},
|
|
{"choices": [{"message": {"role": "assistant", "content": "已补全模型。", "tool_calls": []}}]},
|
|
]
|
|
|
|
async def _complete(self, messages: list[dict[str, object]], *args: object, **kwargs: object) -> dict[str, object]:
|
|
self.seen_messages.append([dict(message) for message in messages])
|
|
self.required_tools.append(kwargs.get("required_tool_name") if "required_tool_name" in kwargs else (args[3] if len(args) > 3 else None))
|
|
return self.responses.pop(0)
|
|
|
|
async def _run_tool(self, name: str, arguments: dict[str, object], *args: object, **kwargs: object) -> tuple[dict[str, object], dict[str, object] | None]:
|
|
if name == "generate_cdsl_model" and arguments.get("cdsl", {}).get("features"):
|
|
return {"ok": True, "summary": "complete"}, None
|
|
return await super()._run_tool(name, arguments, *args, **kwargs)
|
|
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
settings = Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
agent = RetryAgent(settings, WorkspaceStore(settings), CdslLibrary(settings))
|
|
message = ChatMessage(id="user_1", role="user", parts=[MessagePart(type="text", text="生成零件")])
|
|
|
|
async def collect_events() -> None:
|
|
async for _ in agent.stream([message], None, None):
|
|
pass
|
|
|
|
asyncio.run(collect_events())
|
|
|
|
tool_result = agent.seen_messages[1][-1]
|
|
self.assertEqual(tool_result["role"], "tool")
|
|
self.assertEqual(json.loads(str(tool_result["content"]))["code"], "DESIGN_BRIEF_REQUIRED")
|
|
self.assertEqual(agent.required_tools, [None, None, None])
|
|
self.assertEqual(list(settings.task_root.glob("cad_*")), [])
|
|
|
|
|
|
class StructuredResultResponseTests(unittest.TestCase):
|
|
def test_structured_result_does_not_add_a_duplicate_success_message(self) -> None:
|
|
class StructuredResultAgent(AgentService):
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.responses = [
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "design_brief",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "describe_design_intent",
|
|
"arguments": json.dumps({"plan": "建立带中心孔的法兰。", "assumptions": []}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": [{
|
|
"id": "build_cdsl",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "generate_cdsl_model",
|
|
"arguments": json.dumps({"cdsl": {}, "summary": "带中心孔的法兰", "assumptions": []}),
|
|
},
|
|
}]}}]},
|
|
{"choices": [{"message": {"role": "assistant", "content": "", "tool_calls": []}}]},
|
|
]
|
|
|
|
async def _complete(self, *args: object, **kwargs: object) -> dict[str, object]:
|
|
return self.responses.pop(0)
|
|
|
|
async def _run_tool(self, name: str, *args: object, **kwargs: object) -> tuple[dict[str, object], dict[str, object] | None]:
|
|
if name == "describe_design_intent":
|
|
return {"ok": True, "summary": "设计说明已记录"}, None
|
|
if name == "generate_cdsl_model":
|
|
return {"ok": True, "summary": "带中心孔的法兰"}, {
|
|
"task_id": "cad_000000000001",
|
|
"revision_id": "rev_001",
|
|
"cdsl_path": "model.cdsl.json",
|
|
"step_path": "model.step",
|
|
"glb_path": "model.glb",
|
|
"report_path": "report.json",
|
|
"summary": "带中心孔的法兰",
|
|
"reference_ids": [],
|
|
"engine": "cdsl_only",
|
|
}
|
|
raise AssertionError(f"unexpected tool: {name}")
|
|
|
|
backend_root = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
temporary_root = Path(temporary_directory)
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
settings = Settings(
|
|
task_root=temporary_root / "tasks",
|
|
conversation_root=temporary_root / "conversations",
|
|
library_root=backend_root / "cdsl_library",
|
|
engine_root=backend_root / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
store = WorkspaceStore(settings)
|
|
agent = StructuredResultAgent(settings, store, CdslLibrary(settings))
|
|
message = ChatMessage(id="user_result", role="user", parts=[MessagePart(type="text", text="生成一个带中心孔的法兰")])
|
|
|
|
async def collect_events() -> list[dict[str, object]]:
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in agent.stream([message], None, None):
|
|
events.append(json.loads(chunk.decode("utf-8").split("data: ", 1)[1]))
|
|
return events
|
|
|
|
events = asyncio.run(collect_events())
|
|
conversation_id = store.read_conversation(next(settings.conversation_root.iterdir()).name)["conversation_id"]
|
|
assistant_parts = store.read_conversation(conversation_id)["messages"][-1]["parts"]
|
|
diagnostics = settings.conversation_root / conversation_id / "diagnostics"
|
|
attempts = list(diagnostics.glob("cdsl_attempt_*.json"))
|
|
|
|
self.assertEqual([part["type"] for part in assistant_parts], ["data-cad-result"])
|
|
self.assertTrue(any(event.get("taskId") == "cad_000000000001" for event in events))
|
|
self.assertFalse(any("已生成：" in str(event.get("text", "")) for event in events))
|
|
self.assertEqual(len(attempts), 1)
|
|
self.assertEqual(json.loads(attempts[0].read_text(encoding="utf-8")), {})
|
|
self.assertEqual(list(diagnostics.glob("cdsl_validation_*.json")), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|