change

backend/tests/test_structured_llm.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+import sys
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "backend"))
+
+from app.cad_agent.adapters.structured_llm import StructuredModelError, StructuredModelGateway  # noqa: E402
+from app.settings import ProviderConfig, ProviderModel, Settings  # noqa: E402
+
+
+def _settings(*, api_style: str = "responses", reasoning_effort: str = "medium") -> Settings:
+    provider = ProviderConfig(
+        "provider", "Provider", "https://example.invalid/v1", "key",
+        (ProviderModel("model"),), reasoning_effort=reasoning_effort, api_style=api_style,
+    )
+    return Settings(
+        task_root=ROOT / "tmp-tasks",
+        conversation_root=ROOT / "tmp-conversations",
+        library_root=ROOT / "backend" / "cdsl_library",
+        engine_root=ROOT / "backend" / "engine" / "cdsl_engine",
+        llm_base_url=provider.base_url,
+        llm_api_key=provider.api_key,
+        llm_model="model",
+        llm_timeout_s=1,
+        default_provider_id="provider",
+        providers=(provider,),
+    )
+
+
+def _tool() -> dict:
+    return {
+        "type": "function",
+        "function": {
+            "name": "write_document",
+            "description": "Write one document.",
+            "parameters": {"type": "object", "properties": {}, "additionalProperties": False},
+        },
+    }
+
+
+def _response(api_style: str) -> dict:
+    if api_style == "responses":
+        return {
+            "output": [{"type": "function_call", "name": "write_document", "arguments": "{}"}],
+            "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
+        }
+    return {
+        "choices": [{"message": {"tool_calls": [{"function": {"name": "write_document", "arguments": "{}"}}]}}],
+        "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
+    }
+
+
+class StructuredModelGatewayCompatibilityTests(unittest.TestCase):
+    def test_thinking_tool_choice_retries_without_reasoning_before_relaxing_tool_choice(self) -> None:
+        gateway = StructuredModelGateway(_settings())
+        payloads: list[dict] = []
+
+        async def request(_provider: object, payload: dict) -> dict:
+            payloads.append(payload)
+            if len(payloads) == 1:
+                raise StructuredModelError("Provider rejected structured request (400): Thinking mode does not support this tool_choice")
+            return _response("responses")
+
+        gateway._request = request  # type: ignore[method-assign]
+        result = asyncio.run(gateway.call_tool(
+            messages=[{"role": "system", "content": "Call the tool."}],
+            tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
+        ))
+
+        self.assertEqual(len(payloads), 2)
+        self.assertEqual(payloads[0]["tool_choice"], {"type": "function", "name": "write_document"})
+        self.assertEqual(payloads[0]["reasoning"], {"effort": "medium"})
+        self.assertEqual(payloads[1]["tool_choice"], {"type": "function", "name": "write_document"})
+        self.assertNotIn("reasoning", payloads[1])
+        self.assertEqual(result["usage"]["structured_compatibility_mode"], "reasoning_disabled")
+
+    def test_persistent_thinking_rejection_uses_auto_with_the_same_single_tool(self) -> None:
+        gateway = StructuredModelGateway(_settings(api_style="chat_completions", reasoning_effort=""))
+        payloads: list[dict] = []
+
+        async def request(_provider: object, payload: dict) -> dict:
+            payloads.append(payload)
+            if len(payloads) < 3:
+                raise StructuredModelError("Thinking mode does not support this tool_choice")
+            return _response("chat_completions")
+
+        gateway._request = request  # type: ignore[method-assign]
+        result = asyncio.run(gateway.call_tool(
+            messages=[{"role": "system", "content": "Call the tool."}],
+            tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
+        ))
+
+        self.assertEqual(len(payloads), 3)
+        self.assertEqual(payloads[0]["tool_choice"]["function"]["name"], "write_document")
+        self.assertEqual(payloads[1]["tool_choice"]["function"]["name"], "write_document")
+        self.assertEqual(payloads[2]["tool_choice"], "auto")
+        self.assertEqual(len(payloads[2]["tools"]), 1)
+        self.assertEqual(result["usage"]["structured_compatibility_mode"], "single_tool_auto")
+
+    def test_unrelated_provider_rejection_is_not_retried(self) -> None:
+        gateway = StructuredModelGateway(_settings())
+        payloads: list[dict] = []
+
+        async def request(_provider: object, payload: dict) -> dict:
+            payloads.append(payload)
+            raise StructuredModelError("Provider rejected structured request (400): invalid model")
+
+        gateway._request = request  # type: ignore[method-assign]
+        with self.assertRaisesRegex(StructuredModelError, "invalid model"):
+            asyncio.run(gateway.call_tool(
+                messages=[{"role": "system", "content": "Call the tool."}],
+                tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
+            ))
+        self.assertEqual(len(payloads), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()