from __future__ import annotations import asyncio import base64 from hashlib import sha256 import json from pathlib import Path import sqlite3 import sys import tempfile import unittest from unittest.mock import AsyncMock, patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "backend")) from app.cad_agent.adapters.artifact_store import FileArtifactStore from app.cad_agent.adapters.event_publisher import IdempotentInProcessPublisher from app.cad_agent.adapters.review_gateway import RenderedReviewGateway from app.cad_agent.adapters.runtime import ProfileCadRuntime from app.cad_agent.adapters.sqlite_repository import SqliteTaskRepository from app.cad_agent.application.capabilities import cached_model_capability, conformance_hash, conformance_tools, verify_model_capability from app.cad_agent.application.llm_contracts import ( RequirementsAuthorOutput, StatelessCandidateReview, requirements_spec_schema, stateless_final_review_schema, stateless_next_action_schema, stateless_rollback_checkpoint_schema, ) from app.cad_agent.application.outbox import OutboxDispatcher from app.cad_agent.application.requirements import RequirementsCommandHandler from app.cad_agent.application.results import Accepted, Rejected, Waiting from app.cad_agent.application.workflow import ModelIdentity, WorkflowConfig, WorkflowCoordinator from app.cad_agent.domain.errors import ErrorCode from app.cad_agent.domain.operation_contract import fragment_schema from app.cad_agent.domain.state import TaskPhase, TaskState, legal_transitions, retry_resume_event, transition from app.cad_agent.domain.verifier_registry import default_registry from app.models.contracts import ChatMessage from app.services.agent_service import AgentService from app.services.library import CdslLibrary from app.services.storage import WorkspaceStore from app.settings import ProviderConfig, ProviderModel, Settings def settings(root: Path) -> Settings: author = ProviderConfig("author", "Author", "https://author.invalid/v1", "author-key", (ProviderModel("author-model"),)) reviewer = ProviderConfig("reviewer", "Reviewer", "https://reviewer.invalid/v1", "reviewer-key", (ProviderModel("reviewer-model", vision=True),)) return Settings( task_root=root / "tasks", conversation_root=root / "conversations", library_root=ROOT / "backend" / "cdsl_library", engine_root=ROOT / "backend" / "engine" / "cdsl_engine", llm_base_url=author.base_url, llm_api_key=author.api_key, llm_model="author-model", llm_timeout_s=1, default_provider_id="author", providers=(author, reviewer), review_provider_id="reviewer", review_model_id="reviewer-model", ) def ready_spec(*, spacing: float = 60.0) -> RequirementsAuthorOutput: return RequirementsAuthorOutput.model_validate({ "outcome": "ready", "summary": "Circular flange with an eight-hole pattern.", "assumptions": ["Dimensions use millimetres."], "requirements": [{ "statement": f"Use eight holes with a declared spacing of {spacing:g} degrees.", "assumptions": [], "acceptance_claims": [{ "claim_kind": "visual", "expected": {"description": f"Eight holes are shown with the requested {spacing:g} degree declaration."}, }], }], }) def walk_keys(value: object) -> set[str]: keys: set[str] = set() if isinstance(value, dict): keys.update(str(key) for key in value) for item in value.values(): keys.update(walk_keys(item)) elif isinstance(value, list): for item in value: keys.update(walk_keys(item)) return keys class CadV3ProtocolTests(unittest.TestCase): def test_state_machine_has_no_requirements_review_phase(self) -> None: self.assertNotIn("REVIEWING_REQUIREMENTS", {phase.value for phase in TaskPhase}) state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS, 0) approved = transition(state, "requirements_approved", requirements_contract_path="documents/contract.json") self.assertEqual(approved.phase, TaskPhase.AWAITING_ACTION) self.assertNotIn("requirements_finalized", {event for _phase, event in legal_transitions()}) def test_waiting_retry_resumes_exact_source_phase(self) -> None: state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS, 0) waiting = transition(state, "waiting_retry", error=ErrorCode.MODEL_PROTOCOL_CHECK_PENDING) self.assertEqual(retry_resume_event(waiting), "resume_drafting_requirements") resumed = transition(waiting, retry_resume_event(waiting) or "") self.assertEqual(resumed.phase, TaskPhase.DRAFTING_REQUIREMENTS) self.assertIsNone(resumed.retry_from_phase) def test_llm_schemas_exclude_server_owned_runtime_ids(self) -> None: with tempfile.TemporaryDirectory() as temporary: runtime = ProfileCadRuntime(settings(Path(temporary))) schemas = [ requirements_spec_schema(default_registry().expected_one_of_schema()), stateless_next_action_schema(list(runtime.supported_atomic_ids())), StatelessCandidateReview.model_json_schema(), stateless_final_review_schema(2), ] forbidden = { "task_id", "working_head", "requirement_id", "requirement_ids", "claim_id", "candidate_id", "action_id", "evidence_id", "evidence_refs", "source_id", "source_ids", "draft_id", "attachment_id", } for schema in schemas: self.assertFalse(walk_keys(schema) & forbidden, walk_keys(schema) & forbidden) def test_dynamic_tokens_are_enum_constrained(self) -> None: rollback = stateless_rollback_checkpoint_schema(["checkpoint_one"]) self.assertEqual(rollback["properties"]["checkpoint_token"], {"enum": ["checkpoint_one"]}) with tempfile.TemporaryDirectory() as temporary: runtime = ProfileCadRuntime(settings(Path(temporary))) schema = fragment_schema(runtime.operation_contract("hole_blind"), selector_tokens=["selector_one"], reference_tokens=[]) selector = schema["properties"]["feature"]["properties"]["selector_tokens"]["items"] self.assertEqual(selector, {"enum": ["selector_one"]}) def test_requirements_spec_freezes_once_and_preserves_user_value(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") artifacts = FileArtifactStore(root / "tasks") handler = RequirementsCommandHandler(repository, artifacts, default_registry()) task_id = "cad_123456abcdef" repository.create_task(task_id, "Use 8 holes around a full circle at 60 degrees.") artifacts.initialize_task(task_id, "Use 8 holes around a full circle at 60 degrees.") result = handler.submit_spec(task_id, ready_spec(spacing=60), invocation_id="requirements_once") self.assertIsInstance(result, Accepted) state = repository.get_state(task_id) self.assertEqual(state.phase, TaskPhase.AWAITING_ACTION) contract = artifacts.read_requirements_contract(task_id, state.requirements_contract_path) or {} requirement = contract["requirements"][0] self.assertIn("60 degrees", requirement["statement"]) self.assertIn("60 degree", requirement["acceptance_claims"][0]["expected"]["description"]) self.assertNotIn("45 degrees", requirement["statement"]) self.assertNotIn("applied_normalizations", contract) self.assertTrue((artifacts.task_dir(task_id) / "requirements.md").is_file()) self.assertTrue((artifacts.task_dir(task_id) / "completion-target.md").is_file()) self.assertFalse((artifacts.task_dir(task_id) / "completion.md").exists()) def test_invalid_verifier_contract_is_rejected_without_state_change(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") artifacts = FileArtifactStore(root / "tasks") handler = RequirementsCommandHandler(repository, artifacts, default_registry()) task_id = "cad_123456abcdef" initial = repository.create_task(task_id, "Create one solid.") artifacts.initialize_task(task_id, "Create one solid.") value = RequirementsAuthorOutput.model_validate({ "outcome": "ready", "summary": "One solid", "assumptions": [], "requirements": [{"statement": "One solid", "assumptions": [], "acceptance_claims": [ {"claim_kind": "solid_count_equals", "expected": {"value": 0}}, ]}], }) result = handler.submit_spec(task_id, value, invocation_id="invalid_spec") self.assertIsInstance(result, Rejected) self.assertEqual(result.error.code, ErrorCode.REQUIREMENTS_SPEC_INVALID) self.assertEqual(repository.get_state(task_id), initial) def test_clarification_uses_same_task_and_one_question(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") artifacts = FileArtifactStore(root / "tasks") runtime = ProfileCadRuntime(settings(root)) requirements = RequirementsCommandHandler(repository, artifacts, default_registry()) workflow = WorkflowCoordinator(WorkflowConfig(4, 2), repository, artifacts, runtime, None, None, requirements, None) task_id = "cad_123456abcdef" repository.create_task(task_id, "Use four holes and use six holes.") artifacts.initialize_task(task_id, "Use four holes and use six holes.") value = RequirementsAuthorOutput.model_validate({ "outcome": "clarification", "source_quotes": ["four holes", "six holes"], "question": "Should the part use four holes or six holes?", }) result = requirements.submit_spec(task_id, value, invocation_id="clarify_once") self.assertIsInstance(result, Waiting) waiting = repository.get_state(task_id) terminal = workflow.waiting_for_user_terminal(task_id, waiting) self.assertEqual(terminal["questions"], ["Should the part use four holes or six holes?"]) self.assertTrue(terminal["userActionRequired"]) self.assertTrue(workflow.resume_with_user_clarification(task_id, "Use six holes.", message_id="user_reply")) self.assertEqual(repository.get_state(task_id).phase, TaskPhase.DRAFTING_REQUIREMENTS) def test_contract_views_and_completion_result_share_one_spec(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") artifacts = FileArtifactStore(root / "tasks") handler = RequirementsCommandHandler(repository, artifacts, default_registry()) task_id = "cad_123456abcdef" repository.create_task(task_id, "Create a coherent flange.") artifacts.initialize_task(task_id, "Create a coherent flange.") handler.submit_spec(task_id, ready_spec(), invocation_id="requirements_once") state = repository.get_state(task_id) path = handler.write_completion_result( task_id, state, claim_results=[{"claim_id": "claim_001", "deterministic": False, "status": "pending"}], review={"visual_claims": [{"status": "pass", "evidence": "Reference and render match."}]}, ) self.assertEqual(path, "completion-result.md") result = (artifacts.task_dir(task_id) / path).read_text(encoding="utf-8") target = (artifacts.task_dir(task_id) / "completion-target.md").read_text(encoding="utf-8") requirements = (artifacts.task_dir(task_id) / "requirements.md").read_text(encoding="utf-8") self.assertIn("eight-hole pattern", requirements) self.assertIn("visual", target) self.assertIn("visual: pass", result) def test_sqlite_schema_contains_only_current_requirement_paths(self) -> None: with tempfile.TemporaryDirectory() as temporary: database = Path(temporary) / "state.sqlite3" SqliteTaskRepository(database) with sqlite3.connect(database) as connection: columns = {row[1] for row in connection.execute("PRAGMA table_info(tasks)")} self.assertIn("requirements_spec_path", columns) self.assertIn("clarification_path", columns) self.assertNotIn("requirements_draft_path", columns) self.assertNotIn("requirements_review_path", columns) def test_role_specific_capability_tools_have_no_review_loop(self) -> None: with tempfile.TemporaryDirectory() as temporary: runtime = ProfileCadRuntime(settings(Path(temporary))) author = conformance_tools(runtime, role="author") reviewer = conformance_tools(runtime, role="reviewer") author_names = {item["function"]["name"] for item in author} reviewer_names = {item["function"]["name"] for item in reviewer} self.assertIn("submit_requirements_spec", author_names) self.assertNotIn("review_requirements", author_names | reviewer_names) self.assertNotIn("get_cdsl_operation_contract", author_names) self.assertEqual(reviewer_names, {"observe_images", "review_candidate", "review_final"}) def test_capability_cache_is_role_scoped(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") runtime = ProfileCadRuntime(settings(root)) class Models: def __init__(self) -> None: self.calls = 0 async def conformance(self, **_kwargs: object) -> dict[str, object]: self.calls += 1 return {"supported": True, "failures": [], "probe_unavailable": False} models = Models() first = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="author")) second = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="author")) reviewer = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="reviewer")) self.assertFalse(first.get("cached", False)) self.assertTrue(second["cached"]) self.assertNotEqual(first["schema_hash"], reviewer["schema_hash"]) self.assertEqual(models.calls, 2) self.assertIsNotNone(cached_model_capability(repository, runtime, provider_id="p", model_id="m", role="author")) def test_unsupported_capability_is_cached_but_transport_failure_is_not(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) repository = SqliteTaskRepository(root / "state.sqlite3") runtime = ProfileCadRuntime(settings(root)) schema_hash = conformance_hash(conformance_tools(runtime, role="reviewer"), role="reviewer") class Unsupported: async def conformance(self, **_kwargs: object) -> dict[str, object]: return {"supported": False, "failures": [{"message": "schema"}], "probe_unavailable": False} asyncio.run(verify_model_capability(repository, runtime, Unsupported(), provider_id="p", model_id="m", role="reviewer")) self.assertIsNotNone(repository.model_capability("p", "m", schema_hash)) class Unavailable: async def conformance(self, **_kwargs: object) -> dict[str, object]: return {"supported": False, "failures": [{"message": "network"}], "probe_unavailable": True} asyncio.run(verify_model_capability(repository, runtime, Unavailable(), provider_id="p2", model_id="m", role="reviewer")) self.assertIsNone(repository.model_capability("p2", "m", schema_hash)) def test_task_started_is_emitted_before_capability_work(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) configured = settings(root) service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured)) message = ChatMessage.model_validate({"id": "user_1", "role": "user", "parts": [{"type": "text", "text": "Create a plate."}]}) async def first_chunk() -> bytes: stream = service.stream([message], None, None) chunk = await anext(stream) await stream.aclose() return chunk with patch("app.services.agent_service.verify_model_capability", AsyncMock()) as capability: chunk = asyncio.run(first_chunk()).decode("utf-8") self.assertIn("task_started", chunk) capability.assert_not_awaited() self.assertEqual(len(service.v3.repository.running_task_ids()), 1) def test_cached_capabilities_skip_normal_request_probe(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) configured = settings(root) service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured)) task_id = "cad_123456abcdef" service.v3.workflow.create_task(task_id, "Create a plate.") queue: asyncio.Queue = asyncio.Queue() with patch("app.services.agent_service.cached_model_capability", return_value={"supported": True}), patch( "app.services.agent_service.verify_model_capability", AsyncMock() ) as verify: result = asyncio.run(service._ensure_task_capabilities( task_id, ModelIdentity("author", "author-model"), ModelIdentity("reviewer", "reviewer-model"), queue, )) self.assertIsNone(result) verify.assert_not_awaited() self.assertTrue(queue.empty()) def test_missing_cache_is_visible_and_auto_resumes_same_task(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) configured = settings(root) service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured)) task_id = "cad_123456abcdef" service.v3.workflow.create_task(task_id, "Create a plate.") queue: asyncio.Queue = asyncio.Queue() with patch("app.services.agent_service.cached_model_capability", return_value=None), patch( "app.services.agent_service.verify_model_capability", AsyncMock(return_value={"supported": True, "probe_unavailable": False}) ) as verify: result = asyncio.run(service._ensure_task_capabilities( task_id, ModelIdentity("author", "author-model"), ModelIdentity("reviewer", "reviewer-model"), queue, )) self.assertIsNone(result) self.assertEqual(verify.await_count, 2) self.assertEqual(service.v3.repository.get_state(task_id).phase, TaskPhase.DRAFTING_REQUIREMENTS) events = [queue.get_nowait(), queue.get_nowait()] self.assertEqual([item[1]["status"] for item in events], ["waiting", "success"]) def test_image_bytes_are_frozen_and_sent_to_vision(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) artifacts = FileArtifactStore(root / "tasks") task_id = "cad_123456abcdef" source = root / "reference.png" png = b"\x89PNG\r\n\x1a\nreference-bytes" source.write_bytes(png) artifacts.initialize_task(task_id, "Match the image.", image_inputs=[{ "path": str(source), "mime": "image/png", "sha256": sha256(png).hexdigest(), }]) frozen = Path(artifacts.source_image_paths(task_id)[0]) class Models: def __init__(self) -> None: self.messages: list[list[dict[str, object]]] = [] async def call_tool(self, *, messages: list[dict[str, object]], **_kwargs: object) -> dict[str, object]: self.messages.append(messages) return {"tool_calls": [], "usage": {}} models = Models() gateway = RenderedReviewGateway(models) tool = {"type": "function", "function": {"name": "observe_images", "parameters": {"type": "object"}}} asyncio.run(gateway.review(kind="image_observation", payload={"reference_image_paths": [str(frozen)]}, tool=tool, provider_id="p", model_id="m")) image_part = models.messages[0][1]["content"][1] encoded = image_part["image_url"]["url"].split(",", 1)[1] self.assertEqual(base64.b64decode(encoded), png) def test_final_review_schema_has_ordered_visual_decision_count(self) -> None: visual = stateless_final_review_schema(2)["properties"]["visual_claims"] self.assertEqual((visual["minItems"], visual["maxItems"]), (2, 2)) def test_sqlite_cas_and_outbox_are_atomic(self) -> None: with tempfile.TemporaryDirectory() as temporary: repository = SqliteTaskRepository(Path(temporary) / "state.sqlite3") initial = repository.create_task("cad_123456abcdef", "Create a plate.") changed = transition(initial, "image_observed") self.assertTrue(repository.compare_and_swap(changed, events=[{"event": "image_observation_ready"}])) self.assertFalse(repository.compare_and_swap(changed, events=[{"event": "duplicate"}])) self.assertEqual(len(repository.pending_outbox()), 1) delivered = asyncio.run(OutboxDispatcher(repository, IdempotentInProcessPublisher()).dispatch_pending()) self.assertEqual(len(delivered), 1) self.assertEqual(repository.pending_outbox(), []) if __name__ == "__main__": unittest.main()