388 lines
22 KiB
Python
388 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.cad_agent.adapters.artifact_store import FileArtifactStore
|
|
from app.cad_agent.adapters.event_publisher import IdempotentInProcessPublisher
|
|
from app.cad_agent.adapters.review_gateway import RenderedReviewGateway
|
|
from app.cad_agent.adapters.runtime import ProfileCadRuntime
|
|
from app.cad_agent.adapters.sqlite_repository import SqliteTaskRepository
|
|
from app.cad_agent.application.capabilities import cached_model_capability, conformance_hash, conformance_tools, verify_model_capability
|
|
from app.cad_agent.application.llm_contracts import (
|
|
RequirementsAuthorOutput,
|
|
StatelessCandidateReview,
|
|
requirements_spec_schema,
|
|
stateless_final_review_schema,
|
|
stateless_next_action_schema,
|
|
stateless_rollback_checkpoint_schema,
|
|
)
|
|
from app.cad_agent.application.outbox import OutboxDispatcher
|
|
from app.cad_agent.application.requirements import RequirementsCommandHandler
|
|
from app.cad_agent.application.results import Accepted, Rejected, Waiting
|
|
from app.cad_agent.application.workflow import ModelIdentity, WorkflowConfig, WorkflowCoordinator
|
|
from app.cad_agent.domain.errors import ErrorCode
|
|
from app.cad_agent.domain.operation_contract import fragment_schema
|
|
from app.cad_agent.domain.state import TaskPhase, TaskState, legal_transitions, retry_resume_event, transition
|
|
from app.cad_agent.domain.verifier_registry import default_registry
|
|
from app.models.contracts import ChatMessage
|
|
from app.services.agent_service import AgentService
|
|
from app.services.library import CdslLibrary
|
|
from app.services.storage import WorkspaceStore
|
|
from app.settings import ProviderConfig, ProviderModel, Settings
|
|
|
|
|
|
def settings(root: Path) -> Settings:
|
|
author = ProviderConfig("author", "Author", "https://author.invalid/v1", "author-key", (ProviderModel("author-model"),))
|
|
reviewer = ProviderConfig("reviewer", "Reviewer", "https://reviewer.invalid/v1", "reviewer-key", (ProviderModel("reviewer-model", vision=True),))
|
|
return Settings(
|
|
task_root=root / "tasks",
|
|
conversation_root=root / "conversations",
|
|
library_root=ROOT / "backend" / "cdsl_library",
|
|
engine_root=ROOT / "backend" / "engine" / "cdsl_engine",
|
|
llm_base_url=author.base_url,
|
|
llm_api_key=author.api_key,
|
|
llm_model="author-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="author",
|
|
providers=(author, reviewer),
|
|
review_provider_id="reviewer",
|
|
review_model_id="reviewer-model",
|
|
)
|
|
|
|
|
|
def ready_spec(*, spacing: float = 60.0) -> RequirementsAuthorOutput:
|
|
return RequirementsAuthorOutput.model_validate({
|
|
"outcome": "ready",
|
|
"summary": "Circular flange with an eight-hole pattern.",
|
|
"assumptions": ["Dimensions use millimetres."],
|
|
"requirements": [{
|
|
"statement": f"Use eight holes with a declared spacing of {spacing:g} degrees.",
|
|
"assumptions": [],
|
|
"acceptance_claims": [{
|
|
"claim_kind": "visual",
|
|
"expected": {"description": f"Eight holes are shown with the requested {spacing:g} degree declaration."},
|
|
}],
|
|
}],
|
|
})
|
|
|
|
|
|
def walk_keys(value: object) -> set[str]:
|
|
keys: set[str] = set()
|
|
if isinstance(value, dict):
|
|
keys.update(str(key) for key in value)
|
|
for item in value.values():
|
|
keys.update(walk_keys(item))
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
keys.update(walk_keys(item))
|
|
return keys
|
|
|
|
|
|
class CadV3ProtocolTests(unittest.TestCase):
|
|
def test_state_machine_has_no_requirements_review_phase(self) -> None:
|
|
self.assertNotIn("REVIEWING_REQUIREMENTS", {phase.value for phase in TaskPhase})
|
|
state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS, 0)
|
|
approved = transition(state, "requirements_approved", requirements_contract_path="documents/contract.json")
|
|
self.assertEqual(approved.phase, TaskPhase.AWAITING_ACTION)
|
|
self.assertNotIn("requirements_finalized", {event for _phase, event in legal_transitions()})
|
|
|
|
def test_waiting_retry_resumes_exact_source_phase(self) -> None:
|
|
state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS, 0)
|
|
waiting = transition(state, "waiting_retry", error=ErrorCode.MODEL_PROTOCOL_CHECK_PENDING)
|
|
self.assertEqual(retry_resume_event(waiting), "resume_drafting_requirements")
|
|
resumed = transition(waiting, retry_resume_event(waiting) or "")
|
|
self.assertEqual(resumed.phase, TaskPhase.DRAFTING_REQUIREMENTS)
|
|
self.assertIsNone(resumed.retry_from_phase)
|
|
|
|
def test_llm_schemas_exclude_server_owned_runtime_ids(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
runtime = ProfileCadRuntime(settings(Path(temporary)))
|
|
schemas = [
|
|
requirements_spec_schema(default_registry().expected_one_of_schema()),
|
|
stateless_next_action_schema(list(runtime.supported_atomic_ids())),
|
|
StatelessCandidateReview.model_json_schema(),
|
|
stateless_final_review_schema(2),
|
|
]
|
|
forbidden = {
|
|
"task_id", "working_head", "requirement_id", "requirement_ids", "claim_id",
|
|
"candidate_id", "action_id", "evidence_id", "evidence_refs", "source_id", "source_ids",
|
|
"draft_id", "attachment_id",
|
|
}
|
|
for schema in schemas:
|
|
self.assertFalse(walk_keys(schema) & forbidden, walk_keys(schema) & forbidden)
|
|
|
|
def test_dynamic_tokens_are_enum_constrained(self) -> None:
|
|
rollback = stateless_rollback_checkpoint_schema(["checkpoint_one"])
|
|
self.assertEqual(rollback["properties"]["checkpoint_token"], {"enum": ["checkpoint_one"]})
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
runtime = ProfileCadRuntime(settings(Path(temporary)))
|
|
schema = fragment_schema(runtime.operation_contract("hole_blind"), selector_tokens=["selector_one"], reference_tokens=[])
|
|
selector = schema["properties"]["feature"]["properties"]["selector_tokens"]["items"]
|
|
self.assertEqual(selector, {"enum": ["selector_one"]})
|
|
|
|
def test_requirements_spec_freezes_once_and_preserves_user_value(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
artifacts = FileArtifactStore(root / "tasks")
|
|
handler = RequirementsCommandHandler(repository, artifacts, default_registry())
|
|
task_id = "cad_123456abcdef"
|
|
repository.create_task(task_id, "Use 8 holes around a full circle at 60 degrees.")
|
|
artifacts.initialize_task(task_id, "Use 8 holes around a full circle at 60 degrees.")
|
|
result = handler.submit_spec(task_id, ready_spec(spacing=60), invocation_id="requirements_once")
|
|
self.assertIsInstance(result, Accepted)
|
|
state = repository.get_state(task_id)
|
|
self.assertEqual(state.phase, TaskPhase.AWAITING_ACTION)
|
|
contract = artifacts.read_requirements_contract(task_id, state.requirements_contract_path) or {}
|
|
requirement = contract["requirements"][0]
|
|
self.assertIn("60 degrees", requirement["statement"])
|
|
self.assertIn("60 degree", requirement["acceptance_claims"][0]["expected"]["description"])
|
|
self.assertNotIn("45 degrees", requirement["statement"])
|
|
self.assertNotIn("applied_normalizations", contract)
|
|
self.assertTrue((artifacts.task_dir(task_id) / "requirements.md").is_file())
|
|
self.assertTrue((artifacts.task_dir(task_id) / "completion-target.md").is_file())
|
|
self.assertFalse((artifacts.task_dir(task_id) / "completion.md").exists())
|
|
|
|
def test_invalid_verifier_contract_is_rejected_without_state_change(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
artifacts = FileArtifactStore(root / "tasks")
|
|
handler = RequirementsCommandHandler(repository, artifacts, default_registry())
|
|
task_id = "cad_123456abcdef"
|
|
initial = repository.create_task(task_id, "Create one solid.")
|
|
artifacts.initialize_task(task_id, "Create one solid.")
|
|
value = RequirementsAuthorOutput.model_validate({
|
|
"outcome": "ready", "summary": "One solid", "assumptions": [],
|
|
"requirements": [{"statement": "One solid", "assumptions": [], "acceptance_claims": [
|
|
{"claim_kind": "solid_count_equals", "expected": {"value": 0}},
|
|
]}],
|
|
})
|
|
result = handler.submit_spec(task_id, value, invocation_id="invalid_spec")
|
|
self.assertIsInstance(result, Rejected)
|
|
self.assertEqual(result.error.code, ErrorCode.REQUIREMENTS_SPEC_INVALID)
|
|
self.assertEqual(repository.get_state(task_id), initial)
|
|
|
|
def test_clarification_uses_same_task_and_one_question(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
artifacts = FileArtifactStore(root / "tasks")
|
|
runtime = ProfileCadRuntime(settings(root))
|
|
requirements = RequirementsCommandHandler(repository, artifacts, default_registry())
|
|
workflow = WorkflowCoordinator(WorkflowConfig(4, 2), repository, artifacts, runtime, None, None, requirements, None)
|
|
task_id = "cad_123456abcdef"
|
|
repository.create_task(task_id, "Use four holes and use six holes.")
|
|
artifacts.initialize_task(task_id, "Use four holes and use six holes.")
|
|
value = RequirementsAuthorOutput.model_validate({
|
|
"outcome": "clarification", "source_quotes": ["four holes", "six holes"],
|
|
"question": "Should the part use four holes or six holes?",
|
|
})
|
|
result = requirements.submit_spec(task_id, value, invocation_id="clarify_once")
|
|
self.assertIsInstance(result, Waiting)
|
|
waiting = repository.get_state(task_id)
|
|
terminal = workflow.waiting_for_user_terminal(task_id, waiting)
|
|
self.assertEqual(terminal["questions"], ["Should the part use four holes or six holes?"])
|
|
self.assertTrue(terminal["userActionRequired"])
|
|
self.assertTrue(workflow.resume_with_user_clarification(task_id, "Use six holes.", message_id="user_reply"))
|
|
self.assertEqual(repository.get_state(task_id).phase, TaskPhase.DRAFTING_REQUIREMENTS)
|
|
|
|
def test_contract_views_and_completion_result_share_one_spec(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
artifacts = FileArtifactStore(root / "tasks")
|
|
handler = RequirementsCommandHandler(repository, artifacts, default_registry())
|
|
task_id = "cad_123456abcdef"
|
|
repository.create_task(task_id, "Create a coherent flange.")
|
|
artifacts.initialize_task(task_id, "Create a coherent flange.")
|
|
handler.submit_spec(task_id, ready_spec(), invocation_id="requirements_once")
|
|
state = repository.get_state(task_id)
|
|
path = handler.write_completion_result(
|
|
task_id, state,
|
|
claim_results=[{"claim_id": "claim_001", "deterministic": False, "status": "pending"}],
|
|
review={"visual_claims": [{"status": "pass", "evidence": "Reference and render match."}]},
|
|
)
|
|
self.assertEqual(path, "completion-result.md")
|
|
result = (artifacts.task_dir(task_id) / path).read_text(encoding="utf-8")
|
|
target = (artifacts.task_dir(task_id) / "completion-target.md").read_text(encoding="utf-8")
|
|
requirements = (artifacts.task_dir(task_id) / "requirements.md").read_text(encoding="utf-8")
|
|
self.assertIn("eight-hole pattern", requirements)
|
|
self.assertIn("visual", target)
|
|
self.assertIn("visual: pass", result)
|
|
|
|
def test_sqlite_schema_contains_only_current_requirement_paths(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
database = Path(temporary) / "state.sqlite3"
|
|
SqliteTaskRepository(database)
|
|
with sqlite3.connect(database) as connection:
|
|
columns = {row[1] for row in connection.execute("PRAGMA table_info(tasks)")}
|
|
self.assertIn("requirements_spec_path", columns)
|
|
self.assertIn("clarification_path", columns)
|
|
self.assertNotIn("requirements_draft_path", columns)
|
|
self.assertNotIn("requirements_review_path", columns)
|
|
|
|
def test_role_specific_capability_tools_have_no_review_loop(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
runtime = ProfileCadRuntime(settings(Path(temporary)))
|
|
author = conformance_tools(runtime, role="author")
|
|
reviewer = conformance_tools(runtime, role="reviewer")
|
|
author_names = {item["function"]["name"] for item in author}
|
|
reviewer_names = {item["function"]["name"] for item in reviewer}
|
|
self.assertIn("submit_requirements_spec", author_names)
|
|
self.assertNotIn("review_requirements", author_names | reviewer_names)
|
|
self.assertNotIn("get_cdsl_operation_contract", author_names)
|
|
self.assertEqual(reviewer_names, {"observe_images", "review_candidate", "review_final"})
|
|
|
|
def test_capability_cache_is_role_scoped(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
runtime = ProfileCadRuntime(settings(root))
|
|
class Models:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
async def conformance(self, **_kwargs: object) -> dict[str, object]:
|
|
self.calls += 1
|
|
return {"supported": True, "failures": [], "probe_unavailable": False}
|
|
models = Models()
|
|
first = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="author"))
|
|
second = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="author"))
|
|
reviewer = asyncio.run(verify_model_capability(repository, runtime, models, provider_id="p", model_id="m", role="reviewer"))
|
|
self.assertFalse(first.get("cached", False))
|
|
self.assertTrue(second["cached"])
|
|
self.assertNotEqual(first["schema_hash"], reviewer["schema_hash"])
|
|
self.assertEqual(models.calls, 2)
|
|
self.assertIsNotNone(cached_model_capability(repository, runtime, provider_id="p", model_id="m", role="author"))
|
|
|
|
def test_unsupported_capability_is_cached_but_transport_failure_is_not(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = SqliteTaskRepository(root / "state.sqlite3")
|
|
runtime = ProfileCadRuntime(settings(root))
|
|
schema_hash = conformance_hash(conformance_tools(runtime, role="reviewer"), role="reviewer")
|
|
class Unsupported:
|
|
async def conformance(self, **_kwargs: object) -> dict[str, object]:
|
|
return {"supported": False, "failures": [{"message": "schema"}], "probe_unavailable": False}
|
|
asyncio.run(verify_model_capability(repository, runtime, Unsupported(), provider_id="p", model_id="m", role="reviewer"))
|
|
self.assertIsNotNone(repository.model_capability("p", "m", schema_hash))
|
|
class Unavailable:
|
|
async def conformance(self, **_kwargs: object) -> dict[str, object]:
|
|
return {"supported": False, "failures": [{"message": "network"}], "probe_unavailable": True}
|
|
asyncio.run(verify_model_capability(repository, runtime, Unavailable(), provider_id="p2", model_id="m", role="reviewer"))
|
|
self.assertIsNone(repository.model_capability("p2", "m", schema_hash))
|
|
|
|
def test_task_started_is_emitted_before_capability_work(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
configured = settings(root)
|
|
service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured))
|
|
message = ChatMessage.model_validate({"id": "user_1", "role": "user", "parts": [{"type": "text", "text": "Create a plate."}]})
|
|
async def first_chunk() -> bytes:
|
|
stream = service.stream([message], None, None)
|
|
chunk = await anext(stream)
|
|
await stream.aclose()
|
|
return chunk
|
|
with patch("app.services.agent_service.verify_model_capability", AsyncMock()) as capability:
|
|
chunk = asyncio.run(first_chunk()).decode("utf-8")
|
|
self.assertIn("task_started", chunk)
|
|
capability.assert_not_awaited()
|
|
self.assertEqual(len(service.v3.repository.running_task_ids()), 1)
|
|
|
|
def test_cached_capabilities_skip_normal_request_probe(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
configured = settings(root)
|
|
service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured))
|
|
task_id = "cad_123456abcdef"
|
|
service.v3.workflow.create_task(task_id, "Create a plate.")
|
|
queue: asyncio.Queue = asyncio.Queue()
|
|
with patch("app.services.agent_service.cached_model_capability", return_value={"supported": True}), patch(
|
|
"app.services.agent_service.verify_model_capability", AsyncMock()
|
|
) as verify:
|
|
result = asyncio.run(service._ensure_task_capabilities(
|
|
task_id, ModelIdentity("author", "author-model"), ModelIdentity("reviewer", "reviewer-model"), queue,
|
|
))
|
|
self.assertIsNone(result)
|
|
verify.assert_not_awaited()
|
|
self.assertTrue(queue.empty())
|
|
|
|
def test_missing_cache_is_visible_and_auto_resumes_same_task(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
configured = settings(root)
|
|
service = AgentService(configured, WorkspaceStore(configured), CdslLibrary(configured))
|
|
task_id = "cad_123456abcdef"
|
|
service.v3.workflow.create_task(task_id, "Create a plate.")
|
|
queue: asyncio.Queue = asyncio.Queue()
|
|
with patch("app.services.agent_service.cached_model_capability", return_value=None), patch(
|
|
"app.services.agent_service.verify_model_capability", AsyncMock(return_value={"supported": True, "probe_unavailable": False})
|
|
) as verify:
|
|
result = asyncio.run(service._ensure_task_capabilities(
|
|
task_id, ModelIdentity("author", "author-model"), ModelIdentity("reviewer", "reviewer-model"), queue,
|
|
))
|
|
self.assertIsNone(result)
|
|
self.assertEqual(verify.await_count, 2)
|
|
self.assertEqual(service.v3.repository.get_state(task_id).phase, TaskPhase.DRAFTING_REQUIREMENTS)
|
|
events = [queue.get_nowait(), queue.get_nowait()]
|
|
self.assertEqual([item[1]["status"] for item in events], ["waiting", "success"])
|
|
|
|
def test_image_bytes_are_frozen_and_sent_to_vision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
artifacts = FileArtifactStore(root / "tasks")
|
|
task_id = "cad_123456abcdef"
|
|
source = root / "reference.png"
|
|
png = b"\x89PNG\r\n\x1a\nreference-bytes"
|
|
source.write_bytes(png)
|
|
artifacts.initialize_task(task_id, "Match the image.", image_inputs=[{
|
|
"path": str(source), "mime": "image/png", "sha256": sha256(png).hexdigest(),
|
|
}])
|
|
frozen = Path(artifacts.source_image_paths(task_id)[0])
|
|
class Models:
|
|
def __init__(self) -> None:
|
|
self.messages: list[list[dict[str, object]]] = []
|
|
async def call_tool(self, *, messages: list[dict[str, object]], **_kwargs: object) -> dict[str, object]:
|
|
self.messages.append(messages)
|
|
return {"tool_calls": [], "usage": {}}
|
|
models = Models()
|
|
gateway = RenderedReviewGateway(models)
|
|
tool = {"type": "function", "function": {"name": "observe_images", "parameters": {"type": "object"}}}
|
|
asyncio.run(gateway.review(kind="image_observation", payload={"reference_image_paths": [str(frozen)]}, tool=tool, provider_id="p", model_id="m"))
|
|
image_part = models.messages[0][1]["content"][1]
|
|
encoded = image_part["image_url"]["url"].split(",", 1)[1]
|
|
self.assertEqual(base64.b64decode(encoded), png)
|
|
|
|
def test_final_review_schema_has_ordered_visual_decision_count(self) -> None:
|
|
visual = stateless_final_review_schema(2)["properties"]["visual_claims"]
|
|
self.assertEqual((visual["minItems"], visual["maxItems"]), (2, 2))
|
|
|
|
def test_sqlite_cas_and_outbox_are_atomic(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
repository = SqliteTaskRepository(Path(temporary) / "state.sqlite3")
|
|
initial = repository.create_task("cad_123456abcdef", "Create a plate.")
|
|
changed = transition(initial, "image_observed")
|
|
self.assertTrue(repository.compare_and_swap(changed, events=[{"event": "image_observation_ready"}]))
|
|
self.assertFalse(repository.compare_and_swap(changed, events=[{"event": "duplicate"}]))
|
|
self.assertEqual(len(repository.pending_outbox()), 1)
|
|
delivered = asyncio.run(OutboxDispatcher(repository, IdempotentInProcessPublisher()).dispatch_pending())
|
|
self.assertEqual(len(delivered), 1)
|
|
self.assertEqual(repository.pending_outbox(), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|