from __future__ import annotations import asyncio import json import subprocess import sys import tempfile import unittest from pathlib import Path from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "backend")) from app.services.agent_service import AgentService, _validate_snapshot_selectors # noqa: E402 from app.services.cdsl_patch import CdslPatchError, apply_cdsl_patch # noqa: E402 from app.services.engine_service import QualityVerificationError, build_revision # noqa: E402 from app.services.library import CdslLibrary # noqa: E402 from app.services.part_skills import PartSkillLibrary # noqa: E402 from app.services.quality import evaluate_quality, validate_verification # noqa: E402 from app.services.storage import WorkspaceStore, write_json # noqa: E402 from app.settings import ProviderConfig, ProviderModel, Settings # noqa: E402 BACKEND = ROOT / "backend" PART_SKILL_ROOT = BACKEND / "agent" / "skills" / "cad-engine" / "references" / "part-skills" def workplane(z: float = 0.0) -> dict[str, list[float]]: return { "origin_mm": [0.0, 0.0, z], "x_dir": [1.0, 0.0, 0.0], "y_dir": [0.0, 1.0, 0.0], "normal": [0.0, 0.0, 1.0], } def fixture() -> dict: return { "schema": "cad.cdsl.llm.v1", "schema_version": "1.0", "kind": "part", "part_id": "direct-cdsl-fixture", "geometry": { "sketches": [{ "id": "base", "workplane": workplane(), "profile": {"type": "polygon", "vertices": [[-10, -5], [10, -5], [10, 5], [-10, 5]]}, }], }, "features": [{ "id": "base_add", "atomic_id": "extrude_add_blind", "depends_on": [], "sketch_id": "base", "params": {"distance_mm": 4}, }], } def settings(root: Path) -> Settings: provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),)) return Settings( task_root=root / "tasks", conversation_root=root / "conversations", library_root=BACKEND / "cdsl_library", engine_root=BACKEND / "engine" / "cdsl_engine", llm_base_url=provider.base_url, llm_api_key=provider.api_key, llm_model="test-model", llm_timeout_s=1, default_provider_id="test", providers=(provider,), ) class PatchTests(unittest.TestCase): def test_rfc6902_patch_updates_cdsl_without_replacing_its_root(self) -> None: document = fixture() patched = apply_cdsl_patch(document, [{"op": "replace", "path": "/features/0/params/distance_mm", "value": 8}]) self.assertEqual(patched["features"][0]["params"]["distance_mm"], 8) self.assertEqual(document["features"][0]["params"]["distance_mm"], 4) with self.assertRaisesRegex(CdslPatchError, "complete CDSL"): apply_cdsl_patch(document, [{"op": "replace", "path": "", "value": {}}]) def test_invalid_patch_path_is_rejected(self) -> None: with self.assertRaisesRegex(CdslPatchError, "out of range"): apply_cdsl_patch(fixture(), [{"op": "replace", "path": "/features/3/id", "value": "x"}]) class VerificationTests(unittest.TestCase): def test_bbox_accepts_dimension_vector(self) -> None: rules = validate_verification({"rules": [ {"id": "size", "type": "bbox", "expected": [20, 10, 4]}, ]}, fixture()) report = evaluate_quality(rules, fixture(), { "bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]}, }) self.assertEqual(report["status"], "passed") self.assertEqual(report["results"][0]["actual"]["dimensions"], [20.0, 10.0, 4.0]) def test_bbox_accepts_coordinate_ranges(self) -> None: rules = validate_verification({"rules": [ {"id": "range", "type": "bbox", "expected": { "x_min": -10, "x_max": 10, "y_min": -5, "y_max": 5, "z_min": 0, "z_max": 4, }}, ]}, fixture()) report = evaluate_quality(rules, fixture(), { "bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]}, }) self.assertEqual(report["status"], "passed") def test_feature_bbox_uses_owned_topology_records(self) -> None: rules = validate_verification({"rules": [ {"id": "base_range", "type": "bbox", "feature": "base_add", "expected": [10, 8, 2]}, ]}, fixture()) report = evaluate_quality(rules, fixture(), { "bbox_mm": {"min": [0, 0, 0], "max": [20, 20, 20]}, "topology_records": [{ "feature_id": "base_add", "owner_feature_ids": ["base_add"], "geometry": {"bbox_mm": [0, 0, 0, 10, 8, 2]}, }], }) self.assertEqual(report["status"], "passed") self.assertEqual(report["results"][0]["source"], "runtime.topology_records[base_add].bbox_mm") def test_feature_bbox_excludes_inherited_runtime_records(self) -> None: document = fixture() document["features"].append({ "id": "boss", "atomic_id": "extrude_add_blind", "depends_on": ["base_add"], "sketch_id": "base", "params": {"distance_mm": 8}, }) rules = validate_verification({"rules": [ {"id": "boss_range", "type": "bbox", "feature": "boss", "expected": [8, 58, 58]}, ]}, document) report = evaluate_quality(rules, document, { "bbox_mm": {"min": [0, 0, 0], "max": [128, 70, 90]}, "topology_records": [ { "feature_id": "boss", "owner_feature_ids": ["base"], "geometry": {"bbox_mm": [0, 0, 0, 128, 70, 90]}, }, { "feature_id": "boss", "owner_feature_ids": ["boss"], "geometry": {"bbox_mm": [60, -29, 16, 68, 29, 74]}, }, ], }) self.assertEqual(report["status"], "passed") self.assertEqual(report["results"][0]["actual"]["dimensions"], [8.0, 58.0, 58.0]) def test_feature_diameter_uses_target_circle(self) -> None: document = fixture() document["features"][0]["id"] = "boss" document["features"][0]["sketch_id"] = "boss_sketch" document["geometry"]["sketches"][0]["id"] = "boss_sketch" document["geometry"]["sketches"][0]["profile"] = {"type": "circle", "center": [0, 0], "radius_mm": 29} rules = validate_verification({"rules": [ {"id": "boss_dia", "type": "overall_diameter", "feature": "boss", "expected": 58}, ]}, document) report = evaluate_quality(rules, document, { "bbox_mm": {"min": [0, 0, 0], "max": [128, 70, 90]}, }) self.assertEqual(report["status"], "passed") self.assertEqual(report["results"][0]["actual"], 58.0) def test_overall_width_and_height_are_supported(self) -> None: rules = validate_verification({"rules": [ {"id": "width", "type": "overall_width", "expected": 70}, {"id": "height", "type": "overall_height", "expected": 90}, ]}, fixture()) report = evaluate_quality(rules, fixture(), { "bbox_mm": {"min": [-64, -35, 0], "max": [64, 35, 90]}, }) self.assertEqual(report["status"], "passed") self.assertEqual([item["actual"] for item in report["results"]], [70.0, 90.0]) def test_feature_scoped_verification_requires_feature(self) -> None: with self.assertRaisesRegex(ValueError, "feature is required for hole_count"): validate_verification({"rules": [ {"id": "holes", "type": "hole_count", "expected": 1}, ]}, fixture()) def test_bbox_shape_is_rejected_before_execution(self) -> None: with self.assertRaisesRegex(ValueError, "expected for bbox"): validate_verification({"rules": [ {"id": "bad", "type": "bbox", "expected": {"width": 10}}, ]}, fixture()) def test_unknown_feature_reference_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "must reference a CDSL feature ID"): validate_verification({"rules": [{"id": "holes", "type": "hole_count", "feature": "missing", "expected": 4}]}, fixture()) def test_invalid_expected_value_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "expected for bbox"): validate_verification({"rules": [{"id": "size", "type": "bbox", "expected": [10, 5]}]}, fixture()) def test_warning_does_not_block_generic_quality(self) -> None: rules = validate_verification({"rules": [ {"id": "solids", "type": "solid_count", "expected": 1, "severity": "blocking"}, {"id": "length", "type": "overall_length", "expected": 999, "severity": "warning"}, ]}, fixture()) report = evaluate_quality(rules, fixture(), { "solid_count": 1, "bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]}, }) self.assertEqual(report["status"], "passed") self.assertEqual(len(report["blocking_failures"]), 0) self.assertEqual(len(report["warnings"]), 1) def test_blocking_failure_requires_repair(self) -> None: rules = validate_verification({"rules": [{"id": "solids", "type": "solid_count", "expected": 2}]}, fixture()) report = evaluate_quality(rules, fixture(), {"solid_count": 1, "bbox_mm": {"min": [0, 0, 0], "max": [1, 1, 1]}}) self.assertEqual(report["status"], "failed") self.assertEqual(report["blocking_failures"][0]["id"], "solids") class AgentPatchFlowTests(unittest.TestCase): def _seed_revision(self, store: WorkspaceStore) -> tuple[str, str]: task = store.ensure_task(None, "Create a direct CDSL fixture") revision_id = "rev_001" revision_dir = store.task_dir(task["task_id"]) / "revisions" / revision_id revision_dir.mkdir(parents=True) write_json(revision_dir / "model.cdsl.json", fixture()) store.update_task(task["task_id"], { "revision_id": revision_id, "status": "success", "cdsl_path": f"revisions/{revision_id}/model.cdsl.json", }) return task["task_id"], revision_id def test_patch_creates_a_child_revision_from_explicit_parent(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, parent = self._seed_revision(store) agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) captured: dict[str, object] = {} def fake_build_revision(**kwargs: object) -> dict[str, object]: captured.update(kwargs) return {"task_id": task_id, "revision_id": "rev_002"} with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision): result, _ = asyncio.run(agent._run_tool( "patch_cdsl_model", { "base_revision_id": parent, "patches": [{"op": "replace", "path": "/features/0/params/distance_mm", "value": 8}], "summary": "Increase thickness", "assumptions": [], }, task_id, "Increase thickness", [], planning_state={"phase": "PLANNED", "current_model_read": True}, )) self.assertTrue(result["ok"]) self.assertEqual(captured["parent_revision_id"], parent) self.assertEqual(captured["operation"]["type"], "cdsl_patch") self.assertEqual(captured["cdsl"]["features"][0]["params"]["distance_mm"], 8) def test_completed_feature_can_keep_selector_from_historical_snapshot(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, parent = self._seed_revision(store) parent_dir = store.task_dir(task_id) / "revisions" / parent parent_snapshot_id = f"{task_id}/{parent}" write_json(parent_dir / "model.topology.json", { "schema_version": "cad.topology.v1", "task_id": task_id, "revision_id": parent, "snapshot_id": parent_snapshot_id, "records": [{"record_id": "body:base:edge:0", "kind": "edge", "executable": True, "geometry": {}}], }) task = store.read_task(task_id) task["revisions"][0]["topology_path"] = f"revisions/{parent}/model.topology.json" write_json(store.task_path(task_id), task) child = "rev_002" child_dir = store.task_dir(task_id) / "revisions" / child child_dir.mkdir(parents=True) write_json(child_dir / "model.cdsl.json", fixture()) write_json(child_dir / "model.topology.json", { "schema_version": "cad.topology.v1", "task_id": task_id, "revision_id": child, "snapshot_id": f"{task_id}/{child}", "records": [{"record_id": "body:base:edge:1", "kind": "edge", "executable": True, "geometry": {}}], }) store.update_task(task_id, { "revision_id": child, "status": "success", "cdsl_path": f"revisions/{child}/model.cdsl.json", "topology_path": f"revisions/{child}/model.topology.json", }) document = fixture() document["features"][0]["selectors"] = [{ "kind": "edge", "stable_id": "body:base:edge:0", "source": "runtime_snapshot", "snapshot_id": parent_snapshot_id, "confidence": 1.0, }] _validate_snapshot_selectors(store, task_id, document) document["features"][0]["selectors"][0]["snapshot_id"] = f"{task_id}/rev_999" with self.assertRaisesRegex(ValueError, "TOPOLOGY_SNAPSHOT_STALE"): _validate_snapshot_selectors(store, task_id, document) def test_topology_inspection_unlocks_waiting_plan_nodes(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, revision_id = self._seed_revision(store) revision_dir = store.task_dir(task_id) / "revisions" / revision_id snapshot_id = f"{task_id}/{revision_id}" write_json(revision_dir / "model.topology.json", { "schema_version": "cad.topology.v1", "task_id": task_id, "revision_id": revision_id, "snapshot_id": snapshot_id, "records": [{ "record_id": "body:base_add", "kind": "body", "body_id": "body:base_add", "feature_id": "base_add", "owner_feature_ids": ["base_add"], "geometry": {}, "executable": True, }], }) task = store.read_task(task_id) task["revisions"][0]["topology_path"] = f"revisions/{revision_id}/model.topology.json" write_json(store.task_path(task_id), task) state = { "phase": "TOPOLOGY_READY", "feature_plan": { "schema_version": "cad.feature-plan.v1", "plan_id": "plan_1", "task_id": task_id, "nodes": [ {"id": "base", "atomic_id": "extrude_add_blind", "depends_on": [], "cdsl_feature_ids": ["base_add"]}, {"id": "round", "atomic_id": "fillet", "depends_on": ["base"], "cdsl_feature_ids": ["round"]}, ], }, } agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) result, _ = asyncio.run(agent._run_tool( "inspect_current_topology", {}, task_id, "Round the edges", [], planning_state=state, )) self.assertTrue(result["ok"]) self.assertEqual(state["feature_plan"]["topology_snapshot_id"], snapshot_id) self.assertEqual(state["feature_plan"]["ready_nodes"], ["round"]) def test_successful_build_automatically_binds_topology_for_waiting_nodes(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) state = { "phase": "PLAN_READY", "feature_plan": { "schema_version": "cad.feature-plan.v1", "plan_id": "plan_1", "task_id": "", "nodes": [ { "id": "base", "atomic_id": "extrude_add_blind", "depends_on": [], "cdsl_feature_ids": ["base_add"], "status": "ready", }, { "id": "round", "atomic_id": "fillet", "depends_on": ["base"], "cdsl_feature_ids": ["round"], }, ], }, } result, _ = asyncio.run(agent._run_tool( "generate_cdsl_model", {"cdsl": fixture(), "summary": "Base feature", "assumptions": []}, None, "Create a base with a later fillet", [], planning_state=state, )) self.assertTrue(result["ok"]) self.assertEqual(result["required_action"], "patch_cdsl_model") self.assertEqual(result["plan_status"]["ready_nodes"], ["round"]) self.assertEqual(result["plan_status"]["waiting_nodes"], []) self.assertEqual(state["feature_plan"]["topology_snapshot_id"], "{}/{}".format(result["task_id"], result["revision_id"])) def test_bad_patch_creates_no_revision(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, parent = self._seed_revision(store) agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) with self.assertRaisesRegex(ValueError, "INVALID_CDSL_PATCH"): asyncio.run(agent._run_tool( "patch_cdsl_model", {"base_revision_id": parent, "patches": [{"op": "replace", "path": "/missing", "value": 8}], "summary": "bad", "assumptions": []}, task_id, "bad patch", [], planning_state={"phase": "PLANNED"}, )) task = store.read_task(task_id) self.assertEqual([item["revision_id"] for item in task["revisions"]], [parent]) def test_patch_rejects_non_current_parent_revision(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, parent = self._seed_revision(store) child = "rev_002" child_dir = store.task_dir(task_id) / "revisions" / child child_dir.mkdir(parents=True) write_json(child_dir / "model.cdsl.json", fixture()) store.update_task(task_id, { "revision_id": child, "status": "success", "cdsl_path": f"revisions/{child}/model.cdsl.json", }) agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) with self.assertRaisesRegex(ValueError, "TOPOLOGY_SNAPSHOT_STALE"): asyncio.run(agent._run_tool( "patch_cdsl_model", {"base_revision_id": parent, "patches": [{"op": "replace", "path": "/features/0/params/distance_mm", "value": 8}], "summary": "stale", "assumptions": []}, task_id, "stale patch", [], planning_state={"phase": "PLANNED"}, )) def test_complete_replacement_uses_current_revision_as_parent(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) task_id, parent = self._seed_revision(store) agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) captured: dict[str, object] = {} def fake_build_revision(**kwargs: object) -> dict[str, object]: captured.update(kwargs) return {"task_id": task_id, "revision_id": "rev_002"} with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision): result, _ = asyncio.run(agent._run_tool( "generate_cdsl_model", {"cdsl": fixture(), "summary": "Replacement", "assumptions": []}, task_id, "replace geometry", [], planning_state={"phase": "PLANNED", "current_model_read": True}, )) self.assertTrue(result["ok"]) self.assertEqual(captured["parent_revision_id"], parent) self.assertEqual(captured["operation"]["type"], "cdsl_replacement") def test_blocking_quality_failure_is_repairable_and_readable(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) with self.assertRaises(QualityVerificationError) as context: build_revision( settings=config, store=store, task_id=None, request="Create fixture", cdsl=fixture(), reference_ids=[], summary="fixture", verification={"rules": [{"id": "wrong-solid-count", "type": "solid_count", "expected": 2}]}, ) error = context.exception self.assertTrue(error.task_id) self.assertEqual(error.revision_id, "rev_001") task = store.read_task(error.task_id) self.assertEqual(task["revisions"][0]["status"], "needs_repair") self.assertEqual(task["revisions"][0]["quality_status"], "needs_repair") agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT)) read, _ = asyncio.run(agent._run_tool( "read_current_cdsl", {}, error.task_id, "Repair fixture", [], planning_state={"phase": "INTAKE"} )) self.assertTrue(read["ok"]) self.assertEqual(read["revision_id"], error.revision_id) def test_runtime_build_failure_does_not_leave_a_revision(self) -> None: with tempfile.TemporaryDirectory() as directory: config = settings(Path(directory)) store = WorkspaceStore(config) class BrokenEngine: def run_cdsl_only(self, _cdsl: dict, _step: Path) -> dict: raise RuntimeError("intentional runtime failure") with patch("app.services.engine_service.load_engine", return_value=BrokenEngine()), patch( "app.services.engine_service.validate_cdsl", return_value=None ), self.assertRaisesRegex(RuntimeError, "intentional runtime failure"): build_revision( settings=config, store=store, task_id=None, request="broken fixture", cdsl=fixture(), reference_ids=[], summary="broken", ) tasks = list(config.task_root.glob("cad_*")) self.assertEqual(len(tasks), 1) task = store.read_task(tasks[0].name) self.assertEqual(task["revisions"], []) class CleanupAndBoundaryTests(unittest.TestCase): def test_cleanup_is_dry_run_then_removes_only_retired_artifacts(self) -> None: with tempfile.TemporaryDirectory() as directory: data_root = Path(directory) / "data" task_dir = data_root / "tasks" / "cad_aaaaaaaaaaaa" / "revisions" / "rev_001" conversation_dir = data_root / "conversations" / "conv_aaaaaaaaaaaa" / "diagnostics" task_dir.mkdir(parents=True) conversation_dir.mkdir(parents=True) write_json(task_dir / "generation-spec.json", {"schema": "cad.generation-spec.v1"}) write_json(task_dir / "model.cdsl.json", fixture()) (task_dir / "model.step").write_bytes(b"step") (task_dir / "model.glb").write_bytes(b"glb") write_json(data_root / "tasks" / "cad_aaaaaaaaaaaa" / "task.json", { "generation_spec_path": "revisions/rev_001/generation-spec.json", "current_design_intent_id": "intent_aaaaaaaaaaaa", "revisions": [{"acceptance_path": "revisions/rev_001/acceptance-report.json"}], }) write_json(conversation_dir / "generation_spec_validation_deadbeef.json", {"kind": "old"}) script = BACKEND / "scripts" / "remove_generation_spec_artifacts.py" dry = subprocess.run([sys.executable, str(script), "--data-root", str(data_root)], check=True, capture_output=True, text=True) self.assertIn("Dry run only", dry.stdout) self.assertTrue((task_dir / "generation-spec.json").is_file()) subprocess.run([sys.executable, str(script), "--data-root", str(data_root), "--apply"], check=True, capture_output=True, text=True) self.assertFalse((task_dir / "generation-spec.json").exists()) self.assertFalse((conversation_dir / "generation_spec_validation_deadbeef.json").exists()) self.assertTrue((task_dir / "model.cdsl.json").is_file()) self.assertTrue((task_dir / "model.step").is_file()) self.assertTrue((task_dir / "model.glb").is_file()) cleaned = json.loads((data_root / "tasks" / "cad_aaaaaaaaaaaa" / "task.json").read_text(encoding="utf-8")) self.assertNotIn("generation_spec_path", cleaned) self.assertNotIn("current_design_intent_id", cleaned) self.assertNotIn("acceptance_path", cleaned["revisions"][0]) def test_engine_does_not_import_application_or_dispatch_part_families(self) -> None: engine_root = BACKEND / "engine" / "cdsl_engine" source = "\n".join(path.read_text(encoding="utf-8") for path in engine_root.rglob("*.py")) lowered = source.casefold() self.assertNotIn("from app", lowered) self.assertNotIn("import app", lowered) for business_family in ("mounting_bracket", "mounting_plate", "flange_sleeve", "bearing_housing", "hex_nut"): self.assertNotIn(business_family, lowered)