361 lines
17 KiB
Python
361 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.services.agent_service import AgentService # noqa: E402
|
|
from app.services.cdsl_patch import CdslPatchError, apply_cdsl_patch # noqa: E402
|
|
from app.services.engine_service import QualityVerificationError, build_revision # noqa: E402
|
|
from app.services.library import CdslLibrary # noqa: E402
|
|
from app.services.part_skills import PartSkillLibrary # noqa: E402
|
|
from app.services.quality import evaluate_quality, validate_verification # noqa: E402
|
|
from app.services.storage import WorkspaceStore, write_json # noqa: E402
|
|
from app.settings import ProviderConfig, ProviderModel, Settings # noqa: E402
|
|
|
|
|
|
BACKEND = ROOT / "backend"
|
|
PART_SKILL_ROOT = BACKEND / "agent" / "skills" / "cad-engine" / "references" / "part-skills"
|
|
|
|
|
|
def workplane(z: float = 0.0) -> dict[str, list[float]]:
|
|
return {
|
|
"origin_mm": [0.0, 0.0, z],
|
|
"x_dir": [1.0, 0.0, 0.0],
|
|
"y_dir": [0.0, 1.0, 0.0],
|
|
"normal": [0.0, 0.0, 1.0],
|
|
}
|
|
|
|
|
|
def fixture() -> dict:
|
|
return {
|
|
"schema": "cad.cdsl.llm.v1",
|
|
"schema_version": "1.0",
|
|
"kind": "part",
|
|
"part_id": "direct-cdsl-fixture",
|
|
"geometry": {
|
|
"sketches": [{
|
|
"id": "base",
|
|
"workplane": workplane(),
|
|
"profile": {"type": "polygon", "vertices": [[-10, -5], [10, -5], [10, 5], [-10, 5]]},
|
|
}],
|
|
},
|
|
"features": [{
|
|
"id": "base_add",
|
|
"atomic_id": "extrude_add_blind",
|
|
"depends_on": [],
|
|
"sketch_id": "base",
|
|
"params": {"distance_mm": 4},
|
|
}],
|
|
}
|
|
|
|
|
|
def settings(root: Path) -> Settings:
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
return Settings(
|
|
task_root=root / "tasks",
|
|
conversation_root=root / "conversations",
|
|
library_root=BACKEND / "cdsl_library",
|
|
engine_root=BACKEND / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
|
|
|
|
class PatchTests(unittest.TestCase):
|
|
def test_rfc6902_patch_updates_cdsl_without_replacing_its_root(self) -> None:
|
|
document = fixture()
|
|
patched = apply_cdsl_patch(document, [{"op": "replace", "path": "/features/0/params/distance_mm", "value": 8}])
|
|
|
|
self.assertEqual(patched["features"][0]["params"]["distance_mm"], 8)
|
|
self.assertEqual(document["features"][0]["params"]["distance_mm"], 4)
|
|
with self.assertRaisesRegex(CdslPatchError, "complete CDSL"):
|
|
apply_cdsl_patch(document, [{"op": "replace", "path": "", "value": {}}])
|
|
|
|
def test_invalid_patch_path_is_rejected(self) -> None:
|
|
with self.assertRaisesRegex(CdslPatchError, "out of range"):
|
|
apply_cdsl_patch(fixture(), [{"op": "replace", "path": "/features/3/id", "value": "x"}])
|
|
|
|
|
|
class VerificationTests(unittest.TestCase):
|
|
def test_bbox_accepts_dimension_vector(self) -> None:
|
|
rules = validate_verification({"rules": [
|
|
{"id": "size", "type": "bbox", "expected": [20, 10, 4]},
|
|
]}, fixture())
|
|
report = evaluate_quality(rules, fixture(), {
|
|
"bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]},
|
|
})
|
|
|
|
self.assertEqual(report["status"], "passed")
|
|
self.assertEqual(report["results"][0]["actual"]["dimensions"], [20.0, 10.0, 4.0])
|
|
|
|
def test_bbox_accepts_coordinate_ranges(self) -> None:
|
|
rules = validate_verification({"rules": [
|
|
{"id": "range", "type": "bbox", "expected": {
|
|
"x_min": -10, "x_max": 10, "y_min": -5, "y_max": 5, "z_min": 0, "z_max": 4,
|
|
}},
|
|
]}, fixture())
|
|
report = evaluate_quality(rules, fixture(), {
|
|
"bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]},
|
|
})
|
|
|
|
self.assertEqual(report["status"], "passed")
|
|
|
|
def test_feature_bbox_uses_owned_topology_records(self) -> None:
|
|
rules = validate_verification({"rules": [
|
|
{"id": "base_range", "type": "bbox", "feature": "base_add", "expected": [10, 8, 2]},
|
|
]}, fixture())
|
|
report = evaluate_quality(rules, fixture(), {
|
|
"bbox_mm": {"min": [0, 0, 0], "max": [20, 20, 20]},
|
|
"topology_records": [{
|
|
"feature_id": "base_add", "owner_feature_ids": ["base_add"],
|
|
"geometry": {"bbox_mm": [0, 0, 0, 10, 8, 2]},
|
|
}],
|
|
})
|
|
|
|
self.assertEqual(report["status"], "passed")
|
|
self.assertEqual(report["results"][0]["source"], "runtime.topology_records[base_add].bbox_mm")
|
|
|
|
def test_bbox_shape_is_rejected_before_execution(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "expected for bbox"):
|
|
validate_verification({"rules": [
|
|
{"id": "bad", "type": "bbox", "expected": {"width": 10}},
|
|
]}, fixture())
|
|
|
|
def test_unknown_feature_reference_is_rejected(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "must reference a CDSL feature ID"):
|
|
validate_verification({"rules": [{"id": "holes", "type": "hole_count", "feature": "missing", "expected": 4}]}, fixture())
|
|
|
|
def test_invalid_expected_value_is_rejected(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "expected for bbox"):
|
|
validate_verification({"rules": [{"id": "size", "type": "bbox", "expected": [10, 5]}]}, fixture())
|
|
|
|
def test_warning_does_not_block_generic_quality(self) -> None:
|
|
rules = validate_verification({"rules": [
|
|
{"id": "solids", "type": "solid_count", "expected": 1, "severity": "blocking"},
|
|
{"id": "length", "type": "overall_length", "expected": 999, "severity": "warning"},
|
|
]}, fixture())
|
|
report = evaluate_quality(rules, fixture(), {
|
|
"solid_count": 1,
|
|
"bbox_mm": {"min": [-10, -5, 0], "max": [10, 5, 4]},
|
|
})
|
|
|
|
self.assertEqual(report["status"], "passed")
|
|
self.assertEqual(len(report["blocking_failures"]), 0)
|
|
self.assertEqual(len(report["warnings"]), 1)
|
|
|
|
def test_blocking_failure_requires_repair(self) -> None:
|
|
rules = validate_verification({"rules": [{"id": "solids", "type": "solid_count", "expected": 2}]}, fixture())
|
|
report = evaluate_quality(rules, fixture(), {"solid_count": 1, "bbox_mm": {"min": [0, 0, 0], "max": [1, 1, 1]}})
|
|
|
|
self.assertEqual(report["status"], "failed")
|
|
self.assertEqual(report["blocking_failures"][0]["id"], "solids")
|
|
|
|
|
|
class AgentPatchFlowTests(unittest.TestCase):
|
|
def _seed_revision(self, store: WorkspaceStore) -> tuple[str, str]:
|
|
task = store.ensure_task(None, "Create a direct CDSL fixture")
|
|
revision_id = "rev_001"
|
|
revision_dir = store.task_dir(task["task_id"]) / "revisions" / revision_id
|
|
revision_dir.mkdir(parents=True)
|
|
write_json(revision_dir / "model.cdsl.json", fixture())
|
|
store.update_task(task["task_id"], {
|
|
"revision_id": revision_id,
|
|
"status": "success",
|
|
"cdsl_path": f"revisions/{revision_id}/model.cdsl.json",
|
|
})
|
|
return task["task_id"], revision_id
|
|
|
|
def test_patch_creates_a_child_revision_from_explicit_parent(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config = settings(Path(directory))
|
|
store = WorkspaceStore(config)
|
|
task_id, parent = self._seed_revision(store)
|
|
agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT))
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
|
captured.update(kwargs)
|
|
return {"task_id": task_id, "revision_id": "rev_002"}
|
|
|
|
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
|
result, _ = asyncio.run(agent._run_tool(
|
|
"patch_cdsl_model",
|
|
{
|
|
"base_revision_id": parent,
|
|
"patches": [{"op": "replace", "path": "/features/0/params/distance_mm", "value": 8}],
|
|
"summary": "Increase thickness",
|
|
"assumptions": [],
|
|
},
|
|
task_id,
|
|
"Increase thickness",
|
|
[],
|
|
planning_state={"phase": "PLANNED", "current_model_read": True},
|
|
))
|
|
|
|
self.assertTrue(result["ok"])
|
|
self.assertEqual(captured["parent_revision_id"], parent)
|
|
self.assertEqual(captured["operation"]["type"], "cdsl_patch")
|
|
self.assertEqual(captured["cdsl"]["features"][0]["params"]["distance_mm"], 8)
|
|
|
|
def test_bad_patch_creates_no_revision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config = settings(Path(directory))
|
|
store = WorkspaceStore(config)
|
|
task_id, parent = self._seed_revision(store)
|
|
agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT))
|
|
with self.assertRaisesRegex(ValueError, "INVALID_CDSL_PATCH"):
|
|
asyncio.run(agent._run_tool(
|
|
"patch_cdsl_model",
|
|
{"base_revision_id": parent, "patches": [{"op": "replace", "path": "/missing", "value": 8}], "summary": "bad", "assumptions": []},
|
|
task_id,
|
|
"bad patch",
|
|
[],
|
|
planning_state={"phase": "PLANNED"},
|
|
))
|
|
|
|
task = store.read_task(task_id)
|
|
self.assertEqual([item["revision_id"] for item in task["revisions"]], [parent])
|
|
|
|
def test_complete_replacement_uses_current_revision_as_parent(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config = settings(Path(directory))
|
|
store = WorkspaceStore(config)
|
|
task_id, parent = self._seed_revision(store)
|
|
agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT))
|
|
captured: dict[str, object] = {}
|
|
|
|
def fake_build_revision(**kwargs: object) -> dict[str, object]:
|
|
captured.update(kwargs)
|
|
return {"task_id": task_id, "revision_id": "rev_002"}
|
|
|
|
with patch("app.services.agent_service.build_revision", side_effect=fake_build_revision):
|
|
result, _ = asyncio.run(agent._run_tool(
|
|
"generate_cdsl_model",
|
|
{"cdsl": fixture(), "summary": "Replacement", "assumptions": []},
|
|
task_id,
|
|
"replace geometry",
|
|
[],
|
|
planning_state={"phase": "PLANNED", "current_model_read": True},
|
|
))
|
|
|
|
self.assertTrue(result["ok"])
|
|
self.assertEqual(captured["parent_revision_id"], parent)
|
|
self.assertEqual(captured["operation"]["type"], "cdsl_replacement")
|
|
|
|
def test_blocking_quality_failure_is_repairable_and_readable(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config = settings(Path(directory))
|
|
store = WorkspaceStore(config)
|
|
with self.assertRaises(QualityVerificationError) as context:
|
|
build_revision(
|
|
settings=config,
|
|
store=store,
|
|
task_id=None,
|
|
request="Create fixture",
|
|
cdsl=fixture(),
|
|
reference_ids=[],
|
|
summary="fixture",
|
|
verification={"rules": [{"id": "wrong-solid-count", "type": "solid_count", "expected": 2}]},
|
|
)
|
|
|
|
error = context.exception
|
|
self.assertTrue(error.task_id)
|
|
self.assertEqual(error.revision_id, "rev_001")
|
|
task = store.read_task(error.task_id)
|
|
self.assertEqual(task["revisions"][0]["status"], "needs_repair")
|
|
self.assertEqual(task["revisions"][0]["quality_status"], "needs_repair")
|
|
|
|
agent = AgentService(config, store, CdslLibrary(config), PartSkillLibrary(PART_SKILL_ROOT))
|
|
read, _ = asyncio.run(agent._run_tool(
|
|
"read_current_cdsl", {}, error.task_id, "Repair fixture", [], planning_state={"phase": "INTAKE"}
|
|
))
|
|
self.assertTrue(read["ok"])
|
|
self.assertEqual(read["revision_id"], error.revision_id)
|
|
|
|
def test_runtime_build_failure_does_not_leave_a_revision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config = settings(Path(directory))
|
|
store = WorkspaceStore(config)
|
|
|
|
class BrokenEngine:
|
|
def run_cdsl_only(self, _cdsl: dict, _step: Path) -> dict:
|
|
raise RuntimeError("intentional runtime failure")
|
|
|
|
with patch("app.services.engine_service.load_engine", return_value=BrokenEngine()), patch(
|
|
"app.services.engine_service.validate_cdsl", return_value=None
|
|
), self.assertRaisesRegex(RuntimeError, "intentional runtime failure"):
|
|
build_revision(
|
|
settings=config,
|
|
store=store,
|
|
task_id=None,
|
|
request="broken fixture",
|
|
cdsl=fixture(),
|
|
reference_ids=[],
|
|
summary="broken",
|
|
)
|
|
|
|
tasks = list(config.task_root.glob("cad_*"))
|
|
self.assertEqual(len(tasks), 1)
|
|
task = store.read_task(tasks[0].name)
|
|
self.assertEqual(task["revisions"], [])
|
|
|
|
|
|
class CleanupAndBoundaryTests(unittest.TestCase):
|
|
def test_cleanup_is_dry_run_then_removes_only_retired_artifacts(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
data_root = Path(directory) / "data"
|
|
task_dir = data_root / "tasks" / "cad_aaaaaaaaaaaa" / "revisions" / "rev_001"
|
|
conversation_dir = data_root / "conversations" / "conv_aaaaaaaaaaaa" / "diagnostics"
|
|
task_dir.mkdir(parents=True)
|
|
conversation_dir.mkdir(parents=True)
|
|
write_json(task_dir / "generation-spec.json", {"schema": "cad.generation-spec.v1"})
|
|
write_json(task_dir / "model.cdsl.json", fixture())
|
|
(task_dir / "model.step").write_bytes(b"step")
|
|
(task_dir / "model.glb").write_bytes(b"glb")
|
|
write_json(data_root / "tasks" / "cad_aaaaaaaaaaaa" / "task.json", {
|
|
"generation_spec_path": "revisions/rev_001/generation-spec.json",
|
|
"current_design_intent_id": "intent_aaaaaaaaaaaa",
|
|
"revisions": [{"acceptance_path": "revisions/rev_001/acceptance-report.json"}],
|
|
})
|
|
write_json(conversation_dir / "generation_spec_validation_deadbeef.json", {"kind": "old"})
|
|
|
|
script = BACKEND / "scripts" / "remove_generation_spec_artifacts.py"
|
|
dry = subprocess.run([sys.executable, str(script), "--data-root", str(data_root)], check=True, capture_output=True, text=True)
|
|
self.assertIn("Dry run only", dry.stdout)
|
|
self.assertTrue((task_dir / "generation-spec.json").is_file())
|
|
|
|
subprocess.run([sys.executable, str(script), "--data-root", str(data_root), "--apply"], check=True, capture_output=True, text=True)
|
|
self.assertFalse((task_dir / "generation-spec.json").exists())
|
|
self.assertFalse((conversation_dir / "generation_spec_validation_deadbeef.json").exists())
|
|
self.assertTrue((task_dir / "model.cdsl.json").is_file())
|
|
self.assertTrue((task_dir / "model.step").is_file())
|
|
self.assertTrue((task_dir / "model.glb").is_file())
|
|
cleaned = json.loads((data_root / "tasks" / "cad_aaaaaaaaaaaa" / "task.json").read_text(encoding="utf-8"))
|
|
self.assertNotIn("generation_spec_path", cleaned)
|
|
self.assertNotIn("current_design_intent_id", cleaned)
|
|
self.assertNotIn("acceptance_path", cleaned["revisions"][0])
|
|
|
|
def test_engine_does_not_import_application_or_dispatch_part_families(self) -> None:
|
|
engine_root = BACKEND / "engine" / "cdsl_engine"
|
|
source = "\n".join(path.read_text(encoding="utf-8") for path in engine_root.rglob("*.py"))
|
|
lowered = source.casefold()
|
|
|
|
self.assertNotIn("from app", lowered)
|
|
self.assertNotIn("import app", lowered)
|
|
for business_family in ("mounting_bracket", "mounting_plate", "flange_sleeve", "bearing_housing", "hex_nut"):
|
|
self.assertNotIn(business_family, lowered)
|