Files
cdsl-cad/backend/tests/test_feature_plan.py
T

358 lines
24 KiB
Python

from __future__ import annotations
from pathlib import Path
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "backend"))
from app.cad_agent.domain.feature_plan import FeaturePlan, FeatureScheduler, node_hash, plan_hash, validate_feature_plan
from app.cad_agent.adapters.artifact_store import FileArtifactStore
from app.cad_agent.adapters.runtime import ProfileCadRuntime
from app.cad_agent.adapters.sqlite_repository import SqliteTaskRepository
from app.cad_agent.adapters.verifier import RegistryVerifierExecutor
from app.cad_agent.application.action_handlers import ActionCommandHandler
from app.cad_agent.application.llm_contracts import CompiledRequirementsSpec, MarkdownDocument
from app.cad_agent.application.requirements import RequirementsCommandHandler
from app.cad_agent.application.results import Accepted, Rejected
from app.cad_agent.domain.state import TaskPhase, transition
from app.cad_agent.domain.verifier_registry import default_registry
from app.settings import ProviderConfig, ProviderModel, Settings
def runtime_settings(root: Path) -> Settings:
provider = ProviderConfig("test", "Test", "https://test.invalid/v1", "key", (ProviderModel("test-model"),))
return Settings(
task_root=root / "tasks", conversation_root=root / "conversations",
library_root=ROOT / "backend" / "cdsl_library", engine_root=ROOT / "backend" / "engine" / "cdsl_engine",
llm_base_url=provider.base_url, llm_api_key=provider.api_key, llm_model="test-model", llm_timeout_s=1,
default_provider_id="test", providers=(provider,),
)
def contract() -> dict[str, object]:
return {
"requirements": [
{
"requirement_id": "req_001",
"acceptance_claims": [
{"claim_id": "claim_base", "verification_mode": "deterministic"},
{"claim_id": "claim_visual", "verification_mode": "visual"},
],
},
{
"requirement_id": "req_002",
"acceptance_claims": [{"claim_id": "claim_bore", "verification_mode": "deterministic"}],
},
{
"requirement_id": "req_003",
"acceptance_claims": [{"claim_id": "claim_pattern", "verification_mode": "deterministic"}],
},
],
}
def initial_plan() -> FeaturePlan:
return FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1",
"parent_plan_hash": "",
"replaces_node_ids": [],
"nodes": [
{"node_id": "base", "priority": 10, "intent": "Create the base.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_base"], "expected_change": "One base solid."},
{"node_id": "bore", "priority": 20, "intent": "Cut the bore.", "atomic_id": "hole_blind", "depends_on": ["base"], "claim_ids": ["claim_bore"], "expected_change": "One through bore."},
{"node_id": "pattern", "priority": 30, "intent": "Add the pattern.", "atomic_id": "hole_blind", "depends_on": ["base"], "claim_ids": ["claim_pattern"], "expected_change": "Mounting holes."},
],
"final_claim_ids": ["claim_visual"],
})
class FeaturePlanTests(unittest.TestCase):
def test_plan_requires_exact_claim_ownership(self) -> None:
plan = initial_plan()
self.assertEqual(validate_feature_plan(plan, contract(), {"extrude_add_blind", "hole_blind"}), [])
broken = plan.model_copy(deep=True)
broken.nodes[1].claim_ids = ["claim_base"]
messages = [item["message"] for item in validate_feature_plan(broken, contract(), {"extrude_add_blind", "hole_blind"})]
self.assertTrue(any("already owned" in message for message in messages))
self.assertTrue(any("has no owner" in message for message in messages))
def test_visual_claim_repeated_on_a_node_is_removed_before_persisting_the_plan(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
repository = SqliteTaskRepository(root / "state.sqlite3")
artifacts = FileArtifactStore(root / "tasks")
handler = RequirementsCommandHandler(repository, artifacts, default_registry(), atomic_ids=lambda: ("extrude_add_blind", "hole_blind"))
task_id = "cad_123456abcdef"
repository.create_task(task_id, "Create a part.")
artifacts.initialize_task(task_id, "Create a part.")
self.assertIsInstance(handler.submit_requirements_document(task_id, MarkdownDocument(markdown="Create a part."), invocation_id="requirements"), Accepted)
self.assertIsInstance(handler.submit_completion_target(task_id, MarkdownDocument(markdown="- [ ] A base and a visual edge treatment."), invocation_id="target"), Accepted)
compiled = CompiledRequirementsSpec.model_validate({"requirements": [{"assumptions": [], "acceptance_claims": [
{"claim_kind": "single_connected_body", "expected": {}},
{"claim_kind": "visual", "expected": {"description": "A visible edge treatment."}},
]}]})
self.assertIsInstance(handler.submit_compiled_spec(task_id, compiled, invocation_id="compile"), Accepted)
submitted = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1", "parent_plan_hash": "", "replaces_node_ids": [],
"nodes": [{"node_id": "base", "priority": 10, "intent": "Create the base.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_001", "claim_002"], "expected_change": "One base solid."}],
"final_claim_ids": ["claim_002"],
})
accepted = handler.submit_feature_plan(task_id, submitted, invocation_id="plan")
self.assertIsInstance(accepted, Accepted)
state = repository.get_state(task_id)
persisted = artifacts.read_json(task_id, state.feature_plan_path)
self.assertEqual(persisted["nodes"][0]["claim_ids"], ["claim_001"])
self.assertEqual(persisted["final_claim_ids"], ["claim_002"])
def test_unowned_global_health_claim_is_bound_to_the_unique_root_add_feature(self) -> None:
plan = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1", "parent_plan_hash": "", "replaces_node_ids": [],
"nodes": [
{"node_id": "base", "priority": 10, "intent": "Create the base.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_bore"], "expected_change": "One base solid."},
{"node_id": "finish", "priority": 20, "intent": "Finish the part.", "atomic_id": "hole_blind", "depends_on": ["base"], "claim_ids": ["claim_pattern"], "expected_change": "One bore."},
],
"final_claim_ids": ["claim_visual"],
})
health_contract = contract()
health_contract["requirements"][0]["acceptance_claims"][0]["claim_kind"] = "single_connected_body"
normalized = RequirementsCommandHandler._assign_unowned_global_health_claims(plan, health_contract)
self.assertEqual(normalized.nodes[0].claim_ids, ["claim_bore", "claim_base"])
self.assertEqual(validate_feature_plan(normalized, health_contract, {"extrude_add_blind", "hole_blind"}), [])
def test_scheduler_uses_ready_nodes_and_fixed_priority(self) -> None:
plan = initial_plan()
scheduler = FeatureScheduler(plan, [])
self.assertEqual(scheduler.next_ready().node_id, "base")
base = plan.nodes[0]
events = [{
"event": "feature_node_verified", "node_id": "base", "node_hash": node_hash(base),
"feature_id": "feature_001", "revision_id": "rev_001",
}]
scheduler = FeatureScheduler(plan, events)
self.assertEqual(scheduler.statuses()["base"], "done")
# Both bore and pattern are ready; priority decides deterministically.
self.assertEqual(scheduler.next_ready().node_id, "bore")
self.assertEqual(scheduler.feature_ids(), {"base": "feature_001"})
def test_revision_cannot_change_done_node_and_replaces_failed_subgraph(self) -> None:
previous = initial_plan()
base = previous.nodes[0]
bore = previous.nodes[1]
events = [
{"event": "feature_node_verified", "node_id": "base", "node_hash": node_hash(base), "feature_id": "feature_001", "revision_id": "rev_001"},
{"event": "feature_node_failed", "node_id": "bore", "node_hash": node_hash(bore), "failure_class": "engine_build", "attempt": 2, "terminal": True},
]
completed = FeatureScheduler(previous, events).completed_node_hashes()
revision = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1",
"parent_plan_hash": plan_hash(previous),
"replaces_node_ids": ["bore"],
"nodes": [
base.model_dump(mode="json"),
{"node_id": "bore_revised", "priority": 20, "intent": "Cut the bore with revised operation.", "atomic_id": "hole_blind", "depends_on": ["base"], "claim_ids": ["claim_bore"], "expected_change": "One through bore."},
previous.nodes[2].model_dump(mode="json"),
],
"final_claim_ids": ["claim_visual"],
})
self.assertEqual(
validate_feature_plan(revision, contract(), {"extrude_add_blind", "hole_blind"}, previous_plan=previous, completed_node_hashes=completed, required_replacements={"bore"}),
[],
)
changed = revision.model_copy(deep=True)
changed.nodes[0].intent = "Changed completed node."
self.assertTrue(any("completed node 'base' was modified" in item["message"] for item in validate_feature_plan(changed, contract(), {"extrude_add_blind", "hole_blind"}, previous_plan=previous, completed_node_hashes=completed, required_replacements={"bore"})))
def test_materialized_feature_uses_direct_dag_dependencies_not_previous_history_item(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
runtime = ProfileCadRuntime(runtime_settings(Path(temporary)))
base = {
"schema": "cad.cdsl.llm.v1", "schema_version": "1.1.0", "kind": "part", "part_id": "dag_test",
"geometry": {"sketches": [{
"id": "sketch_001", "workplane": {"origin_mm": [0, 0, 0], "normal": [0, 0, 1], "x_dir": [1, 0, 0]},
"profile": {"type": "circle", "center": [0, 0], "radius_mm": 10},
}]},
"features": [
{"id": "feature_001", "atomic_id": "extrude_add_blind", "params": {"distance_mm": 5}, "depends_on": [], "sketch_id": "sketch_001"},
{"id": "feature_002", "atomic_id": "reference_plane", "params": {"plane": {"origin_mm": [0, 0, 5], "normal": [0, 0, 1], "x_dir": [1, 0, 0]}}, "depends_on": ["feature_001"]},
],
}
contract = runtime.operation_contract("reference_axis")
document, audit = runtime.materialize_fragment(
base,
{"feature": {"atomic_id": "reference_axis", "params": {"axis": {"origin_mm": [0, 0, 0], "direction": [0, 0, 1]}}}},
contract,
{},
runtime.reference_tokens(base),
depends_on_feature_ids=("feature_001",),
)
self.assertEqual(document["features"][-1]["depends_on"], ["feature_001"])
self.assertEqual(audit["depends_on_feature_ids"], ["feature_001"])
def test_required_through_cut_gets_a_server_recorded_exit_allowance(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
runtime = ProfileCadRuntime(runtime_settings(Path(temporary)))
base = {
"schema": "cad.cdsl.llm.v1", "schema_version": "1.1.0", "kind": "part", "part_id": "through_cut",
"geometry": {"sketches": [{
"id": "sketch_001",
"workplane": {"origin_mm": [0, 0, 0], "normal": [0, 0, 1], "x_dir": [1, 0, 0]},
"profile": {"type": "circle", "center": [0, 0], "radius_mm": 10},
}]},
"features": [{"id": "feature_001", "atomic_id": "extrude_add_blind", "params": {"distance_mm": 5}, "depends_on": [], "sketch_id": "sketch_001"}],
}
document, audit = runtime.materialize_fragment(
base,
{
"sketch": {
"workplane": {"origin_mm": [0, 0, 5], "normal": [0, 0, 1], "x_dir": [1, 0, 0]},
"profile": {"type": "circle", "center": [0, 0], "radius_mm": 2},
},
"feature": {"atomic_id": "extrude_cut_blind", "params": {"distance_mm": 5, "reverse": True}},
},
runtime.operation_contract("extrude_cut_blind"),
{"body": {"kind": "body", "geometry": {"bbox_mm": [-10, -10, 0, 10, 10, 5]}}},
runtime.reference_tokens(base),
require_through=True,
depends_on_feature_ids=("feature_001",),
)
self.assertAlmostEqual(document["features"][-1]["params"]["distance_mm"], 5.01)
self.assertEqual(audit["server_normalizations"][0]["submitted_mm"], 5.0)
self.assertEqual(audit["server_normalizations"][0]["reason"], "required through-cut exit allowance")
def test_verified_node_publishes_without_a_render_bundle_or_candidate_review(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
registry = default_registry()
runtime = ProfileCadRuntime(runtime_settings(root))
repository = SqliteTaskRepository(root / "state.sqlite3")
artifacts = FileArtifactStore(root / "tasks")
requirements = RequirementsCommandHandler(repository, artifacts, registry, atomic_ids=runtime.supported_atomic_ids)
actions = ActionCommandHandler(repository, artifacts, runtime, RegistryVerifierExecutor(registry))
task_id = "cad_123456abcdef"
repository.create_task(task_id, "Create a disk.")
artifacts.initialize_task(task_id, "Create a disk.")
self.assertIsInstance(requirements.submit_requirements_document(task_id, MarkdownDocument(markdown="Create a disk."), invocation_id="requirements"), Accepted)
self.assertIsInstance(requirements.submit_completion_target(task_id, MarkdownDocument(markdown="- [ ] One connected disk with 20 mm diameter and 5 mm thickness."), invocation_id="target"), Accepted)
compiled = CompiledRequirementsSpec.model_validate({"requirements": [{"assumptions": [], "acceptance_claims": [
{"claim_kind": "single_connected_body", "expected": {}},
{"claim_kind": "outer_cylindrical_surface", "expected": {"diameter_mm": 20, "axial_span_mm": 5, "tolerance_mm": 0.1}},
]}]})
self.assertIsInstance(requirements.submit_compiled_spec(task_id, compiled, invocation_id="compile"), Accepted)
plan = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1", "parent_plan_hash": "", "replaces_node_ids": [],
"nodes": [{"node_id": "base", "priority": 10, "intent": "Create disk.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_001", "claim_002"], "expected_change": "One disk solid."}],
"final_claim_ids": [],
})
self.assertIsInstance(requirements.submit_feature_plan(task_id, plan, invocation_id="plan"), Accepted)
self.assertEqual(repository.get_state(task_id).phase, TaskPhase.SCHEDULING_FEATURE)
self.assertIsInstance(actions.schedule_next_feature(task_id), Accepted)
self.assertEqual(repository.get_state(task_id).phase, TaskPhase.FEATURE_PENDING)
result = actions.submit_feature_fragment(task_id, {
"sketch": {"workplane": {"origin_mm": [0, 0, 0], "normal": [0, 0, 1], "x_dir": [1, 0, 0]}, "profile": {"type": "circle", "center": [0, 0], "radius_mm": 10}},
"feature": {"atomic_id": "extrude_add_blind", "params": {"distance_mm": 5}},
}, invocation_id="fragment")
self.assertIsInstance(result, Accepted)
state = repository.get_state(task_id)
self.assertEqual(state.phase, TaskPhase.SCHEDULING_FEATURE)
self.assertEqual(state.active_revision, "rev_001")
self.assertFalse((artifacts.task_dir(task_id) / "revisions" / "rev_001" / "renders").exists())
events = repository.ledger_events(task_id)
self.assertTrue(any(event.get("event") == "feature_node_verified" for event in events))
self.assertFalse(any(event.get("event") == "candidate_built" for event in events))
final_gate = actions.schedule_next_feature(task_id)
self.assertIsInstance(final_gate, Accepted)
self.assertEqual(repository.get_state(task_id).phase, TaskPhase.FINAL_VALIDATION)
self.assertTrue(any(event.get("event") == "feature_plan_complete" for event in repository.ledger_events(task_id)))
def test_feature_plan_schema_binds_plan_revision_lineage_to_state(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
runtime = ProfileCadRuntime(runtime_settings(root))
repository = SqliteTaskRepository(root / "state.sqlite3")
artifacts = FileArtifactStore(root / "tasks")
requirements = RequirementsCommandHandler(repository, artifacts, default_registry(), atomic_ids=runtime.supported_atomic_ids)
task_id = "cad_123456abcdef"
repository.create_task(task_id, "Create a disk.")
artifacts.initialize_task(task_id, "Create a disk.")
self.assertIsInstance(requirements.submit_requirements_document(task_id, MarkdownDocument(markdown="Create a disk."), invocation_id="requirements"), Accepted)
self.assertIsInstance(requirements.submit_completion_target(task_id, MarkdownDocument(markdown="- [ ] One disk."), invocation_id="target"), Accepted)
compiled = CompiledRequirementsSpec.model_validate({"requirements": [{"assumptions": [], "acceptance_claims": [
{"claim_kind": "single_connected_body", "expected": {}},
]}]})
self.assertIsInstance(requirements.submit_compiled_spec(task_id, compiled, invocation_id="compile"), Accepted)
plan = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1", "parent_plan_hash": "", "replaces_node_ids": [],
"nodes": [{"node_id": "base", "priority": 10, "intent": "Create disk.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_001"], "expected_change": "One disk solid."}],
"final_claim_ids": [],
})
self.assertIsInstance(requirements.submit_feature_plan(task_id, plan, invocation_id="plan"), Accepted)
scheduled = transition(repository.get_state(task_id), "feature_scheduled")
self.assertTrue(repository.compare_and_swap(scheduled))
replanning = transition(repository.get_state(task_id), "feature_replan")
self.assertTrue(repository.compare_and_swap(replanning))
schema = requirements.feature_plan_schema(task_id)
properties = schema["properties"]
self.assertEqual(properties["parent_plan_hash"]["enum"], [plan_hash(plan)])
self.assertEqual(properties["replaces_node_ids"]["minItems"], 0)
self.assertEqual(properties["replaces_node_ids"]["maxItems"], 0)
node_schema = schema["$defs"]["FeatureNode"]["properties"]
self.assertEqual(node_schema["claim_ids"]["items"], {"enum": ["claim_001"]})
self.assertEqual(properties["final_claim_ids"]["items"], {"enum": []})
def test_pending_assigned_claim_rejects_the_node_checkpoint(self) -> None:
class PendingAssignedClaimVerifier:
def evaluate(self, claims: list[dict[str, object]], _facts: dict[str, object]) -> list[dict[str, object]]:
return [
{
"claim_id": str(claim.get("claim_id") or ""),
"claim_kind": str(claim.get("claim_kind") or ""),
"deterministic": True,
"status": "pending" if claim.get("claim_id") == "claim_002" else "pass",
"evidence": {"reason": "target geometry has not been introduced"},
}
for claim in claims
]
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
runtime = ProfileCadRuntime(runtime_settings(root))
repository = SqliteTaskRepository(root / "state.sqlite3")
artifacts = FileArtifactStore(root / "tasks")
requirements = RequirementsCommandHandler(repository, artifacts, default_registry(), atomic_ids=runtime.supported_atomic_ids)
actions = ActionCommandHandler(repository, artifacts, runtime, PendingAssignedClaimVerifier())
task_id = "cad_123456abcdef"
repository.create_task(task_id, "Create a disk.")
artifacts.initialize_task(task_id, "Create a disk.")
self.assertIsInstance(requirements.submit_requirements_document(task_id, MarkdownDocument(markdown="Create a disk."), invocation_id="requirements"), Accepted)
self.assertIsInstance(requirements.submit_completion_target(task_id, MarkdownDocument(markdown="- [ ] One disk."), invocation_id="target"), Accepted)
compiled = CompiledRequirementsSpec.model_validate({"requirements": [{"assumptions": [], "acceptance_claims": [
{"claim_kind": "single_connected_body", "expected": {}},
{"claim_kind": "outer_cylindrical_surface", "expected": {"diameter_mm": 20, "tolerance_mm": 0.1}},
]}]})
self.assertIsInstance(requirements.submit_compiled_spec(task_id, compiled, invocation_id="compile"), Accepted)
plan = FeaturePlan.model_validate({
"schema_version": "cad.v3.2.feature-plan.v1", "parent_plan_hash": "", "replaces_node_ids": [],
"nodes": [{"node_id": "base", "priority": 10, "intent": "Create disk.", "atomic_id": "extrude_add_blind", "depends_on": [], "claim_ids": ["claim_001", "claim_002"], "expected_change": "One disk solid."}],
"final_claim_ids": [],
})
self.assertIsInstance(requirements.submit_feature_plan(task_id, plan, invocation_id="plan"), Accepted)
self.assertIsInstance(actions.schedule_next_feature(task_id), Accepted)
result = actions.submit_feature_fragment(task_id, {
"sketch": {"workplane": {"origin_mm": [0, 0, 0], "normal": [0, 0, 1], "x_dir": [1, 0, 0]}, "profile": {"type": "circle", "center": [0, 0], "radius_mm": 10}},
"feature": {"atomic_id": "extrude_add_blind", "params": {"distance_mm": 5}},
}, invocation_id="fragment")
self.assertIsInstance(result, Rejected)
self.assertEqual(repository.get_state(task_id).phase, TaskPhase.FEATURE_PENDING)
events = repository.ledger_events(task_id)
failure = next(event for event in reversed(events) if event.get("event") == "feature_node_failed")
self.assertEqual(failure["failure_class"], "node_validation")
self.assertTrue(any(item.get("claim_id") == "claim_002" and item.get("status") == "pending" for item in failure["blockers"]))
self.assertFalse(any(event.get("event") == "feature_node_verified" for event in events))
if __name__ == "__main__":
unittest.main()