1902 lines
108 KiB
Python
1902 lines
108 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.services.autonomous_cdsl_generation import ( # noqa: E402
|
|
AutonomousCdslGenerationRunner,
|
|
AutonomousGenerationError,
|
|
_format_correction_card,
|
|
_fragment_selector_tokens,
|
|
_requires_edge_selector_recovery,
|
|
_final_repair_card,
|
|
_checkpoint_token,
|
|
_geometry_fingerprint,
|
|
_has_material_volume_change,
|
|
_material_volume_tolerance,
|
|
_operation_contract_payload,
|
|
parse_completion_audit,
|
|
parse_completion_checklist,
|
|
_preferred_tool_call,
|
|
_is_author_quota_error,
|
|
_is_author_transport_error,
|
|
autonomous_tools,
|
|
build_candidate,
|
|
parse_tool_arguments,
|
|
)
|
|
from app.services.cdsl_fragment import ( # noqa: E402
|
|
AutonomousFragmentError,
|
|
autonomous_candidate_prompt_tokens,
|
|
autonomous_selector_tokens,
|
|
materialize_autonomous_fragment,
|
|
)
|
|
from app.services.engine_service import load_engine, validate_cdsl # noqa: E402
|
|
from app.services.storage import WorkspaceStore, write_json # noqa: E402
|
|
from app.settings import ProviderConfig, ProviderModel, Settings # noqa: E402
|
|
from app.services.agent_service import conversation_user_context # noqa: E402
|
|
from app.services.visual_review import VisualReviewError # noqa: E402
|
|
|
|
|
|
BACKEND = ROOT / "backend"
|
|
|
|
|
|
def settings(root: Path) -> Settings:
|
|
provider = ProviderConfig("test", "Test", "https://example.invalid/v1", "test-key", (ProviderModel("test-model"),))
|
|
return Settings(
|
|
task_root=root / "tasks",
|
|
conversation_root=root / "conversations",
|
|
library_root=BACKEND / "cdsl_library",
|
|
engine_root=BACKEND / "engine" / "cdsl_engine",
|
|
llm_base_url=provider.base_url,
|
|
llm_api_key=provider.api_key,
|
|
llm_model="test-model",
|
|
llm_timeout_s=1,
|
|
default_provider_id="test",
|
|
providers=(provider,),
|
|
)
|
|
|
|
|
|
def base_fragment() -> dict:
|
|
return {
|
|
"sketch": {
|
|
"workplane": {"origin_mm": [0, 0, 0], "x_dir": [1, 0, 0], "normal": [0, 0, 1]},
|
|
"profile": {"type": "polygon", "vertices": [[-10, -5], [10, -5], [10, 5], [-10, 5]]},
|
|
},
|
|
"feature": {"atomic_id": "extrude_add_blind", "params": {"distance_mm": 4}},
|
|
}
|
|
|
|
|
|
def completion_checklist() -> str:
|
|
return "- [ ] rectangular plate has the frozen 20 x 10 x 4 mm envelope"
|
|
|
|
|
|
def completed_plate_audit() -> str:
|
|
return "- [x] rectangular plate has the frozen 20 x 10 x 4 mm envelope :: rebuilt bbox is 20 x 10 x 4 mm"
|
|
|
|
|
|
def accepted_plate_batch_review() -> dict[str, object]:
|
|
return {
|
|
"schema_version": "cad.candidate-review.v1",
|
|
"node_id": "candidate",
|
|
"model": "deepseek-v4-flash-vision-exp",
|
|
"verdict": "accept",
|
|
"confidence": 0.98,
|
|
"batch_goal": "Create the rectangular base plate.",
|
|
"batch_goal_status": "achieved",
|
|
"coverage": [{
|
|
"item": "rectangular plate has the frozen 20 x 10 x 4 mm envelope",
|
|
"status": "complete",
|
|
"evidence": "The canonical views show the requested rectangular plate.",
|
|
}],
|
|
"evidence": ["The batch goal is visibly achieved."],
|
|
}
|
|
|
|
|
|
class AutonomousStorageTests(unittest.TestCase):
|
|
def test_completion_checklist_is_free_markdown_but_has_unique_pending_items(self) -> None:
|
|
items = parse_completion_checklist(
|
|
"# Completion\n- [ ] one coherent solid\n- [ ] six holes in each flange\n"
|
|
)
|
|
|
|
self.assertEqual(items, ["one coherent solid", "six holes in each flange"])
|
|
with self.assertRaisesRegex(AutonomousGenerationError, "begin with every checklist item unchecked"):
|
|
parse_completion_checklist("- [x] already done", require_unchecked=True)
|
|
with self.assertRaisesRegex(AutonomousGenerationError, "duplicate"):
|
|
parse_completion_checklist("- [ ] one solid\n- [ ] One solid")
|
|
|
|
def test_completion_audit_requires_every_frozen_item_and_evidence(self) -> None:
|
|
checklist = ["one coherent solid", "six holes in each flange"]
|
|
audit = parse_completion_audit(
|
|
"- [x] one coherent solid :: engine reports one solid\n- [ ] six holes in each flange :: not yet modelled",
|
|
checklist,
|
|
)
|
|
|
|
self.assertEqual([item["status"] for item in audit], ["complete", "missing"])
|
|
with self.assertRaisesRegex(AutonomousGenerationError, "missing: six holes"):
|
|
parse_completion_audit("- [x] one coherent solid :: engine reports one solid", checklist)
|
|
with self.assertRaisesRegex(AutonomousGenerationError, "needs evidence"):
|
|
parse_completion_audit(
|
|
"- [x] one coherent solid\n- [x] six holes in each flange :: inferred",
|
|
checklist,
|
|
)
|
|
|
|
def test_conversation_context_preserves_earlier_user_intent_without_keyword_rules(self) -> None:
|
|
context = conversation_user_context({"messages": [
|
|
{"id": "u1", "role": "user", "parts": [{"type": "text", "text": "Build a V-shaped guide block with mounting holes."}]},
|
|
{"id": "a1", "role": "assistant", "parts": [{"type": "text", "text": "Incorrect generated summary."}]},
|
|
{"id": "u2", "role": "user", "parts": [{"type": "text", "text": "retry"}]},
|
|
]})
|
|
|
|
self.assertEqual(len(context), 1)
|
|
self.assertIn("V-shaped guide block", context[0]["content"])
|
|
self.assertIn("retry", context[0]["content"])
|
|
self.assertNotIn("Incorrect generated summary", context[0]["content"])
|
|
|
|
def test_multi_tool_response_prioritizes_completion_over_observation(self) -> None:
|
|
inspect = {"function": {"name": "inspect_model", "arguments": "{}"}}
|
|
complete = {"function": {"name": "complete_task", "arguments": "{\"self_review\":\"done\"}"}}
|
|
|
|
self.assertIs(_preferred_tool_call([inspect, complete]), complete)
|
|
|
|
def test_multi_tool_priority_never_bypasses_current_tool_state(self) -> None:
|
|
requirements = {"function": {"name": "write_requirements_document", "arguments": "{}"}}
|
|
fragment = {"function": {"name": "submit_cdsl_fragment", "arguments": "{}"}}
|
|
|
|
self.assertIs(
|
|
_preferred_tool_call([requirements, fragment], allowed_names={"write_requirements_document"}),
|
|
requirements,
|
|
)
|
|
|
|
def test_no_progress_does_not_hide_recovery_tools(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "complete a checked model")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Checked model")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint"})
|
|
store.set_active_revision(task_id, "rev_001")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
loaded = store.read_task(task_id) or {}
|
|
|
|
tools = runner._author_tools(loaded, requirements_frozen=True, state={"no_progress": 3, "recent_events": [{"kind": "tool_error"}]})
|
|
|
|
names = {tool["function"]["name"] for tool in tools}
|
|
self.assertIn("inspect_topology", names)
|
|
self.assertIn("submit_cdsl_fragment", names)
|
|
self.assertIn("complete_task", names)
|
|
|
|
def test_format_correction_forces_a_direct_runtime_contract_rewrite(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
fragment = {
|
|
"sketch": base_fragment()["sketch"],
|
|
"feature": {"atomic_id": "extrude_cut_blind", "params": {"depth_mm": 4}},
|
|
}
|
|
card = _format_correction_card(
|
|
engine,
|
|
fragment_json=json.dumps(fragment),
|
|
error_message="CDSL schema violation at $.features[2].params: 'distance_mm' is a required property",
|
|
head="main:rev_002",
|
|
)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
tools = runner._author_tools(
|
|
{"active_revision": "rev_002", "active_branch_id": "main"},
|
|
requirements_frozen=True,
|
|
state={"format_correction": card},
|
|
)
|
|
|
|
self.assertEqual(card["atomic_id"], "extrude_cut_blind")
|
|
self.assertEqual(card["required_params"], ["distance_mm"])
|
|
self.assertEqual(card["optional_params"], ["reverse"])
|
|
self.assertTrue(card["additional_params_forbidden"])
|
|
self.assertEqual(card["unsupported_attempted_params"], ["depth_mm"])
|
|
self.assertEqual(card["valid_feature_shape"]["params"], {"distance_mm": "positive number"})
|
|
self.assertEqual([tool["function"]["name"] for tool in tools], ["submit_cdsl_fragment"])
|
|
|
|
def test_identical_format_error_increments_only_its_own_signature(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = json.dumps({"feature": {"atomic_id": "extrude_cut_blind", "params": {"depth_mm": 4}}})
|
|
first = _format_correction_card(engine, fragment_json=fragment, error_message="missing distance_mm", head="main:rev_002")
|
|
second = _format_correction_card(engine, fragment_json=fragment, error_message="missing distance_mm", head="main:rev_002", previous=first)
|
|
changed = _format_correction_card(engine, fragment_json=fragment, error_message="distance must be positive", head="main:rev_002", previous=second)
|
|
|
|
self.assertEqual(first["repeat_count"], 1)
|
|
self.assertEqual(second["repeat_count"], 2)
|
|
self.assertEqual(changed["repeat_count"], 1)
|
|
|
|
def test_finish_selector_error_requires_one_edge_lookup_before_resubmission(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
correction = _format_correction_card(
|
|
engine,
|
|
fragment_json=json.dumps({"feature": {"atomic_id": "chamfer", "params": {"distance_mm": 1.5}}}),
|
|
error_message="chamfer requires 1..64 selector token(s)",
|
|
head="main:rev_002",
|
|
)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_002", "active_branch_id": "main"}
|
|
|
|
self.assertTrue(_requires_edge_selector_recovery(correction))
|
|
self.assertTrue(correction["selector_tokens_required"])
|
|
self.assertEqual(correction["selector_token_kind"], "edge")
|
|
self.assertIn("selector_tokens", correction["valid_feature_shape"])
|
|
|
|
lookup_tools = runner._author_tools(
|
|
task,
|
|
requirements_frozen=True,
|
|
state={"candidate_action_required": {"working_head": "main:rev_002", "reason": "edge_selector_recovery"}},
|
|
)
|
|
submit_tools = runner._author_tools(
|
|
task,
|
|
requirements_frozen=True,
|
|
state={"candidate_action_required": {"working_head": "main:rev_002", "reason": "edge_selector_recovery_submit"}},
|
|
)
|
|
|
|
self.assertEqual([tool["function"]["name"] for tool in lookup_tools], ["inspect_topology"])
|
|
self.assertEqual(
|
|
[tool["function"]["name"] for tool in submit_tools],
|
|
["submit_cdsl_fragment", "rollback_checkpoint"],
|
|
)
|
|
|
|
def test_finish_recovery_keeps_the_exact_edge_token_bank_in_author_context(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "finish recovery")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.set_active_revision(task_id, "rev_001", branch_id="main")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {
|
|
"recent_events": [],
|
|
"initial_context_delivered": True,
|
|
"candidate_action_required": {"working_head": "main:rev_001", "reason": "edge_selector_recovery_submit"},
|
|
"edge_selector_recovery": {
|
|
"working_head": "main:rev_001",
|
|
"tokens": [{"token": "sel_exact_edge", "kind": "edge", "geometry": {"length_mm": 20}}],
|
|
},
|
|
}
|
|
|
|
prompt = runner._prompt_messages(task_id, state, load_engine(current_settings))[-1]["content"]
|
|
tools = runner._author_tools(store.read_task(task_id) or {}, requirements_frozen=True, state=state)
|
|
|
|
self.assertIn("sel_exact_edge", prompt)
|
|
self.assertEqual([tool["function"]["name"] for tool in tools], ["submit_cdsl_fragment", "rollback_checkpoint"])
|
|
self.assertEqual(_fragment_selector_tokens({"feature": {"selector_tokens": ["sel_exact_edge"]}}), ["sel_exact_edge"])
|
|
|
|
def test_multi_feature_fragment_feedback_states_the_batch_limit(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = {
|
|
"features": [
|
|
{"atomic_id": "extrude_cut_blind", "params": {"distance_mm": 4}},
|
|
{"atomic_id": "extrude_cut_blind", "params": {"distance_mm": 4}},
|
|
],
|
|
}
|
|
card = _format_correction_card(
|
|
engine,
|
|
fragment_json=json.dumps(fragment),
|
|
error_message="A fragment may add at most 6 feature(s)",
|
|
head="main:rev_002",
|
|
)
|
|
|
|
self.assertEqual(card["attempted_feature_count"], 2)
|
|
self.assertEqual(card["max_features_per_fragment"], 6)
|
|
self.assertIn("at most 6 feature", card["instruction"])
|
|
|
|
def test_author_context_explains_that_one_hole_feature_can_hold_a_repeated_position_group(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "a repeated hole pattern")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nCreate a repeated hole group.")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id,
|
|
request="a repeated hole pattern",
|
|
conversation_id="conv_abcdef123456",
|
|
provider=current_settings.providers[0],
|
|
model=current_settings.providers[0].models[0],
|
|
initial_messages=[],
|
|
frozen_attachment_ids=[],
|
|
fresh=True,
|
|
)
|
|
context = json.loads(runner._prompt_messages(task_id, state, load_engine(current_settings))[-1]["content"])
|
|
rules = context["cdsl_authoring_basics"]["rules"]
|
|
contract = _operation_contract_payload(load_engine(current_settings), "hole_blind")
|
|
|
|
self.assertTrue(any("multiple positions" in rule for rule in rules))
|
|
self.assertEqual(contract["author_required_params"], ["diameter_mm", "depth_mm", "positions"])
|
|
self.assertEqual(contract["server_injected_params"], ["host_face"])
|
|
self.assertEqual(contract["selector_rule"], {"kind": "face", "min_items": 1, "max_items": 1})
|
|
|
|
def test_final_repair_recommends_only_the_immediate_predecessor_for_a_bad_last_feature(self) -> None:
|
|
task = {
|
|
"active_branch_id": "main",
|
|
"active_revision": "rev_006",
|
|
"revisions": [
|
|
{"revision_id": "rev_005", "parent_revision_id": "rev_004"},
|
|
{"revision_id": "rev_006", "parent_revision_id": "rev_005"},
|
|
],
|
|
}
|
|
review = {
|
|
"verdict": "repair",
|
|
"confidence": 0.9,
|
|
"evidence": ["The last extrude_cut makes a large rectangular slot rather than the required circular holes."],
|
|
}
|
|
|
|
card = _final_repair_card(task, review)
|
|
|
|
self.assertEqual(card["working_head"], "main:rev_006")
|
|
self.assertTrue(card["rollback_guidance"]["recommended_checkpoint_token"].startswith("checkpoint_"))
|
|
self.assertIn("immediate predecessor", card["rollback_guidance"]["reason"])
|
|
|
|
def test_geometry_fingerprint_ignores_runtime_identities_but_detects_shape_changes(self) -> None:
|
|
base = {
|
|
"records": [{
|
|
"record_id": "body:feature_001:face:0",
|
|
"feature_id": "feature_001",
|
|
"body_id": "body:feature_001",
|
|
"owner_feature_ids": ["feature_001"],
|
|
"kind": "face",
|
|
"geometry": {"center_mm": [0.0, 0.0, 4.0000000001], "area_mm2": 20.0},
|
|
"synthetic": False,
|
|
}],
|
|
}
|
|
equivalent = {
|
|
"records": [{
|
|
"record_id": "body:feature_002:face:9",
|
|
"feature_id": "feature_002",
|
|
"body_id": "body:feature_002",
|
|
"owner_feature_ids": ["feature_002"],
|
|
"kind": "face",
|
|
"geometry": {"center_mm": [0.0, -0.0, 4.0], "area_mm2": 20.0},
|
|
"synthetic": False,
|
|
}],
|
|
}
|
|
changed = {"records": [{**equivalent["records"][0], "geometry": {"center_mm": [0.0, 0.0, 3.0], "area_mm2": 20.0}}]}
|
|
forward_edge = {"records": [{
|
|
"kind": "edge",
|
|
"geometry": {"curve_type": "circle", "start_mm": [2, 0, 0], "end_mm": [-2, 0, 0], "length_mm": 6.28},
|
|
"synthetic": False,
|
|
}]}
|
|
reverse_edge = {"records": [{
|
|
"kind": "edge",
|
|
"geometry": {"curve_type": "circle", "start_mm": [-2, 0, 0], "end_mm": [2, 0, 0], "length_mm": 6.28},
|
|
"synthetic": False,
|
|
}]}
|
|
|
|
self.assertEqual(_geometry_fingerprint(base), _geometry_fingerprint(equivalent))
|
|
self.assertNotEqual(_geometry_fingerprint(base), _geometry_fingerprint(changed))
|
|
self.assertEqual(_geometry_fingerprint(forward_edge), _geometry_fingerprint(reverse_edge))
|
|
|
|
def test_sub_tolerance_volume_noise_is_not_a_material_candidate_change(self) -> None:
|
|
health = {
|
|
"bbox_mm": {"dimensions": [70.0, 70.0, 120.0]},
|
|
"volume_mm3": 243855.7778136896,
|
|
}
|
|
|
|
tolerance = _material_volume_tolerance(health)
|
|
|
|
self.assertEqual(tolerance, 0.01)
|
|
self.assertFalse(_has_material_volume_change(health, 243855.78125688632))
|
|
self.assertTrue(_has_material_volume_change(health, 243855.9))
|
|
|
|
def test_no_geometry_change_candidate_is_rejected_before_checkpoint(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "unchanged candidate")
|
|
task_id = str(task["task_id"])
|
|
engine = load_engine(current_settings)
|
|
base_cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
first = build_candidate(settings=current_settings, store=store, task_id=task_id, cdsl=base_cdsl, fragment_audit=audit, parent_revision_id="")
|
|
revision_dir = store.revision_dir(task_id, "rev_001")
|
|
shutil.copytree(store.candidate_dir(task_id, str(first["candidate_id"])), revision_dir)
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint"})
|
|
store.clear_active_candidate(task_id, str(first["candidate_id"]))
|
|
duplicate_cdsl, duplicate_audit = materialize_autonomous_fragment(base_cdsl, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
with self.assertRaisesRegex(AutonomousGenerationError, "CANDIDATE_GEOMETRY_UNCHANGED"):
|
|
build_candidate(
|
|
settings=current_settings,
|
|
store=store,
|
|
task_id=task_id,
|
|
cdsl=duplicate_cdsl,
|
|
fragment_audit=duplicate_audit,
|
|
parent_revision_id="rev_001",
|
|
)
|
|
|
|
after = store.read_task(task_id) or {}
|
|
self.assertEqual([item["revision_id"] for item in after["revisions"]], ["rev_001"])
|
|
self.assertEqual(after["active_candidate_id"], "")
|
|
|
|
def test_repeated_identical_format_error_reaches_a_terminal_limit(self) -> None:
|
|
async def exercise() -> dict:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
current_settings = Settings(**{**current_settings.__dict__, "agent_format_error_repeat_limit": 2})
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "format loop")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
invalid = {
|
|
"sketch": base_fragment()["sketch"],
|
|
"feature": {"atomic_id": "extrude_add_blind", "params": {"travel_mm": 4}},
|
|
}
|
|
responses = iter([
|
|
("write_requirements_document", {"markdown": "# Frozen\nBuild a plate."}),
|
|
("write_completion_checklist", {"markdown": completion_checklist()}),
|
|
("submit_cdsl_fragment", {"fragment_json": json.dumps(invalid)}),
|
|
("submit_cdsl_fragment", {"fragment_json": json.dumps(invalid)}),
|
|
])
|
|
|
|
async def complete(_messages: list[dict], _tools: list[dict], _provider: ProviderConfig, _model: ProviderModel, _forced: str | None) -> dict:
|
|
name, arguments = next(responses)
|
|
return {"choices": [{"message": {"tool_calls": [{"id": name, "function": {"name": name, "arguments": json.dumps(arguments)}}]}}]}
|
|
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, complete)
|
|
events = [item async for item in runner.run(
|
|
task_id=str(task["task_id"]), request="format loop", conversation_id="conv_abcdef123456",
|
|
provider=provider, model=model, initial_messages=[],
|
|
)]
|
|
return {"events": events, "task": store.read_task(str(task["task_id"])) or {}, "state": store.read_agent_state(str(task["task_id"])) or {}}
|
|
|
|
result = asyncio.run(exercise())
|
|
terminal = result["events"][-1]
|
|
self.assertEqual(terminal[0], "task_terminal")
|
|
self.assertEqual(terminal[1]["lifecycle"], "failed")
|
|
self.assertIn("AUTHORING_FORMAT_LOOP", terminal[1]["message"])
|
|
self.assertEqual(result["task"]["lifecycle"], "failed")
|
|
self.assertEqual(result["state"]["format_correction"]["repeat_count"], 2)
|
|
|
|
def test_transport_failure_selects_another_configured_author(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
primary = ProviderConfig("primary", "Primary", "https://primary.invalid", "key", (ProviderModel("primary-model"),))
|
|
alternate = ProviderConfig("alternate", "Alternate", "https://alternate.invalid", "key", (ProviderModel("alternate-model"),))
|
|
current_settings = settings(Path(temporary))
|
|
current_settings = Settings(
|
|
**{**current_settings.__dict__, "default_provider_id": "primary", "providers": (primary, alternate), "llm_model": "primary-model"},
|
|
)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
|
|
fallback = runner._transport_fallback_author(primary, {"author_transport_failed_providers": ["primary"]})
|
|
|
|
self.assertIsNotNone(fallback)
|
|
assert fallback is not None
|
|
self.assertEqual((fallback[0].id, fallback[1].id), ("alternate", "alternate-model"))
|
|
|
|
def test_requirements_document_is_written_once_and_frozen(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
store = WorkspaceStore(settings(Path(temporary)))
|
|
task = store.ensure_task(None, "make a bracket")
|
|
task_id = str(task["task_id"])
|
|
path = store.write_requirements_document(task_id, "# Frozen requirements\nUse millimetres.")
|
|
|
|
self.assertEqual(path.name, "requirements.md")
|
|
self.assertEqual(store.read_source_requirements(task_id), "make a bracket\n")
|
|
self.assertEqual((store.task_dir(task_id) / "source-requirements.md").read_text(encoding="utf-8"), "make a bracket\n")
|
|
self.assertIn("Use millimetres", store.read_requirements_document(task_id))
|
|
with self.assertRaisesRegex(ValueError, "frozen"):
|
|
store.write_requirements_document(task_id, "replace it")
|
|
|
|
def test_legacy_task_is_not_migrated_into_a_resumable_autonomous_task(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
store = WorkspaceStore(settings(Path(temporary)))
|
|
task = store.ensure_task(None, "legacy task")
|
|
path = store.task_path(str(task["task_id"]))
|
|
record = json.loads(path.read_text(encoding="utf-8"))
|
|
record.update({"schema_version": "1.3", "lifecycle": "running", "generation_spec_path": "generation-spec.json"})
|
|
path.write_text(json.dumps(record), encoding="utf-8")
|
|
|
|
loaded = store.read_task(str(task["task_id"]))
|
|
|
|
self.assertEqual(loaded["schema_version"], "1.3")
|
|
self.assertEqual(loaded["lifecycle"], "running")
|
|
|
|
def test_root_rollback_creates_a_new_branch_and_supersedes_descendants(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
store = WorkspaceStore(settings(Path(temporary)))
|
|
task = store.ensure_task(None, "rollback")
|
|
task_id = str(task["task_id"])
|
|
for revision_id, parent in (("rev_001", ""), ("rev_002", "rev_001")):
|
|
store.update_task(task_id, {
|
|
"revision_id": revision_id,
|
|
"status": "success",
|
|
"parent_revision_id": parent,
|
|
"visibility": "checkpoint",
|
|
})
|
|
|
|
rolled = store.rollback_to_revision(task_id, "", branch_id="branch_test")
|
|
|
|
self.assertEqual(rolled["active_revision"], "")
|
|
self.assertEqual(rolled["active_branch_id"], "branch_test")
|
|
self.assertEqual({item["visibility"] for item in rolled["revisions"]}, {"superseded"})
|
|
|
|
def test_rollback_leaves_other_branch_checkpoints_intact(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
store = WorkspaceStore(settings(Path(temporary)))
|
|
task = store.ensure_task(None, "branch rollback")
|
|
task_id = str(task["task_id"])
|
|
for revision_id, parent, branch_id in (
|
|
("rev_001", "", "main"),
|
|
("rev_002", "rev_001", "main"),
|
|
("rev_003", "rev_001", "other_branch"),
|
|
):
|
|
store.update_task(task_id, {
|
|
"revision_id": revision_id,
|
|
"status": "success",
|
|
"parent_revision_id": parent,
|
|
"branch_id": branch_id,
|
|
"visibility": "checkpoint",
|
|
})
|
|
store.set_active_revision(task_id, "rev_002", branch_id="main")
|
|
|
|
rolled = store.rollback_to_revision(task_id, "rev_001", branch_id="branch_new")
|
|
visibility = {item["revision_id"]: item["visibility"] for item in rolled["revisions"]}
|
|
|
|
self.assertEqual(visibility["rev_002"], "superseded")
|
|
self.assertEqual(visibility["rev_003"], "checkpoint")
|
|
|
|
|
|
class AutonomousFragmentTests(unittest.TestCase):
|
|
def test_fragment_limit_allows_a_coherent_two_feature_batch_and_rejects_seven(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = {
|
|
"sketches": [base_fragment()["sketch"]],
|
|
"features": [
|
|
base_fragment()["feature"],
|
|
{"atomic_id": "reference_axis", "params": {"axis": {"origin_mm": [0, 0, 0], "direction": [0, 0, 1]}}},
|
|
],
|
|
}
|
|
|
|
_, audit = materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
self.assertEqual(len(audit["assigned_feature_ids"]), 2)
|
|
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "at most 6"):
|
|
materialize_autonomous_fragment(
|
|
None,
|
|
{"features": [{"atomic_id": "reference_axis", "params": {"axis": {"origin_mm": [0, 0, 0], "direction": [0, 0, 1]}}}] * 7},
|
|
engine=engine,
|
|
selector_tokens={},
|
|
max_features=6,
|
|
)
|
|
|
|
def test_candidate_health_counts_current_solids_not_feature_history(self) -> None:
|
|
from app.services.autonomous_cdsl_generation import _candidate_health
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
step_path = root / "model.step"
|
|
glb_path = root / "model.glb"
|
|
step_path.write_bytes(b"step")
|
|
glb_path.write_bytes(b"glb")
|
|
health = _candidate_health({
|
|
"bbox_mm": {"min": [0, 0, 0], "max": [10, 10, 2]},
|
|
"volume_mm3": 100,
|
|
"solid_count": 1,
|
|
"feature_results": [{"body_id": "body:base"}, {"body_id": "body:cut"}],
|
|
}, step_path, glb_path)
|
|
|
|
self.assertEqual(health["solid_count"], 1)
|
|
|
|
def test_author_cannot_provide_committed_identity_or_raw_selectors(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
invalid = base_fragment()
|
|
invalid["feature"]["id"] = "invented_feature"
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "server-owned"):
|
|
materialize_autonomous_fragment(None, invalid, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
def test_one_free_fragment_materializes_to_valid_complete_cdsl(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
cdsl["part_id"] = "autonomous-test"
|
|
|
|
validate_cdsl(cdsl, engine)
|
|
self.assertEqual(audit["assigned_feature_ids"], ["feature_001"])
|
|
self.assertEqual(audit["assigned_sketch_ids"], ["sketch_001"])
|
|
self.assertEqual(cdsl["features"][0]["depends_on"], [])
|
|
|
|
def test_blind_extrude_depth_alias_is_normalized_before_cdsl_validation(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
fragment = base_fragment()
|
|
fragment["feature"]["params"] = {"depth_mm": 4}
|
|
|
|
cdsl, audit = materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=1)
|
|
cdsl["part_id"] = "depth-alias-test"
|
|
|
|
validate_cdsl(cdsl, engine)
|
|
self.assertEqual(cdsl["features"][0]["params"], {"distance_mm": 4})
|
|
self.assertEqual(
|
|
audit["compatibility_fixes"],
|
|
[{"path": "feature.params", "from": "depth_mm", "to": "distance_mm", "action": "renamed_equivalent"}],
|
|
)
|
|
self.assertEqual(fragment["feature"]["params"], {"depth_mm": 4})
|
|
|
|
def test_conflicting_depth_and_distance_are_not_silently_overwritten(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = base_fragment()
|
|
fragment["feature"]["params"] = {"depth_mm": 4, "distance_mm": 6}
|
|
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "CONFLICTING_PARAMETER_ALIASES"):
|
|
materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
def test_explicit_axis_origin_alias_is_normalized_without_guessing_an_axis(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = {
|
|
"feature": {
|
|
"atomic_id": "reference_axis",
|
|
"params": {"axis": {"origin": [0, 0, 0], "direction": [0, 0, 1]}},
|
|
},
|
|
}
|
|
|
|
cdsl, audit = materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
self.assertEqual(cdsl["features"][0]["params"]["axis"], {"origin_mm": [0, 0, 0], "direction": [0, 0, 1]})
|
|
self.assertEqual(audit["compatibility_fixes"][0]["from"], "origin")
|
|
|
|
def test_revolve_axis_is_author_required_and_materializes_without_axis_tokens(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
contract = _operation_contract_payload(engine, "revolve_cut")
|
|
base_cdsl, _ = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=6)
|
|
fragment = {
|
|
"sketch": {
|
|
"workplane": {"origin_mm": [0, 0, 0], "x_dir": [1, 0, 0], "normal": [0, 1, 0]},
|
|
"profile": {"type": "polygon", "vertices": [[-2, 1], [2, 1], [2, 3], [-2, 3]]},
|
|
},
|
|
"feature": {
|
|
"atomic_id": "revolve_cut",
|
|
"params": {"angle_deg": 360, "axis": {"origin_mm": [0, 0, 0], "direction": [1, 0, 0]}},
|
|
},
|
|
}
|
|
|
|
cdsl, _ = materialize_autonomous_fragment(base_cdsl, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
cdsl["part_id"] = "revolve-axis-authoring"
|
|
validate_cdsl(cdsl, engine)
|
|
|
|
self.assertEqual(contract["author_required_params"], ["angle_deg", "axis"])
|
|
self.assertEqual(contract["server_injected_params"], [])
|
|
self.assertIsNone(contract["selector_rule"])
|
|
self.assertEqual(cdsl["features"][-1]["params"]["axis"]["direction"], [1, 0, 0])
|
|
|
|
def test_revolve_without_explicit_axis_is_rejected_before_schema_retry_loop(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = {
|
|
"sketch": base_fragment()["sketch"],
|
|
"feature": {"atomic_id": "revolve_cut", "params": {"angle_deg": 360}},
|
|
}
|
|
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "requires params.axis"):
|
|
materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
|
|
def test_revolve_common_axis_aliases_are_normalized_losslessly(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
fragment = {
|
|
"sketch": base_fragment()["sketch"],
|
|
"feature": {
|
|
"atomic_id": "revolve_add",
|
|
"params": {
|
|
"angle_degrees": 360,
|
|
"axis_point_mm": [0, 0, 0],
|
|
"axis_dir": [1, 0, 0],
|
|
},
|
|
},
|
|
}
|
|
|
|
cdsl, audit = materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
cdsl["part_id"] = "revolve-axis-aliases"
|
|
validate_cdsl(cdsl, engine)
|
|
|
|
self.assertEqual(cdsl["features"][0]["params"], {
|
|
"angle_deg": 360,
|
|
"axis": {"origin_mm": [0, 0, 0], "direction": [1, 0, 0]},
|
|
})
|
|
self.assertEqual(
|
|
[(item["from"], item["to"]) for item in audit["compatibility_fixes"]],
|
|
[("angle_degrees", "angle_deg"), ("axis_point_mm", "axis.origin_mm"), ("axis_dir", "axis.direction")],
|
|
)
|
|
|
|
def test_batch_revolve_axis_and_feature_local_sketches_materialize(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
fragment = {
|
|
"revolve_axis": {"origin_mm": [0, 0, 0], "direction": [1, 0, 0]},
|
|
"features": [
|
|
{
|
|
"atomic_id": "revolve_add",
|
|
"params": {"angle_deg": 360},
|
|
"sketches": [base_fragment()["sketch"]],
|
|
},
|
|
{
|
|
"atomic_id": "revolve_cut",
|
|
"params": {"angle_deg": 360},
|
|
"sketches": [{
|
|
"workplane": {"origin_mm": [0, 0, 0], "x_dir": [1, 0, 0], "normal": [0, 0, 1]},
|
|
"profile": {"type": "polygon", "points": [[-5, 0], [5, 0], [5, 2], [-5, 2]]},
|
|
}],
|
|
},
|
|
],
|
|
}
|
|
|
|
cdsl, audit = materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
cdsl["part_id"] = "batch-revolve-axis"
|
|
validate_cdsl(cdsl, engine)
|
|
|
|
self.assertEqual(len(cdsl["geometry"]["sketches"]), 2)
|
|
self.assertEqual(len(cdsl["features"]), 2)
|
|
self.assertTrue(all(feature["params"]["axis"]["direction"] == [1, 0, 0] for feature in cdsl["features"]))
|
|
self.assertTrue(any(item["action"] == "copied_explicit_batch_axis" for item in audit["compatibility_fixes"]))
|
|
self.assertTrue(any(item["from"] == "points" and item["to"] == "vertices" for item in audit["compatibility_fixes"]))
|
|
|
|
def test_submit_tool_exposes_a_structured_shared_revolve_axis(self) -> None:
|
|
tool = next(item for item in autonomous_tools() if item["function"]["name"] == "submit_cdsl_fragment")
|
|
axis = tool["function"]["parameters"]["properties"]["shared_revolve_axis"]
|
|
|
|
self.assertEqual(axis["required"], ["origin_mm", "direction"])
|
|
self.assertFalse(axis["additionalProperties"])
|
|
|
|
def test_conflicting_revolve_axis_aliases_are_not_guessed(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
engine = load_engine(settings(Path(temporary)))
|
|
fragment = {
|
|
"sketch": base_fragment()["sketch"],
|
|
"feature": {
|
|
"atomic_id": "revolve_add",
|
|
"params": {
|
|
"angle_deg": 360,
|
|
"axis": {"origin_mm": [0, 0, 0], "direction": [1, 0, 0]},
|
|
"axis_dir": [0, 1, 0],
|
|
},
|
|
},
|
|
}
|
|
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "CONFLICTING_PARAMETER_ALIASES"):
|
|
materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=6)
|
|
|
|
def test_selector_tokens_are_opaque_and_snapshot_scoped(self) -> None:
|
|
tokens = autonomous_selector_tokens({
|
|
"snapshot_id": "cad_test/rev_001",
|
|
"records": [{
|
|
"record_id": "body:base:edge:7",
|
|
"kind": "edge",
|
|
"feature_id": "base",
|
|
"owner_feature_ids": ["base"],
|
|
"geometry": {"curve_type": "line", "length_mm": 10},
|
|
"executable": True,
|
|
}],
|
|
})
|
|
token, record = next(iter(tokens.items()))
|
|
self.assertTrue(token.startswith("sel_"))
|
|
self.assertEqual(record["kind"], "edge")
|
|
self.assertNotIn("stable_id", {"token": token, "kind": record["kind"], "geometry": record["geometry"]})
|
|
|
|
def test_selector_prompt_tokens_exclude_runtime_only_geometry_details(self) -> None:
|
|
tokens = {
|
|
"sel_compact": {
|
|
"kind": "edge",
|
|
"geometry": {
|
|
"center_mm": [2, 3, 4],
|
|
"curve_type": "line",
|
|
"length_mm": 12.5,
|
|
"adjacency_signature": "expensive-runtime-detail",
|
|
"start_mm": [0, 0, 0],
|
|
"end_mm": [5, 5, 5],
|
|
},
|
|
},
|
|
}
|
|
|
|
prompt_tokens = autonomous_candidate_prompt_tokens(tokens)
|
|
|
|
self.assertEqual(prompt_tokens[0]["geometry"], {
|
|
"center_mm": [2, 3, 4], "curve_type": "line", "length_mm": 12.5,
|
|
})
|
|
self.assertNotIn("adjacency_signature", json.dumps(prompt_tokens))
|
|
self.assertNotIn("start_mm", json.dumps(prompt_tokens))
|
|
self.assertNotIn("end_mm", json.dumps(prompt_tokens))
|
|
|
|
def test_forged_selector_token_is_rejected_before_candidate_staging(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
fragment = base_fragment()
|
|
fragment["feature"]["selector_tokens"] = ["sel_not_from_this_snapshot"]
|
|
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "TOPOLOGY_TOKEN_INVALID"):
|
|
materialize_autonomous_fragment(None, fragment, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
def test_hole_uses_server_injected_host_face_without_a_sketch(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
base_cdsl, _ = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
face_tokens = autonomous_selector_tokens({
|
|
"snapshot_id": "cad_test/rev_001",
|
|
"records": [{
|
|
"record_id": "face:top", "kind": "face", "feature_id": "feature_001",
|
|
"owner_feature_ids": ["feature_001"], "geometry": {"center_mm": [0, 0, 4], "normal": [0, 0, 1]},
|
|
"executable": True,
|
|
}],
|
|
})
|
|
token = next(iter(face_tokens))
|
|
fragment = {
|
|
"feature": {
|
|
"atomic_id": "hole_blind",
|
|
"params": {"diameter_mm": 4, "depth_mm": 6, "positions": [{"mm": [0, 0, 4]}]},
|
|
"selector_tokens": [token],
|
|
},
|
|
}
|
|
|
|
cdsl, _ = materialize_autonomous_fragment(base_cdsl, fragment, engine=engine, selector_tokens=face_tokens, max_features=1)
|
|
cdsl["part_id"] = "hole-contract-test"
|
|
|
|
validate_cdsl(cdsl, engine)
|
|
self.assertNotIn("sketch_id", cdsl["features"][-1])
|
|
self.assertIn("host_face", cdsl["features"][-1]["params"])
|
|
|
|
def test_single_feature_accepts_safe_top_level_selector_token_shorthand(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
base_cdsl, _ = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
face_tokens = autonomous_selector_tokens({
|
|
"snapshot_id": "cad_test/rev_001",
|
|
"records": [{
|
|
"record_id": "face:top", "kind": "face", "feature_id": "feature_001",
|
|
"owner_feature_ids": ["feature_001"], "geometry": {"center_mm": [0, 0, 4], "normal": [0, 0, 1]},
|
|
"executable": True,
|
|
}],
|
|
})
|
|
token = next(iter(face_tokens))
|
|
fragment = {
|
|
"feature": {
|
|
"atomic_id": "hole_blind",
|
|
"params": {"diameter_mm": 4, "depth_mm": 6, "positions": [{"mm": [0, 0, 4]}]},
|
|
},
|
|
"selector_tokens": [token],
|
|
}
|
|
|
|
cdsl, _ = materialize_autonomous_fragment(base_cdsl, fragment, engine=engine, selector_tokens=face_tokens, max_features=1)
|
|
|
|
self.assertIn("host_face", cdsl["features"][-1]["params"])
|
|
|
|
def test_transport_error_is_retryable_but_quota_error_is_not_transport(self) -> None:
|
|
self.assertTrue(_is_author_transport_error(RuntimeError("LLM connection failed after 3 attempts: timeout")))
|
|
self.assertFalse(_is_author_transport_error(RuntimeError("LLM request failed (429): quota exhausted")))
|
|
|
|
def test_hole_raw_host_face_is_rejected_with_actionable_guidance(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
engine = load_engine(current_settings)
|
|
invalid = {
|
|
"feature": {
|
|
"atomic_id": "hole_blind",
|
|
"params": {"diameter_mm": 4, "depth_mm": 6, "positions": [{"mm": [0, 0, 0]}], "host_face": "top"},
|
|
},
|
|
}
|
|
with self.assertRaisesRegex(AutonomousFragmentError, "selector_tokens"):
|
|
materialize_autonomous_fragment(None, invalid, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
def test_failed_candidate_build_never_creates_a_revision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "invalid candidate")
|
|
malformed = base_fragment()
|
|
malformed["feature"]["params"] = {}
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, malformed, engine=engine, selector_tokens={}, max_features=1)
|
|
|
|
with self.assertRaises(ValueError):
|
|
build_candidate(
|
|
settings=current_settings,
|
|
store=store,
|
|
task_id=str(task["task_id"]),
|
|
cdsl=cdsl,
|
|
fragment_audit=audit,
|
|
parent_revision_id="",
|
|
)
|
|
|
|
after = store.read_task(str(task["task_id"])) or {}
|
|
self.assertEqual(after["revisions"], [])
|
|
self.assertEqual(after["active_candidate_id"], "")
|
|
candidates = list(store.task_dir(str(task["task_id"])).glob("candidates/*/candidate.json"))
|
|
self.assertEqual(len(candidates), 1)
|
|
self.assertEqual(json.loads(candidates[0].read_text(encoding="utf-8"))["status"], "failed")
|
|
|
|
|
|
class AutonomousToolTests(unittest.TestCase):
|
|
def test_rejected_independent_review_keeps_checkpoint_unchanged_and_auditable(self) -> None:
|
|
async def exercise() -> tuple[dict, dict, dict, list[str], list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "reject the wrong base")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="reject the wrong base")
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.write_completion_checklist(task_id, completion_checklist())
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="reject the wrong base", conversation_id="conv_abcdef123456", provider=provider,
|
|
model=model, initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
rejected_review = {**accepted_plate_batch_review(), "verdict": "reject", "batch_goal_status": "failed", "evidence": ["The plate is visibly the wrong shape."]}
|
|
with patch("app.services.autonomous_cdsl_generation.render_checkpoint", return_value={"views": []}), patch(
|
|
"app.services.autonomous_cdsl_generation.review_candidate_batch",
|
|
AsyncMock(return_value=rejected_review),
|
|
):
|
|
events, progressed = await runner._execute_tool(
|
|
task_id, "reject the wrong base", state, load_engine(current_settings), "submit_cdsl_fragment",
|
|
{"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())},
|
|
)
|
|
candidates = list(store.task_dir(task_id).glob("candidates/*/candidate.json"))
|
|
tools = [item["function"]["name"] for item in runner._author_tools(
|
|
store.read_task(task_id) or {}, requirements_frozen=True, state=state,
|
|
)]
|
|
return store.read_task(task_id) or {}, json.loads(candidates[0].read_text(encoding="utf-8")), state, [name for name, _ in events], tools
|
|
|
|
task, candidate, state, events, tools = asyncio.run(exercise())
|
|
self.assertEqual(task["active_revision"], "")
|
|
self.assertEqual(task["active_candidate_id"], "")
|
|
self.assertEqual(candidate["status"], "rejected")
|
|
self.assertIn("candidate_review_path", candidate)
|
|
self.assertIn("candidate_review", events)
|
|
self.assertEqual(state["candidate_action_required"]["reason"], "candidate_review_rejected")
|
|
self.assertEqual(tools, ["submit_cdsl_fragment", "rollback_checkpoint"])
|
|
|
|
def test_reviewer_failure_is_reported_without_advancing_the_checkpoint(self) -> None:
|
|
async def exercise() -> tuple[dict, dict, list[tuple[str, dict]]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "review outage")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="review outage")
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.write_completion_checklist(task_id, completion_checklist())
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="review outage", conversation_id="conv_abcdef123456", provider=provider,
|
|
model=model, initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
with patch("app.services.autonomous_cdsl_generation.render_checkpoint", return_value={"views": []}), patch(
|
|
"app.services.autonomous_cdsl_generation.review_candidate_batch",
|
|
AsyncMock(side_effect=VisualReviewError("review endpoint timed out")),
|
|
):
|
|
events, _ = await runner._execute_tool(
|
|
task_id, "review outage", state, load_engine(current_settings), "submit_cdsl_fragment",
|
|
{"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())},
|
|
)
|
|
candidate = json.loads(next(store.task_dir(task_id).glob("candidates/*/candidate.json")).read_text(encoding="utf-8"))
|
|
return store.read_task(task_id) or {}, candidate, events
|
|
|
|
task, candidate, events = asyncio.run(exercise())
|
|
self.assertEqual(task["active_revision"], "")
|
|
self.assertEqual(task["active_candidate_id"], "")
|
|
self.assertEqual(candidate["status"], "review_failed")
|
|
self.assertIn("CANDIDATE_REVIEW_FAILED", events[-1][1]["message"])
|
|
|
|
def test_completion_audit_blocks_publication_until_current_checkpoint_is_verified(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, str | None]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "completion gate")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="completion gate")
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a checked plate.")
|
|
store.write_completion_checklist(task_id, "- [ ] checked plate")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.set_active_revision(task_id, "rev_001", branch_id="main")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="completion gate", conversation_id="conv_abcdef123456",
|
|
provider=current_settings.providers[0], model=current_settings.providers[0].models[0],
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=False,
|
|
)
|
|
events, _ = await runner._execute_tool(
|
|
task_id, "completion gate", state, load_engine(current_settings), "complete_task", {"self_review": "done"},
|
|
)
|
|
await runner._execute_tool(
|
|
task_id, "completion gate", state, load_engine(current_settings), "audit_completion_checklist",
|
|
{"markdown": "- [x] checked plate :: rebuilt bbox and single-solid report verified"},
|
|
)
|
|
loaded = store.read_task(task_id) or {}
|
|
return events, state, runner._completion_gate_error(task_id, loaded, state)
|
|
|
|
events, state, gate_error = asyncio.run(exercise())
|
|
self.assertIn("INDEPENDENT_REVIEW_REQUIRED", events[0][1]["message"])
|
|
self.assertIn("AUTHOR_SELF_REVIEW_DISABLED", (state.get("recent_events") or [])[-1]["message"])
|
|
self.assertIn("INDEPENDENT_REVIEW_REQUIRED", gate_error or "")
|
|
|
|
def test_author_completion_audit_is_disabled(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "incomplete completion gate")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a checked plate.")
|
|
store.write_completion_checklist(task_id, "- [ ] checked plate")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.set_active_revision(task_id, "rev_001", branch_id="main")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {"completion_checklist_written": True, "completion_ledger": {}}
|
|
|
|
events, progressed = await runner._execute_tool(
|
|
task_id,
|
|
"incomplete completion gate",
|
|
state,
|
|
load_engine(current_settings),
|
|
"audit_completion_checklist",
|
|
{"markdown": "- [?] checked plate :: edge relief is not evidenced"},
|
|
)
|
|
tools = runner._author_tools(store.read_task(task_id) or {}, requirements_frozen=True, state=state)
|
|
self.assertFalse(progressed)
|
|
return events, state, [item["function"]["name"] for item in tools]
|
|
|
|
events, state, tools = asyncio.run(exercise())
|
|
|
|
self.assertIn("AUTHOR_SELF_REVIEW_DISABLED", events[0][1]["message"])
|
|
self.assertEqual(state["completion_ledger"], {})
|
|
self.assertIn("inspect_topology", tools)
|
|
|
|
def test_checkpoint_invalidates_a_previous_completion_audit(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "stale completion")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.write_completion_checklist(task_id, "- [ ] checked plate")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {"completion_ledger": {"items": [{"item": "checked plate", "status": "complete", "evidence": "old", "updated_at": ""}], "verified_revision": "rev_001", "audited_at": "now"}}
|
|
|
|
runner._invalidate_completion_audit(task_id, state, reason="checkpoint_changed")
|
|
|
|
self.assertEqual(state["completion_ledger"]["verified_revision"], "")
|
|
self.assertEqual(state["completion_ledger"]["items"][0]["status"], "complete")
|
|
|
|
def test_only_the_newest_detailed_tool_result_is_repeated_to_the_author(self) -> None:
|
|
state = {
|
|
"recent_events": [
|
|
{"kind": "measure_model", "message": "old measurement", "result": {"records": ["x" * 1000]}},
|
|
{"kind": "inspect_topology", "message": "new topology", "result": {"tokens": ["sel_current"]}},
|
|
],
|
|
}
|
|
|
|
events = AutonomousCdslGenerationRunner._author_events(state)
|
|
|
|
self.assertNotIn("result", events[0])
|
|
self.assertEqual(events[0]["message"], "old measurement")
|
|
self.assertEqual(events[1]["result"], {"tokens": ["sel_current"]})
|
|
|
|
def test_operation_contract_turn_goes_straight_to_authoring(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
tools = runner._author_tools(
|
|
{"active_revision": "rev_001", "active_branch_id": "main"},
|
|
requirements_frozen=True,
|
|
state={"recent_events": [{"kind": "get_cdsl_operation_contract"}]},
|
|
)
|
|
|
|
self.assertEqual(
|
|
[tool["function"]["name"] for tool in tools],
|
|
["submit_cdsl_fragment", "rollback_checkpoint", "complete_task"],
|
|
)
|
|
def test_current_checkpoint_is_not_a_valid_rollback_target(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, dict]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "strict rollback")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="strict rollback")
|
|
store.write_requirements_document(task_id, "# Frozen\nRepair it.")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.set_active_revision(task_id, "rev_001", branch_id="main")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {"final_repair": {"working_head": "main:rev_001", "topology_observed": True}}
|
|
token = _checkpoint_token("main", "rev_001")
|
|
events, _ = await runner._execute_tool(
|
|
task_id, "strict rollback", state, load_engine(current_settings), "rollback_checkpoint",
|
|
{"checkpoint_token": token, "reason": "no-op rollback"},
|
|
)
|
|
return events, state, store.read_task(task_id) or {}
|
|
|
|
events, state, task = asyncio.run(exercise())
|
|
self.assertIn("strict ancestor", events[0][1]["message"])
|
|
self.assertEqual(task["active_revision"], "rev_001")
|
|
self.assertEqual(task["active_branch_id"], "main")
|
|
self.assertEqual(state["final_repair"]["working_head"], "main:rev_001")
|
|
|
|
def test_strict_rollback_preserves_final_repair_gate_on_the_new_branch(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, dict]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "rollback repair")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="rollback repair")
|
|
store.write_requirements_document(task_id, "# Frozen\nRepair it.")
|
|
store.update_task(task_id, {"revision_id": "rev_001", "status": "success", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.update_task(task_id, {"revision_id": "rev_002", "status": "success", "parent_revision_id": "rev_001", "visibility": "checkpoint", "branch_id": "main"})
|
|
store.set_active_revision(task_id, "rev_002", branch_id="main")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {"final_repair": {"working_head": "main:rev_002", "topology_observed": True, "evidence": ["missing bolt group"]}}
|
|
token = _checkpoint_token("main", "rev_001")
|
|
events, _ = await runner._execute_tool(
|
|
task_id, "rollback repair", state, load_engine(current_settings), "rollback_checkpoint",
|
|
{"checkpoint_token": token, "reason": "remove bad feature"},
|
|
)
|
|
return events, state, store.read_task(task_id) or {}
|
|
|
|
events, state, task = asyncio.run(exercise())
|
|
self.assertEqual(events[0][0], "rollback")
|
|
self.assertEqual(task["active_revision"], "rev_001")
|
|
self.assertTrue(task["active_branch_id"].startswith("branch_"))
|
|
self.assertEqual(state["final_repair"]["working_head"], f"{task['active_branch_id']}:rev_001")
|
|
self.assertFalse(state["final_repair"]["topology_observed"])
|
|
self.assertTrue(state["final_repair"]["evidence_stale_after_checkpoint"])
|
|
|
|
def test_candidate_state_exposes_a_single_forced_review_transition(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
tools = runner._author_tools(
|
|
{"active_revision": "rev_001", "active_candidate_id": "candidate_0123456789ab"},
|
|
requirements_frozen=True,
|
|
state={"last_review": {}},
|
|
)
|
|
self.assertEqual([item["function"]["name"] for item in tools], ["rollback_checkpoint"])
|
|
|
|
def test_staged_candidate_precedes_final_repair_and_format_correction(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
tools = runner._author_tools(
|
|
{"active_revision": "rev_001", "active_branch_id": "main", "active_candidate_id": "candidate_0123456789ab"},
|
|
requirements_frozen=True,
|
|
state={
|
|
"last_review": {},
|
|
"format_correction": {"working_head": "main:rev_001"},
|
|
"final_repair": {"working_head": "main:rev_001", "topology_observed": True},
|
|
},
|
|
)
|
|
self.assertEqual([item["function"]["name"] for item in tools], ["rollback_checkpoint"])
|
|
|
|
def test_unchanged_geometry_requires_bounded_diagnostics_then_conclusion(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_020", "active_branch_id": "branch_repair"}
|
|
state = {
|
|
"final_repair": {"working_head": "branch_repair:rev_020", "topology_observed": True},
|
|
"geometry_rejection": {"working_head": "branch_repair:rev_020", "geometry_fingerprint": "fingerprint", "conclusion_required": True},
|
|
"geometry_diagnoses_by_fingerprint": {"fingerprint": {"observations": {}, "conclusion": None}},
|
|
}
|
|
|
|
tools = runner._author_tools(task, requirements_frozen=True, state=state)
|
|
names = {item["function"]["name"] for item in tools}
|
|
|
|
self.assertNotIn("submit_cdsl_fragment", names)
|
|
self.assertIn("inspect_model", names)
|
|
self.assertIn("measure_model", names)
|
|
self.assertIn("render_views", names)
|
|
self.assertIn("record_geometry_conclusion", names)
|
|
self.assertNotIn("rollback_checkpoint", names)
|
|
|
|
state["geometry_diagnoses_by_fingerprint"]["fingerprint"]["observations"] = {
|
|
"inspect": {"ref": "diag_fingerprint_inspect"},
|
|
"measure": {"ref": "diag_fingerprint_measure"},
|
|
}
|
|
tools = runner._author_tools(task, requirements_frozen=True, state=state)
|
|
self.assertEqual(
|
|
[item["function"]["name"] for item in tools],
|
|
["render_views", "record_geometry_conclusion"],
|
|
)
|
|
|
|
state["geometry_diagnoses_by_fingerprint"]["fingerprint"]["conclusion"] = {"decision": "modify"}
|
|
self.assertEqual(
|
|
[item["function"]["name"] for item in runner._author_tools(task, requirements_frozen=True, state=state)],
|
|
["submit_cdsl_fragment", "rollback_checkpoint"],
|
|
)
|
|
|
|
def test_geometry_conclusion_requires_known_evidence_and_unlocks_only_its_decision(self) -> None:
|
|
async def exercise() -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "diagnose unchanged candidate")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen requirements")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {
|
|
"geometry_rejection": {"working_head": "main:root", "geometry_fingerprint": "root", "conclusion_required": True},
|
|
"geometry_diagnoses_by_fingerprint": {"root": {"observations": {"inspect": {"ref": "diag_root_inspect"}}, "conclusion": None}},
|
|
}
|
|
engine = load_engine(current_settings)
|
|
events, progressed = await runner._execute_tool(
|
|
task_id, "", state, engine, "record_geometry_conclusion",
|
|
{"root_cause": "duplicate_feature", "evidence_refs": ["not-real"], "decision": "rollback"},
|
|
)
|
|
self.assertFalse(progressed)
|
|
self.assertIn("EVIDENCE_UNKNOWN", events[0][1]["message"])
|
|
|
|
events, progressed = await runner._execute_tool(
|
|
task_id, "", state, engine, "record_geometry_conclusion",
|
|
{"root_cause": "duplicate_feature", "evidence_refs": ["diag_root_inspect"], "decision": "modify", "optimization_plan": {"action": "change the next feature", "reason": "the current geometry is unchanged"}},
|
|
)
|
|
self.assertTrue(progressed)
|
|
self.assertEqual(events[0][0], "geometry_conclusion")
|
|
tools = [item["function"]["name"] for item in runner._author_tools(store.read_task(task_id) or {}, requirements_frozen=True, state=state)]
|
|
self.assertEqual(tools, ["submit_cdsl_fragment", "rollback_checkpoint"])
|
|
|
|
asyncio.run(exercise())
|
|
|
|
def test_incomplete_current_audit_reopens_a_noop_complete_decision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_020", "active_branch_id": "main"}
|
|
state = {
|
|
"completion_checklist_required": False,
|
|
"completion_checklist_written": True,
|
|
"completion_ledger": {
|
|
"verified_revision": "rev_020",
|
|
"items": [{"item": "four mounting holes", "status": "missing", "evidence": "not modelled"}],
|
|
},
|
|
"geometry_rejection": {"working_head": "main:rev_020", "geometry_fingerprint": "fingerprint", "conclusion_required": False},
|
|
"geometry_diagnoses_by_fingerprint": {
|
|
"fingerprint": {
|
|
"observations": {"measure": {"ref": "diag_fingerprint_measure"}},
|
|
"conclusion": {"decision": "complete"},
|
|
},
|
|
},
|
|
}
|
|
|
|
names = [item["function"]["name"] for item in runner._author_tools(task, requirements_frozen=True, state=state)]
|
|
|
|
self.assertEqual(names, ["record_geometry_conclusion"])
|
|
self.assertIsNone(state["geometry_diagnoses_by_fingerprint"]["fingerprint"]["conclusion"])
|
|
self.assertTrue(state["geometry_rejection"]["conclusion_required"])
|
|
|
|
def test_modify_geometry_conclusion_accepts_next_action_without_reason(self) -> None:
|
|
async def exercise() -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "conclusion spelling")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = {
|
|
"geometry_rejection": {"working_head": "main:root", "geometry_fingerprint": "root"},
|
|
"geometry_diagnoses_by_fingerprint": {"root": {"observations": {"inspect": {"ref": "diag_root_inspect"}}, "conclusion": None}},
|
|
}
|
|
events, progressed = await runner._execute_tool(
|
|
task_id, "", state, load_engine(current_settings), "record_geometry_conclusion",
|
|
{"root_cause": "unknown", "evidence_refs": ["diag_root_inspect"], "decision": "modify", "optimization_plan": {"next_action": "use another small feature"}},
|
|
)
|
|
self.assertTrue(progressed)
|
|
self.assertEqual(events[0][0], "geometry_conclusion")
|
|
conclusion = state["geometry_diagnoses_by_fingerprint"]["root"]["conclusion"]
|
|
self.assertEqual(conclusion["optimization_plan"]["action"], "use another small feature")
|
|
|
|
asyncio.run(exercise())
|
|
|
|
def test_final_repair_suppresses_repeated_no_argument_observation_and_bounds_evidence_phase(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_020", "active_branch_id": "branch_repair"}
|
|
state = {
|
|
"final_repair": {"working_head": "branch_repair:rev_020", "topology_observed": True},
|
|
"completed_repair_observations_by_head": {"branch_repair:rev_020": ["inspect_model"]},
|
|
"repair_observation_counts_by_head": {"branch_repair:rev_020": 1},
|
|
}
|
|
|
|
names = {item["function"]["name"] for item in runner._author_tools(task, requirements_frozen=True, state=state)}
|
|
|
|
self.assertNotIn("inspect_model", names)
|
|
self.assertIn("read_cdsl_slice", names)
|
|
self.assertIn("submit_cdsl_fragment", names)
|
|
|
|
state["repair_observation_counts_by_head"]["branch_repair:rev_020"] = current_settings.agent_tool_calls_per_cycle
|
|
names = [item["function"]["name"] for item in runner._author_tools(task, requirements_frozen=True, state=state)]
|
|
self.assertEqual(names, ["get_cdsl_operation_contract", "submit_cdsl_fragment", "rollback_checkpoint"])
|
|
|
|
def test_duplicate_parameterized_repair_observation_requires_action(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_020", "active_branch_id": "branch_repair"}
|
|
state = {
|
|
"final_repair": {"working_head": "branch_repair:rev_020", "topology_observed": True},
|
|
"repair_observation_counts_by_head": {},
|
|
"repair_observation_keys_by_head": {},
|
|
}
|
|
arguments = {"feature_ids": ["feature_003"], "sketch_ids": ["sketch_002"]}
|
|
|
|
self.assertTrue(runner._record_repair_observation(task, state, "read_cdsl_slice", arguments))
|
|
self.assertFalse(runner._record_repair_observation(task, state, "read_cdsl_slice", arguments))
|
|
self.assertEqual(state["candidate_action_required"]["reason"], "duplicate_repair_observation")
|
|
|
|
names = [item["function"]["name"] for item in runner._author_tools(task, requirements_frozen=True, state=state)]
|
|
self.assertEqual(names, ["submit_cdsl_fragment", "rollback_checkpoint"])
|
|
|
|
def test_unchanged_candidate_attempts_are_not_compensated(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
state = {
|
|
"candidate_attempts_by_head": {"branch_repair:rev_020": 3},
|
|
"geometry_rejection": {
|
|
"working_head": "branch_repair:rev_020",
|
|
"repeat_count": 2,
|
|
},
|
|
}
|
|
self.assertEqual(state["candidate_attempts_by_head"]["branch_repair:rev_020"], 3)
|
|
|
|
def test_candidate_attempt_limit_is_not_a_format_retry(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_020", "active_branch_id": "branch_repair"}
|
|
state = {
|
|
"format_correction": {
|
|
"working_head": "branch_repair:rev_020",
|
|
"error": "CANDIDATE_ATTEMPT_LIMIT: head branch_repair:rev_020 exhausted 3 candidate build attempts",
|
|
},
|
|
}
|
|
|
|
tools = runner._author_tools(task, requirements_frozen=True, state=state)
|
|
names = {item["function"]["name"] for item in tools}
|
|
|
|
self.assertEqual(state["format_correction"], {})
|
|
self.assertEqual(state["candidate_action_required"]["reason"], "candidate_attempt_limit")
|
|
self.assertNotIn("submit_cdsl_fragment", names)
|
|
self.assertIn("rollback_checkpoint", names)
|
|
self.assertIn("complete_task", names)
|
|
|
|
def test_author_cannot_commit_repair_candidate(self) -> None:
|
|
async def exercise() -> tuple[dict, dict, list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "repair carry forward")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="repair carry forward")
|
|
store.write_requirements_document(task_id, "# Frozen\nRepair the model.")
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="repair carry forward", conversation_id="conv_abcdef123456",
|
|
provider=current_settings.providers[0], model=current_settings.providers[0].models[0],
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
await asyncio.to_thread(build_candidate, settings=current_settings, store=store, task_id=task_id, cdsl=cdsl, fragment_audit=audit, parent_revision_id="")
|
|
state["final_repair"] = {"working_head": "main:root", "topology_observed": True, "evidence": ["missing feature"]}
|
|
await runner._execute_tool(task_id, "repair carry forward", state, engine, "record_step_review", {"markdown": "Valid repair candidate.", "decision": "commit"})
|
|
loaded = store.read_task(task_id) or {}
|
|
tools = [item["function"]["name"] for item in runner._author_tools(loaded, requirements_frozen=True, state=state)]
|
|
return loaded, state, tools
|
|
|
|
task, state, tools = asyncio.run(exercise())
|
|
self.assertEqual(task["active_revision"], "")
|
|
self.assertEqual(state["final_repair"]["working_head"], "main:root")
|
|
self.assertTrue(state["final_repair"]["topology_observed"])
|
|
self.assertEqual(tools, ["rollback_checkpoint"])
|
|
|
|
def test_pending_candidate_submission_is_not_treated_as_a_format_loop(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], bool, dict, list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "pending candidate")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nReview the staged candidate.")
|
|
store.new_candidate(task_id)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="pending candidate", conversation_id="conv_abcdef123456",
|
|
provider=current_settings.providers[0], model=current_settings.providers[0].models[0],
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
events, progressed = await runner._execute_tool(
|
|
task_id, "pending candidate", state, load_engine(current_settings), "submit_cdsl_fragment",
|
|
{"fragment_json": json.dumps(base_fragment())},
|
|
)
|
|
loaded = store.read_task(task_id) or {}
|
|
tools = [item["function"]["name"] for item in runner._author_tools(loaded, requirements_frozen=True, state=state)]
|
|
return events, progressed, state, tools
|
|
|
|
events, progressed, state, tools = asyncio.run(exercise())
|
|
self.assertFalse(progressed)
|
|
self.assertEqual(events[0][0], "candidate_result")
|
|
self.assertEqual(state["format_correction"], {})
|
|
self.assertTrue(state["candidate_action_required"])
|
|
self.assertEqual(tools, ["rollback_checkpoint"])
|
|
|
|
def test_operation_contract_explains_countersunk_face_and_position_protocol(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
contract = _operation_contract_payload(load_engine(current_settings), "hole_countersink")
|
|
|
|
self.assertEqual(
|
|
contract["author_required_params"],
|
|
["diameter_mm", "depth_mm", "positions", "countersink_diameter_mm", "countersink_angle_rad"],
|
|
)
|
|
self.assertEqual(contract["server_injected_params"], ["host_face"])
|
|
self.assertEqual(contract["selector_rule"], {"kind": "face", "min_items": 1, "max_items": 1})
|
|
self.assertIn('positions is a non-empty array of objects', contract["position_format"])
|
|
self.assertIn("server_injected_params", contract["authoring_rule"])
|
|
|
|
def test_nonstandard_provider_tool_objects_are_recovered_without_evaluation(self) -> None:
|
|
self.assertEqual(parse_tool_arguments('{kind: "face", limit: 12}'), ({"kind": "face", "limit": 12}, True))
|
|
self.assertEqual(parse_tool_arguments("{'kind': 'edge', 'limit': 3}"), ({"kind": "edge", "limit": 3}, True))
|
|
self.assertEqual(parse_tool_arguments('{"kind":"face"}'), ({"kind": "face"}, False))
|
|
with self.assertRaisesRegex(ValueError, "Expecting property name"):
|
|
parse_tool_arguments('{kind: call_that_must_not_run()}')
|
|
|
|
def test_only_explicit_quota_exhaustion_is_eligible_for_author_failover(self) -> None:
|
|
self.assertTrue(_is_author_quota_error(RuntimeError("LLM request failed (429): API_KEY_QUOTA_EXHAUSTED")))
|
|
self.assertFalse(_is_author_quota_error(RuntimeError("LLM request failed (429): rate limit exceeded")))
|
|
self.assertFalse(_is_author_quota_error(RuntimeError("LLM request failed (500): quota backend unavailable")))
|
|
|
|
def test_authoring_tools_do_not_request_strict_provider_schema(self) -> None:
|
|
tools = autonomous_tools()
|
|
submit = next(tool for tool in tools if tool["function"]["name"] == "submit_cdsl_fragment")
|
|
self.assertEqual(submit["function"]["parameters"]["properties"]["fragment_json"]["type"], "string")
|
|
self.assertFalse(any(tool["function"].get("strict") for tool in tools))
|
|
self.assertEqual(tools[0]["function"]["name"], "write_requirements_document")
|
|
|
|
def test_frozen_requirements_writer_is_removed_from_follow_up_author_tools(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
|
|
first_turn = {item["function"]["name"] for item in runner._author_tools({}, requirements_frozen=False)}
|
|
root_follow_up = {item["function"]["name"] for item in runner._author_tools({}, requirements_frozen=True)}
|
|
checkpoint_follow_up = {
|
|
item["function"]["name"]
|
|
for item in runner._author_tools(
|
|
{"active_revision": "rev_001"},
|
|
requirements_frozen=True,
|
|
state={"recent_events": [{"kind": "checkpoint"}]},
|
|
)
|
|
}
|
|
|
|
self.assertIn("write_requirements_document", first_turn)
|
|
self.assertEqual(root_follow_up, {"submit_cdsl_fragment"})
|
|
self.assertNotIn("write_requirements_document", checkpoint_follow_up)
|
|
self.assertEqual(checkpoint_follow_up, {"inspect_topology", "complete_task"})
|
|
self.assertNotIn("inspect_model", checkpoint_follow_up)
|
|
|
|
def test_topology_observation_allows_the_next_fragment_and_renders(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
names = {
|
|
item["function"]["name"]
|
|
for item in runner._author_tools(
|
|
{"active_revision": "rev_001"},
|
|
requirements_frozen=True,
|
|
state={"recent_events": [{"kind": "inspect_topology"}]},
|
|
)
|
|
}
|
|
|
|
self.assertIn("submit_cdsl_fragment", names)
|
|
self.assertIn("render_views", names)
|
|
self.assertNotIn("inspect_model", names)
|
|
|
|
def test_staged_candidate_forces_review_then_its_decision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
runner = AutonomousCdslGenerationRunner(current_settings, WorkspaceStore(current_settings), AsyncMock())
|
|
task = {"active_revision": "rev_001", "active_candidate_id": "candidate_0123456789ab"}
|
|
|
|
needs_review = runner._author_tools(task, requirements_frozen=True, state={"last_review": {}})
|
|
commit = runner._author_tools(
|
|
task,
|
|
requirements_frozen=True,
|
|
state={"last_review": {"candidate_id": "candidate_0123456789ab", "decision": "commit"}},
|
|
)
|
|
discard = runner._author_tools(
|
|
task,
|
|
requirements_frozen=True,
|
|
state={"last_review": {"candidate_id": "candidate_0123456789ab", "decision": "discard"}},
|
|
)
|
|
|
|
self.assertEqual({item["function"]["name"] for item in needs_review}, {"rollback_checkpoint"})
|
|
self.assertEqual({item["function"]["name"] for item in commit}, {"rollback_checkpoint"})
|
|
self.assertEqual({item["function"]["name"] for item in discard}, {"rollback_checkpoint"})
|
|
|
|
def test_empty_head_prompt_contains_a_format_reference_for_the_first_solid(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "root reference")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a 40 by 20 by 8 mm plate.")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="root reference", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
|
|
prompt = str(runner._prompt_messages(task_id, state, load_engine(current_settings))[-1]["content"])
|
|
|
|
self.assertIn("root_fragment_reference", prompt)
|
|
self.assertIn("extrude_add_blind", prompt)
|
|
self.assertIn("polygon", prompt)
|
|
self.assertIn("cdsl_authoring_basics", prompt)
|
|
self.assertIn("get_cdsl_operation_contract", prompt)
|
|
self.assertNotIn("sketch_required_features", prompt)
|
|
self.assertNotIn("constant_thickness_bend_profiles", prompt)
|
|
|
|
def test_schema_preflight_does_not_consume_an_engine_candidate_attempt(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, list[Path]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "schema preflight")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.start_generation(task_id, request="schema preflight")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="schema preflight", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
invalid = {
|
|
"sketch": {
|
|
"workplane": "XY",
|
|
"profile": {"type": "circle", "radius_mm": 5},
|
|
},
|
|
"feature": {"atomic_id": "extrude_add_blind", "params": {"distance_mm": 4}},
|
|
}
|
|
events, _ = await runner._execute_tool(
|
|
task_id, "schema preflight", state, load_engine(current_settings), "submit_cdsl_fragment",
|
|
{"batch_goal": "Create the initial circular plate.", "fragment_json": json.dumps(invalid)},
|
|
)
|
|
candidates = list(store.task_dir(task_id).glob("candidates/*/candidate.json"))
|
|
return events, state, candidates
|
|
|
|
events, state, candidates = asyncio.run(exercise())
|
|
self.assertEqual(state["candidate_attempts_by_head"], {})
|
|
self.assertEqual(candidates, [])
|
|
self.assertIn("workplane", events[0][1]["message"])
|
|
|
|
def test_author_step_review_cannot_commit_a_staged_candidate(self) -> None:
|
|
async def exercise() -> tuple[bool, list[str], dict]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "candidate progress")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.start_generation(task_id, request="candidate progress")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="candidate progress", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=6)
|
|
await asyncio.to_thread(
|
|
build_candidate,
|
|
settings=current_settings,
|
|
store=store,
|
|
task_id=task_id,
|
|
cdsl=cdsl,
|
|
fragment_audit=audit,
|
|
parent_revision_id="",
|
|
)
|
|
review_events, reviewed = await runner._execute_tool(
|
|
task_id, "candidate progress", state, engine, "record_step_review",
|
|
{"markdown": "Candidate is valid.", "decision": "commit"},
|
|
)
|
|
return reviewed, [name for name, _ in review_events], store.read_task(task_id) or {}
|
|
|
|
reviewed, event_names, task = asyncio.run(exercise())
|
|
self.assertFalse(reviewed)
|
|
self.assertEqual(event_names, ["tool_call"])
|
|
self.assertTrue(task["active_candidate_id"])
|
|
self.assertEqual(task["active_revision"], "")
|
|
|
|
def test_context_budget_never_truncates_frozen_requirements(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
current_settings = Settings(**{**current_settings.__dict__, "agent_context_char_limit": 4000})
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "context")
|
|
task_id = str(task["task_id"])
|
|
requirements = "# Frozen\n" + ("exact engineering requirement\n" * 500)
|
|
store.write_requirements_document(task_id, requirements)
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="context", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
state["recent_events"] = [{"kind": "large", "message": "x" * 20000}]
|
|
messages = runner._prompt_messages(task_id, state, load_engine(current_settings))
|
|
prompt = str(messages[-1]["content"])
|
|
|
|
self.assertIn("exact engineering requirement", prompt)
|
|
self.assertIn("normal cross x_dir", prompt)
|
|
self.assertIn("get_cdsl_operation_contract for that exact atomic_id", prompt)
|
|
self.assertNotIn("context truncated", prompt)
|
|
|
|
def test_text_only_author_never_receives_render_image_content(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "text-only")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
image_path = Path(temporary) / "render.png"
|
|
image_path.write_bytes(b"not-an-image-needed-for-this-routing-test")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="text-only", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
state["pending_images"] = [str(image_path)]
|
|
|
|
messages = runner._prompt_messages(task_id, state, load_engine(current_settings))
|
|
|
|
self.assertFalse(any(
|
|
isinstance(message.get("content"), list)
|
|
and any(isinstance(part, dict) and part.get("type") == "image_url" for part in message["content"])
|
|
for message in messages
|
|
))
|
|
self.assertIn("text-only", str(messages[-1]["content"]))
|
|
|
|
def test_successful_candidate_cannot_be_discarded_without_a_step_review(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], bool]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "discard review")
|
|
task_id = str(task["task_id"])
|
|
store.write_requirements_document(task_id, "# Frozen\nBuild a plate.")
|
|
store.start_generation(task_id, request="discard review")
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
await asyncio.to_thread(
|
|
build_candidate,
|
|
settings=current_settings,
|
|
store=store,
|
|
task_id=task_id,
|
|
cdsl=cdsl,
|
|
fragment_audit=audit,
|
|
parent_revision_id="",
|
|
)
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="discard review", conversation_id="conv_abcdef123456", provider=provider, model=model,
|
|
initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
events, progressed = await runner._execute_tool(task_id, "discard review", state, engine, "discard_candidate", {"reason": "retry"})
|
|
return events, progressed
|
|
|
|
events, progressed = asyncio.run(exercise())
|
|
self.assertFalse(progressed)
|
|
self.assertEqual(events[0][0], "tool_call")
|
|
self.assertIn("AUTHOR_SELF_REVIEW_DISABLED", events[0][1]["message"])
|
|
|
|
def test_runner_stages_then_commits_before_final_publication(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, list[set[str]]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "make a plate")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
calls = iter([
|
|
("write_requirements_document", {"markdown": "# Plate\nUse a 20 x 10 x 4 mm rectangular plate."}),
|
|
("write_completion_checklist", {"markdown": completion_checklist()}),
|
|
("submit_cdsl_fragment", {"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())}),
|
|
("complete_task", {"self_review": "All frozen requirements are satisfied."}),
|
|
])
|
|
author_tool_names: list[set[str]] = []
|
|
|
|
async def complete(_messages: list[dict], _tools: list[dict], _provider: ProviderConfig, _model: ProviderModel, _forced: str | None) -> dict:
|
|
author_tool_names.append({item["function"]["name"] for item in _tools})
|
|
name, arguments = next(calls)
|
|
return {"choices": [{"message": {"tool_calls": [{"id": name, "function": {"name": name, "arguments": json.dumps(arguments)}}]}}]}
|
|
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, complete)
|
|
runner._final_review = AsyncMock(return_value=({"verdict": "pass", "confidence": 1, "evidence": []}, {"solid_count": 1})) # type: ignore[method-assign]
|
|
with patch("app.services.autonomous_cdsl_generation.render_checkpoint", return_value={"views": []}), patch(
|
|
"app.services.autonomous_cdsl_generation.review_candidate_batch",
|
|
AsyncMock(return_value=accepted_plate_batch_review()),
|
|
):
|
|
events = [item async for item in runner.run(
|
|
task_id=str(task["task_id"]), request="make a plate", conversation_id="conv_abcdef123456",
|
|
provider=provider, model=model, initial_messages=[{"role": "user", "content": "make a plate"}],
|
|
)]
|
|
return events, store.read_task(str(task["task_id"])) or {}, author_tool_names
|
|
|
|
events, task, author_tool_names = asyncio.run(exercise())
|
|
names = [name for name, _ in events]
|
|
self.assertIn("candidate_result", names)
|
|
self.assertIn("checkpoint", names)
|
|
self.assertEqual(events[-1][0], "task_terminal")
|
|
self.assertEqual(task["lifecycle"], "completed")
|
|
self.assertTrue(task["published_revision"])
|
|
self.assertTrue(task["revisions"][0]["candidate_review_path"].startswith("revisions/rev_001/reviews/"))
|
|
self.assertIn("write_requirements_document", author_tool_names[0])
|
|
self.assertEqual(author_tool_names[1], {"write_completion_checklist"})
|
|
self.assertEqual(author_tool_names[2], {"submit_cdsl_fragment"})
|
|
self.assertTrue(all("write_requirements_document" not in names for names in author_tool_names[1:]))
|
|
|
|
def test_final_warning_keeps_task_running_and_returns_to_author(self) -> None:
|
|
async def exercise() -> tuple[list[tuple[str, dict]], dict, dict, list[str], list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "final warning")
|
|
task_id = str(task["task_id"])
|
|
store.start_generation(task_id, request="final warning")
|
|
store.write_requirements_document(task_id, "# Frozen\nA complete plate.")
|
|
store.write_completion_checklist(task_id, "- [ ] complete plate")
|
|
engine = load_engine(current_settings)
|
|
cdsl, audit = materialize_autonomous_fragment(None, base_fragment(), engine=engine, selector_tokens={}, max_features=1)
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, AsyncMock())
|
|
state = runner._state(
|
|
task_id, request="final warning", conversation_id="conv_abcdef123456", provider=current_settings.providers[0],
|
|
model=current_settings.providers[0].models[0], initial_messages=[], frozen_attachment_ids=[], fresh=True,
|
|
)
|
|
candidate = await asyncio.to_thread(build_candidate, settings=current_settings, store=store, task_id=task_id, cdsl=cdsl, fragment_audit=audit, parent_revision_id="")
|
|
candidate_id = str(candidate["candidate_id"])
|
|
review_path = store.candidate_dir(task_id, candidate_id) / "reviews" / "candidate-review" / "candidate-review.json"
|
|
candidate_review = {
|
|
"verdict": "accept", "batch_goal_status": "achieved", "coverage": [{
|
|
"item": "complete plate", "status": "complete", "evidence": "single solid rebuilt",
|
|
}], "evidence": ["base plate accepted"],
|
|
}
|
|
write_json(review_path, candidate_review)
|
|
candidate_data = json.loads((store.candidate_dir(task_id, candidate_id) / "candidate.json").read_text(encoding="utf-8"))
|
|
candidate_data["candidate_review_path"] = f"candidates/{candidate_id}/reviews/candidate-review/candidate-review.json"
|
|
write_json(store.candidate_dir(task_id, candidate_id) / "candidate.json", candidate_data)
|
|
await runner._commit_reviewed_candidate(task_id, task, state, candidate_id, candidate_review=candidate_review)
|
|
runner._final_review = AsyncMock(return_value=({"verdict": "warning", "confidence": 0.1, "evidence": ["missing chamfer"]}, {"solid_count": 1})) # type: ignore[method-assign]
|
|
events, progressed = await runner._execute_tool(task_id, "final warning", state, engine, "complete_task", {"self_review": "done"})
|
|
loaded = store.read_task(task_id) or {}
|
|
first_tools = [item["function"]["name"] for item in runner._author_tools(loaded, requirements_frozen=True, state=state)]
|
|
state["final_repair"]["topology_observed"] = True
|
|
follow_up_tools = [item["function"]["name"] for item in runner._author_tools(loaded, requirements_frozen=True, state=state)]
|
|
return events, loaded, state, first_tools, follow_up_tools
|
|
|
|
events, task, state, first_tools, follow_up_tools = asyncio.run(exercise())
|
|
self.assertFalse(any(name == "task_terminal" for name, _ in events))
|
|
self.assertTrue(any(name == "agent_thinking" for name, _ in events))
|
|
self.assertEqual(task["lifecycle"], "running")
|
|
self.assertEqual(task["published_revision"], "")
|
|
self.assertEqual(state["final_repair"]["review_verdict"], "warning")
|
|
self.assertEqual(first_tools, ["inspect_topology", "rollback_checkpoint"])
|
|
self.assertEqual(
|
|
follow_up_tools,
|
|
[
|
|
"inspect_model", "read_cdsl_slice", "measure_model", "render_views", "render_section",
|
|
"get_cdsl_operation_contract", "submit_cdsl_fragment", "rollback_checkpoint",
|
|
],
|
|
)
|
|
|
|
def test_runner_defers_extra_calls_until_the_next_stateful_turn(self) -> None:
|
|
async def exercise() -> tuple[dict, list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "defer calls")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
responses = iter([
|
|
[
|
|
("write_requirements_document", {"markdown": "# Plate\nUse a 20 x 10 x 4 mm rectangular plate."}),
|
|
("submit_cdsl_fragment", {"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())}),
|
|
],
|
|
[("write_completion_checklist", {"markdown": completion_checklist()})],
|
|
[("submit_cdsl_fragment", {"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())})],
|
|
[("complete_task", {"self_review": "Complete."})],
|
|
])
|
|
|
|
async def complete(_messages: list[dict], _tools: list[dict], _provider: ProviderConfig, _model: ProviderModel, _forced: str | None) -> dict:
|
|
calls = [
|
|
{"id": name, "function": {"name": name, "arguments": json.dumps(arguments)}}
|
|
for name, arguments in next(responses)
|
|
]
|
|
return {"choices": [{"message": {"tool_calls": calls}}]}
|
|
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, complete)
|
|
runner._final_review = AsyncMock(return_value=({"verdict": "pass", "confidence": 1, "evidence": []}, {"solid_count": 1})) # type: ignore[method-assign]
|
|
with patch("app.services.autonomous_cdsl_generation.render_checkpoint", return_value={"views": []}), patch(
|
|
"app.services.autonomous_cdsl_generation.review_candidate_batch",
|
|
AsyncMock(return_value=accepted_plate_batch_review()),
|
|
):
|
|
events = [item async for item in runner.run(
|
|
task_id=str(task["task_id"]), request="defer calls", conversation_id="conv_abcdef123456",
|
|
provider=provider, model=model, initial_messages=[],
|
|
)]
|
|
return store.read_agent_state(str(task["task_id"])) or {}, [name for name, _ in events]
|
|
|
|
state, event_names = asyncio.run(exercise())
|
|
self.assertIn("extra_tool_calls_deferred", [item["kind"] for item in state["recent_events"]])
|
|
self.assertEqual(event_names.count("candidate_result"), 1)
|
|
self.assertEqual(event_names[-1], "task_terminal")
|
|
|
|
def test_staged_candidate_rejects_an_unexposed_observation_tool(self) -> None:
|
|
async def exercise() -> tuple[dict, list[str]]:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
current_settings = settings(Path(temporary))
|
|
store = WorkspaceStore(current_settings)
|
|
task = store.ensure_task(None, "candidate state")
|
|
provider, model = current_settings.resolve_model("test", "test-model")
|
|
responses = iter([
|
|
[("write_requirements_document", {"markdown": "# Plate"})],
|
|
[("write_completion_checklist", {"markdown": completion_checklist()})],
|
|
[("submit_cdsl_fragment", {"batch_goal": "Create the rectangular base plate.", "fragment_json": json.dumps(base_fragment())})],
|
|
[("inspect_model", {})],
|
|
[("complete_task", {"self_review": "done"})],
|
|
])
|
|
|
|
async def complete(_messages: list[dict], _tools: list[dict], _provider: ProviderConfig, _model: ProviderModel, _forced: str | None) -> dict:
|
|
calls = [{"id": name, "function": {"name": name, "arguments": json.dumps(arguments)}} for name, arguments in next(responses)]
|
|
return {"choices": [{"message": {"tool_calls": calls}}]}
|
|
|
|
runner = AutonomousCdslGenerationRunner(current_settings, store, complete)
|
|
runner._final_review = AsyncMock(return_value=({"verdict": "pass", "confidence": 1, "evidence": []}, {"solid_count": 1})) # type: ignore[method-assign]
|
|
with patch("app.services.autonomous_cdsl_generation.render_checkpoint", return_value={"views": []}), patch(
|
|
"app.services.autonomous_cdsl_generation.review_candidate_batch",
|
|
AsyncMock(return_value=accepted_plate_batch_review()),
|
|
):
|
|
events = [item async for item in runner.run(
|
|
task_id=str(task["task_id"]), request="candidate state", conversation_id="conv_abcdef123456",
|
|
provider=provider, model=model, initial_messages=[],
|
|
)]
|
|
return store.read_agent_state(str(task["task_id"])) or {}, [name for name, _ in events]
|
|
|
|
state, event_names = asyncio.run(exercise())
|
|
self.assertIn("tool_error", [item["kind"] for item in state["recent_events"]])
|
|
self.assertEqual(event_names.count("checkpoint"), 1)
|
|
self.assertEqual(event_names[-1], "task_terminal")
|