likang/cdsl-cad
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change

Browse Source
This commit is contained in:
李康
2026-09-04 11:16:23 +08:00
parent 9e27f11bb7
commit 921ba5eab3
36 changed files with 2665 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_author_guidance.py
+158
View File
@@ -0,0 +1,158 @@
from __future__ import annotations
import asyncio
import json
from pathlib import Path
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "backend"))
from app.cad_agent.adapters.author_guidance import FileAuthorGuidance # noqa: E402
from app.cad_agent.application.workflow import ModelIdentity, WorkflowConfig, WorkflowCoordinator # noqa: E402
from app.cad_agent.domain.errors import ErrorCode, WorkflowError # noqa: E402
from app.cad_agent.domain.state import TaskPhase, TaskState # noqa: E402
GUIDANCE_ROOT = ROOT / "backend" / "agent" / "skills" / "cdsl-author-guidance"
PROFILE = ROOT / "backend" / "engine" / "cdsl_engine" / "profile_schema.json"
def atomic_ids() -> tuple[str, ...]:
return tuple(json.loads(PROFILE.read_text(encoding="utf-8"))["operation_contracts"])
class _Repository:
def __init__(self, state: TaskState) -> None:
self.state = state
self.usage_records: list[dict] = []
def get_state(self, _task_id: str) -> TaskState:
return self.state
def ledger_events(self, _task_id: str) -> list[dict]:
return []
def record_usage(self, _task_id: str, payload: dict) -> None:
self.usage_records.append(payload)
def record_tool_audit(self, _task_id: str, _payload: dict) -> None:
pass
class _Artifacts:
def read_source_requirements(self, _task_id: str) -> str:
return "Create a symmetric mounting plate."
def read_json(self, *_args: object) -> None:
return None
class _Runtime:
def supported_atomic_ids(self) -> tuple[str, ...]:
return atomic_ids()
class AuthorGuidanceTests(unittest.TestCase):
def test_manifest_covers_every_runtime_atomic_and_keeps_coordinate_core_at_minimum_budget(self) -> None:
guidance = FileAuthorGuidance(GUIDANCE_ROOT, max_chars=1_200)
covered: set[str] = set()
for atomic_id in atomic_ids():
selection = guidance.select(
phase=TaskPhase.FEATURE_PENDING,
atomic_id=atomic_id,
repair_required=False,
supported_atomic_ids=atomic_ids(),
)
self.assertTrue(selection.enabled, selection.fallback_reason)
self.assertIn("00-author-contract", selection.section_ids)
self.assertIn("03-coordinate-system-and-datums", selection.section_ids)
self.assertLessEqual(len(selection.content), 1_200)
self.assertIn("世界坐标", selection.content)
covered.update(section_id for section_id in selection.section_ids if section_id.startswith("op-"))
self.assertEqual(covered, {"op-extrude-add", "op-extrude-cut", "op-revolve", "op-hole", "op-reference", "op-pattern", "op-finish", "op-sphere"})
def test_phase_repair_and_budget_selection_are_stable(self) -> None:
guidance = FileAuthorGuidance(GUIDANCE_ROOT, max_chars=3_600)
planning = guidance.select(
phase=TaskPhase.COMPILING_FEATURE_PLAN,
atomic_id="",
repair_required=False,
supported_atomic_ids=atomic_ids(),
)
repair = guidance.select(
phase=TaskPhase.AWAITING_ACTION,
atomic_id="fillet",
repair_required=True,
supported_atomic_ids=atomic_ids(),
)
self.assertEqual(planning.section_ids[:2], ("00-author-contract", "03-coordinate-system-and-datums"))
self.assertIn("02-parameters-and-derived-dimensions", planning.section_ids)
self.assertEqual(repair.section_ids[:3], ("00-author-contract", "03-coordinate-system-and-datums", "op-finish"))
self.assertIn("10-repair-and-best-effort", repair.section_ids)
def test_disabled_missing_and_invalid_corpus_fall_back_without_authoring_failure(self) -> None:
common = {
"phase": TaskPhase.FEATURE_PENDING,
"atomic_id": "extrude_add_blind",
"repair_required": False,
"supported_atomic_ids": atomic_ids(),
}
self.assertEqual(FileAuthorGuidance(GUIDANCE_ROOT, enabled=False).select(**common).fallback_reason, "guidance_disabled")
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
self.assertEqual(FileAuthorGuidance(root).select(**common).fallback_reason, "guidance_load_failed:FileNotFoundError")
(root / "manifest.json").write_text("{}", encoding="utf-8")
self.assertEqual(FileAuthorGuidance(root).select(**common).fallback_reason, "guidance_load_failed:ValueError")
def test_author_context_receives_guidance_but_keeps_the_existing_tool_instruction(self) -> None:
state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS_DOCUMENT, 1)
workflow = WorkflowCoordinator(
WorkflowConfig(max_turns=8, format_error_limit=2),
_Repository(state),
_Artifacts(),
_Runtime(),
object(),
object(),
object(),
object(),
FileAuthorGuidance(GUIDANCE_ROOT),
)
messages, selection = workflow._author_context(state.task_id, [])
system = str(messages[0]["content"])
self.assertTrue(selection.enabled)
self.assertIn("Use exactly one offered structured tool call", system)
self.assertIn("Coordinate System And Datums", system)
self.assertIn("世界坐标", system)
def test_invalid_author_tool_call_retains_guidance_usage_metadata(self) -> None:
state = TaskState("cad_123456abcdef", TaskPhase.DRAFTING_REQUIREMENTS_DOCUMENT, 1)
repository = _Repository(state)
class _Models:
async def call_tool(self, **_kwargs: object) -> dict:
return {"tool_calls": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}}
workflow = WorkflowCoordinator(
WorkflowConfig(max_turns=8, format_error_limit=2),
repository,
_Artifacts(),
_Runtime(),
_Models(),
object(),
object(),
object(),
FileAuthorGuidance(GUIDANCE_ROOT),
)
tool = {"type": "function", "function": {"name": "write_requirements_document", "parameters": {"type": "object"}}}
result = asyncio.run(workflow._author_turn(state.task_id, ModelIdentity("provider", "model"), [tool], []))
self.assertIsInstance(result, WorkflowError)
self.assertEqual(result.code, ErrorCode.AUTHOR_FORMAT_INVALID)
self.assertEqual(repository.usage_records[0]["guidance_enabled"], True)
self.assertIn("03-coordinate-system-and-datums", repository.usage_records[0]["guidance_section_ids"])
self.assertEqual(repository.usage_records[0]["retry_reason"], "invalid_tool_call")
if __name__ == "__main__":
unittest.main()
backend/tests/test_live_guidance_comparison.py
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
from pathlib import Path
import sys
import unittest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "backend"))
from app.cad_agent.evals.live import _fixture, compare_guidance_reports # noqa: E402
def _result(*, scenario: str, repetition: int, author_calls: int, context_chars: int, schema_rejections: int, failure_layer: str = "") -> dict:
return {
"scenario": scenario,
"repetition": repetition,
"outcome": "passed",
"revision_ids": ["revision_001"],
"projection": {"phase": "COMPLETED"},
"checks": {"deterministic_claims_pass": True},
"schema_rejection_count": schema_rejections,
"failure_attribution": {"layer": failure_layer} if failure_layer else None,
"scenario_budget": {"max_author_turns": 8, "max_reviewer_turns": 2, "max_total_calls": 10, "max_total_tokens": 1000},
"usage": {"records": [{"context_chars": context_chars} for _ in range(author_calls)]},
}
def _report(results: list[dict]) -> dict:
return {
"author": {"provider": "author", "model": "model"},
"reviewer": {"provider": "reviewer", "model": "review"},
"runtime_profile_sha256": "a" * 64,
"operation_contracts": [{"atomic_id": "extrude_add_blind", "contract_hash": "b" * 64}],
"author_guidance": {"enabled": False, "max_chars": 3600},
"results": results,
}
class LiveGuidanceComparisonTests(unittest.TestCase):
def test_fixture_accepts_multiple_stable_scenarios_in_fixture_order(self) -> None:
selected = _fixture("comprehensive", ["l_bracket", "circular_flange_pcd"])
self.assertEqual([item["id"] for item in selected], ["circular_flange_pcd", "l_bracket"])
def test_comparison_enforces_matched_budget_and_quality_gates(self) -> None:
control = _report([
_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=2, failure_layer="cdsl_expression"),
_result(scenario="part_a", repetition=2, author_calls=10, context_chars=1000, schema_rejections=1),
])
treatment = _report([
_result(scenario="part_a", repetition=1, author_calls=11, context_chars=1300, schema_rejections=0),
_result(scenario="part_a", repetition=2, author_calls=11, context_chars=1300, schema_rejections=0),
])
treatment["author_guidance"]["enabled"] = True
comparison = compare_guidance_reports(control, treatment)
self.assertEqual(comparison["status"], "passed")
self.assertTrue(comparison["gates"]["median_author_calls_within_ten_percent"])
self.assertTrue(comparison["gates"]["model_or_cdsl_failure_improved"])
def test_comparison_excludes_explicit_unsupported_runtime_capability(self) -> None:
control = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=1)])
treatment = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1200, schema_rejections=0)])
treatment["author_guidance"]["enabled"] = True
treatment["results"][0]["ledger"] = [{"operation_failures": [{"message": "unsupported_draft"}]}]
comparison = compare_guidance_reports(control, treatment)
self.assertEqual(comparison["treatment"]["eligible_runs"], 0)
self.assertEqual(comparison["excluded_capability_gaps"]["treatment"], [{"scenario": "part_a", "repetition": 1}])
if __name__ == "__main__":
unittest.main()
backend/tests/test_structured_llm.py
+123
View File
@@ -0,0 +1,123 @@
from __future__ import annotations
import asyncio
from pathlib import Path
import sys
import unittest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "backend"))
from app.cad_agent.adapters.structured_llm import StructuredModelError, StructuredModelGateway # noqa: E402
from app.settings import ProviderConfig, ProviderModel, Settings # noqa: E402
def _settings(*, api_style: str = "responses", reasoning_effort: str = "medium") -> Settings:
provider = ProviderConfig(
"provider", "Provider", "https://example.invalid/v1", "key",
(ProviderModel("model"),), reasoning_effort=reasoning_effort, api_style=api_style,
)
return Settings(
task_root=ROOT / "tmp-tasks",
conversation_root=ROOT / "tmp-conversations",
library_root=ROOT / "backend" / "cdsl_library",
engine_root=ROOT / "backend" / "engine" / "cdsl_engine",
llm_base_url=provider.base_url,
llm_api_key=provider.api_key,
llm_model="model",
llm_timeout_s=1,
default_provider_id="provider",
providers=(provider,),
)
def _tool() -> dict:
return {
"type": "function",
"function": {
"name": "write_document",
"description": "Write one document.",
"parameters": {"type": "object", "properties": {}, "additionalProperties": False},
},
}
def _response(api_style: str) -> dict:
if api_style == "responses":
return {
"output": [{"type": "function_call", "name": "write_document", "arguments": "{}"}],
"usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
}
return {
"choices": [{"message": {"tool_calls": [{"function": {"name": "write_document", "arguments": "{}"}}]}}],
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
}
class StructuredModelGatewayCompatibilityTests(unittest.TestCase):
def test_thinking_tool_choice_retries_without_reasoning_before_relaxing_tool_choice(self) -> None:
gateway = StructuredModelGateway(_settings())
payloads: list[dict] = []
async def request(_provider: object, payload: dict) -> dict:
payloads.append(payload)
if len(payloads) == 1:
raise StructuredModelError("Provider rejected structured request (400): Thinking mode does not support this tool_choice")
return _response("responses")
gateway._request = request # type: ignore[method-assign]
result = asyncio.run(gateway.call_tool(
messages=[{"role": "system", "content": "Call the tool."}],
tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
))
self.assertEqual(len(payloads), 2)
self.assertEqual(payloads[0]["tool_choice"], {"type": "function", "name": "write_document"})
self.assertEqual(payloads[0]["reasoning"], {"effort": "medium"})
self.assertEqual(payloads[1]["tool_choice"], {"type": "function", "name": "write_document"})
self.assertNotIn("reasoning", payloads[1])
self.assertEqual(result["usage"]["structured_compatibility_mode"], "reasoning_disabled")
def test_persistent_thinking_rejection_uses_auto_with_the_same_single_tool(self) -> None:
gateway = StructuredModelGateway(_settings(api_style="chat_completions", reasoning_effort=""))
payloads: list[dict] = []
async def request(_provider: object, payload: dict) -> dict:
payloads.append(payload)
if len(payloads) < 3:
raise StructuredModelError("Thinking mode does not support this tool_choice")
return _response("chat_completions")
gateway._request = request # type: ignore[method-assign]
result = asyncio.run(gateway.call_tool(
messages=[{"role": "system", "content": "Call the tool."}],
tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
))
self.assertEqual(len(payloads), 3)
self.assertEqual(payloads[0]["tool_choice"]["function"]["name"], "write_document")
self.assertEqual(payloads[1]["tool_choice"]["function"]["name"], "write_document")
self.assertEqual(payloads[2]["tool_choice"], "auto")
self.assertEqual(len(payloads[2]["tools"]), 1)
self.assertEqual(result["usage"]["structured_compatibility_mode"], "single_tool_auto")
def test_unrelated_provider_rejection_is_not_retried(self) -> None:
gateway = StructuredModelGateway(_settings())
payloads: list[dict] = []
async def request(_provider: object, payload: dict) -> dict:
payloads.append(payload)
raise StructuredModelError("Provider rejected structured request (400): invalid model")
gateway._request = request # type: ignore[method-assign]
with self.assertRaisesRegex(StructuredModelError, "invalid model"):
asyncio.run(gateway.call_tool(
messages=[{"role": "system", "content": "Call the tool."}],
tool=_tool(), provider_id="provider", model_id="model", required_tool_name="write_document",
))
self.assertEqual(len(payloads), 1)
if __name__ == "__main__":
unittest.main()