121 lines
6.2 KiB
Python
121 lines
6.2 KiB
Python
"""Transport and artifact tests; no paid model calls in the test suite."""
|
|
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
SPEC = importlib.util.spec_from_file_location("cad_ab_runner", Path(__file__).with_name("run.py"))
|
|
runner = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(runner)
|
|
|
|
|
|
class Response:
|
|
def __init__(self, events):
|
|
self.events = events
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
pass
|
|
|
|
def __iter__(self):
|
|
for event in self.events:
|
|
yield ("data: " + json.dumps(event) + "\n\n").encode()
|
|
yield b"data: [DONE]\n\n"
|
|
|
|
|
|
class RunnerTests(unittest.TestCase):
|
|
def test_stream_assembles_fragmented_tool_calls_without_request_deadline(self):
|
|
config = {"model": "doubao-test", "max_tokens": 100, "thinking": "disabled",
|
|
"api_key": "private-test-key", "base_url": "https://example.invalid/v3"}
|
|
events = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call-1", "function": {"name": "execute", "arguments": '{"command":'}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"solid.box","params":{}}'}}]}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"total_tokens": 12}},
|
|
]
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with patch.object(runner.urllib.request, "urlopen", return_value=Response(events)) as opener:
|
|
result = runner.stream_completion(config, [], [], directory, 1)
|
|
self.assertIsNone(opener.call_args.kwargs["timeout"])
|
|
request = opener.call_args.args[0]
|
|
self.assertEqual(request.get_header("Authorization"), "Bearer private-test-key")
|
|
self.assertEqual(json.loads(request.data)["model"], "doubao-test")
|
|
self.assertEqual(result["message"]["tool_calls"][0]["function"]["name"], "execute")
|
|
self.assertEqual(json.loads(result["message"]["tool_calls"][0]["function"]["arguments"])["command"], "solid.box")
|
|
self.assertEqual(result["usage"]["total_tokens"], 12)
|
|
for path in Path(directory).iterdir():
|
|
self.assertNotIn("private-test-key", path.read_text())
|
|
|
|
def test_no_family_examples_leak_into_base_or_condition_context(self):
|
|
kb = runner.KnowledgeBase(runner.ROOT)
|
|
prompt = runner.base_prompt(kb, "engine reference")
|
|
self.assertNotIn("example-mounting-plate", prompt)
|
|
self.assertNotIn("parameter_schema", prompt)
|
|
context = kb.build_context(["involute-spur-gear"], include_examples=False)
|
|
ids = [item["doc_id"] for item in context["manifest"]]
|
|
self.assertIn("involute-spur-gear", ids)
|
|
self.assertFalse(any(item.startswith("example-") for item in ids))
|
|
|
|
def test_incomplete_stream_is_not_treated_as_a_final_answer(self):
|
|
config = {"model": "doubao-test", "max_tokens": 100, "thinking": "disabled",
|
|
"api_key": "private-test-key", "base_url": "https://example.invalid/v3"}
|
|
events = [{"choices": [{"delta": {"content": "partial answer"}}]}]
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with patch.object(runner.urllib.request, "urlopen", return_value=Response(events)):
|
|
with self.assertRaisesRegex(RuntimeError, "without finish_reason"):
|
|
runner.stream_completion(config, [], [], directory, 1)
|
|
self.assertTrue((Path(directory) / "response-001.jsonl").exists())
|
|
self.assertFalse((Path(directory) / "completion-001.json").exists())
|
|
|
|
def test_truncated_response_never_executes_tool_calls(self):
|
|
class CAD:
|
|
tools = []
|
|
|
|
def call(self, *_args):
|
|
raise AssertionError("A truncated response must not execute any tool")
|
|
|
|
completion = {"message": {"role": "assistant", "content": None, "tool_calls": [
|
|
{"id": "partial", "function": {"name": "execute", "arguments": "{}"}}]},
|
|
"finish_reason": "length"}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with patch.object(runner, "stream_completion", return_value=completion):
|
|
result = runner.run_agent({"max_model_turns": 2}, CAD(), [], directory, "test")
|
|
self.assertFalse(result["ended"])
|
|
self.assertEqual(result["stop_reason"], "length")
|
|
|
|
def test_cylinder_check_compares_axes_not_arbitrary_axis_origin_coordinate(self):
|
|
cylinder = {"radius": 3, "axis": [0, 0, -1], "origin": [35, 15, 20], "extent": [12, 20]}
|
|
self.assertTrue(runner.cylinder_matches(cylinder, 3, [0, 0, 1], [35, 15, 0], 8))
|
|
self.assertFalse(runner.cylinder_matches(cylinder, 3, [0, 0, 1], [34, 15, 0], 8))
|
|
self.assertFalse(runner.cylinder_matches(cylinder, 3, [0, 1, 0], [35, 15, 0], 8))
|
|
|
|
def test_geometry_gates_fail_empty_design_and_keep_manual_checks(self):
|
|
kb = runner.KnowledgeBase(runner.ROOT)
|
|
params = kb.documents["example-mounting-plate"]["meta"]["params"]
|
|
derived = kb.validate_params("mounting-plate", params)["derived"]
|
|
result = runner.assess("mounting-plate", params, derived, {"bodies": [], "timeline": []}, {"bodies": []}, {})
|
|
self.assertFalse(result["automatic_checks_passed"])
|
|
self.assertTrue(result["manual_review_required"])
|
|
self.assertFalse(result["full_geometry_certified"])
|
|
|
|
def test_environment_file_is_parsed_as_data_not_executed(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / ".env"
|
|
path.write_text("ARK_API_KEY='key'\nMODEL=doubao-test\nNOEXEC=$(touch forbidden)\n")
|
|
parsed = runner.env_file(path)
|
|
self.assertEqual(parsed["ARK_API_KEY"], "key")
|
|
self.assertEqual(parsed["NOEXEC"], "$(touch forbidden)")
|
|
self.assertFalse((Path(directory) / "forbidden").exists())
|
|
|
|
def test_public_manifest_removes_api_key(self):
|
|
self.assertEqual(runner.public_config({"api_key": "secret", "model": "doubao-test"}), {"model": "doubao-test"})
|
|
self.assertEqual(runner.redact({"detail": ["echo secret"]}, "secret"), {"detail": ["echo [REDACTED]"]})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|