feat: release v1.0.1 CADWorld 网站与 LeKiwi 智能抓放
web-platform-ci / Standalone decision service (no cloud credentials) (push) Has been cancelled
web-platform-ci / TypeScript, lint, unit, build (push) Has been cancelled
web-platform-ci / Playwright E2E (push) Has been cancelled
lekiwi-compatibility / cpu-compatibility (push) Has been cancelled
web-platform-ci / Standalone decision service (no cloud credentials) (pull_request) Has been cancelled
web-platform-ci / TypeScript, lint, unit, build (pull_request) Has been cancelled
web-platform-ci / Playwright E2E (pull_request) Has been cancelled
lekiwi-compatibility / cpu-compatibility (pull_request) Has been cancelled
web-platform-ci / Standalone decision service (no cloud credentials) (push) Has been cancelled
web-platform-ci / TypeScript, lint, unit, build (push) Has been cancelled
web-platform-ci / Playwright E2E (push) Has been cancelled
lekiwi-compatibility / cpu-compatibility (push) Has been cancelled
web-platform-ci / Standalone decision service (no cloud credentials) (pull_request) Has been cancelled
web-platform-ci / TypeScript, lint, unit, build (pull_request) Has been cancelled
web-platform-ci / Playwright E2E (pull_request) Has been cancelled
lekiwi-compatibility / cpu-compatibility (pull_request) Has been cancelled
集成同源 BYOK 会话隔离、精简模型设置、官方订阅入口和 HTTPS 发布运维;保留本地训练/调参与控制能力。同步 npm 版本及 CHANGELOG,记录公网真实 API 验收仍待用户凭据。
This commit is contained in:
@@ -0,0 +1,280 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestServer
|
||||
|
||||
from decision_server.credentials import deepseek_llm, openrouter_jev
|
||||
from decision_server.protocol import DecisionError
|
||||
from decision_server.providers.codex import CodexAccount
|
||||
from decision_server.providers.codex_gate import verify_no_tools
|
||||
from decision_server.providers.jev import answer
|
||||
from decision_server.tests.test_service import plan_value, request_value
|
||||
|
||||
|
||||
class CredentialTests(unittest.TestCase):
|
||||
def test_explicit_single_variable_no_eval(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "env"
|
||||
path.write_text(
|
||||
"OTHER_SECRET=do-not-import\n"
|
||||
'export OPENROUTER_API_KEY="fixture-key-at-least-16" # note\n'
|
||||
)
|
||||
conn = openrouter_jev(path)
|
||||
self.assertEqual(conn.protocol, "openrouter-decisions")
|
||||
self.assertEqual(conn.model, "typesafe/jev-1.13")
|
||||
self.assertNotIn("OTHER_SECRET", os.environ)
|
||||
path.write_text("DEEPSEEK_API_KEY=another-fixture-key-16\n")
|
||||
self.assertEqual(deepseek_llm(path).model, "deepseek-flash")
|
||||
self.assertEqual(deepseek_llm(path).base_url, "https://api.deepseek.com")
|
||||
for value in ("$(touch PWNED)", "`some-command`", "short"):
|
||||
path.write_text("OPENROUTER_API_KEY=" + value)
|
||||
with self.assertRaises(DecisionError):
|
||||
openrouter_jev(path)
|
||||
path.write_text(
|
||||
"OPENROUTER_API_KEY=fixture-key-at-least-16\nOPENROUTER_API_KEY=duplicate-fixture-16"
|
||||
)
|
||||
with self.assertRaises(DecisionError):
|
||||
openrouter_jev(path)
|
||||
|
||||
def test_official_choice_not_reordered(self):
|
||||
result = {"answers": {"test": {"choice": "a", "probabilities": {"a": 0.1, "b": 0.9}}}}
|
||||
self.assertEqual(answer(result, "test", ["a", "b"]), "a")
|
||||
for value in (float("nan"), True, -1, 1.01):
|
||||
result["answers"]["test"]["probabilities"]["a"] = value
|
||||
with self.assertRaises(DecisionError):
|
||||
answer(result, "test", ["a", "b"])
|
||||
|
||||
|
||||
class CodexTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.client = CodexAccount(Path(self.temp.name))
|
||||
self.client.status = AsyncMock(return_value={"loggedIn": True})
|
||||
self.client.check_model = AsyncMock(return_value={"toolGatePassed": True})
|
||||
self.client.close = AsyncMock()
|
||||
self.calls = []
|
||||
self.mode = "success"
|
||||
self.started = asyncio.Event()
|
||||
|
||||
async def rpc(method, params):
|
||||
self.calls.append((method, params))
|
||||
if method == "thread/start":
|
||||
return {"thread": {"id": "thread"}}
|
||||
if method == "turn/start":
|
||||
queue = self.client.queues["thread"]
|
||||
self.started.set()
|
||||
if self.mode != "wait":
|
||||
item = {"type": "agentMessage", "text": json.dumps(plan_value())}
|
||||
if self.mode == "tool":
|
||||
item = {"type": "commandExecution"}
|
||||
queue.put_nowait(
|
||||
{
|
||||
"method": "item/completed",
|
||||
"params": {
|
||||
"threadId": "thread",
|
||||
"turnId": "turn",
|
||||
"item": item,
|
||||
},
|
||||
}
|
||||
)
|
||||
queue.put_nowait(
|
||||
{
|
||||
"method": "turn/completed",
|
||||
"params": {
|
||||
"threadId": "thread",
|
||||
"turn": {
|
||||
"id": "turn",
|
||||
"status": self.mode if self.mode == "failed" else "completed",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
return {"turn": {"id": "turn"}}
|
||||
return {}
|
||||
|
||||
self.client.rpc = AsyncMock(side_effect=rpc)
|
||||
self.client.process = SimpleNamespace(returncode=None)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
async def test_plan_is_structured_and_ephemeral(self):
|
||||
value, _ = await self.client.plan(request_value(), "allowed")
|
||||
self.assertEqual(value, plan_value())
|
||||
self.assertTrue(self.calls[0][1]["ephemeral"])
|
||||
self.assertIn("outputSchema", self.calls[1][1])
|
||||
self.assertEqual(self.calls[-1][0], "thread/unsubscribe")
|
||||
self.assertFalse(self.client.queues)
|
||||
|
||||
async def test_tool_error_and_failed_turn_interrupt(self):
|
||||
for mode in ("tool", "failed"):
|
||||
self.mode = mode
|
||||
self.calls.clear()
|
||||
with self.assertRaises(DecisionError):
|
||||
await self.client.plan(request_value(), "allowed")
|
||||
self.assertIn("turn/interrupt", [method for method, _ in self.calls])
|
||||
|
||||
async def test_cancel_uses_exact_turn_interrupt(self):
|
||||
self.mode = "wait"
|
||||
task = asyncio.create_task(self.client.plan(request_value(), "allowed"))
|
||||
await self.started.wait()
|
||||
task.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
self.assertIn(("turn/interrupt", {"threadId": "thread", "turnId": "turn"}), self.calls)
|
||||
self.assertFalse(self.client.queues)
|
||||
|
||||
async def test_failed_gate_never_starts_turn(self):
|
||||
self.client.check_model.side_effect = DecisionError("codex_tool_gate_failed")
|
||||
with self.assertRaises(DecisionError):
|
||||
await self.client.plan(request_value(), "allowed")
|
||||
self.assertFalse(self.calls)
|
||||
|
||||
async def test_no_login_no_turn(self):
|
||||
self.client.status.return_value = {"loggedIn": False}
|
||||
with self.assertRaises(DecisionError):
|
||||
await self.client.plan(request_value(), "allowed")
|
||||
self.assertFalse(self.calls)
|
||||
|
||||
async def test_login_cancel_and_logout_are_named_operations(self):
|
||||
self.client.start = AsyncMock()
|
||||
self.client.rpc = AsyncMock(
|
||||
return_value={
|
||||
"authUrl": "https://auth.openai.com/oauth/authorize?state=fixture",
|
||||
"loginId": "login",
|
||||
}
|
||||
)
|
||||
result = await self.client.login()
|
||||
self.assertEqual(result["storage"], "session-only")
|
||||
self.client.rpc.assert_awaited_with("account/login/start", {"type": "chatgpt"})
|
||||
await self.client.cancel_login()
|
||||
self.client.rpc.assert_awaited_with("account/login/cancel", {"loginId": "login"})
|
||||
self.assertIsNone(self.client.login_id)
|
||||
await self.client.logout()
|
||||
self.client.rpc.assert_awaited_with("account/logout", {})
|
||||
self.client.close.assert_awaited()
|
||||
|
||||
async def test_device_login_uses_official_protocol(self):
|
||||
self.client.start = AsyncMock()
|
||||
self.client.rpc = AsyncMock(
|
||||
return_value={
|
||||
"verificationUrl": "https://auth.openai.com/codex/device",
|
||||
"userCode": "ABCD-1234",
|
||||
"loginId": "device-login",
|
||||
}
|
||||
)
|
||||
result = await self.client.login(device=True)
|
||||
self.client.rpc.assert_awaited_with("account/login/start", {"type": "chatgptDeviceCode"})
|
||||
self.assertEqual(result["userCode"], "ABCD-1234")
|
||||
self.assertNotIn("authUrl", result)
|
||||
self.assertEqual(self.client.login_id, "device-login")
|
||||
|
||||
async def test_unexpected_auth_url_rejected(self):
|
||||
self.client.start = AsyncMock()
|
||||
self.client.rpc = AsyncMock(return_value={"authUrl": "https://evil.test/login"})
|
||||
with self.assertRaisesRegex(DecisionError, "codex_unexpected_login_url"):
|
||||
await self.client.login()
|
||||
self.client.close.assert_awaited()
|
||||
|
||||
async def test_hidden_or_unknown_model_rejected_before_gate(self):
|
||||
native = CodexAccount(Path(self.temp.name))
|
||||
native.models = AsyncMock(return_value={"models": [{"id": "current"}]})
|
||||
with self.assertRaisesRegex(DecisionError, "codex_model_unavailable"):
|
||||
await native.check_model("gpt-5.4")
|
||||
self.assertFalse(native.checked)
|
||||
|
||||
async def test_unknown_rpc_forbidden(self):
|
||||
native = CodexAccount(Path(self.temp.name))
|
||||
with self.assertRaises(DecisionError):
|
||||
await native.rpc("command/exec", {})
|
||||
with (
|
||||
patch("decision_server.providers.codex.shutil.which", return_value=None),
|
||||
self.assertRaisesRegex(DecisionError, "codex_not_installed"),
|
||||
):
|
||||
await native.start()
|
||||
|
||||
|
||||
class NativeGates(unittest.IsolatedAsyncioTestCase):
|
||||
@unittest.skipUnless(
|
||||
os.environ.get("DECISION_CODEX_SMOKE") == "1", "native fake inference is opt-in"
|
||||
)
|
||||
async def test_native_structured_turn_and_unsubscribe_without_login(self):
|
||||
async def respond(request):
|
||||
incoming = await request.json()
|
||||
self.assertEqual(incoming.get("tools", []), [])
|
||||
item = {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"id": "msg_plan",
|
||||
"status": "completed",
|
||||
"phase": "final_answer",
|
||||
"content": [{"type": "output_text", "text": json.dumps(plan_value())}],
|
||||
}
|
||||
events = [
|
||||
{
|
||||
"type": "response.created",
|
||||
"response": {"id": "resp_plan", "status": "in_progress"},
|
||||
},
|
||||
{"type": "response.output_item.done", "output_index": 0, "item": item},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_plan",
|
||||
"status": "completed",
|
||||
"output": [item],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
},
|
||||
]
|
||||
return web.Response(
|
||||
content_type="text/event-stream",
|
||||
text="".join(
|
||||
"event: " + e["type"] + "\ndata: " + json.dumps(e) + "\n\n" for e in events
|
||||
),
|
||||
)
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_post("/v1/responses", respond)
|
||||
server = TestServer(app)
|
||||
await server.start_server()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
client = CodexAccount(Path(directory), probe_url=str(server.make_url("/v1")))
|
||||
try:
|
||||
await client.start()
|
||||
# Fake-inference fixture only: no OAuth or cloud inference.
|
||||
with (
|
||||
patch.object(client, "status", AsyncMock(return_value={"loggedIn": True})),
|
||||
patch.object(client, "check_model", AsyncMock()),
|
||||
):
|
||||
result, _ = await client.plan(request_value(), "gpt-5.6-terra")
|
||||
self.assertEqual(result, plan_value())
|
||||
self.assertIsNotNone(client.process)
|
||||
self.assertIsNone(client.process.returncode)
|
||||
finally:
|
||||
await client.close()
|
||||
finally:
|
||||
await server.close()
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get("DECISION_CODEX_SMOKE") == "1", "native offline gate is opt-in"
|
||||
)
|
||||
async def test_all_visible_models_no_tools_and_injected_calls_rejected(self):
|
||||
catalog = Path(__file__).parents[1] / "providers/codex_models_0_147.json"
|
||||
models = json.loads(catalog.read_text())["models"]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
for model in models:
|
||||
if model["visibility"] != "list":
|
||||
continue
|
||||
with self.subTest(model=model["slug"]):
|
||||
evidence = {}
|
||||
await verify_no_tools(Path(directory), model["slug"], evidence)
|
||||
self.assertTrue(evidence["passed"])
|
||||
self.assertEqual(evidence["tools"], [])
|
||||
@@ -0,0 +1,425 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from decision_server.connections import Connection, Connections, endpoint
|
||||
from decision_server.protocol import (
|
||||
PRECONDITIONS,
|
||||
SKILLS,
|
||||
VERSION,
|
||||
DecisionError,
|
||||
loads,
|
||||
validate,
|
||||
validate_plan,
|
||||
)
|
||||
from decision_server.providers.codex import CodexAccount
|
||||
from decision_server.server import PREFIX, STATE, create_app
|
||||
|
||||
|
||||
def plan_value():
|
||||
return {
|
||||
"version": VERSION,
|
||||
"objectId": "block",
|
||||
"goalId": "placement",
|
||||
"summary": "搬运方块",
|
||||
"steps": [
|
||||
{"skill": s, "precondition": PRECONDITIONS[s], "onFailure": "stop"} for s in SKILLS
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def observation(request_id="r1"):
|
||||
return {
|
||||
"version": VERSION,
|
||||
"stamp": {
|
||||
"runId": "run1",
|
||||
"sceneRevision": 1,
|
||||
"sequence": 0,
|
||||
"planRevision": 0,
|
||||
"requestId": request_id,
|
||||
},
|
||||
"source": "mujoco-ground-truth",
|
||||
"units": "SI",
|
||||
"frame": "world-z-up",
|
||||
"time": 0,
|
||||
"phase": "open",
|
||||
"base": {"position": [0, 0, 0.09], "yaw": 0},
|
||||
"joints": [0] * 5,
|
||||
"opening": 1,
|
||||
"tcp": [0.2, 0, 0.2],
|
||||
"object": {"id": "block", "position": [0.257, 0.015, 0.128], "speed": 0},
|
||||
"goal": {"id": "placement", "position": [0.257, 0.615, 0.128]},
|
||||
"evidence": {
|
||||
"fingerForces": [0, 0],
|
||||
"supported": True,
|
||||
"onGoalSupport": False,
|
||||
"secure": False,
|
||||
"transported": 0,
|
||||
},
|
||||
"safety": [],
|
||||
}
|
||||
|
||||
|
||||
def request_value(ident="r1"):
|
||||
return {"observation": observation(ident), "instruction": "把方块搬到目标", "remaining": SKILLS}
|
||||
|
||||
|
||||
class ProtocolTests(unittest.TestCase):
|
||||
def test_shared_schema_and_semantics(self):
|
||||
self.assertEqual(validate("Observation", observation()), observation())
|
||||
self.assertEqual(validate_plan(plan_value(), SKILLS), plan_value())
|
||||
for mutate in (
|
||||
lambda p: p.update(objectId="other"),
|
||||
lambda p: p["steps"].reverse(),
|
||||
lambda p: p["steps"][0].update(precondition="released"),
|
||||
lambda p: p["steps"].append(p["steps"][0]),
|
||||
lambda p: p.update(command="shell"),
|
||||
):
|
||||
value = plan_value()
|
||||
mutate(value)
|
||||
with self.assertRaises(DecisionError):
|
||||
validate_plan(value, SKILLS)
|
||||
for number in (float("nan"), float("inf"), True, 1e10):
|
||||
value = observation()
|
||||
value["time"] = number
|
||||
with self.assertRaises(DecisionError):
|
||||
validate("Observation", value)
|
||||
for text in ('{"a":1,"a":2}', '{"a":NaN}', "no json"):
|
||||
with self.assertRaises(DecisionError):
|
||||
loads(text)
|
||||
|
||||
def test_endpoint_and_credentials(self):
|
||||
for url in (
|
||||
"http://evil.test/v1",
|
||||
"https://host/?key=secret",
|
||||
"https://user:key@host",
|
||||
"file:///etc/passwd",
|
||||
"http://[bad",
|
||||
"https://x:99999",
|
||||
"https://x/\\evil",
|
||||
):
|
||||
with self.assertRaises(DecisionError, msg=url):
|
||||
endpoint(url)
|
||||
for url in ("http://127.0.0.1:9000/v1", "http://localhost/v1", "https://api.openai.com/v1"):
|
||||
self.assertEqual(endpoint(url), url)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = Connections(Path(directory))
|
||||
data = {
|
||||
"role": "llm",
|
||||
"protocol": "responses",
|
||||
"baseUrl": "https://api.openai.com/v1",
|
||||
"model": "test-model",
|
||||
"apiKey": "test-secret",
|
||||
}
|
||||
store.set(data)
|
||||
self.assertNotIn("test-secret", store.path.read_text())
|
||||
self.assertEqual(store.path.stat().st_mode & 0o777, 0o600)
|
||||
self.assertFalse(Connections(Path(directory)).values["llm"].key)
|
||||
store.set({**data, "baseUrl": "http://localhost:9000", "apiKey": ""})
|
||||
with self.assertRaises(DecisionError):
|
||||
store.get("llm")
|
||||
with self.assertRaises(DecisionError):
|
||||
store.set({**data, "protocol": "codex"})
|
||||
|
||||
|
||||
class ServerTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.responses = []
|
||||
self.received = []
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
self.block = False
|
||||
|
||||
async def upstream(request):
|
||||
self.received.append({"path": request.path, "body": await request.json()})
|
||||
self.started.set()
|
||||
if self.block:
|
||||
await self.release.wait()
|
||||
if self.responses:
|
||||
return self.responses.pop(0)
|
||||
return web.json_response(
|
||||
{
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [{"type": "output_text", "text": json.dumps(plan_value())}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 20, "secret": "test-secret"},
|
||||
}
|
||||
)
|
||||
|
||||
upstream_app = web.Application()
|
||||
upstream_app.router.add_post("/{path:.*}", upstream)
|
||||
self.upstream = TestServer(upstream_app)
|
||||
await self.upstream.start_server()
|
||||
app = create_app(Path(self.temp.name), "token", ["http://localhost:5173"])
|
||||
self.client = TestClient(TestServer(app))
|
||||
await self.client.start_server()
|
||||
self.service = app[STATE]
|
||||
self.service.hosts = {f"127.0.0.1:{self.client.port}"}
|
||||
self.headers = {"Authorization": "Bearer token", "Origin": "http://localhost:5173"}
|
||||
self.conn = {
|
||||
"role": "llm",
|
||||
"protocol": "responses",
|
||||
"baseUrl": str(self.upstream.make_url("/v1")),
|
||||
"model": "fixture",
|
||||
"apiKey": "test-secret",
|
||||
}
|
||||
self.service.connections.set(self.conn)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self.release.set()
|
||||
await self.client.close()
|
||||
await self.upstream.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
async def post(self, path, data):
|
||||
return await self.client.post(PREFIX + path, json=data, headers=self.headers)
|
||||
|
||||
async def test_host_origin_token_and_body(self):
|
||||
cases = [
|
||||
({}, 401),
|
||||
({**self.headers, "Host": "evil.test"}, 403),
|
||||
({**self.headers, "Origin": "https://evil.test"}, 403),
|
||||
(self.headers, 200),
|
||||
]
|
||||
for headers, status in cases:
|
||||
response = await self.client.get(PREFIX + "/status", headers=headers)
|
||||
self.assertEqual(response.status, status)
|
||||
response = await self.client.options(
|
||||
PREFIX + "/plan", headers={"Origin": "http://localhost:5173"}
|
||||
)
|
||||
self.assertEqual(response.status, 204)
|
||||
self.assertEqual(response.headers["Access-Control-Allow-Origin"], "http://localhost:5173")
|
||||
response = await self.client.post(
|
||||
PREFIX + "/plan",
|
||||
data="x" * 70000,
|
||||
headers={**self.headers, "Content-Type": "application/json"},
|
||||
)
|
||||
self.assertEqual(response.status, 413)
|
||||
response = await self.post("/codex/turn", {})
|
||||
self.assertEqual(response.status, 405)
|
||||
|
||||
async def test_responses_stamp_usage_and_duplicate(self):
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
value = await response.json()
|
||||
self.assertEqual(value["stamp"], observation()["stamp"])
|
||||
self.assertEqual(value["value"], plan_value())
|
||||
self.assertEqual(value["usage"], {"input_tokens": 10, "output_tokens": 20})
|
||||
self.assertEqual(self.received[0]["body"]["tools"], [])
|
||||
self.assertEqual(self.received[0]["body"]["tool_choice"], "none")
|
||||
self.assertEqual((await self.post("/plan", request_value())).status, 409)
|
||||
state = await (await self.client.get(PREFIX + "/status", headers=self.headers)).text()
|
||||
self.assertNotIn("test-secret", state)
|
||||
self.assertNotIn("instruction", state)
|
||||
|
||||
async def test_explicit_chat_protocol(self):
|
||||
self.service.connections.set({**self.conn, "protocol": "chat-completions"})
|
||||
self.responses.append(
|
||||
web.json_response(
|
||||
{
|
||||
"choices": [
|
||||
{"finish_reason": "stop", "message": {"content": json.dumps(plan_value())}}
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
self.assertEqual(self.received[0]["path"], "/v1/chat/completions")
|
||||
self.assertIn("response_format", self.received[0]["body"])
|
||||
|
||||
async def test_typesafe_choices_and_probabilities(self):
|
||||
self.service.connections.set(
|
||||
{
|
||||
**self.conn,
|
||||
"role": "jev",
|
||||
"protocol": "typesafe",
|
||||
"baseUrl": str(self.upstream.make_url("/v1/systemone")),
|
||||
}
|
||||
)
|
||||
chosen = {
|
||||
"choice": "open",
|
||||
"grasp": "uncertain",
|
||||
"diagnosis": "none",
|
||||
"recovery": "continue",
|
||||
}
|
||||
self.responses.append(
|
||||
web.json_response({"answers": {k: {"choice": v} for k, v in chosen.items()}})
|
||||
)
|
||||
response = await self.post(
|
||||
"/decide", {"observation": observation(), "candidates": ["open", "stop"]}
|
||||
)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
self.assertEqual((await response.json())["value"], {"version": VERSION, **chosen})
|
||||
self.assertIn("questions", self.received[0]["body"])
|
||||
self.assertNotIn("messages", self.received[0]["body"])
|
||||
bad = {k: {"choice": v} for k, v in chosen.items()}
|
||||
bad["choice"] = {"choice": "carry"}
|
||||
self.responses.append(web.json_response({"answers": bad}))
|
||||
response = await self.post(
|
||||
"/decide", {"observation": observation("r2"), "candidates": ["open", "stop"]}
|
||||
)
|
||||
self.assertEqual((await response.json())["error"], "jev_invalid_choice")
|
||||
|
||||
async def test_openrouter_explicit_decisions_and_real_usage_only(self):
|
||||
self.service.connections.set(
|
||||
{
|
||||
**self.conn,
|
||||
"role": "jev",
|
||||
"protocol": "openrouter-decisions",
|
||||
"baseUrl": str(self.upstream.make_url("/api/alpha/decisions")),
|
||||
}
|
||||
)
|
||||
selected = {"choice": "open", "grasp": "empty", "diagnosis": "none", "recovery": "continue"}
|
||||
self.responses.append(
|
||||
web.json_response(
|
||||
{
|
||||
"answers": {k: {"choice": v} for k, v in selected.items()},
|
||||
"usage": {"cost": 0.00004, "input_tokens": 100, "untrusted": "secret"},
|
||||
}
|
||||
)
|
||||
)
|
||||
response = await self.post(
|
||||
"/decide", {"observation": observation(), "candidates": ["open", "stop"]}
|
||||
)
|
||||
result = await response.json()
|
||||
self.assertEqual(response.status, 200, result)
|
||||
self.assertEqual(result["usage"], {"cost": 0.00004, "input_tokens": 100})
|
||||
self.assertEqual(self.received[0]["body"]["provider"], {"allow_fallbacks": False})
|
||||
self.assertEqual(self.received[0]["path"], "/api/alpha/decisions")
|
||||
|
||||
async def test_credentials_not_in_prompts_and_tools_not_executed(self):
|
||||
payload = request_value()
|
||||
payload["instruction"] = "do not disclose test-secret"
|
||||
await self.post("/plan", payload)
|
||||
self.assertNotIn("test-secret", json.dumps(self.received[0]["body"]))
|
||||
self.responses.append(
|
||||
web.json_response(
|
||||
{
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{"type": "function_call", "name": "shell", "arguments": "untrusted"}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
result = await (await self.post("/plan", request_value("r2"))).json()
|
||||
self.assertEqual(result["error"], "llm_tool_or_unknown_output")
|
||||
|
||||
async def test_http_and_bad_json_no_retry_no_secret_echo(self):
|
||||
for index, status in enumerate((401, 429, 302)):
|
||||
self.responses.append(
|
||||
web.Response(status=status, text="test-secret", headers={"Location": "/stolen"})
|
||||
)
|
||||
response = await self.post("/plan", request_value(str(index)))
|
||||
self.assertEqual((await response.json())["error"], f"upstream_http_{status}")
|
||||
self.assertEqual(len(self.received), index + 1)
|
||||
self.assertEqual((await self.post("/plan", request_value("budget"))).status, 429)
|
||||
self.assertEqual(len(self.received), 3)
|
||||
|
||||
async def test_bad_contract_no_fallback(self):
|
||||
self.responses.append(web.Response(text="not JSON test-secret"))
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual((await response.json())["error"], "invalid_json")
|
||||
value = plan_value()
|
||||
value["steps"].reverse()
|
||||
self.responses.append(
|
||||
web.json_response(
|
||||
{
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"content": [{"type": "output_text", "text": json.dumps(value)}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
response = await self.post("/plan", request_value("r2"))
|
||||
self.assertEqual((await response.json())["error"], "invalid_plan_order")
|
||||
self.assertEqual(len(self.received), 2)
|
||||
|
||||
async def test_cancel_reconfigure_and_count_failed_requests(self):
|
||||
self.block = True
|
||||
pending = asyncio.create_task(self.post("/plan", request_value()))
|
||||
await asyncio.wait_for(self.started.wait(), 3)
|
||||
self.assertEqual((await self.post("/plan", request_value("parallel"))).status, 409)
|
||||
response = await self.post("/cancel", {"runId": "run1", "requestId": "r1"})
|
||||
self.assertTrue((await response.json())["cancelled"])
|
||||
result = await pending
|
||||
self.assertEqual((await result.json())["error"], "request_cancelled")
|
||||
self.assertEqual(self.service.runs["run1"]["llm"], 1)
|
||||
self.assertFalse(self.service.active)
|
||||
self.started.clear()
|
||||
pending = asyncio.create_task(self.post("/plan", request_value("r2")))
|
||||
await asyncio.wait_for(self.started.wait(), 3)
|
||||
response = await self.client.put(
|
||||
PREFIX + "/connections", json=self.conn, headers=self.headers
|
||||
)
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual((await (await pending).json())["error"], "request_cancelled")
|
||||
|
||||
async def test_cancel_before_post_prevents_late_launch(self):
|
||||
response = await self.post("/cancel", {"runId": "run1", "requestId": "r1"})
|
||||
self.assertEqual(response.status, 200)
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual((await response.json())["error"], "request_cancelled")
|
||||
self.assertEqual(self.received, [])
|
||||
|
||||
async def test_timeout_budgets_and_redaction(self):
|
||||
with patch(
|
||||
"decision_server.providers.openai.plan", new=AsyncMock(side_effect=TimeoutError)
|
||||
):
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual((await response.json())["error"], "request_timeout")
|
||||
value = plan_value()
|
||||
value["summary"] = "test-secret"
|
||||
with patch(
|
||||
"decision_server.providers.openai.plan", new=AsyncMock(return_value=(value, {}))
|
||||
):
|
||||
response = await self.post("/plan", request_value("r2"))
|
||||
self.assertEqual((await response.json())["value"]["summary"], "[redacted]")
|
||||
for i in range(60):
|
||||
stamp = {**observation()["stamp"], "runId": "jev-run", "requestId": str(i)}
|
||||
self.service.admit("jev", stamp)
|
||||
with self.assertRaises(DecisionError):
|
||||
self.service.admit("jev", {**stamp, "requestId": "61"})
|
||||
|
||||
async def test_codex_planning_fails_closed(self):
|
||||
self.service.connections.values["llm"] = Connection("codex", "", "account-model")
|
||||
with patch.object(
|
||||
self.service.codex, "status", AsyncMock(return_value={"loggedIn": False})
|
||||
):
|
||||
response = await self.post("/plan", request_value())
|
||||
self.assertEqual((await response.json())["error"], "codex_chatgpt_login_required")
|
||||
self.assertEqual(self.received, [])
|
||||
with self.assertRaises(DecisionError):
|
||||
await self.service.codex.rpc("command/exec", {})
|
||||
|
||||
|
||||
class NativeCodexTests(unittest.IsolatedAsyncioTestCase):
|
||||
@unittest.skipUnless(
|
||||
os.environ.get("DECISION_CODEX_SMOKE") == "1", "opt-in: isolated native CLI, no login/turn"
|
||||
)
|
||||
async def test_isolated_status_and_models(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
client = CodexAccount(Path(directory))
|
||||
try:
|
||||
self.assertFalse((await client.status())["loggedIn"])
|
||||
self.assertFalse((await client.models())["planningAvailable"])
|
||||
self.assertFalse((Path(directory) / "home/auth.json").exists())
|
||||
finally:
|
||||
await client.close()
|
||||
@@ -0,0 +1,58 @@
|
||||
import unittest
|
||||
|
||||
from decision_server.protocol import DecisionError
|
||||
from decision_server.web_config import MemoryConnections, configure, public_config, website_origin
|
||||
|
||||
|
||||
def config(provider="deepseek", model="deepseek-flash"):
|
||||
return {
|
||||
"llm": {"provider": provider, "model": model, "apiKey": "llm-secret-fixture"},
|
||||
"jev": {"apiKey": "jev-secret-fixture"},
|
||||
}
|
||||
|
||||
|
||||
class WebsiteConfigTests(unittest.TestCase):
|
||||
def test_atomic_memory_only_and_no_secret_response(self):
|
||||
store = MemoryConnections()
|
||||
result = configure(store.values, config(), ())
|
||||
self.assertEqual(store.values, {})
|
||||
self.assertNotIn("secret-fixture", str(public_config(result)))
|
||||
self.assertEqual(result["jev"].model, "typesafe/jev-1.13")
|
||||
self.assertFalse(hasattr(store, "path"))
|
||||
bad = config()
|
||||
bad["jev"]["apiKey"] = "bad key"
|
||||
with self.assertRaises(DecisionError):
|
||||
configure(result, bad, ())
|
||||
self.assertEqual(result["jev"].key, "jev-secret-fixture")
|
||||
|
||||
def test_reject_arbitrary_urls_unknown_models_and_providers(self):
|
||||
for field in ("baseUrl", "protocol", "command"):
|
||||
data = config()
|
||||
data["llm"][field] = "http://169.254.169.254"
|
||||
with self.assertRaises(DecisionError):
|
||||
configure({}, data, ())
|
||||
for provider, model in (("other", "x"), ("openrouter", "unknown"), ("codex", "x")):
|
||||
with self.assertRaises(DecisionError):
|
||||
configure({}, config(provider, model), ())
|
||||
|
||||
def test_key_reuse_only_same_provider_no_cross_role_reuse(self):
|
||||
values = configure({}, config(), ())
|
||||
draft = {"llm": {"provider": "deepseek", "model": "deepseek-flash"}, "jev": {}}
|
||||
self.assertEqual(configure(values, draft, ()), values)
|
||||
draft["llm"] = {"provider": "openrouter", "model": "vendor/model"}
|
||||
with self.assertRaisesRegex(DecisionError, "api_key_required"):
|
||||
configure(values, draft, ("vendor/model",))
|
||||
draft["llm"]["apiKey"] = "new-router-key"
|
||||
result = configure(values, draft, ("vendor/model",))
|
||||
self.assertEqual(result["jev"].key, values["jev"].key)
|
||||
|
||||
def test_public_origin_requires_https(self):
|
||||
self.assertEqual(
|
||||
website_origin("https://cadworld-sim.robotquan.com"), "cadworld-sim.robotquan.com"
|
||||
)
|
||||
for value in ("http://public.test", "https://x/path", "https://user@x", "https://x?key=x"):
|
||||
with self.assertRaises(DecisionError):
|
||||
website_origin(value)
|
||||
self.assertEqual(
|
||||
website_origin("http://localhost:5173", development=True), "localhost:5173"
|
||||
)
|
||||
@@ -0,0 +1,163 @@
|
||||
import asyncio
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from decision_server.server import PREFIX
|
||||
from decision_server.tests.test_service import plan_value, request_value
|
||||
from decision_server.tests.test_web_config import config
|
||||
from decision_server.web_config import COOKIE
|
||||
from decision_server.web_server import CATALOG, MANAGER, create_website_app
|
||||
from decision_server.web_sessions import Limits
|
||||
|
||||
|
||||
class WebsiteTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.origin = "https://site.test"
|
||||
self.app = create_website_app(self.origin)
|
||||
self.client = TestClient(TestServer(self.app))
|
||||
await self.client.start_server()
|
||||
self.manager = self.app[MANAGER]
|
||||
self.app[CATALOG].refresh = AsyncMock()
|
||||
self.headers = {"Host": "site.test", "Origin": self.origin}
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.client.close()
|
||||
|
||||
async def visitor(self):
|
||||
response = await self.client.post(PREFIX + "/session", json={}, headers=self.headers)
|
||||
self.assertEqual(response.status, 200)
|
||||
value = await response.json()
|
||||
cookie = response.cookies[COOKIE]
|
||||
self.assertTrue(cookie["httponly"])
|
||||
self.assertTrue(cookie["secure"])
|
||||
self.assertEqual(cookie["samesite"], "Strict")
|
||||
self.assertEqual(cookie["domain"], "")
|
||||
return {
|
||||
**self.headers,
|
||||
"Cookie": COOKIE + "=" + cookie.value,
|
||||
"X-CSRF-Token": value["csrfToken"],
|
||||
"X-Config-Version": "0",
|
||||
}, self.manager.values[cookie.value]
|
||||
|
||||
async def save(self, headers):
|
||||
response = await self.client.put(PREFIX + "/configuration", json=config(), headers=headers)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
headers["X-Config-Version"] = response.headers["X-Config-Version"]
|
||||
return await response.json()
|
||||
|
||||
async def test_boundary_no_cookie_csrf_origin_host_or_cross_site(self):
|
||||
response = await self.client.get(PREFIX + "/status", headers=self.headers)
|
||||
self.assertEqual(response.status, 401)
|
||||
headers, _ = await self.visitor()
|
||||
for patch_headers in (
|
||||
{"Origin": "https://evil.test"},
|
||||
{"Origin": ""},
|
||||
{"Host": "evil.test"},
|
||||
{"X-CSRF-Token": "bad"},
|
||||
{"Sec-Fetch-Site": "same-site"},
|
||||
):
|
||||
response = await self.client.put(
|
||||
PREFIX + "/configuration", json=config(), headers={**headers, **patch_headers}
|
||||
)
|
||||
self.assertEqual(response.status, 403)
|
||||
self.assertNotIn("Access-Control-Allow-Origin", response.headers)
|
||||
self.assertEqual(response.headers["Cache-Control"], "no-store")
|
||||
response = await self.client.post(
|
||||
PREFIX + "/session", json={}, headers={"Host": "site.test"}
|
||||
)
|
||||
self.assertEqual(response.status, 403)
|
||||
|
||||
async def test_atomic_credentials_no_metadata_files_and_stale_tab(self):
|
||||
a, av = await self.visitor()
|
||||
b, bv = await self.visitor()
|
||||
result = await self.save(a)
|
||||
self.assertNotIn("secret-fixture", str(result))
|
||||
self.assertFalse(bv.service.connections.values)
|
||||
self.assertFalse(hasattr(av.service.connections, "path"))
|
||||
response = await self.client.put(
|
||||
PREFIX + "/configuration", json=config(), headers={**a, "X-Config-Version": "0"}
|
||||
)
|
||||
self.assertEqual(response.status, 409)
|
||||
bad = config()
|
||||
bad["jev"]["apiKey"] = "invalid key"
|
||||
response = await self.client.put(PREFIX + "/configuration", json=bad, headers=a)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(av.service.connections.values["jev"].key, "jev-secret-fixture")
|
||||
response = await self.client.get(PREFIX + "/status", headers=b)
|
||||
self.assertFalse((await response.json())["ready"])
|
||||
|
||||
async def test_two_visitors_identical_request_ids_and_cancel_isolation(self):
|
||||
a, av = await self.visitor()
|
||||
b, bv = await self.visitor()
|
||||
await self.save(a)
|
||||
await self.save(b)
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def provider(*_):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return plan_value(), {}
|
||||
|
||||
with patch("decision_server.providers.openai.plan", side_effect=provider):
|
||||
pending = asyncio.create_task(
|
||||
self.client.post(PREFIX + "/plan", json=request_value(), headers=b)
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
response = await self.client.post(
|
||||
PREFIX + "/cancel", json={"runId": "run1", "requestId": "r1"}, headers=a
|
||||
)
|
||||
self.assertFalse((await response.json())["cancelled"])
|
||||
self.assertEqual(len(bv.service.active), 1)
|
||||
await self.client.delete(PREFIX + "/session", headers=a)
|
||||
self.assertTrue(av.closed)
|
||||
self.assertTrue(bv.service.connections.values)
|
||||
release.set()
|
||||
self.assertEqual((await pending).status, 200)
|
||||
self.assertEqual(self.manager.inference, 0)
|
||||
|
||||
async def test_ttl_status_does_not_refresh_and_credentials_destroyed(self):
|
||||
headers, visitor = await self.visitor()
|
||||
await self.save(headers)
|
||||
touched = visitor.touched
|
||||
await self.client.get(PREFIX + "/status", headers=headers)
|
||||
await self.client.post(PREFIX + "/session", json={}, headers=headers)
|
||||
self.assertEqual(visitor.touched, touched)
|
||||
visitor.touched = time.monotonic() - 1801
|
||||
response = await self.client.get(PREFIX + "/status", headers=headers)
|
||||
self.assertEqual(response.status, 401)
|
||||
self.assertFalse(visitor.service.connections.values)
|
||||
self.assertTrue(visitor.closed)
|
||||
|
||||
async def test_limits_ip_spoof_does_not_bypass_and_no_implicit_cli(self):
|
||||
self.manager.limits = Limits(ip_sessions=2, codex=1)
|
||||
a, av = await self.visitor()
|
||||
b, bv = await self.visitor()
|
||||
response = await self.client.post(
|
||||
PREFIX + "/session", json={}, headers={**self.headers, "X-Real-IP": "1.2.3.4"}
|
||||
)
|
||||
self.assertEqual(response.status, 429)
|
||||
self.manager.reserve_codex(av)
|
||||
with self.assertRaisesRegex(Exception, "subscription_capacity"):
|
||||
self.manager.reserve_codex(bv)
|
||||
av.codex_reserved = False
|
||||
with patch.object(av.service.codex, "start", new_callable=AsyncMock) as start:
|
||||
response = await self.client.get(PREFIX + "/codex/status", headers=a)
|
||||
self.assertFalse((await response.json())["loggedIn"])
|
||||
start.assert_not_called()
|
||||
await self.save(b)
|
||||
self.manager.inference = self.manager.limits.inference
|
||||
response = await self.client.post(PREFIX + "/plan", json=request_value(), headers=b)
|
||||
self.assertEqual(response.status, 429)
|
||||
self.manager.inference = 0
|
||||
|
||||
async def test_no_arbitrary_rpc_or_local_connection_or_queries(self):
|
||||
headers, _ = await self.visitor()
|
||||
for path in ("/connections", "/codex/exec", "/codex/rpc"):
|
||||
response = await self.client.post(PREFIX + path, json={}, headers=headers)
|
||||
self.assertIn(response.status, (404, 405))
|
||||
response = await self.client.get(PREFIX + "/status?key=fixture", headers=headers)
|
||||
self.assertEqual(response.status, 400)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Explicit opt-in E2E fixture: real gateway, loopback fake HTTP upstream; never deployed."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from dataclasses import replace
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from decision_server.providers import http, jev, openai
|
||||
from decision_server.tests.test_service import plan_value
|
||||
from decision_server.web_server import CATALOG, create_website_app
|
||||
|
||||
|
||||
async def main():
|
||||
if os.environ.get("CADWORLD_E2E") != "1":
|
||||
raise RuntimeError("fixture_requires_explicit_opt_in")
|
||||
upstream = web.Application()
|
||||
|
||||
async def respond(request):
|
||||
body = await request.json()
|
||||
if request.path == "/decisions":
|
||||
return web.json_response(
|
||||
{
|
||||
"answers": {
|
||||
name: {"choice": next(iter(q["criteria"]))}
|
||||
for name, q in body["questions"].items()
|
||||
},
|
||||
"usage": {"input_tokens": 1},
|
||||
}
|
||||
)
|
||||
import json
|
||||
|
||||
value = '{"ok":true}' if '"ok"' in str(body) else json.dumps(plan_value())
|
||||
if request.path == "/chat/completions":
|
||||
return web.json_response(
|
||||
{"choices": [{"finish_reason": "stop", "message": {"content": value}}]}
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{"type": "message", "content": [{"type": "output_text", "text": value}]}
|
||||
],
|
||||
"usage": {"input_tokens": 1},
|
||||
}
|
||||
)
|
||||
|
||||
upstream.router.add_post("/{path:.*}", respond)
|
||||
runner = web.AppRunner(upstream, access_log=None)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, "127.0.0.1", 0)
|
||||
await site.start()
|
||||
port = site._server.sockets[0].getsockname()[1]
|
||||
|
||||
async def fake_post(session, connection, path, payload):
|
||||
url = f"http://127.0.0.1:{port}" + ("/decisions" if not path else "")
|
||||
return await http.post(session, replace(connection, base_url=url), path, payload)
|
||||
|
||||
openai.post = jev.post = fake_post
|
||||
# test_connection also references the bounded helper directly.
|
||||
from decision_server import server
|
||||
|
||||
server.post = fake_post
|
||||
app = create_website_app("http://127.0.0.1:4180", development=True)
|
||||
catalog = app[CATALOG]
|
||||
catalog.models = {"fixture/structured": "Fixture structured model (not real)"}
|
||||
catalog.available = True
|
||||
|
||||
async def refresh(_):
|
||||
catalog.checked_at = time.monotonic()
|
||||
|
||||
catalog.refresh = refresh
|
||||
gateway = web.AppRunner(app, access_log=None, handler_cancellation=True)
|
||||
await gateway.setup()
|
||||
await web.TCPSite(gateway, "127.0.0.1", 8769).start()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
await gateway.cleanup()
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user