194 lines
8.0 KiB
Python
194 lines
8.0 KiB
Python
"""Native error propagation and stalled-model handling, without model requests."""
|
|
import asyncio
|
|
import importlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
app = importlib.import_module('codex_pcb.app')
|
|
|
|
|
|
def notification(method, **params):
|
|
return {'method': method, 'params': {'threadId': 'test-thread', 'turnId': 'test-turn', **params}}
|
|
|
|
|
|
class FakeAppServer:
|
|
def __init__(self, messages):
|
|
self.events = asyncio.Queue()
|
|
self.requests = set()
|
|
for message in messages:
|
|
self.events.put_nowait(message)
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *args):
|
|
return False
|
|
|
|
async def call(self, method, params):
|
|
if method == 'thread/start':
|
|
return {'thread': {'id': 'test-thread'}, 'model': app.MODEL}
|
|
if method == 'turn/start':
|
|
return {'turn': {'id': 'test-turn'}}
|
|
raise AssertionError('Unexpected call: ' + method)
|
|
|
|
|
|
class ModelProgress(unittest.IsolatedAsyncioTestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.directory = Path(self.temp.name)
|
|
self.entries = []
|
|
self.task = {'id': 'test-task', 'state': 'planning', 'busy': True}
|
|
self.patches = [
|
|
patch.object(app, 'require_runtime_ready', new=AsyncMock(return_value={'ok': True})),
|
|
patch.object(app, 'workspace', return_value=self.directory),
|
|
patch.object(app, 'save'),
|
|
patch.object(app, 'event', side_effect=lambda t, kind, data: self.entries.append((kind, data))),
|
|
patch.object(app, 'fingerprint', return_value='test-source'),
|
|
patch.object(app, 'CODEX_IDLE_TIMEOUT', 0.04),
|
|
]
|
|
for p in self.patches:
|
|
p.start()
|
|
self.addCleanup(p.stop)
|
|
|
|
async def run_work(self, messages, prepare=None):
|
|
client = FakeAppServer(messages)
|
|
if prepare:
|
|
prepare(client)
|
|
with patch.object(app, 'AppServer', return_value=client):
|
|
await asyncio.wait_for(app.work(self.task, 'Publish a proposal only.', False), 1)
|
|
return client
|
|
|
|
async def test_native_retries_and_terminal_error_reach_public_state(self):
|
|
retry = notification('error', willRetry=True, error={
|
|
'message': 'Reconnecting... 1/5', 'additionalDetails': 'upstream returned 403 Forbidden'})
|
|
terminal = notification('error', willRetry=False, error={
|
|
'message': '403 Forbidden', 'additionalDetails': 'Country, region, or territory not supported'})
|
|
await self.run_work([retry, terminal])
|
|
self.assertEqual(self.task['state'], 'failed')
|
|
self.assertFalse(self.task['busy'])
|
|
self.assertIn('Country, region', self.task['error'])
|
|
retries = [data for kind, data in self.entries if kind == 'session.reconnecting']
|
|
self.assertEqual(len(retries), 1)
|
|
self.assertIn('upstream returned 403', retries[0]['message'])
|
|
trace = [json.loads(line) for line in (self.directory / 'evidence/codex-events.jsonl').read_text().splitlines()]
|
|
self.assertEqual(trace, [retry, terminal])
|
|
|
|
async def test_silent_model_fails_and_retains_thread_for_retry(self):
|
|
await self.run_work([])
|
|
self.assertEqual(self.task['state'], 'failed')
|
|
self.assertFalse(self.task['busy'])
|
|
self.assertEqual(self.task['thread_id'], 'test-thread')
|
|
self.assertIn('no progress', self.task['error'])
|
|
self.assertNotIn('test-task', app.CLIENTS)
|
|
self.assertFalse(any(kind == 'session.reconnecting' for kind, _ in self.entries))
|
|
|
|
async def test_status_and_retry_noise_do_not_keep_turn_alive(self):
|
|
producer = None
|
|
|
|
def prepare(client):
|
|
async def send_noise():
|
|
for _ in range(40):
|
|
await client.events.put(notification('thread/status/changed', status={'type': 'active'}))
|
|
await client.events.put(notification('error', willRetry=True, error={'message': 'Retrying'}))
|
|
await asyncio.sleep(0.01)
|
|
nonlocal producer
|
|
producer = asyncio.create_task(send_noise())
|
|
|
|
try:
|
|
await self.run_work([], prepare)
|
|
self.assertEqual(self.task['state'], 'failed')
|
|
self.assertIn('no progress', self.task['error'])
|
|
self.assertIn('Last upstream error: Retrying', self.task['error'])
|
|
self.assertFalse(producer.done())
|
|
finally:
|
|
producer.cancel()
|
|
await asyncio.gather(producer, return_exceptions=True)
|
|
|
|
async def test_real_model_deltas_reset_idle_deadline(self):
|
|
async def send_progress(client):
|
|
for _ in range(6):
|
|
await asyncio.sleep(0.015)
|
|
await client.events.put(notification('item/reasoning/textDelta', delta='progress'))
|
|
self.task['proposal'] = {'revision': 1}
|
|
await client.events.put(notification('turn/completed', turn={'id': 'test-turn', 'status': 'completed'}))
|
|
|
|
producer = None
|
|
|
|
def prepare(client):
|
|
nonlocal producer
|
|
producer = asyncio.create_task(send_progress(client))
|
|
|
|
await self.run_work([], prepare)
|
|
await producer
|
|
self.assertEqual(self.task['state'], 'review')
|
|
self.assertFalse(self.task['busy'])
|
|
self.assertNotIn('error', self.task)
|
|
|
|
async def test_active_native_tool_is_not_cut_off_by_model_idle_deadline(self):
|
|
producer = None
|
|
|
|
def prepare(client):
|
|
async def finish_tool():
|
|
await asyncio.sleep(0.1)
|
|
client.requests.clear()
|
|
self.task['proposal'] = {'revision': 1}
|
|
await client.events.put(notification('turn/completed', turn={'id': 'test-turn', 'status': 'completed'}))
|
|
nonlocal producer
|
|
producer = asyncio.create_task(finish_tool())
|
|
client.requests.add(producer)
|
|
|
|
await self.run_work([], prepare)
|
|
await producer
|
|
self.assertEqual(self.task['state'], 'review')
|
|
|
|
async def test_error_from_another_turn_is_ignored(self):
|
|
stale = notification('error', turnId='old-turn', willRetry=False, error={'message': 'old error'})
|
|
self.task['proposal'] = {'revision': 1}
|
|
done = notification('turn/completed', turn={'id': 'test-turn', 'status': 'completed'})
|
|
await self.run_work([stale, done])
|
|
self.assertEqual(self.task['state'], 'review')
|
|
self.assertNotIn('error', self.task)
|
|
|
|
async def test_long_native_shell_uses_its_own_execution_timeout(self):
|
|
producer = None
|
|
|
|
def prepare(client):
|
|
async def finish_command():
|
|
await asyncio.sleep(0.1)
|
|
await client.events.put(notification('item/completed', item={
|
|
'id': 'command-1', 'type': 'commandExecution', 'status': 'completed'}))
|
|
self.task['proposal'] = {'revision': 1}
|
|
await client.events.put(notification('turn/completed', turn={'id': 'test-turn', 'status': 'completed'}))
|
|
nonlocal producer
|
|
producer = asyncio.create_task(finish_command())
|
|
|
|
start = notification('item/started', item={
|
|
'id': 'command-1', 'type': 'commandExecution', 'status': 'inProgress'})
|
|
await self.run_work([start], prepare)
|
|
await producer
|
|
self.assertEqual(self.task['state'], 'review')
|
|
|
|
async def test_idle_deadline_resumes_after_native_tool_finishes(self):
|
|
producer = None
|
|
|
|
def prepare(client):
|
|
async def finish_tool():
|
|
await asyncio.sleep(0.1)
|
|
client.requests.clear()
|
|
nonlocal producer
|
|
producer = asyncio.create_task(finish_tool())
|
|
client.requests.add(producer)
|
|
|
|
await self.run_work([], prepare)
|
|
await producer
|
|
self.assertEqual(self.task['state'], 'failed')
|
|
self.assertIn('no progress', self.task['error'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|