Files
vibemake-pcb-studio/backend/tests/test_model_errors.py
T
2026-11-10 19:04:27 +08:00

99 lines
4.9 KiB
Python

"""Provider error attribution and permanent failure recovery; entirely offline."""
import json
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import AsyncMock, patch
BACKEND = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(BACKEND), str(BACKEND / 'api'), str(BACKEND / 'easyeda_backend')]
from providers.errors import ModelError, ModelUnavailableError, known_model_unavailable, provider_failure
from providers.codex import Codex
from app.config import Settings
from app.engine import Engine
from app.store import Store
from design_runtime.service import DesignService
RAW = {'type': 'error', 'status': 400, 'error': {'type': 'invalid_request_error',
'message': "The 'gpt-6.1-sol' model is not supported when using Codex with a ChatGPT account.",
'code': 'model_not_found'}}
LEGACY = 'KiCadError: Codex 请求未完成:' + json.dumps(RAW)
class ModelErrorTests(unittest.TestCase):
def test_nested_model_rejection_is_a_permanent_model_error(self):
for detail in [RAW, {'message': json.dumps(RAW)}, json.dumps(RAW)]:
error = provider_failure(detail)
self.assertIsInstance(error, ModelUnavailableError)
self.assertTrue(error.permanent)
self.assertEqual(error.code, 'model_not_found')
self.assertNotIn('KiCad', str(error))
self.assertIn('gpt-6.1-sol', str(error))
def test_codex_does_not_retry_or_reclassify_model_rejection(self):
error = provider_failure(RAW)
with tempfile.TemporaryDirectory() as tmp, patch.object(Codex, '_turn', new_callable=AsyncMock, side_effect=error) as turn:
model = Codex(tmp, guidance='offline rules')
with self.assertRaises(ModelUnavailableError):
model.call('intake', {}, {'type': 'object'}, retry=1)
self.assertEqual(turn.await_count, 1)
failure = json.loads(next(Path(tmp).glob('*/failure.json')).read_text())
self.assertEqual(failure['code'], 'model_not_found')
self.assertTrue(failure['permanent'])
def test_easyeda_fails_without_requesting_operator_recovery(self):
with tempfile.TemporaryDirectory() as tmp:
service = DesignService(tmp, agent=object())
task = {'task_id': 'pcb-' + 'b' * 20, 'eda_engine': 'easyeda', 'state': 'assessing'}
try:
service._record_exception(task, provider_failure(RAW))
self.assertEqual(task['state'], 'failed')
self.assertEqual(task['error_code'], 'model_not_found')
self.assertEqual(service.public(task)['eda_engine'], 'easyeda')
self.assertIsNone(task['active_agent_call'])
finally:
service.close()
def test_historical_model_rejection_is_shown_without_kicad_label(self):
with tempfile.TemporaryDirectory() as tmp:
settings = Settings(Path(tmp), Path(tmp) / 'unused', mode='real')
store = Store(settings.root)
engine = Engine(settings, store)
tid = 'pcb-' + 'c' * 20
engine.update_task({'task_id': tid, 'state': 'needs_operator', 'eda_engine': 'easyeda', 'error': LEGACY})
engine._auto_recover(tid, {'action': 'create'})
actual = engine.public_task(store.task(tid))
self.assertEqual(actual['state'], 'failed')
self.assertNotIn('KiCad', actual['error'])
self.assertEqual(actual['error_code'], 'model_not_found')
with store.db() as db:
self.assertEqual(db.execute('SELECT count(*) FROM operations').fetchone()[0], 0)
def test_restart_does_not_resubmit_known_unsupported_model(self):
with tempfile.TemporaryDirectory() as tmp:
settings = Settings(Path(tmp), Path(tmp) / 'unused', mode='real')
store = Store(settings.root)
engine = Engine(settings, store)
tid = 'pcb-' + 'd' * 20
path = settings.core_root / 'tasks' / tid / 'task.json'
path.parent.mkdir(parents=True)
path.write_text(json.dumps({'task_id': tid, 'state': 'needs_operator', 'eda_engine': 'easyeda', 'error': LEGACY}))
with patch.object(engine, 'loop'), patch.object(engine.provider, 'call') as call:
engine.start()
engine.shutdown()
call.assert_not_called()
snapshot = json.loads(path.read_text())
self.assertEqual(snapshot['state'], 'failed')
self.assertEqual(snapshot['provider_failure'], LEGACY)
self.assertEqual(store.task(tid)['state'], 'failed')
def test_native_eda_failures_are_not_renamed_or_treated_as_model_rejection(self):
self.assertIsNone(known_model_unavailable('KiCadError: PCB DRC failed'))
self.assertIsNone(known_model_unavailable('DesignError: footprint model_not_found'))
self.assertFalse(provider_failure({'message': 'Connection reset'}).permanent)
if __name__ == '__main__':
unittest.main()