145 lines
8.0 KiB
Python
145 lines
8.0 KiB
Python
"""Exercise the complete Responses/tool protocol with deterministic responses."""
|
|
import copy
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from test_workflow import ROOT, OPS, brief, review
|
|
from engine.config import QualityConfig
|
|
from engine.quality_pipeline import generate_quality, resume_quality
|
|
from engine.pipeline import PipelineError
|
|
|
|
|
|
class Item(SimpleNamespace):
|
|
def model_dump(self, **kwargs): return vars(self).copy()
|
|
|
|
|
|
class Response(SimpleNamespace):
|
|
def model_dump(self, **kwargs):
|
|
return {'id':self.id,'status':'completed','output':[x.model_dump() for x in self.output],
|
|
'usage':{'input_tokens':10,'output_tokens':5,'total_tokens':15}}
|
|
|
|
|
|
def response(index,text='',tool=None,args=None):
|
|
items=[] if tool is None else [Item(type='function_call',name=tool,arguments=json.dumps(args),call_id=f'call_{index}')]
|
|
return Response(id=f'resp_{index}',output_text=text,output=items)
|
|
|
|
|
|
class Stream:
|
|
def __init__(self,response): self.response=response
|
|
def __enter__(self): return self
|
|
def __exit__(self,*args): pass
|
|
def __iter__(self): return iter([SimpleNamespace(type='response.created',response=self.response)])
|
|
def get_final_response(self): return self.response
|
|
|
|
|
|
class Client:
|
|
def __init__(self,responses):
|
|
self.responses=self; self.queue=list(responses); self.requests=[]
|
|
def stream(self,**kwargs):
|
|
self.requests.append(copy.deepcopy(kwargs))
|
|
return Stream(self.queue.pop(0))
|
|
|
|
|
|
class ApiLoopTests(unittest.TestCase):
|
|
def test_old_operation_replay_failure_reaches_agent_for_repair(self):
|
|
old=copy.deepcopy(OPS)
|
|
old[1]['op']='removed_old_operation'
|
|
seed={'requirements':brief().model_dump(),'stage_index':0,'parent_job':'old',
|
|
'operations':old,'previous_output_id':'body','previous_errors':[]}
|
|
responses=[response(1,tool='rebuild_model',args={'operations':OPS})]
|
|
for i in range(3):
|
|
responses.extend([response(2+i*2,tool='submit_stage',args={'output_id':'body'}),response(3+i*2,review().model_dump_json())])
|
|
client=Client(responses)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base=Path(folder);(base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
|
|
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
|
|
result=generate_quality('box',base,config=QualityConfig(max_calls=7),client=client,seed=seed)
|
|
self.assertEqual(result['quality_status'],'passed')
|
|
first=json.dumps(client.requests[0]['input'],ensure_ascii=False)
|
|
self.assertIn('旧操作重放失败',first)
|
|
self.assertIn('removed_old_operation',first)
|
|
self.assertNotIn('removed_old_operation',json.dumps(result['operations']))
|
|
|
|
def test_resume_discards_stale_parameter_links_after_contract_renumbering(self):
|
|
seed={'requirements':brief().model_dump(),'stage_index':0,'parent_job':'old',
|
|
'operations':OPS,'previous_output_id':'body','previous_errors':[],
|
|
'parameters':[{'name':'width','value':10,'unit':'mm','source':'explicit',
|
|
'evidence':'historical width','requirement_ids':['removed_old_id']} ]}
|
|
responses=[]
|
|
for i in range(3):
|
|
responses.extend([response(1+i*2,tool='submit_stage',args={'output_id':'body'}),response(2+i*2,review().model_dump_json())])
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base=Path(folder);(base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
|
|
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
|
|
result=generate_quality('box',base,config=QualityConfig(max_calls=6),client=Client(responses),seed=seed)
|
|
self.assertEqual(result['quality_status'],'passed')
|
|
self.assertEqual(result['candidate_parameters'][0]['requirement_ids'],[])
|
|
|
|
def test_full_loop_persists_and_reviews_in_separate_context(self):
|
|
responses=[response(1,brief().model_dump_json()),response(2,tool='rebuild_model',args={'operations':OPS})]
|
|
for i in range(3):
|
|
responses.extend([response(3+i*2,tool='submit_stage',args={'output_id':'body'}),
|
|
response(4+i*2,review().model_dump_json())])
|
|
client=Client(responses)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base=Path(folder); (base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
|
|
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
|
|
report=generate_quality('10 x 20 x 30 box',base,config=QualityConfig(max_calls=8),client=client)
|
|
self.assertEqual(report['quality_status'],'passed')
|
|
self.assertEqual(report['step_count'],8)
|
|
self.assertEqual(report['usage']['total_tokens'],120)
|
|
output=Path(report['step_file']).parent
|
|
self.assertTrue((output/'response_008.json').exists())
|
|
self.assertEqual(len(report['reviews']),3)
|
|
original_report=(output/'report.json').read_bytes()
|
|
resumed_client=Client([response(9,tool='submit_stage',args={'output_id':'body'}),
|
|
response(10,review().model_dump_json())])
|
|
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
|
|
resumed=resume_quality(report['part_name'],base,QualityConfig(max_calls=2),resumed_client)
|
|
self.assertEqual(resumed['quality_status'],'passed')
|
|
self.assertEqual(resumed['resumed_from'],report['part_name'])
|
|
self.assertNotEqual(resumed['part_name'],report['part_name'])
|
|
self.assertEqual(resumed['source_description'],'10 x 20 x 30 box')
|
|
self.assertEqual(resumed['requirements'],report['requirements'])
|
|
self.assertEqual((output/'report.json').read_bytes(),original_report)
|
|
|
|
# Reviewer receives a fresh system/user request, never builder tool history.
|
|
for index in (3,5,7):
|
|
self.assertEqual([x['role'] for x in client.requests[index]['input']],['system','user'])
|
|
self.assertNotIn('tools',client.requests[index])
|
|
|
|
def test_incomplete_response_is_saved_and_never_executed(self):
|
|
r=response(1)
|
|
r.model_dump=lambda **kwargs: {'id':r.id,'status':'incomplete',
|
|
'incomplete_details':{'reason':'max_output_tokens'},'usage':{'input_tokens':10,'output_tokens':5,'total_tokens':15}}
|
|
class IncompleteStream(Stream):
|
|
def __iter__(self): return iter([SimpleNamespace(type='response.incomplete',response=self.response)])
|
|
client=SimpleNamespace(responses=SimpleNamespace(stream=lambda **kwargs: IncompleteStream(r)))
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base=Path(folder)
|
|
with self.assertRaises(PipelineError):
|
|
generate_quality('box',base,config=QualityConfig(),client=client)
|
|
job=next((base/'output').iterdir())
|
|
self.assertEqual(json.loads((job/'response_001.json').read_text())['status'],'incomplete')
|
|
self.assertIn('max_output_tokens',json.loads((job/'job.json').read_text())['reason'])
|
|
self.assertFalse((job/'model.step').exists())
|
|
|
|
def test_budget_before_candidate_fails_without_placeholder(self):
|
|
client=Client([response(1,brief().model_dump_json())])
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
base=Path(folder); (base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
|
|
with self.assertRaises(PipelineError):
|
|
generate_quality('box',base,config=QualityConfig(max_calls=1),client=client)
|
|
jobs=list((base/'output').iterdir())
|
|
self.assertEqual(len(jobs),1)
|
|
self.assertEqual(json.loads((jobs[0]/'job.json').read_text())['status'],'failed')
|
|
self.assertFalse((jobs[0]/'model.step').exists())
|
|
self.assertFalse((jobs[0]/'report.json').exists())
|
|
|
|
|
|
if __name__=='__main__': unittest.main()
|