Files
2026-09-24 10:40:57 +08:00

145 lines
8.0 KiB
Python

"""Exercise the complete Responses/tool protocol with deterministic responses."""
import copy
import json
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from test_workflow import ROOT, OPS, brief, review
from engine.config import QualityConfig
from engine.quality_pipeline import generate_quality, resume_quality
from engine.pipeline import PipelineError
class Item(SimpleNamespace):
def model_dump(self, **kwargs): return vars(self).copy()
class Response(SimpleNamespace):
def model_dump(self, **kwargs):
return {'id':self.id,'status':'completed','output':[x.model_dump() for x in self.output],
'usage':{'input_tokens':10,'output_tokens':5,'total_tokens':15}}
def response(index,text='',tool=None,args=None):
items=[] if tool is None else [Item(type='function_call',name=tool,arguments=json.dumps(args),call_id=f'call_{index}')]
return Response(id=f'resp_{index}',output_text=text,output=items)
class Stream:
def __init__(self,response): self.response=response
def __enter__(self): return self
def __exit__(self,*args): pass
def __iter__(self): return iter([SimpleNamespace(type='response.created',response=self.response)])
def get_final_response(self): return self.response
class Client:
def __init__(self,responses):
self.responses=self; self.queue=list(responses); self.requests=[]
def stream(self,**kwargs):
self.requests.append(copy.deepcopy(kwargs))
return Stream(self.queue.pop(0))
class ApiLoopTests(unittest.TestCase):
def test_old_operation_replay_failure_reaches_agent_for_repair(self):
old=copy.deepcopy(OPS)
old[1]['op']='removed_old_operation'
seed={'requirements':brief().model_dump(),'stage_index':0,'parent_job':'old',
'operations':old,'previous_output_id':'body','previous_errors':[]}
responses=[response(1,tool='rebuild_model',args={'operations':OPS})]
for i in range(3):
responses.extend([response(2+i*2,tool='submit_stage',args={'output_id':'body'}),response(3+i*2,review().model_dump_json())])
client=Client(responses)
with tempfile.TemporaryDirectory() as folder:
base=Path(folder);(base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
result=generate_quality('box',base,config=QualityConfig(max_calls=7),client=client,seed=seed)
self.assertEqual(result['quality_status'],'passed')
first=json.dumps(client.requests[0]['input'],ensure_ascii=False)
self.assertIn('旧操作重放失败',first)
self.assertIn('removed_old_operation',first)
self.assertNotIn('removed_old_operation',json.dumps(result['operations']))
def test_resume_discards_stale_parameter_links_after_contract_renumbering(self):
seed={'requirements':brief().model_dump(),'stage_index':0,'parent_job':'old',
'operations':OPS,'previous_output_id':'body','previous_errors':[],
'parameters':[{'name':'width','value':10,'unit':'mm','source':'explicit',
'evidence':'historical width','requirement_ids':['removed_old_id']} ]}
responses=[]
for i in range(3):
responses.extend([response(1+i*2,tool='submit_stage',args={'output_id':'body'}),response(2+i*2,review().model_dump_json())])
with tempfile.TemporaryDirectory() as folder:
base=Path(folder);(base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
result=generate_quality('box',base,config=QualityConfig(max_calls=6),client=Client(responses),seed=seed)
self.assertEqual(result['quality_status'],'passed')
self.assertEqual(result['candidate_parameters'][0]['requirement_ids'],[])
def test_full_loop_persists_and_reviews_in_separate_context(self):
responses=[response(1,brief().model_dump_json()),response(2,tool='rebuild_model',args={'operations':OPS})]
for i in range(3):
responses.extend([response(3+i*2,tool='submit_stage',args={'output_id':'body'}),
response(4+i*2,review().model_dump_json())])
client=Client(responses)
with tempfile.TemporaryDirectory() as folder:
base=Path(folder); (base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
report=generate_quality('10 x 20 x 30 box',base,config=QualityConfig(max_calls=8),client=client)
self.assertEqual(report['quality_status'],'passed')
self.assertEqual(report['step_count'],8)
self.assertEqual(report['usage']['total_tokens'],120)
output=Path(report['step_file']).parent
self.assertTrue((output/'response_008.json').exists())
self.assertEqual(len(report['reviews']),3)
original_report=(output/'report.json').read_bytes()
resumed_client=Client([response(9,tool='submit_stage',args={'output_id':'body'}),
response(10,review().model_dump_json())])
with patch('engine.quality_pipeline.render_shape_png',return_value=b'png'):
resumed=resume_quality(report['part_name'],base,QualityConfig(max_calls=2),resumed_client)
self.assertEqual(resumed['quality_status'],'passed')
self.assertEqual(resumed['resumed_from'],report['part_name'])
self.assertNotEqual(resumed['part_name'],report['part_name'])
self.assertEqual(resumed['source_description'],'10 x 20 x 30 box')
self.assertEqual(resumed['requirements'],report['requirements'])
self.assertEqual((output/'report.json').read_bytes(),original_report)
# Reviewer receives a fresh system/user request, never builder tool history.
for index in (3,5,7):
self.assertEqual([x['role'] for x in client.requests[index]['input']],['system','user'])
self.assertNotIn('tools',client.requests[index])
def test_incomplete_response_is_saved_and_never_executed(self):
r=response(1)
r.model_dump=lambda **kwargs: {'id':r.id,'status':'incomplete',
'incomplete_details':{'reason':'max_output_tokens'},'usage':{'input_tokens':10,'output_tokens':5,'total_tokens':15}}
class IncompleteStream(Stream):
def __iter__(self): return iter([SimpleNamespace(type='response.incomplete',response=self.response)])
client=SimpleNamespace(responses=SimpleNamespace(stream=lambda **kwargs: IncompleteStream(r)))
with tempfile.TemporaryDirectory() as folder:
base=Path(folder)
with self.assertRaises(PipelineError):
generate_quality('box',base,config=QualityConfig(),client=client)
job=next((base/'output').iterdir())
self.assertEqual(json.loads((job/'response_001.json').read_text())['status'],'incomplete')
self.assertIn('max_output_tokens',json.loads((job/'job.json').read_text())['reason'])
self.assertFalse((job/'model.step').exists())
def test_budget_before_candidate_fails_without_placeholder(self):
client=Client([response(1,brief().model_dump_json())])
with tempfile.TemporaryDirectory() as folder:
base=Path(folder); (base/'capabilities').symlink_to(ROOT/'capabilities',target_is_directory=True)
with self.assertRaises(PipelineError):
generate_quality('box',base,config=QualityConfig(max_calls=1),client=client)
jobs=list((base/'output').iterdir())
self.assertEqual(len(jobs),1)
self.assertEqual(json.loads((jobs[0]/'job.json').read_text())['status'],'failed')
self.assertFalse((jobs[0]/'model.step').exists())
self.assertFalse((jobs[0]/'report.json').exists())
if __name__=='__main__': unittest.main()