Files
2026-09-24 10:40:57 +08:00

252 lines
16 KiB
Python

"""Source/geometry separation, bounded retries and safe public outcomes; no network."""
import copy
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from build123d import Box
from test_workflow import ROOT, REGISTRY, OPS, brief, review
from test_api_loop import Client, response
from engine.config import QualityConfig
from engine.job import QualityJob, write_json
from engine.pipeline import PipelineError
from engine.quality_pipeline import QualityWorkflow, generate_quality, resume_quality, structured_call
from engine.constraint_audit import ConstraintAudit, audit_constraints, NeedsClarification, needs_audit
from engine.quality_models import Brief, dimension_errors
from web.messages import failure_response
TEXT='底板150x90x16,顶部法兰外径110厚14,支柱高度50。'
def flange_brief():
b=brief().model_dump()
def req(id, desc, axis, value):
return dict(id=id,description=desc,stage='base',source='explicit',evidence=TEXT,critical=True,
envelope_dimension=dict(axis=axis,expected_mm=value,tolerance_mm=.1) if axis else None)
b['requirements']=[req('plate','底板150x90x16','X',150),req('flange','法兰外径110厚14',None,None),
req('invented','整件Y向总包络90','Y',90)]
return Brief.model_validate(b)
def audit_result(actions=None):
return ConstraintAudit(dimensions=[dict(requirement_id=id,action=action,evidence='尺寸引出对象为底板或法兰,不是整件限宽',
source_quote=TEXT,question='') for id,action in (actions or [('plate','local_only'),('invented','remove_unsupported')])])
class ReliabilityTests(unittest.TestCase):
def test_missing_measurement_preserves_geometry_instead_of_pausing_for_sources(self):
from engine.quality_models import ReviewFinding
with tempfile.TemporaryDirectory() as d:
b=brief()
b.requirements[0].envelope_dimension=None
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,b,'box',[])
for op in OPS:w.session.apply_op(op['op'],op['params'],op.get('target'),op['id'])
original=w.session.context['body']
r=review('unverified')
r.findings=[ReviewFinding(category='measurement',requirement_id='body',message='need section',evidence='not yet measured')]
with patch('engine.quality_pipeline.structured_call',return_value=r):
result=w.submit({'output_id':'body'})
self.assertEqual(result['failure_kind'],'protocol')
self.assertEqual(result['failure_keys'],['MEASUREMENT_EVIDENCE'])
self.assertIs(w.session.context['body'],original)
self.assertEqual(w.stage_index,0)
self.assertFalse((w.job.path/'clarification.json').exists())
def test_different_chamfer_selections_are_not_one_repeated_failure(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
result=dict(ok=False,error='EXECUTION_ERROR: kernel failure',failed_index=0)
for index in (1,2,3):
args={'operations':[dict(op='chamfer_edge',target='body',params=dict(radius=.3,edge_selector='by_index',edge_indices=[index]))]}
w.track_failure('execute_batch',args,result)
self.assertEqual(list(w.failure_counts.values()),[1,1,1])
args['operations'][0]['params']['radius']=.2
w.track_failure('execute_batch',args,result)
args['operations'][0]['params']['radius']=.1
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
w.track_failure('execute_batch',args,result)
def test_duplicate_defect_messages_count_once_per_attempt(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
result=dict(ok=False,failure_kind='geometry',failure_keys=['width']*4,errors=['width: mismatch'])
w.track_failure('submit_stage',{},result)
self.assertEqual(list(w.failure_counts.values()),[1])
w.track_failure('submit_stage',{},result)
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
w.track_failure('submit_stage',{},result)
def test_claim_schema_errors_preserve_geometry_and_are_not_geometric_failures(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
for op in OPS:w.session.apply_op(op['op'],op['params'],op.get('target'),op['id'])
original=w.session.context['body']
result=w.submit({'output_id':'body','claims':[{'type':'invented'}]*4})
self.assertEqual(result['failure_kind'],'protocol')
self.assertTrue(result['geometry_preserved'])
self.assertIs(w.session.context['body'],original)
w.track_failure('submit_stage',{},result)
self.assertEqual(w.latest_review_errors,[])
self.assertEqual(list(w.failure_counts.values()),[1])
def test_resume_information_note_keeps_brief_and_completed_stage(self):
with tempfile.TemporaryDirectory() as d:
base=Path(d);folder=base/'output'/'saved';folder.mkdir(parents=True)
b=brief().model_dump()
write_json(folder/'request.json',{'text':'box','images':[]})
write_json(folder/'requirements.json',b)
write_json(folder/'model.json',{'operations':OPS})
write_json(folder/'resume.json',{'stage_index':1})
candidate=folder/'candidate_001';candidate.mkdir()
write_json(candidate/'review.json',{'accepted':False,'stage':'features','errors':['material unknown'],
'review':{'items':[{'requirement_id':'material','status':'unverified'}],
'findings':[{'requirement_id':'material','category':'information'}]}})
with patch('engine.quality_pipeline.generate_quality',return_value={}) as generate:
resume_quality('saved',base)
seed=generate.call_args.kwargs['seed']
self.assertEqual(seed['requirements'],b)
self.assertEqual(seed['stage_index'],1)
self.assertEqual(seed['previous_errors'],['material unknown'])
resume_quality('saved',base,clarification='change width to 12')
self.assertIsNone(generate.call_args.kwargs['seed']['requirements'])
def test_scope_correction_preserves_original_local_requirements(self):
with tempfile.TemporaryDirectory() as d:
b=flange_brief();original=b.model_dump();job=QualityJob(d,TEXT,[],QualityConfig())
self.assertTrue(needs_audit(b))
client=Client([response(1,audit_result().model_dump_json())])
fixed=audit_constraints(client,job,b,TEXT,[],structured_call)
self.assertEqual(b.model_dump(),original)
self.assertEqual([r.id for r in fixed.requirements],['plate','flange'])
self.assertEqual(fixed.requirements[0].description,b.requirements[0].description)
self.assertEqual(fixed.requirements[1],b.requirements[1])
self.assertEqual(dimension_errors(fixed,'base',Box(150,110,80)),[])
self.assertTrue((job.path/'requirements_before_constraint_audit.json').exists())
self.assertNotIn('tools',client.requests[0])
def test_explicit_overall_limit_cannot_be_silently_removed(self):
text='整件总宽90,法兰外径110'
bad=audit_result([('plate','keep'),('invented','remove_unsupported')])
bad.dimensions[1].source_quote=text
with tempfile.TemporaryDirectory() as d:
job=QualityJob(d,text,[],QualityConfig())
client=Client([response(1,bad.model_dump_json()),response(2,bad.model_dump_json())])
with self.assertRaises(RuntimeError):audit_constraints(client,job,flange_brief(),text,[],structured_call)
self.assertEqual(job.calls,2)
def test_audit_rejects_missing_image_and_mixed_requirement_deletion(self):
for mixed in (False, True):
b=flange_brief();audit=audit_result()
if mixed:
b.requirements[-1].description='整件总宽90,另有6个孔'
else:
audit.dimensions[1].source_quote='image_99: 不存在的区域'
with tempfile.TemporaryDirectory() as d:
job=QualityJob(d,TEXT,[],QualityConfig())
client=Client([response(1,audit.model_dump_json()),response(2,audit.model_dump_json())])
with self.assertRaises(RuntimeError): audit_constraints(client,job,b,TEXT,[],structured_call)
def test_true_source_conflict_saves_question_without_building(self):
b=flange_brief();audit=audit_result([('plate','keep'),('invented','conflict')])
audit.dimensions[1].question='整体限宽90与法兰直径110需要调整哪一项?'
with tempfile.TemporaryDirectory() as d:
base=Path(d);client=Client([response(1,b.model_dump_json()),response(2,audit.model_dump_json())])
with patch('engine.quality_pipeline.load_registry',side_effect=AssertionError('must not build')):
with self.assertRaises(PipelineError) as caught: generate_quality(TEXT,base,client=client)
public=failure_response(str(caught.exception),base,caught.exception.job_id)
self.assertEqual(public['status'],'needs_clarification');self.assertFalse(public['can_resume'])
self.assertIn('90',public['questions'][0]);self.assertNotIn('NeedsClarification',public['error'])
def test_repeated_plain_replies_stop_after_three_without_tools(self):
client=Client([response(1,brief().model_dump_json()),response(2,'cannot proceed'),response(3,'cannot proceed'),
response(4,ConstraintAudit(dimensions=[dict(requirement_id='body',action='keep',evidence='explicit',source_quote='',question='')]).model_dump_json()),
response(5,'cannot proceed')])
with tempfile.TemporaryDirectory() as d:
base=Path(d);(base/'capabilities').symlink_to(ROOT/'capabilities')
with self.assertRaises(PipelineError) as caught:generate_quality('box',base,client=client)
self.assertIn('NoProgress',str(caught.exception));self.assertEqual(len(client.requests),5)
def test_repeated_inspection_without_new_evidence_stops(self):
client=Client([response(1,brief().model_dump_json())] + [response(i,tool='inspect_state',args={}) for i in range(2,9)])
with tempfile.TemporaryDirectory() as d:
base=Path(d);(base/'capabilities').symlink_to(ROOT/'capabilities')
with self.assertRaisesRegex(PipelineError, 'NoProgress'):
generate_quality('box',base,client=client)
self.assertEqual(len(client.requests),8)
def test_old_persisted_errors_are_filtered_on_reload(self):
from web.tasks import TaskManager
with tempfile.TemporaryDirectory() as d:
base=Path(d);folder=base/'output'/'_http_tasks';folder.mkdir(parents=True)
write_json(folder/('a'*32+'.json'), {'job_id':None,'response':{'ok':False,'error':'TypeError: unsupported operand /private/path'}})
response=TaskManager(base,lambda r:r).status('a'*32)['response']
self.assertNotIn('TypeError',response['error']);self.assertNotIn('/private',response['error'])
def test_same_dimension_defect_stops_despite_changed_ids_and_parameters(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(None,QualityJob(d,'test',[],QualityConfig()),REGISTRY,brief(),'test',[])
for i in range(2):w.track_failure('submit_stage',{'output_id':str(i)},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':20}))
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
w.track_failure('execute_batch',{'output_id':'other'},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':20}))
def test_measured_improvement_resets_stagnation(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(None,QualityJob(d,'test',[],QualityConfig()),REGISTRY,brief(),'test',[])
for delta in [20,20,10,10,5,5]:
w.track_failure('submit_stage',{},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':delta}))
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
w.track_failure('submit_stage',{},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':5}))
def test_dimension_rejection_keeps_valid_candidate_as_partial(self):
with tempfile.TemporaryDirectory() as d:
w=QualityWorkflow(None,QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
w.constraints_checked=True
ops=copy.deepcopy(OPS);ops[0]['params']['width']=12
w.dispatch('rebuild_model',{'operations':ops})
self.assertFalse(w.submit({'output_id':'body'})['ok'])
report=w.publish(False,'NoProgress')
self.assertEqual(report['quality_status'],'partial');self.assertTrue(Path(report['step_file']).exists())
self.assertTrue(report['geometry_issues']);self.assertEqual(report['reviews'],[])
def test_resume_correction_preserves_original_and_images(self):
with tempfile.TemporaryDirectory() as d:
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig())
job.write('requirements.json',flange_brief().model_dump());job.write('model.json',{'operations':OPS})
old=(job.path/'request.json').read_bytes()
with patch('engine.quality_pipeline.generate_quality',return_value={}) as generate:
resume_quality(job.id,base,clarification='90是底板宽度,不限制整件宽度')
args=generate.call_args
self.assertIn(TEXT,args.args[0]);self.assertIn('90是底板',args.args[0])
self.assertIsNone(args.kwargs['seed']['requirements']);self.assertEqual(args.kwargs['seed']['operations'],OPS)
self.assertEqual(old,(job.path/'request.json').read_bytes())
def test_legacy_resume_runs_source_audit_before_cad(self):
with tempfile.TemporaryDirectory() as d:
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig());job.write('requirements.json',flange_brief().model_dump())
client=Client([response(1,audit_result().model_dump_json())])
with patch('engine.quality_pipeline.load_registry',side_effect=RuntimeError('end offline test')):
with self.assertRaises(PipelineError):resume_quality(job.id,base,client=client)
self.assertEqual(len(client.requests),1)
self.assertEqual(client.requests[0]['text']['format']['name'],'ConstraintAudit')
resumed=next(p for p in (base/'output').iterdir() if p != job.path)
self.assertTrue((resumed/'resume.json').exists())
self.assertTrue((resumed/'model.json').exists())
def test_http_filters_exceptions_and_requires_answer_for_real_conflict(self):
import web.app as web
with tempfile.TemporaryDirectory() as d:
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig())
job.write('clarification.json',{'questions':['确认整体宽度还是底板宽度?']})
with patch.object(web,'BASE_DIR',base),patch.object(web.tasks,'start',return_value='a'*32) as start:
client=web.app.test_client()
self.assertEqual(client.post('/api/tasks',json={'action':'resume','part_name':job.id}).status_code,400)
self.assertEqual(client.post('/api/tasks',json={'action':'resume','part_name':job.id,'clarification':'底板宽90'}).status_code,202)
self.assertEqual(start.call_args.args[1]['clarification'],'底板宽90')
self.assertEqual(client.post('/api/tasks',json={'action':'plan','text':'x'}).status_code,400)
for reason in ['BudgetExceeded: /private/path', "TypeError: unsupported operand str / str", 'RuntimeError: max_output_tokens']:
public=failure_response(reason,base)
self.assertNotIn(reason,public['error']);self.assertNotIn('/private',public['error'])
if __name__=='__main__':unittest.main()