252 lines
16 KiB
Python
252 lines
16 KiB
Python
"""Source/geometry separation, bounded retries and safe public outcomes; no network."""
|
|
import copy
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from build123d import Box
|
|
from test_workflow import ROOT, REGISTRY, OPS, brief, review
|
|
from test_api_loop import Client, response
|
|
from engine.config import QualityConfig
|
|
from engine.job import QualityJob, write_json
|
|
from engine.pipeline import PipelineError
|
|
from engine.quality_pipeline import QualityWorkflow, generate_quality, resume_quality, structured_call
|
|
from engine.constraint_audit import ConstraintAudit, audit_constraints, NeedsClarification, needs_audit
|
|
from engine.quality_models import Brief, dimension_errors
|
|
from web.messages import failure_response
|
|
|
|
TEXT='底板150x90x16,顶部法兰外径110厚14,支柱高度50。'
|
|
|
|
|
|
def flange_brief():
|
|
b=brief().model_dump()
|
|
def req(id, desc, axis, value):
|
|
return dict(id=id,description=desc,stage='base',source='explicit',evidence=TEXT,critical=True,
|
|
envelope_dimension=dict(axis=axis,expected_mm=value,tolerance_mm=.1) if axis else None)
|
|
b['requirements']=[req('plate','底板150x90x16','X',150),req('flange','法兰外径110厚14',None,None),
|
|
req('invented','整件Y向总包络90','Y',90)]
|
|
return Brief.model_validate(b)
|
|
|
|
|
|
def audit_result(actions=None):
|
|
return ConstraintAudit(dimensions=[dict(requirement_id=id,action=action,evidence='尺寸引出对象为底板或法兰,不是整件限宽',
|
|
source_quote=TEXT,question='') for id,action in (actions or [('plate','local_only'),('invented','remove_unsupported')])])
|
|
|
|
|
|
class ReliabilityTests(unittest.TestCase):
|
|
def test_missing_measurement_preserves_geometry_instead_of_pausing_for_sources(self):
|
|
from engine.quality_models import ReviewFinding
|
|
with tempfile.TemporaryDirectory() as d:
|
|
b=brief()
|
|
b.requirements[0].envelope_dimension=None
|
|
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,b,'box',[])
|
|
for op in OPS:w.session.apply_op(op['op'],op['params'],op.get('target'),op['id'])
|
|
original=w.session.context['body']
|
|
r=review('unverified')
|
|
r.findings=[ReviewFinding(category='measurement',requirement_id='body',message='need section',evidence='not yet measured')]
|
|
with patch('engine.quality_pipeline.structured_call',return_value=r):
|
|
result=w.submit({'output_id':'body'})
|
|
self.assertEqual(result['failure_kind'],'protocol')
|
|
self.assertEqual(result['failure_keys'],['MEASUREMENT_EVIDENCE'])
|
|
self.assertIs(w.session.context['body'],original)
|
|
self.assertEqual(w.stage_index,0)
|
|
self.assertFalse((w.job.path/'clarification.json').exists())
|
|
|
|
def test_different_chamfer_selections_are_not_one_repeated_failure(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
|
|
result=dict(ok=False,error='EXECUTION_ERROR: kernel failure',failed_index=0)
|
|
for index in (1,2,3):
|
|
args={'operations':[dict(op='chamfer_edge',target='body',params=dict(radius=.3,edge_selector='by_index',edge_indices=[index]))]}
|
|
w.track_failure('execute_batch',args,result)
|
|
self.assertEqual(list(w.failure_counts.values()),[1,1,1])
|
|
args['operations'][0]['params']['radius']=.2
|
|
w.track_failure('execute_batch',args,result)
|
|
args['operations'][0]['params']['radius']=.1
|
|
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
|
|
w.track_failure('execute_batch',args,result)
|
|
|
|
def test_duplicate_defect_messages_count_once_per_attempt(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
|
|
result=dict(ok=False,failure_kind='geometry',failure_keys=['width']*4,errors=['width: mismatch'])
|
|
w.track_failure('submit_stage',{},result)
|
|
self.assertEqual(list(w.failure_counts.values()),[1])
|
|
w.track_failure('submit_stage',{},result)
|
|
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
|
|
w.track_failure('submit_stage',{},result)
|
|
|
|
def test_claim_schema_errors_preserve_geometry_and_are_not_geometric_failures(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(Client([]),QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
|
|
for op in OPS:w.session.apply_op(op['op'],op['params'],op.get('target'),op['id'])
|
|
original=w.session.context['body']
|
|
result=w.submit({'output_id':'body','claims':[{'type':'invented'}]*4})
|
|
self.assertEqual(result['failure_kind'],'protocol')
|
|
self.assertTrue(result['geometry_preserved'])
|
|
self.assertIs(w.session.context['body'],original)
|
|
w.track_failure('submit_stage',{},result)
|
|
self.assertEqual(w.latest_review_errors,[])
|
|
self.assertEqual(list(w.failure_counts.values()),[1])
|
|
|
|
def test_resume_information_note_keeps_brief_and_completed_stage(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);folder=base/'output'/'saved';folder.mkdir(parents=True)
|
|
b=brief().model_dump()
|
|
write_json(folder/'request.json',{'text':'box','images':[]})
|
|
write_json(folder/'requirements.json',b)
|
|
write_json(folder/'model.json',{'operations':OPS})
|
|
write_json(folder/'resume.json',{'stage_index':1})
|
|
candidate=folder/'candidate_001';candidate.mkdir()
|
|
write_json(candidate/'review.json',{'accepted':False,'stage':'features','errors':['material unknown'],
|
|
'review':{'items':[{'requirement_id':'material','status':'unverified'}],
|
|
'findings':[{'requirement_id':'material','category':'information'}]}})
|
|
with patch('engine.quality_pipeline.generate_quality',return_value={}) as generate:
|
|
resume_quality('saved',base)
|
|
seed=generate.call_args.kwargs['seed']
|
|
self.assertEqual(seed['requirements'],b)
|
|
self.assertEqual(seed['stage_index'],1)
|
|
self.assertEqual(seed['previous_errors'],['material unknown'])
|
|
resume_quality('saved',base,clarification='change width to 12')
|
|
self.assertIsNone(generate.call_args.kwargs['seed']['requirements'])
|
|
|
|
def test_scope_correction_preserves_original_local_requirements(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
b=flange_brief();original=b.model_dump();job=QualityJob(d,TEXT,[],QualityConfig())
|
|
self.assertTrue(needs_audit(b))
|
|
client=Client([response(1,audit_result().model_dump_json())])
|
|
fixed=audit_constraints(client,job,b,TEXT,[],structured_call)
|
|
self.assertEqual(b.model_dump(),original)
|
|
self.assertEqual([r.id for r in fixed.requirements],['plate','flange'])
|
|
self.assertEqual(fixed.requirements[0].description,b.requirements[0].description)
|
|
self.assertEqual(fixed.requirements[1],b.requirements[1])
|
|
self.assertEqual(dimension_errors(fixed,'base',Box(150,110,80)),[])
|
|
self.assertTrue((job.path/'requirements_before_constraint_audit.json').exists())
|
|
self.assertNotIn('tools',client.requests[0])
|
|
|
|
def test_explicit_overall_limit_cannot_be_silently_removed(self):
|
|
text='整件总宽90,法兰外径110'
|
|
bad=audit_result([('plate','keep'),('invented','remove_unsupported')])
|
|
bad.dimensions[1].source_quote=text
|
|
with tempfile.TemporaryDirectory() as d:
|
|
job=QualityJob(d,text,[],QualityConfig())
|
|
client=Client([response(1,bad.model_dump_json()),response(2,bad.model_dump_json())])
|
|
with self.assertRaises(RuntimeError):audit_constraints(client,job,flange_brief(),text,[],structured_call)
|
|
self.assertEqual(job.calls,2)
|
|
|
|
def test_audit_rejects_missing_image_and_mixed_requirement_deletion(self):
|
|
for mixed in (False, True):
|
|
b=flange_brief();audit=audit_result()
|
|
if mixed:
|
|
b.requirements[-1].description='整件总宽90,另有6个孔'
|
|
else:
|
|
audit.dimensions[1].source_quote='image_99: 不存在的区域'
|
|
with tempfile.TemporaryDirectory() as d:
|
|
job=QualityJob(d,TEXT,[],QualityConfig())
|
|
client=Client([response(1,audit.model_dump_json()),response(2,audit.model_dump_json())])
|
|
with self.assertRaises(RuntimeError): audit_constraints(client,job,b,TEXT,[],structured_call)
|
|
|
|
def test_true_source_conflict_saves_question_without_building(self):
|
|
b=flange_brief();audit=audit_result([('plate','keep'),('invented','conflict')])
|
|
audit.dimensions[1].question='整体限宽90与法兰直径110需要调整哪一项?'
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);client=Client([response(1,b.model_dump_json()),response(2,audit.model_dump_json())])
|
|
with patch('engine.quality_pipeline.load_registry',side_effect=AssertionError('must not build')):
|
|
with self.assertRaises(PipelineError) as caught: generate_quality(TEXT,base,client=client)
|
|
public=failure_response(str(caught.exception),base,caught.exception.job_id)
|
|
self.assertEqual(public['status'],'needs_clarification');self.assertFalse(public['can_resume'])
|
|
self.assertIn('90',public['questions'][0]);self.assertNotIn('NeedsClarification',public['error'])
|
|
|
|
def test_repeated_plain_replies_stop_after_three_without_tools(self):
|
|
client=Client([response(1,brief().model_dump_json()),response(2,'cannot proceed'),response(3,'cannot proceed'),
|
|
response(4,ConstraintAudit(dimensions=[dict(requirement_id='body',action='keep',evidence='explicit',source_quote='',question='')]).model_dump_json()),
|
|
response(5,'cannot proceed')])
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);(base/'capabilities').symlink_to(ROOT/'capabilities')
|
|
with self.assertRaises(PipelineError) as caught:generate_quality('box',base,client=client)
|
|
self.assertIn('NoProgress',str(caught.exception));self.assertEqual(len(client.requests),5)
|
|
|
|
def test_repeated_inspection_without_new_evidence_stops(self):
|
|
client=Client([response(1,brief().model_dump_json())] + [response(i,tool='inspect_state',args={}) for i in range(2,9)])
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);(base/'capabilities').symlink_to(ROOT/'capabilities')
|
|
with self.assertRaisesRegex(PipelineError, 'NoProgress'):
|
|
generate_quality('box',base,client=client)
|
|
self.assertEqual(len(client.requests),8)
|
|
|
|
def test_old_persisted_errors_are_filtered_on_reload(self):
|
|
from web.tasks import TaskManager
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);folder=base/'output'/'_http_tasks';folder.mkdir(parents=True)
|
|
write_json(folder/('a'*32+'.json'), {'job_id':None,'response':{'ok':False,'error':'TypeError: unsupported operand /private/path'}})
|
|
response=TaskManager(base,lambda r:r).status('a'*32)['response']
|
|
self.assertNotIn('TypeError',response['error']);self.assertNotIn('/private',response['error'])
|
|
|
|
def test_same_dimension_defect_stops_despite_changed_ids_and_parameters(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(None,QualityJob(d,'test',[],QualityConfig()),REGISTRY,brief(),'test',[])
|
|
for i in range(2):w.track_failure('submit_stage',{'output_id':str(i)},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':20}))
|
|
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
|
|
w.track_failure('execute_batch',{'output_id':'other'},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':20}))
|
|
|
|
def test_measured_improvement_resets_stagnation(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(None,QualityJob(d,'test',[],QualityConfig()),REGISTRY,brief(),'test',[])
|
|
for delta in [20,20,10,10,5,5]:
|
|
w.track_failure('submit_stage',{},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':delta}))
|
|
with self.assertRaisesRegex(RuntimeError,'RepairLimit'):
|
|
w.track_failure('submit_stage',{},dict(ok=False,failure_kind='geometry',failure_keys=['width'],errors=['width: mismatch'],progress_metrics={'width':5}))
|
|
|
|
def test_dimension_rejection_keeps_valid_candidate_as_partial(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
w=QualityWorkflow(None,QualityJob(d,'box',[],QualityConfig()),REGISTRY,brief(),'box',[])
|
|
w.constraints_checked=True
|
|
ops=copy.deepcopy(OPS);ops[0]['params']['width']=12
|
|
w.dispatch('rebuild_model',{'operations':ops})
|
|
self.assertFalse(w.submit({'output_id':'body'})['ok'])
|
|
report=w.publish(False,'NoProgress')
|
|
self.assertEqual(report['quality_status'],'partial');self.assertTrue(Path(report['step_file']).exists())
|
|
self.assertTrue(report['geometry_issues']);self.assertEqual(report['reviews'],[])
|
|
|
|
def test_resume_correction_preserves_original_and_images(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig())
|
|
job.write('requirements.json',flange_brief().model_dump());job.write('model.json',{'operations':OPS})
|
|
old=(job.path/'request.json').read_bytes()
|
|
with patch('engine.quality_pipeline.generate_quality',return_value={}) as generate:
|
|
resume_quality(job.id,base,clarification='90是底板宽度,不限制整件宽度')
|
|
args=generate.call_args
|
|
self.assertIn(TEXT,args.args[0]);self.assertIn('90是底板',args.args[0])
|
|
self.assertIsNone(args.kwargs['seed']['requirements']);self.assertEqual(args.kwargs['seed']['operations'],OPS)
|
|
self.assertEqual(old,(job.path/'request.json').read_bytes())
|
|
|
|
def test_legacy_resume_runs_source_audit_before_cad(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig());job.write('requirements.json',flange_brief().model_dump())
|
|
client=Client([response(1,audit_result().model_dump_json())])
|
|
with patch('engine.quality_pipeline.load_registry',side_effect=RuntimeError('end offline test')):
|
|
with self.assertRaises(PipelineError):resume_quality(job.id,base,client=client)
|
|
self.assertEqual(len(client.requests),1)
|
|
self.assertEqual(client.requests[0]['text']['format']['name'],'ConstraintAudit')
|
|
resumed=next(p for p in (base/'output').iterdir() if p != job.path)
|
|
self.assertTrue((resumed/'resume.json').exists())
|
|
self.assertTrue((resumed/'model.json').exists())
|
|
|
|
def test_http_filters_exceptions_and_requires_answer_for_real_conflict(self):
|
|
import web.app as web
|
|
with tempfile.TemporaryDirectory() as d:
|
|
base=Path(d);job=QualityJob(base,TEXT,[],QualityConfig())
|
|
job.write('clarification.json',{'questions':['确认整体宽度还是底板宽度?']})
|
|
with patch.object(web,'BASE_DIR',base),patch.object(web.tasks,'start',return_value='a'*32) as start:
|
|
client=web.app.test_client()
|
|
self.assertEqual(client.post('/api/tasks',json={'action':'resume','part_name':job.id}).status_code,400)
|
|
self.assertEqual(client.post('/api/tasks',json={'action':'resume','part_name':job.id,'clarification':'底板宽90'}).status_code,202)
|
|
self.assertEqual(start.call_args.args[1]['clarification'],'底板宽90')
|
|
self.assertEqual(client.post('/api/tasks',json={'action':'plan','text':'x'}).status_code,400)
|
|
for reason in ['BudgetExceeded: /private/path', "TypeError: unsupported operand str / str", 'RuntimeError: max_output_tokens']:
|
|
public=failure_response(reason,base)
|
|
self.assertNotIn(reason,public['error']);self.assertNotIn('/private',public['error'])
|
|
|
|
if __name__=='__main__':unittest.main()
|