140 lines
8.3 KiB
Python
140 lines
8.3 KiB
Python
"""Keep native CAD crashes and hard timeouts outside the HTTP server process."""
|
|
import json
|
|
import multiprocessing
|
|
import queue
|
|
import shutil
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
def recover_candidate(base_dir, job_id, reason):
|
|
"""Copy an already exported/inspected candidate; never invoke CAD in the host."""
|
|
from .job import write_json
|
|
from .validation import safe_name
|
|
folder = Path(base_dir)/'output'/safe_name(job_id)
|
|
request = json.loads((folder/'request.json').read_text())
|
|
state = json.loads((folder/'job.json').read_text())
|
|
events = []
|
|
event_file = folder/'events.jsonl'
|
|
for line in event_file.read_text().splitlines() if event_file.exists() else []:
|
|
try:
|
|
events.append(json.loads(line))
|
|
except ValueError:
|
|
pass # A killed worker can leave a partially written final event.
|
|
state['calls']=max((e.get('call',0) for e in events),default=state.get('calls',0))
|
|
state['elapsed_s']=max((e.get('elapsed_s',0) for e in events),default=state.get('elapsed_s',0))
|
|
state['elapsed_measurement']='last_recorded_event_before_worker_exit'
|
|
state['usage']={key:sum(int((e.get('usage') or {}).get(key,0) or 0) for e in events if e['event']=='api_done')
|
|
for key in ('input_tokens','output_tokens','total_tokens')}
|
|
brief = json.loads((folder/'requirements.json').read_text()) if (folder/'requirements.json').exists() else {}
|
|
with (folder/'events.jsonl').open('a') as stream:
|
|
stream.write(json.dumps({'event':'worker_interrupted','error':reason,'elapsed_s':state.get('elapsed_s',0)},ensure_ascii=False)+'\n')
|
|
# A submitted whole-part candidate outranks a later scratch component.
|
|
# Emergency checkpoints remain the fallback before the first submission.
|
|
saved = list(folder.glob("candidate_*")) + list(folder.glob("checkpoint_[0-9]*"))
|
|
candidates = [p for p in sorted(saved, key=lambda p: (p.name.startswith("candidate_"), p.stat().st_mtime), reverse=True)
|
|
if all((p/name).exists() for name in ('model.step','model.stl','model.json','geometry.json'))]
|
|
for candidate in candidates:
|
|
geometry = json.loads((candidate/'geometry.json').read_text())
|
|
if not geometry.get('valid') or geometry.get('solid_count',0)<1: continue
|
|
model = json.loads((candidate/'model.json').read_text())
|
|
reviews = [json.loads(p.read_text()) for p in sorted(folder.glob('candidate_*/review.json')) if p.parent.name<=candidate.name]
|
|
for file in candidate.iterdir():
|
|
if file.suffix in ('.png','.step','.stl'): shutil.copy2(file,folder/file.name)
|
|
shutil.copy2(candidate/'model.json',folder/'published_model.json')
|
|
params = json.loads((candidate/'parameters.json').read_text()) if (candidate/'parameters.json').exists() else []
|
|
config = request['config']
|
|
report = {'ok':True,'part_name':job_id,'display_name':brief.get('name',job_id),'output_id':model['output'],
|
|
'operations':model['operations'],'claims':model.get('claims',[]),
|
|
'volume_mm3':geometry['volume_mm3'],'bounding_box_mm':{'min':geometry['bbox_min'],'max':geometry['bbox_max']},
|
|
'step_file':str(folder/'model.step'),'stl_file':str(folder/'model.stl'),
|
|
'published_model_file':str(folder/'published_model.json'),'candidate_parameters':params,
|
|
'source_description':request['text'],'requirements':brief,'llm_assumptions':brief.get('assumptions',[]),
|
|
'unresolved':brief.get('unresolved',[]),'reviews':reviews,
|
|
'remaining_issues':(reviews[-1]['errors'] if reviews else [])+[reason],
|
|
'quality_status':'partial','quality_reason':reason,'quality_mode':True,'agentic':True,
|
|
'llm_model':config['model'],'models':{'read':config.get('read_model') or config['model'],'build':config['model'],'review':config['review_model']},
|
|
'usage':state.get('usage',{}),'step_count':state.get('calls',0),
|
|
'degraded_fallback':False,'is_fallback_placeholder':False,
|
|
'recovered_candidate':candidate.name,'recovery_note':'保留崩溃前已导出并记录有效性的候选;未执行新几何或将结果升级为通过。'}
|
|
write_json(folder/'report.json',report)
|
|
write_json(folder/'job.json',{**state,'status':'partial','reason':reason})
|
|
return report
|
|
write_json(folder/'job.json',{**state,'status':'failed','reason':reason})
|
|
return None
|
|
|
|
|
|
def _worker(channel, action, base_dir, values, config):
|
|
try:
|
|
base_dir = Path(base_dir)
|
|
from .quality_pipeline import generate_quality, resume_quality
|
|
def announce(job_id): channel.put({'type':'job','job_id':job_id})
|
|
if action=='generate':
|
|
report=generate_quality(values['text'],base_dir,values.get('images'),config,job_callback=announce)
|
|
elif action=='resume':
|
|
report=resume_quality(values['part_name'],base_dir,config,job_callback=announce, clarification=values.get('clarification'))
|
|
elif action=='legacy':
|
|
from .pipeline import generate_from_description_best
|
|
report=generate_from_description_best(values['text'],base_dir,image_base64=values.get('images'),mode='legacy')
|
|
elif action=='modify':
|
|
from .pipeline import modify_existing_part
|
|
report=modify_existing_part(values['part_name'],values['text'],base_dir,image_base64=values.get('images'),
|
|
config=config, job_callback=announce)
|
|
else: raise ValueError('Unknown isolated action')
|
|
channel.put({'type':'result','report':report})
|
|
except Exception as exc:
|
|
channel.put({'type':'error','error':str(exc),'job_id':getattr(exc,'job_id',None),
|
|
'input_error':isinstance(exc,ValueError)})
|
|
|
|
|
|
def run_isolated(action, base_dir, *, config=None, _target=None, job_callback=None, cancel_event=None, **values):
|
|
from .config import QualityConfig
|
|
from .pipeline import PipelineError
|
|
config=config or QualityConfig.from_env()
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
raise PipelineError('Cancelled: 用户已停止任务')
|
|
context=multiprocessing.get_context('spawn')
|
|
channel=context.Queue()
|
|
process=context.Process(target=_target or _worker,args=(channel,action,str(base_dir),values,config))
|
|
process.start(); deadline=time.monotonic()+config.max_elapsed_seconds+60
|
|
from .worker_resources import memory_limits, memory_exceeded, kill_tree
|
|
limits = memory_limits()
|
|
resource_error = None
|
|
job_id=None; message=None; timed_out=False; cancelled=False
|
|
try:
|
|
while True:
|
|
try:
|
|
update=channel.get(timeout=.2)
|
|
if update['type']=='job':
|
|
job_id=update['job_id']
|
|
if job_callback: job_callback(job_id)
|
|
else: message=update; break
|
|
except queue.Empty:
|
|
if not process.is_alive(): break
|
|
if cancel_event is not None and cancel_event.is_set():
|
|
cancelled=True; kill_tree(process); break
|
|
if time.monotonic()>=deadline:
|
|
timed_out=True; kill_tree(process); break
|
|
resource_error = memory_exceeded(process.pid, limits)
|
|
if resource_error:
|
|
kill_tree(process); break
|
|
process.join(timeout=3)
|
|
if message:
|
|
if message['type']=='result': return message['report']
|
|
if message.get('input_error'): raise ValueError(message['error'])
|
|
failed_job = message.get('job_id') or job_id
|
|
if failed_job:
|
|
candidate = recover_candidate(base_dir, failed_job, message['error'])
|
|
if candidate:
|
|
return candidate
|
|
raise PipelineError(message['error'],job_id=failed_job)
|
|
reason = resource_error or ('Cancelled: 用户已停止任务,保留已有进度' if cancelled else 'CADWorkerTimeout: 建模进程超时,已停止该进程' if timed_out else
|
|
f'CADWorkerExit: 建模进程异常退出({process.exitcode}),服务仍可继续使用')
|
|
if job_id:
|
|
candidate=recover_candidate(base_dir,job_id,reason)
|
|
if candidate: return candidate
|
|
raise PipelineError(reason,job_id=job_id)
|
|
finally:
|
|
if process.is_alive(): kill_tree(process); process.join(timeout=3)
|
|
channel.close()
|