Files
2026-09-24 10:40:57 +08:00

140 lines
8.3 KiB
Python

"""Keep native CAD crashes and hard timeouts outside the HTTP server process."""
import json
import multiprocessing
import queue
import shutil
import time
from pathlib import Path
def recover_candidate(base_dir, job_id, reason):
"""Copy an already exported/inspected candidate; never invoke CAD in the host."""
from .job import write_json
from .validation import safe_name
folder = Path(base_dir)/'output'/safe_name(job_id)
request = json.loads((folder/'request.json').read_text())
state = json.loads((folder/'job.json').read_text())
events = []
event_file = folder/'events.jsonl'
for line in event_file.read_text().splitlines() if event_file.exists() else []:
try:
events.append(json.loads(line))
except ValueError:
pass # A killed worker can leave a partially written final event.
state['calls']=max((e.get('call',0) for e in events),default=state.get('calls',0))
state['elapsed_s']=max((e.get('elapsed_s',0) for e in events),default=state.get('elapsed_s',0))
state['elapsed_measurement']='last_recorded_event_before_worker_exit'
state['usage']={key:sum(int((e.get('usage') or {}).get(key,0) or 0) for e in events if e['event']=='api_done')
for key in ('input_tokens','output_tokens','total_tokens')}
brief = json.loads((folder/'requirements.json').read_text()) if (folder/'requirements.json').exists() else {}
with (folder/'events.jsonl').open('a') as stream:
stream.write(json.dumps({'event':'worker_interrupted','error':reason,'elapsed_s':state.get('elapsed_s',0)},ensure_ascii=False)+'\n')
# A submitted whole-part candidate outranks a later scratch component.
# Emergency checkpoints remain the fallback before the first submission.
saved = list(folder.glob("candidate_*")) + list(folder.glob("checkpoint_[0-9]*"))
candidates = [p for p in sorted(saved, key=lambda p: (p.name.startswith("candidate_"), p.stat().st_mtime), reverse=True)
if all((p/name).exists() for name in ('model.step','model.stl','model.json','geometry.json'))]
for candidate in candidates:
geometry = json.loads((candidate/'geometry.json').read_text())
if not geometry.get('valid') or geometry.get('solid_count',0)<1: continue
model = json.loads((candidate/'model.json').read_text())
reviews = [json.loads(p.read_text()) for p in sorted(folder.glob('candidate_*/review.json')) if p.parent.name<=candidate.name]
for file in candidate.iterdir():
if file.suffix in ('.png','.step','.stl'): shutil.copy2(file,folder/file.name)
shutil.copy2(candidate/'model.json',folder/'published_model.json')
params = json.loads((candidate/'parameters.json').read_text()) if (candidate/'parameters.json').exists() else []
config = request['config']
report = {'ok':True,'part_name':job_id,'display_name':brief.get('name',job_id),'output_id':model['output'],
'operations':model['operations'],'claims':model.get('claims',[]),
'volume_mm3':geometry['volume_mm3'],'bounding_box_mm':{'min':geometry['bbox_min'],'max':geometry['bbox_max']},
'step_file':str(folder/'model.step'),'stl_file':str(folder/'model.stl'),
'published_model_file':str(folder/'published_model.json'),'candidate_parameters':params,
'source_description':request['text'],'requirements':brief,'llm_assumptions':brief.get('assumptions',[]),
'unresolved':brief.get('unresolved',[]),'reviews':reviews,
'remaining_issues':(reviews[-1]['errors'] if reviews else [])+[reason],
'quality_status':'partial','quality_reason':reason,'quality_mode':True,'agentic':True,
'llm_model':config['model'],'models':{'read':config.get('read_model') or config['model'],'build':config['model'],'review':config['review_model']},
'usage':state.get('usage',{}),'step_count':state.get('calls',0),
'degraded_fallback':False,'is_fallback_placeholder':False,
'recovered_candidate':candidate.name,'recovery_note':'保留崩溃前已导出并记录有效性的候选;未执行新几何或将结果升级为通过。'}
write_json(folder/'report.json',report)
write_json(folder/'job.json',{**state,'status':'partial','reason':reason})
return report
write_json(folder/'job.json',{**state,'status':'failed','reason':reason})
return None
def _worker(channel, action, base_dir, values, config):
try:
base_dir = Path(base_dir)
from .quality_pipeline import generate_quality, resume_quality
def announce(job_id): channel.put({'type':'job','job_id':job_id})
if action=='generate':
report=generate_quality(values['text'],base_dir,values.get('images'),config,job_callback=announce)
elif action=='resume':
report=resume_quality(values['part_name'],base_dir,config,job_callback=announce, clarification=values.get('clarification'))
elif action=='legacy':
from .pipeline import generate_from_description_best
report=generate_from_description_best(values['text'],base_dir,image_base64=values.get('images'),mode='legacy')
elif action=='modify':
from .pipeline import modify_existing_part
report=modify_existing_part(values['part_name'],values['text'],base_dir,image_base64=values.get('images'),
config=config, job_callback=announce)
else: raise ValueError('Unknown isolated action')
channel.put({'type':'result','report':report})
except Exception as exc:
channel.put({'type':'error','error':str(exc),'job_id':getattr(exc,'job_id',None),
'input_error':isinstance(exc,ValueError)})
def run_isolated(action, base_dir, *, config=None, _target=None, job_callback=None, cancel_event=None, **values):
from .config import QualityConfig
from .pipeline import PipelineError
config=config or QualityConfig.from_env()
if cancel_event is not None and cancel_event.is_set():
raise PipelineError('Cancelled: 用户已停止任务')
context=multiprocessing.get_context('spawn')
channel=context.Queue()
process=context.Process(target=_target or _worker,args=(channel,action,str(base_dir),values,config))
process.start(); deadline=time.monotonic()+config.max_elapsed_seconds+60
from .worker_resources import memory_limits, memory_exceeded, kill_tree
limits = memory_limits()
resource_error = None
job_id=None; message=None; timed_out=False; cancelled=False
try:
while True:
try:
update=channel.get(timeout=.2)
if update['type']=='job':
job_id=update['job_id']
if job_callback: job_callback(job_id)
else: message=update; break
except queue.Empty:
if not process.is_alive(): break
if cancel_event is not None and cancel_event.is_set():
cancelled=True; kill_tree(process); break
if time.monotonic()>=deadline:
timed_out=True; kill_tree(process); break
resource_error = memory_exceeded(process.pid, limits)
if resource_error:
kill_tree(process); break
process.join(timeout=3)
if message:
if message['type']=='result': return message['report']
if message.get('input_error'): raise ValueError(message['error'])
failed_job = message.get('job_id') or job_id
if failed_job:
candidate = recover_candidate(base_dir, failed_job, message['error'])
if candidate:
return candidate
raise PipelineError(message['error'],job_id=failed_job)
reason = resource_error or ('Cancelled: 用户已停止任务,保留已有进度' if cancelled else 'CADWorkerTimeout: 建模进程超时,已停止该进程' if timed_out else
f'CADWorkerExit: 建模进程异常退出({process.exitcode}),服务仍可继续使用')
if job_id:
candidate=recover_candidate(base_dir,job_id,reason)
if candidate: return candidate
raise PipelineError(reason,job_id=job_id)
finally:
if process.is_alive(): kill_tree(process); process.join(timeout=3)
channel.close()