181 lines
13 KiB
Python
181 lines
13 KiB
Python
import hashlib,json,mimetypes,threading,time,uuid
|
|
from pathlib import Path
|
|
from .provider import Provider,UnknownOutcome
|
|
from .progress_log import Heartbeat
|
|
from providers.errors import known_model_unavailable, public_model_error
|
|
|
|
PUBLIC_CORE_FIELDS={'error_code','task_id','state','revision','title','eda_engine','delivery_revoked','metrics','questions','proposal','proposal_hash','proposal_version','proposal_feedback','mounting_holes','selected_option','delivery','delivery_review','download_ready','manufacturing_package','error','findings','manufacturing_blockers','native_job_id'}
|
|
|
|
class Engine:
|
|
def __init__(self,settings,store):
|
|
self.settings,self.store=settings,store;self.provider=Provider(settings);self.stop=threading.Event();self.threads=[]
|
|
def start(self):
|
|
with self.store.db() as db:
|
|
resumable={row['task_id'] for row in db.execute("SELECT task_id FROM operations WHERE state IN ('running','queued') AND task_id IS NOT NULL")}
|
|
self.store.recover_operations()
|
|
# Reconcile durable core snapshots before scheduling recovery after a crash.
|
|
for path in self.settings.core_root.glob('tasks/*/task.json'):
|
|
try:
|
|
snapshot=json.loads(path.read_text())
|
|
unavailable=known_model_unavailable(snapshot.get('error'))
|
|
if unavailable and snapshot.get('state') in {'assessing','designing','exporting','needs_operator'}:
|
|
snapshot.update(provider_failure=snapshot.get('error'),state='failed',error=unavailable,
|
|
error_code='model_not_found',active_agent_call=None)
|
|
temporary=path.with_suffix('.recovery.tmp')
|
|
temporary.write_text(json.dumps(snapshot,ensure_ascii=False,indent=2))
|
|
temporary.replace(path)
|
|
self.update_task({k:v for k,v in snapshot.items() if k in PUBLIC_CORE_FIELDS})
|
|
continue
|
|
|
|
if snapshot.get('eda_engine') in {'kicad','easyeda'} and snapshot.get('state') in {'assessing','designing','exporting','needs_operator'}:
|
|
with self.store.db() as db:
|
|
source=db.execute('SELECT id FROM operations WHERE request_id=?',(snapshot.get('origin_request_id'),)).fetchone()
|
|
if source:
|
|
self.store.bind(source['id'],snapshot['task_id'])
|
|
resumable.add(snapshot['task_id'])
|
|
if snapshot.get('task_id') not in resumable:continue
|
|
if snapshot.get('state') in {'assessing','designing','exporting'}:
|
|
snapshot.update(resume_state=snapshot['state'],state='needs_operator',error='Server restarted; reconciling saved progress')
|
|
temporary=path.with_suffix('.recovery.tmp')
|
|
temporary.write_text(json.dumps(snapshot,ensure_ascii=False,indent=2))
|
|
temporary.replace(path)
|
|
self.update_task({k:v for k,v in snapshot.items() if k in PUBLIC_CORE_FIELDS})
|
|
except (OSError,ValueError,KeyError):continue
|
|
# A restart must not leave a design run waiting for a human.
|
|
|
|
for data in self.store.tasks():
|
|
if data.get('task_id') in resumable and data.get('state')=='needs_operator':
|
|
self.store.event(data['task_id'],None,'startup_auto_recover',{})
|
|
self._auto_recover(data['task_id'],{'action':'recover'})
|
|
for control in [False,True]:
|
|
thread=threading.Thread(target=self.loop,args=(control,),daemon=True);thread.start();self.threads.append(thread)
|
|
def shutdown(self):
|
|
self.stop.set();self.provider.shutdown()
|
|
for thread in self.threads:thread.join(timeout=6)
|
|
def loop(self,control):
|
|
while not self.stop.is_set():
|
|
op=self.store.claim(control)
|
|
if not op:self.stop.wait(.15);continue
|
|
command=json.loads(op['command'])
|
|
def event(e):
|
|
tid=e.get('task_id')
|
|
if not tid:return
|
|
self.store.bind(op['id'],tid)
|
|
current=self.store.task(tid)
|
|
if not current:self.store.save_task({'task_id':tid,'state':'assessing','simulation':self.settings.mode=='simulation','title':command.get('payload',{}).get('description','新设计任务')[:80]})
|
|
if e.get('snapshot'):self.update_task(e['snapshot'])
|
|
public={k:v for k,v in e.get('data',{}).items() if k not in {'archive','path','directory','source','code'}}
|
|
self.store.event(tid,op['id'],e.get('kind','progress'),public)
|
|
self.store.event(op.get('task_id'),op['id'],'operation.started',{'action':command['action']})
|
|
try:
|
|
with Heartbeat(op['id'],op.get('task_id')):
|
|
result=self.provider.call(command,event)
|
|
if result.get('task_id'):self.update_task(result)
|
|
state='failed' if result.get('error') and not result.get('state') else 'succeeded'
|
|
self.store.finish(op['id'],result,state)
|
|
self.store.event(result.get('task_id',op.get('task_id')),op['id'],'operation.finished',{'state':state})
|
|
tid_done=result.get('task_id') or op.get('task_id')
|
|
if tid_done and result.get('state') and result['state']!='needs_operator':
|
|
# A healthy step resets the automatic-recovery budget for this task.
|
|
self.store.event(tid_done,op['id'],'auto.recover_cleared',{})
|
|
self._auto_recover(tid_done,command)
|
|
except Exception as exc:
|
|
self.store.finish(op['id'],{'error':type(exc).__name__,'message':str(exc)},'outcome_unknown')
|
|
self.store.event(op.get('task_id'),op['id'],'operation.unknown',{'message':'正在自动核对保存进度'})
|
|
# Provider has stopped its child process before returning an unknown outcome.
|
|
current=self.store.operation(op['id'])
|
|
tid=current.get('task_id') if current else op.get('task_id')
|
|
if tid:
|
|
path=self.settings.core_root/'tasks'/tid/'task.json'
|
|
try:
|
|
data=json.loads(path.read_text())
|
|
if data.get('state') not in {'delivery_ready','draft_ready','completed','failed','cancelled','needs_input','awaiting_selection'}:
|
|
data.update(resume_state=data.get('resume_state') or data['state'],state='needs_operator',error='Execution interrupted; reconciling saved progress')
|
|
temporary=path.with_suffix('.recovery.tmp')
|
|
temporary.write_text(json.dumps(data,ensure_ascii=False,indent=2))
|
|
temporary.replace(path)
|
|
self.update_task({k:v for k,v in data.items() if k in PUBLIC_CORE_FIELDS})
|
|
self._auto_recover(tid,{'action':'recover'})
|
|
except (OSError,ValueError):pass
|
|
|
|
def _auto_recover(self,tid,command):
|
|
"""Users never resolve design failures themselves: retry recovery on the server."""
|
|
if not tid or command.get('action') not in {'create','advance','recover','message','select'}:
|
|
return
|
|
data=self.store.task(tid)
|
|
if not data or data.get('state')!='needs_operator':
|
|
return
|
|
with self.store.db() as db:
|
|
row=db.execute("SELECT count(*) c FROM events WHERE task_id=? AND kind='auto.recover' "
|
|
"AND seq > COALESCE((SELECT MAX(seq) FROM events WHERE task_id=? AND kind='auto.recover_cleared'),0)",
|
|
(tid,tid)).fetchone()
|
|
attempts=row['c']
|
|
with self.store.db() as db:
|
|
pending=db.execute("SELECT 1 FROM operations WHERE task_id=? AND state IN ('queued','running') AND json_extract(command,'$.action')='recover'",(tid,)).fetchone()
|
|
if pending:return
|
|
if attempts and self.stop.wait(min(60,5*attempts)):return
|
|
self.store.event(tid,None,'auto.recover',{'attempt':attempts+1})
|
|
self.store.enqueue({'request_id':f'auto-recover-{tid}-{uuid.uuid4().hex}','action':'recover','payload':{'task_id':tid}},tid)
|
|
def update_task(self,value):
|
|
unavailable=known_model_unavailable(value.get('error'))
|
|
if unavailable and value.get('state') in {'assessing','designing','exporting','needs_operator','failed'}:
|
|
value={**value,'state':'failed','error':unavailable,'error_code':'model_not_found'}
|
|
tid=value['task_id'];old=self.store.task(tid) or {}
|
|
if old.get('state')=='completed' and value.get('state')!='completed' and not value.get('delivery_revoked'):
|
|
return # a delayed generation result cannot revoke a released immutable revision
|
|
if old.get('state')=='failed' and value.get('state')=='needs_operator':
|
|
return # a terminal verdict must not be undone by a status poll
|
|
previous_manifest=(old.get('delivery') or {}).get('manifest_sha256')
|
|
current_manifest=(value.get('delivery') or {}).get('manifest_sha256')
|
|
if current_manifest and previous_manifest and current_manifest!=previous_manifest:
|
|
with self.store.db() as db:db.execute('DELETE FROM artifacts WHERE task_id=?',(tid,))
|
|
data={**old,**value,'simulation':self.settings.mode=='simulation'}
|
|
preview=data.get('render_override') or {}
|
|
if preview.get('source_manifest_sha256')!=(data.get('delivery') or {}).get('manifest_sha256'):
|
|
data.pop('render_override',None)
|
|
if not self.store.save_task(data):return
|
|
delivery=data.get('delivery') or {}
|
|
for item in delivery.get('artifacts',[]):
|
|
path=item.get('absolute_path') or item.get('path')
|
|
if path:self.register(tid,path,item.get('kind','other'),item.get('sha256'),False)
|
|
if delivery.get('archive'):self.register(tid,delivery['archive'],'review_package',delivery.get('archive_sha256'),False)
|
|
preview=data.get('render_override') or {}
|
|
if preview.get('path') and preview.get('sha256'):
|
|
self.register(tid,preview['path'],'render',preview['sha256'],False)
|
|
if data.get('manufacturing_package') and data['state']=='completed':
|
|
item=data['manufacturing_package'];self.register(tid,item['path'],'manufacturing_package',item.get('sha256'),True)
|
|
def register(self,tid,path,kind,sha,release):
|
|
p=Path(path).resolve()
|
|
if not p.is_relative_to(self.settings.core_root.resolve()) or not p.is_file():return
|
|
actual=hashlib.sha256(p.read_bytes()).hexdigest()
|
|
if sha and sha!=actual:return
|
|
aid='art-'+hashlib.sha256((tid+str(p)).encode()).hexdigest()[:24]
|
|
mime=mimetypes.guess_type(p.name)[0] or 'application/octet-stream'
|
|
with self.store.db() as db:
|
|
db.execute('INSERT OR REPLACE INTO artifacts VALUES(?,?,?,?,?,?,?,?)',(aid,tid,str(p),kind,actual,mime,int(self.settings.mode=='simulation'),int(release)))
|
|
def public_task(self,data):
|
|
result={k:v for k,v in data.items() if k not in {'delivery','manufacturing_package','native_job_id','render_override'}}
|
|
if result.get('state')=='delivery_ready' and (result.get('manufacturing_blockers') or result.get('download_ready') is not True):result['state']='draft_ready'
|
|
if result.get('error'):result['error']=public_model_error(result['error'])
|
|
history=self.store.history(data['task_id'])
|
|
if history.get('display_title'):result['title']=history['display_title']
|
|
result['pinned']=bool(history.get('pinned'))
|
|
result['mode']=self.settings.mode
|
|
if data.get('delivery'):
|
|
d=data['delivery'];result['delivery']={'manifest_sha256':d.get('manifest_sha256'),'archive_sha256':d.get('archive_sha256'),'artifacts':self.artifacts(data['task_id'])}
|
|
if data.get('manufacturing_package') and data.get('state')=='completed':
|
|
item=next((x for x in self.artifacts(data['task_id']) if x['kind']=='manufacturing_package'),None)
|
|
if item:result['manufacturing_package']=item
|
|
return result
|
|
def artifacts(self,tid):
|
|
data=self.store.task(tid)
|
|
if not data:return []
|
|
with self.store.db() as db:rows=db.execute('SELECT * FROM artifacts WHERE task_id=?',(tid,)).fetchall()
|
|
# A model-only preview can supplement an immutable delivery without rewriting it.
|
|
preview=data.get('render_override') or {}
|
|
if preview.get('source_manifest_sha256')==(data.get('delivery') or {}).get('manifest_sha256'):
|
|
preferred=next((r['id'] for r in rows if r['kind']=='render' and r['path']==preview.get('path') and r['sha256']==preview.get('sha256')),None)
|
|
if preferred:rows=[r for r in rows if r['kind']!='render' or r['id']==preferred]
|
|
return [{'artifact_id':r['id'],'kind':r['kind'],'name':Path(r['path']).name,'sha256':r['sha256'],'mime_type':r['mime'],
|
|
'simulation':bool(r['simulation']),'download_url':f'/api/v1/tasks/{tid}/artifacts/{r["id"]}'} for r in rows if not r['release_only'] or data['state']=='completed']
|