171 lines
9.0 KiB
Python
171 lines
9.0 KiB
Python
"""Asynchronous HTTP tasks. CAD stays in a cancellable subprocess."""
|
|
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from engine.isolated import run_isolated
|
|
from engine.job import write_json
|
|
from engine.validation import safe_name
|
|
from .messages import failure_response, outcome, issue_text, questions, question_items
|
|
|
|
|
|
def recoverable_error(reason):
|
|
"""One extra worker attempt for transient/protocol interruption, never a design choice."""
|
|
reason = str(reason or '')
|
|
if any(word in reason for word in ('NeedsClarification', 'ReferenceUnavailable', 'Cancelled',
|
|
'BudgetExceeded', 'NoProgress', 'RepairLimit', 'CADMemoryLimit', 'quota', 'credit', '余额',
|
|
'Authentication', 'Permission', '401', '403', '429')):
|
|
return False
|
|
return any(word in reason for word in ('APIConnectionError', 'APITimeoutError',
|
|
'ConnectionError', 'ReadTimeout', '结构化输出纠正后仍不完整'))
|
|
|
|
|
|
def continue_partial(report, base, job_id):
|
|
"""One extra design round for an interrupted real candidate, never a user choice."""
|
|
if (not isinstance(report, dict) or report.get('quality_status') != 'partial'
|
|
or not report.get('ok') or report.get('is_fallback_placeholder') or report.get('degraded_fallback')):
|
|
return False
|
|
reason = str(report.get('quality_reason') or '')
|
|
budget = 'BudgetExceeded' in reason and not any(x in reason for x in ('NoProgress','RepairLimit','Cancelled'))
|
|
if not (budget or recoverable_error(reason)) or not job_id or questions(base, job_id):
|
|
return False
|
|
folder = Path(base)/'output'/safe_name(job_id)
|
|
try:
|
|
model = json.loads((folder/'model.json').read_text())
|
|
request = json.loads((folder/'request.json').read_text())
|
|
except (OSError, ValueError):
|
|
return False
|
|
return bool(model.get('operations') and request.get('text')
|
|
and (folder/'model.step').exists()
|
|
and report.get('export_validation', {}).get('step_roundtrip_valid'))
|
|
|
|
|
|
class TaskManager:
|
|
def __init__(self, base_dir, response_adapter, runner=run_isolated):
|
|
self.base = Path(base_dir)
|
|
self.adapter = response_adapter
|
|
self.runner = runner
|
|
self.lock = threading.Lock()
|
|
self.tasks = {}
|
|
|
|
def start(self, action, values):
|
|
with self.lock:
|
|
if sum(not t['done'] for t in self.tasks.values()) >= 3:
|
|
raise ValueError('已有 3 个任务运行,请等待完成或先停止任务')
|
|
task_id = uuid.uuid4().hex
|
|
task = {'id': task_id, 'created_at': time.time(), 'done': False,
|
|
'job_id': None, 'cancel': threading.Event(), 'response': None}
|
|
self.tasks[task_id] = task
|
|
threading.Thread(target=self._run, args=(task_id, action, values), daemon=True).start()
|
|
return task_id
|
|
|
|
def _run(self, task_id, action, values):
|
|
task = self.tasks[task_id]
|
|
def announce(job_id):
|
|
with self.lock:
|
|
task['job_id'] = job_id
|
|
extra_rounds = 0
|
|
try:
|
|
report = self.runner(action, self.base, job_callback=announce, cancel_event=task['cancel'], **values)
|
|
if not task['cancel'].is_set() and continue_partial(report, self.base, task['job_id']):
|
|
parent = task['job_id'];extra_rounds += 1
|
|
with self.lock:
|
|
task['recovery'] = {'attempt': 1, 'round': 2, 'parent_job': parent,
|
|
'message': '正在进行第 2 轮调整,保留已有模型并继续完成未核对的结构'}
|
|
write_json(self.base/'output'/safe_name(parent)/'http_continuation.json',
|
|
{'task_id': task_id, 'round': 2, 'reason': report.get('quality_reason'),
|
|
'action': 'resume', 'requirements_changed': False})
|
|
report = self.runner('resume', self.base, part_name=parent, config=values.get('config'),
|
|
job_callback=announce, cancel_event=task['cancel'])
|
|
result = self.adapter(report)
|
|
except Exception as exc:
|
|
# Keep the retry inside the same HTTP task. Resume reads the saved
|
|
# original request/images and does not invent a user clarification.
|
|
job_id = task['job_id'] or getattr(exc, 'job_id', None)
|
|
if job_id:
|
|
announce(job_id)
|
|
if (extra_rounds < 1 and job_id and not task['cancel'].is_set() and not questions(self.base, job_id)
|
|
and (self.base/'output'/safe_name(job_id)/'request.json').exists()
|
|
and recoverable_error(exc)):
|
|
task['recovery'] = {'attempt': 1, 'message': '连接或读取异常,正在自动恢复原任务'}
|
|
write_json(self.base/'output'/safe_name(job_id)/'http_recovery.json',
|
|
{'attempt': 1, 'reason': str(exc), 'action': 'resume'})
|
|
try:
|
|
report = self.runner('resume', self.base, part_name=job_id,
|
|
config=values.get('config'), job_callback=announce, cancel_event=task['cancel'])
|
|
result = self.adapter(report)
|
|
except Exception as retry_exc:
|
|
result = failure_response(str(retry_exc), self.base, task['job_id'])
|
|
exc = retry_exc
|
|
else:
|
|
result = failure_response(str(exc), self.base, task['job_id'])
|
|
if task['job_id']:
|
|
try:
|
|
folder = self.base/'output'/safe_name(task['job_id'])
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
write_json(folder/'diagnostic.json', {'error': str(exc)})
|
|
except OSError:
|
|
logging.exception('Failed to save task diagnostic')
|
|
with self.lock:
|
|
task['response'] = result
|
|
task['done'] = True
|
|
# Do not persist input images or credentials a second time.
|
|
folder = self.base / 'output' / '_http_tasks'
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
write_json(folder / (task_id + '.json'), {'id': task_id, 'done': True,
|
|
'job_id': task['job_id'], 'created_at': task['created_at'], 'response': result})
|
|
|
|
def status(self, task_id):
|
|
if len(task_id) != 32 or any(c not in '0123456789abcdef' for c in task_id):
|
|
raise KeyError(task_id)
|
|
with self.lock:
|
|
task = self.tasks.get(task_id)
|
|
if task:
|
|
data = {k: v for k, v in task.items() if k != 'cancel'}
|
|
data['cancelling'] = task['cancel'].is_set() and not task['done']
|
|
else:
|
|
try:
|
|
saved = json.loads((self.base / 'output' / '_http_tasks' / (task_id+'.json')).read_text())
|
|
response = saved.get('response', {})
|
|
if response.get('ok') is False and not response.get('message'):
|
|
saved['response'] = failure_response(response.get('error'), self.base, saved.get('job_id'))
|
|
elif response.get('ok'):
|
|
for key in ('quality_reason', 'modification_failed_reason'):
|
|
if response.get(key): response[key] = outcome(response[key])[1]
|
|
for key in ('geometry_issues', 'review_errors', 'remaining_issues'):
|
|
if response.get(key): response[key] = [issue_text(v) for v in response[key]]
|
|
response['questions'] = questions(self.base, saved.get('job_id'))
|
|
response['question_items'] = question_items(self.base, saved.get('job_id'))
|
|
response['can_clarify'] = bool(response['questions'])
|
|
if response['questions']: response['can_resume'] = False
|
|
if response.get('ok') is False and response.get('message'):
|
|
asks = questions(self.base, saved.get('job_id'))
|
|
response.update(questions=asks, question_items=question_items(self.base, saved.get('job_id')),
|
|
can_clarify=bool(asks))
|
|
if asks:
|
|
response.update(status='needs_clarification', can_resume=False, next_action='answer_questions')
|
|
return saved
|
|
except (OSError, ValueError):
|
|
raise KeyError(task_id)
|
|
data['elapsed_s'] = round(time.time() - data['created_at'])
|
|
if data['job_id']:
|
|
folder = self.base / 'output' / safe_name(data['job_id'])
|
|
for name, key in [('job.json', 'progress'), ('preview.json', 'preview')]:
|
|
try:
|
|
data[key] = json.loads((folder / name).read_text())
|
|
except (OSError, ValueError):
|
|
pass
|
|
if data.get('preview'):
|
|
data['preview_url'] = f"/files/{data['job_id']}/preview.png?v={data['preview']['revision']}"
|
|
return data
|
|
|
|
def cancel(self, task_id):
|
|
with self.lock:
|
|
task = self.tasks.get(task_id)
|
|
if not task:
|
|
raise KeyError(task_id)
|
|
task['cancel'].set()
|