187 lines
10 KiB
Python
187 lines
10 KiB
Python
"""Durable local evidence and bounded API access for a quality run."""
|
||
import hashlib
|
||
import importlib.metadata
|
||
import importlib
|
||
import json
|
||
import subprocess
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from uuid import uuid4
|
||
|
||
from openai import APIConnectionError, APIStatusError
|
||
|
||
# OpenAI releases use either httpx or httpx2. Stream-body failures escape the
|
||
# SDK's request retry handler as transport exceptions, not APIConnectionError.
|
||
_transport_errors = []
|
||
for _module in ("httpx", "httpx2"):
|
||
try:
|
||
_http = importlib.import_module(_module)
|
||
_transport_errors.extend((_http.NetworkError, _http.TimeoutException, _http.RemoteProtocolError))
|
||
except ImportError:
|
||
pass
|
||
|
||
|
||
class InterruptedResponse(RuntimeError):
|
||
pass
|
||
|
||
|
||
def is_transient_error(exc):
|
||
if isinstance(exc, (APIConnectionError, InterruptedResponse, *_transport_errors)):
|
||
return True
|
||
if isinstance(exc, APIStatusError):
|
||
if exc.response.headers.get("x-should-retry") == "false":
|
||
return False
|
||
return exc.status_code in (408, 409, 429) or exc.status_code >= 500
|
||
return False
|
||
|
||
|
||
class TruncatedResponse(RuntimeError):
|
||
pass
|
||
|
||
|
||
class BudgetExceeded(RuntimeError):
|
||
pass
|
||
|
||
|
||
def write_json(path, value):
|
||
path = Path(path)
|
||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False), encoding="utf-8")
|
||
temporary.replace(path)
|
||
|
||
|
||
class QualityJob:
|
||
def __init__(self, base_dir, text, images, config):
|
||
self.id = "quality_" + datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_") + uuid4().hex[:8]
|
||
self.path = Path(base_dir) / "output" / self.id
|
||
self.path.mkdir(parents=True)
|
||
self.config = config
|
||
self.calls = 0
|
||
self.started = time.monotonic()
|
||
self.started_at = time.time()
|
||
self._state = {}
|
||
self.usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
|
||
sources = []
|
||
for i, image in enumerate(images):
|
||
name = f"input_{i + 1}.{image['ext']}"
|
||
(self.path / name).write_bytes(image["data"])
|
||
sources.append({"file": name, "sha256": image["sha256"], "mime": image["mime"]})
|
||
revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=base_dir, capture_output=True, text=True).stdout.strip()
|
||
source_digest = hashlib.sha256()
|
||
for folder in ("engine", "capabilities"):
|
||
for source in sorted((Path(base_dir) / folder).rglob("*")):
|
||
if source.suffix in (".py", ".yaml", ".md"):
|
||
source_digest.update(str(source.relative_to(base_dir)).encode())
|
||
source_digest.update(source.read_bytes())
|
||
self.write("request.json", {"text": text, "text_sha256": hashlib.sha256(text.encode()).hexdigest(),
|
||
"images": sources, "config": config.to_dict(), "code_revision": revision, "source_sha256": source_digest.hexdigest(),
|
||
"build123d": importlib.metadata.version("build123d")})
|
||
self.state("reading")
|
||
|
||
def write(self, name, value):
|
||
write_json(self.path / name, value)
|
||
|
||
def event(self, kind, **data):
|
||
if kind in ("api_start", "api_progress", "api_done", "tool_start", "tool_done", "api_retry"):
|
||
self.state("reviewing" if data.get("role", self._state.get("role")) == "review" else "building",
|
||
activity=kind, **{k: data[k] for k in ("role", "name", "stage") if k in data})
|
||
with (self.path / "events.jsonl").open("a", encoding="utf-8") as stream:
|
||
stream.write(json.dumps({"event": kind, "elapsed_s": round(time.monotonic() - self.started, 2),
|
||
**data}, ensure_ascii=False, allow_nan=False) + "\n")
|
||
|
||
def state(self, status, **extra):
|
||
self._state.update(extra)
|
||
self.write("job.json", {**self._state, "started_at": self.started_at, "id": self.id, "status": status, "calls": self.calls,
|
||
"elapsed_s": round(time.monotonic() - self.started, 2), "usage": self.usage, **extra})
|
||
|
||
def call(self, client, role, **kwargs):
|
||
"""Retry inference only; callers execute tools after a full response returns."""
|
||
for attempt in range(1, self.config.max_api_attempts + 1):
|
||
try:
|
||
return self._call_once(client, role, attempt, **kwargs)
|
||
except Exception as exc:
|
||
if isinstance(exc, TruncatedResponse) and attempt < self.config.max_api_attempts:
|
||
instruction = ("上次模型输出因长度限制被截断,所有不完整工具调用均未执行,当前几何和有效历史保持不变。"
|
||
"请缩短下一次回复:新结构分小批追加execute_batch;修改已存在操作优先revise_model仅提交变化参数,宿主会原子重算下游,避免重抄完整模型。"
|
||
if role == 'build' else
|
||
"上次结构化输出被截断,未作为需求或核对结果采用。请保留全部必需字段和全部需求ID,缩短逐项解释和重复文本后重新输出,不删验收项。")
|
||
kwargs = {**kwargs, 'input': [*kwargs['input'], {'role':'user','content':instruction}]}
|
||
self.event('api_retry',call=self.calls,role=role,next_attempt=attempt+1,delay_s=0,error='输出截断;使用完整旧状态重新请求紧凑响应')
|
||
continue
|
||
if not is_transient_error(exc) or attempt == self.config.max_api_attempts:
|
||
raise
|
||
delay = min(2 ** (attempt - 1), 8)
|
||
if isinstance(exc, APIStatusError):
|
||
try:
|
||
delay = max(delay, float(exc.response.headers.get("retry-after", "0")))
|
||
except ValueError:
|
||
pass
|
||
remaining = self.config.max_elapsed_seconds - (time.monotonic() - self.started)
|
||
if self.calls >= self.config.max_calls or remaining <= delay or delay > 60:
|
||
raise
|
||
self.event("api_retry", call=self.calls, role=role, next_attempt=attempt + 1,
|
||
delay_s=delay, error=f"{type(exc).__name__}: {exc}")
|
||
print(f"[quality] {role} 连接暂时中断,{delay:g} 秒后自动重试({attempt + 1}/{self.config.max_api_attempts})", flush=True)
|
||
time.sleep(delay)
|
||
|
||
def _call_once(self, client, role, attempt, **kwargs):
|
||
if self.calls >= self.config.max_calls or time.monotonic() - self.started >= self.config.max_elapsed_seconds:
|
||
raise BudgetExceeded("任务预算已用完,保留当前候选及未完成项")
|
||
self.calls += 1
|
||
self.event("api_start", call=self.calls, role=role, attempt=attempt, model=kwargs.get("model"))
|
||
remaining = self.config.max_elapsed_seconds - (time.monotonic() - self.started)
|
||
terminal = None
|
||
response_id = None
|
||
event_counts = {}
|
||
partial_output = []
|
||
last_progress = time.monotonic()
|
||
try:
|
||
with client.responses.stream(timeout=min(self.config.timeout_seconds, max(1, remaining)),
|
||
max_output_tokens=self.config.max_output_tokens, **kwargs) as stream:
|
||
for event in stream:
|
||
event_counts[event.type] = event_counts.get(event.type, 0) + 1
|
||
if time.monotonic() - last_progress >= 15:
|
||
self.event("api_progress", call=self.calls, event_counts=event_counts,
|
||
output_chars=sum(map(len, partial_output)))
|
||
last_progress = time.monotonic()
|
||
if time.monotonic() - self.started >= self.config.max_elapsed_seconds:
|
||
raise BudgetExceeded("任务时间预算已用完")
|
||
if event.type == "response.created":
|
||
response_id = event.response.id
|
||
self.event("api_response_created", call=self.calls, response_id=event.response.id)
|
||
elif event.type in ("response.completed", "response.incomplete", "response.failed"):
|
||
terminal = event.response
|
||
elif event.type in ("response.output_text.delta", "response.function_call_arguments.delta"):
|
||
partial_output.append(event.delta)
|
||
elif event.type == "error":
|
||
self.event("api_stream_error", call=self.calls, detail=event.model_dump(exclude_none=True))
|
||
try:
|
||
response = terminal or stream.get_final_response()
|
||
except RuntimeError as exc:
|
||
if str(exc) == "Didn't receive a `response.completed` event.":
|
||
raise InterruptedResponse("连接结束但未收到完整模型响应") from exc
|
||
raise
|
||
except Exception as exc:
|
||
self.write(f"response_{self.calls:03d}_interrupted.json",
|
||
{"event_counts": event_counts, "partial_output": "".join(partial_output),
|
||
"response_id": response_id, "attempt": attempt, "error": f"{type(exc).__name__}: {exc}",
|
||
"note": "不完整的模型输出,仅供诊断,不能作为已执行操作或合格结果"})
|
||
if terminal is None or not is_transient_error(exc):
|
||
raise
|
||
# A complete terminal event is authoritative even if connection
|
||
# teardown fails. Preserve it instead of paying for another response.
|
||
response = terminal
|
||
self.event("api_terminal_recovered", call=self.calls, response_id=response.id)
|
||
payload = response.model_dump(exclude_none=True)
|
||
self.write(f"response_{self.calls:03d}.json", payload)
|
||
for key in self.usage:
|
||
self.usage[key] += int(payload.get("usage", {}).get(key, 0) or 0)
|
||
self.event("api_done", call=self.calls, role=role, response_id=response.id,
|
||
usage=payload.get("usage"), status=payload.get("status"))
|
||
if payload.get("status") == "incomplete" and (payload.get("incomplete_details") or {}).get("reason") == "max_output_tokens":
|
||
raise TruncatedResponse("max_output_tokens: 模型输出截断;不执行任何不完整工具调用")
|
||
if payload.get("status") != "completed":
|
||
raise RuntimeError(f"模型响应未完成: {payload.get('status')}; {payload.get('incomplete_details') or payload.get('error')}")
|
||
return response
|