165 lines
7.9 KiB
Python
165 lines
7.9 KiB
Python
"""Run real-provider CAD tasks with a usable-model success criterion.
|
|
|
|
This evaluator is intentionally narrower than the release gate in ``live.py``.
|
|
It answers one operational question: can the current workflow reliably finish
|
|
ordinary prompts with a downloadable/previewable checkpoint, even if some
|
|
acceptance claims remain best-effort warnings.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
from dataclasses import replace
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import secrets
|
|
import sys
|
|
from typing import Any
|
|
|
|
from app.cad_agent.application.workflow import ModelIdentity
|
|
from app.cad_agent.composition import compose_v3
|
|
from app.settings import BACKEND_ROOT, get_settings
|
|
|
|
|
|
DEFAULT_PROMPTS = (
|
|
"生成一个 80 mm x 50 mm x 8 mm 的简单矩形板，使用毫米，输出一个单一实体。",
|
|
"生成一个简单法兰，外径 100 mm，厚度 10 mm，中间有 30 mm 通孔，使用毫米。",
|
|
"生成一个圆柱垫块，直径 60 mm，高度 20 mm，中间有 20 mm 通孔，使用毫米。",
|
|
)
|
|
|
|
|
|
def _arguments() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run real CAD generation prompts and require usable model artifacts.")
|
|
parser.add_argument("--author-provider")
|
|
parser.add_argument("--author-model")
|
|
parser.add_argument("--review-provider")
|
|
parser.add_argument("--review-model")
|
|
parser.add_argument("--prompt", action="append", help="Prompt to run. Repeat for multiple prompts. Defaults to three simple CAD prompts.")
|
|
parser.add_argument("--max-wall-seconds", type=int, default=1200)
|
|
parser.add_argument("--max-model-calls", type=int, default=60)
|
|
return parser.parse_args()
|
|
|
|
|
|
def _result_code(events: list[dict[str, Any]], projection: dict[str, Any]) -> str:
|
|
terminal = next((item["payload"] for item in reversed(events) if item.get("name") == "task_terminal"), {})
|
|
for source in (terminal, projection):
|
|
code = source.get("code") or source.get("last_error") if isinstance(source, dict) else ""
|
|
if isinstance(code, str) and code:
|
|
return code
|
|
return ""
|
|
|
|
|
|
def _artifact_ok(artifact_root: Path, revision_id: str) -> bool:
|
|
if not revision_id:
|
|
return False
|
|
revision_root = artifact_root / "revisions" / revision_id
|
|
return all((revision_root / name).is_file() for name in ("model.step", "model.glb", "model.cdsl.json", "rebuild-report.json"))
|
|
|
|
|
|
def _classify_failure(code: str, projection: dict[str, Any]) -> str:
|
|
if str(projection.get("lifecycle") or "") == "waiting_retry":
|
|
return "configuration_or_service"
|
|
if code in {"FAILED_INTERNAL", "RUNTIME_CONTRACT_INVALID", "REQUIREMENTS_SPEC_INVALID", "STORAGE_FAILURE", "RENDER_SERVICE_UNAVAILABLE"}:
|
|
return "code_or_flow"
|
|
if code in {"AUTHOR_TRANSPORT_UNAVAILABLE", "REVIEW_SERVICE_UNAVAILABLE", "MODEL_PROTOCOL_CHECK_PENDING"}:
|
|
return "configuration_or_service"
|
|
if code in {"AUTHOR_FORMAT_INVALID", "AUTHOR_DECISION_REJECTED", "CANDIDATE_BUILD_FAILED", "CLAIM_VERIFICATION_FAILED", "CANDIDATE_REVIEW_REJECTED", "NO_PROGRESS_LIMIT", "BEST_EFFORT_COMPLETED"}:
|
|
return "model_output_or_best_effort"
|
|
return "unknown"
|
|
|
|
|
|
async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, Any]:
|
|
settings = get_settings()
|
|
try:
|
|
author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model)
|
|
if arguments.review_provider or arguments.review_model:
|
|
review_provider, review_model = settings.resolve_model(arguments.review_provider, arguments.review_model)
|
|
else:
|
|
review_provider, review_model = settings.resolve_independent_review_model(author_provider, author_model)
|
|
except ValueError as error:
|
|
return {"status": "blocked", "error": str(error), "results": []}
|
|
isolated = replace(settings, task_root=report_root / "artifacts", conversation_root=report_root / "conversations")
|
|
services = compose_v3(isolated)
|
|
services.workflow.config = replace(
|
|
services.workflow.config,
|
|
max_turns=max(8, arguments.max_model_calls * 3),
|
|
max_model_calls=arguments.max_model_calls,
|
|
)
|
|
prompts = tuple(arguments.prompt or DEFAULT_PROMPTS)
|
|
results: list[dict[str, Any]] = []
|
|
for index, prompt in enumerate(prompts, start=1):
|
|
task_id = f"cad_{secrets.token_hex(6)}"
|
|
services.workflow.create_task(task_id, prompt)
|
|
events: list[dict[str, Any]] = []
|
|
started = datetime.now(timezone.utc)
|
|
try:
|
|
async with asyncio.timeout(arguments.max_wall_seconds):
|
|
async for name, payload in services.workflow.run(
|
|
task_id=task_id,
|
|
author=ModelIdentity(author_provider.id, author_model.id),
|
|
reviewer=ModelIdentity(review_provider.id, review_model.id),
|
|
):
|
|
events.append({"name": name, "payload": payload})
|
|
except TimeoutError:
|
|
events.append({"name": "timeout", "payload": {"code": "LIVE_EVAL_TIMEOUT", "message": "Task timed out."}})
|
|
projection = services.repository.get_task_projection(task_id) or {}
|
|
artifact_root = (isolated.task_root / task_id).resolve()
|
|
revision_id = str(projection.get("active_revision") or projection.get("current_revision") or "")
|
|
code = _result_code(events, projection)
|
|
usable = str(projection.get("lifecycle") or "") == "completed" and _artifact_ok(artifact_root, revision_id)
|
|
results.append({
|
|
"index": index,
|
|
"task_id": task_id,
|
|
"prompt": prompt,
|
|
"success": usable,
|
|
"failure_layer": "" if usable else _classify_failure(code, projection),
|
|
"code": code,
|
|
"phase": projection.get("phase"),
|
|
"lifecycle": projection.get("lifecycle"),
|
|
"active_revision": revision_id,
|
|
"verification_status": projection.get("verification_status"),
|
|
"artifact_root": str(artifact_root),
|
|
"duration_ms": round((datetime.now(timezone.utc) - started).total_seconds() * 1000),
|
|
"usage": services.repository.usage_summary(task_id),
|
|
"event_audit": [
|
|
{
|
|
"name": item.get("name"),
|
|
"status": (item.get("payload") or {}).get("status"),
|
|
"lifecycle": (item.get("payload") or {}).get("lifecycle"),
|
|
"code": ((item.get("payload") or {}).get("result") or {}).get("code") if isinstance((item.get("payload") or {}).get("result"), dict) else (item.get("payload") or {}).get("code"),
|
|
"tool": (item.get("payload") or {}).get("tool"),
|
|
}
|
|
for item in events
|
|
],
|
|
})
|
|
return {
|
|
"status": "passed" if results and all(item["success"] for item in results) else "failed",
|
|
"schema_version": "cad.usable-smoke.v1",
|
|
"author": {"provider": author_provider.id, "model": author_model.id},
|
|
"reviewer": {"provider": review_provider.id, "model": review_model.id},
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
arguments = _arguments()
|
|
report_root = BACKEND_ROOT / "live-evals" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
report_root.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
result = asyncio.run(_run(arguments, report_root))
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as error:
|
|
result = {"status": "blocked", "error": f"UNEXPECTED_USABLE_SMOKE_ERROR: {type(error).__name__}: {str(error)[:1000]}", "results": []}
|
|
result.update({"report_root": str(report_root.resolve())})
|
|
report_path = report_root / "usable-smoke-report.json"
|
|
report_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps({"status": result["status"], "report": str(report_path.resolve())}, ensure_ascii=False))
|
|
return 0 if result["status"] == "passed" else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|