142 lines
6.7 KiB
Python
142 lines
6.7 KiB
Python
"""Independent, structured visual review of generated checkpoint renders."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.settings import ProviderConfig, ProviderModel, Settings
|
|
|
|
|
|
VISUAL_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_rendered_checkpoint",
|
|
"description": "Review fixed CAD render views against frozen requirements. Never author or modify CDSL.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["pass", "warning", "repair"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"affected_node_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
"requirement_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 32},
|
|
"evidence": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
},
|
|
"required": ["verdict", "confidence", "affected_node_ids", "requirement_ids", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class VisualReviewError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _image_part(path: Path) -> dict[str, Any]:
|
|
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
|
media = "image/jpeg" if path.suffix.lower() in {".jpg", ".jpeg"} else "image/png"
|
|
return {"type": "image_url", "image_url": {"url": f"data:{media};base64,{encoded}"}}
|
|
|
|
|
|
def _selected_review_views(manifest: dict[str, Any], *, final_checkpoint: bool) -> list[dict[str, Any]]:
|
|
"""Keep each reviewer call bounded while canonical evidence stays archived.
|
|
|
|
A contact sheet establishes global context. Up to two planned detail views
|
|
provide node-specific evidence. The final checkpoint adds full canonical
|
|
views because it is the only point where those extra image tokens pay off.
|
|
"""
|
|
views = [item for item in manifest.get("views") or () if isinstance(item, dict)]
|
|
by_id = {str(item.get("id") or ""): item for item in views}
|
|
selected: list[dict[str, Any]] = []
|
|
contact = Path(str(manifest.get("contact_sheet_path") or ""))
|
|
if contact.is_file():
|
|
selected.append({"id": "contact-sheet", "path": str(contact), "camera": {"projection": "mixed"}})
|
|
for view_id in ("detail-1", "detail-2"):
|
|
item = by_id.get(view_id)
|
|
if item is not None:
|
|
selected.append(item)
|
|
if final_checkpoint:
|
|
selected.extend(by_id[view_id] for view_id in ("top", "bottom", "front", "back", "left", "right", "isometric") if view_id in by_id)
|
|
elif not selected and "isometric" in by_id:
|
|
selected.append(by_id["isometric"])
|
|
return selected or views[:1]
|
|
|
|
|
|
async def review_checkpoint(
|
|
settings: Settings,
|
|
*,
|
|
manifest: dict[str, Any],
|
|
requirements: list[dict[str, Any]],
|
|
node_id: str,
|
|
deterministic_report: dict[str, Any],
|
|
source_images: list[Path] | None = None,
|
|
final_checkpoint: bool = False,
|
|
) -> dict[str, Any]:
|
|
provider, model = settings.resolve_review_model()
|
|
views = _selected_review_views(manifest, final_checkpoint=final_checkpoint)
|
|
paths = [Path(str(item.get("path") or "")) for item in views]
|
|
if not paths or not all(path.is_file() for path in paths):
|
|
raise VisualReviewError("Review render manifest references missing image files")
|
|
content: list[dict[str, Any]] = [{
|
|
"type": "text",
|
|
"text": json.dumps({
|
|
"node_id": node_id,
|
|
"requirements": requirements,
|
|
"deterministic_report": deterministic_report,
|
|
"render_manifest": {
|
|
"renderer": manifest.get("renderer"),
|
|
"source": manifest.get("source"),
|
|
"views": [{"id": item.get("id"), "camera": item.get("camera"), "diagnostics": item.get("diagnostics")} for item in views],
|
|
},
|
|
"instruction": "Identify visible missing geometry, wrong silhouette, orientation, or proportion. Do not infer hidden dimensions. Return repair only for an observable issue.",
|
|
}, ensure_ascii=False),
|
|
}]
|
|
content.extend(_image_part(path) for path in paths)
|
|
# Reference images are only supplementary evidence. Keep this bounded so
|
|
# an attachment-heavy request does not dominate every checkpoint review.
|
|
for path in (source_images or [])[:2]:
|
|
if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"}:
|
|
content.append(_image_part(path))
|
|
tool = json.loads(json.dumps(VISUAL_REVIEW_TOOL))
|
|
if model.strict_tool_schema:
|
|
tool["function"]["strict"] = True
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD visual reviewer. You may only call review_rendered_checkpoint."},
|
|
{"role": "user", "content": content},
|
|
],
|
|
"tools": [tool],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_rendered_checkpoint"}},
|
|
"temperature": 0,
|
|
}
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Visual review request failed ({response.status_code}): {response.text[:500]}")
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_rendered_checkpoint":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise VisualReviewError("Visual reviewer did not return a valid review tool call") from error
|
|
if not isinstance(result, dict) or result.get("verdict") not in {"pass", "warning", "repair"}:
|
|
raise VisualReviewError("Visual reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise VisualReviewError("Visual reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise VisualReviewError("Visual reviewer confidence is outside [0, 1]")
|
|
for key in ("affected_node_ids", "requirement_ids", "evidence"):
|
|
if not isinstance(result.get(key), list) or not all(isinstance(item, str) for item in result[key]):
|
|
raise VisualReviewError(f"Visual reviewer returned an invalid {key}")
|
|
return {"schema_version": "cad.visual-review.v1", "node_id": node_id, "model": model.id, **result}
|