344 lines
18 KiB
Python
344 lines
18 KiB
Python
"""Independent, structured visual review of generated checkpoint renders."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.settings import ProviderConfig, ProviderModel, Settings
|
|
|
|
|
|
VISUAL_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_rendered_checkpoint",
|
|
"description": "Review fixed CAD render views against frozen requirements. Never author or modify CDSL.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["pass", "warning", "repair"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"affected_node_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
"requirement_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 32},
|
|
"evidence": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
},
|
|
"required": ["verdict", "confidence", "affected_node_ids", "requirement_ids", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
CANDIDATE_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_candidate_batch",
|
|
"description": "Independently review one staged CAD batch. Never author, modify, or approve CDSL outside this verdict.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["accept", "reject"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"batch_goal_status": {"enum": ["achieved", "partial", "failed"]},
|
|
"coverage": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"item": {"type": "string"},
|
|
"status": {"enum": ["complete", "pending", "regressed", "uncertain"]},
|
|
"evidence": {"type": "string"},
|
|
},
|
|
"required": ["item", "status", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
"evidence": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
},
|
|
"required": ["verdict", "confidence", "batch_goal_status", "coverage", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class VisualReviewError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class _CandidateReviewFormatError(VisualReviewError):
|
|
"""A locally detected invalid reviewer tool result, eligible for one retry."""
|
|
|
|
|
|
def _thinking_tool_choice_rejected(response: httpx.Response) -> bool:
|
|
"""Recognize the only compatibility error for which retrying is sound.
|
|
|
|
Some OpenAI-compatible reasoning endpoints expose function tools but
|
|
reject a forced function selection while thinking is enabled. Retrying
|
|
without ``tool_choice`` retains the tool contract and the local output
|
|
validation below; it merely lets that endpoint choose the sole available
|
|
review tool itself.
|
|
"""
|
|
return (
|
|
response.status_code == 400
|
|
and "thinking mode does not support this tool_choice" in response.text.lower()
|
|
)
|
|
|
|
|
|
def visual_review_tool() -> dict[str, Any]:
|
|
"""Normal function-call schema; local validation remains authoritative."""
|
|
return json.loads(json.dumps(VISUAL_REVIEW_TOOL))
|
|
|
|
|
|
def candidate_review_tool() -> dict[str, Any]:
|
|
return json.loads(json.dumps(CANDIDATE_REVIEW_TOOL))
|
|
|
|
|
|
def _image_part(path: Path) -> dict[str, Any]:
|
|
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
|
media = "image/jpeg" if path.suffix.lower() in {".jpg", ".jpeg"} else "image/png"
|
|
return {"type": "image_url", "image_url": {"url": f"data:{media};base64,{encoded}"}}
|
|
|
|
|
|
def _selected_review_views(manifest: dict[str, Any], *, final_checkpoint: bool) -> list[dict[str, Any]]:
|
|
"""Keep each reviewer call bounded while canonical evidence stays archived.
|
|
|
|
A contact sheet establishes global context. Up to two planned detail views
|
|
provide node-specific evidence. The final checkpoint adds full canonical
|
|
views because it is the only point where those extra image tokens pay off.
|
|
"""
|
|
views = [item for item in manifest.get("views") or () if isinstance(item, dict)]
|
|
by_id = {str(item.get("id") or ""): item for item in views}
|
|
selected: list[dict[str, Any]] = []
|
|
contact = Path(str(manifest.get("contact_sheet_path") or ""))
|
|
if contact.is_file():
|
|
selected.append({"id": "contact-sheet", "path": str(contact), "camera": {"projection": "mixed"}})
|
|
for view_id in ("detail-1", "detail-2"):
|
|
item = by_id.get(view_id)
|
|
if item is not None:
|
|
selected.append(item)
|
|
if final_checkpoint:
|
|
selected.extend(by_id[view_id] for view_id in ("top", "bottom", "front", "back", "left", "right", "isometric") if view_id in by_id)
|
|
else:
|
|
selected.extend(by_id[view_id] for view_id in ("top", "front", "right", "isometric") if view_id in by_id)
|
|
return selected or views[:1]
|
|
|
|
|
|
async def review_checkpoint(
|
|
settings: Settings,
|
|
*,
|
|
manifest: dict[str, Any],
|
|
requirements: str | list[dict[str, Any]],
|
|
source_requirements: str = "",
|
|
node_id: str = "final",
|
|
deterministic_report: dict[str, Any],
|
|
source_images: list[Path] | None = None,
|
|
final_checkpoint: bool = False,
|
|
) -> dict[str, Any]:
|
|
provider, model = settings.resolve_review_model()
|
|
views = _selected_review_views(manifest, final_checkpoint=final_checkpoint)
|
|
paths = [Path(str(item.get("path") or "")) for item in views]
|
|
if not paths or not all(path.is_file() for path in paths):
|
|
raise VisualReviewError("Review render manifest references missing image files")
|
|
content: list[dict[str, Any]] = [{
|
|
"type": "text",
|
|
"text": json.dumps({
|
|
"node_id": node_id,
|
|
"requirements": requirements,
|
|
"source_requirements": source_requirements,
|
|
"deterministic_report": deterministic_report,
|
|
"render_manifest": {
|
|
"renderer": manifest.get("renderer"),
|
|
"source": manifest.get("source"),
|
|
"views": [{"id": item.get("id"), "camera": item.get("camera"), "diagnostics": item.get("diagnostics")} for item in views],
|
|
},
|
|
"instruction": "Identify visible missing geometry, wrong silhouette, orientation, or proportion. First reconcile every claimed structural element with final_model_evidence: never claim a feature, mirrored copy, slot, cut, or chamfer exists when it is absent from that evidence. Do not infer hidden dimensions. Source requirements are the original user contract. Frozen requirements may add measurable assumptions but may never remove, replace, or weaken source requirements. Return pass only when both contracts are satisfied by the views and evidence; return warning or repair for any unsatisfied or downgraded source requirement.",
|
|
}, ensure_ascii=False),
|
|
}]
|
|
content.extend(_image_part(path) for path in paths)
|
|
# Reference images are only supplementary evidence. Keep this bounded so
|
|
# an attachment-heavy request does not dominate every checkpoint review.
|
|
for path in (source_images or [])[:2]:
|
|
if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"}:
|
|
content.append(_image_part(path))
|
|
tool = visual_review_tool()
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD visual reviewer. You may only call review_rendered_checkpoint."},
|
|
{"role": "user", "content": content},
|
|
],
|
|
"tools": [tool],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_rendered_checkpoint"}},
|
|
"temperature": 0,
|
|
}
|
|
payload.update(provider.chat_completion_options)
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=payload)
|
|
if _thinking_tool_choice_rejected(response):
|
|
# Do not remove the tool itself and do not accept prose as a
|
|
# fallback. This compatibility retry is valid only because the
|
|
# reviewer receives one callable tool and its response is still
|
|
# verified by the strict local validator below.
|
|
payload.pop("tool_choice", None)
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Visual review request failed ({response.status_code}): {response.text[:500]}")
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_rendered_checkpoint":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise VisualReviewError("Visual reviewer did not return a valid review tool call") from error
|
|
allowed_fields = {"verdict", "confidence", "affected_node_ids", "requirement_ids", "evidence"}
|
|
if not isinstance(result, dict) or set(result) != allowed_fields or result.get("verdict") not in {"pass", "warning", "repair"}:
|
|
raise VisualReviewError("Visual reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise VisualReviewError("Visual reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise VisualReviewError("Visual reviewer confidence is outside [0, 1]")
|
|
for key in ("affected_node_ids", "requirement_ids", "evidence"):
|
|
if not isinstance(result.get(key), list) or not all(isinstance(item, str) for item in result[key]):
|
|
raise VisualReviewError(f"Visual reviewer returned an invalid {key}")
|
|
return {"schema_version": "cad.visual-review.v1", "node_id": node_id, "model": model.id, **result}
|
|
|
|
|
|
async def review_candidate_batch(
|
|
settings: Settings,
|
|
*,
|
|
manifest: dict[str, Any],
|
|
requirements: str,
|
|
source_requirements: str = "",
|
|
checklist: list[str],
|
|
batch_goal: str,
|
|
deterministic_report: dict[str, Any],
|
|
node_id: str,
|
|
) -> dict[str, Any]:
|
|
"""Review a staged batch against its local goal and the frozen checklist."""
|
|
provider, model = settings.resolve_review_model()
|
|
views = _selected_review_views(manifest, final_checkpoint=False)
|
|
paths = [Path(str(item.get("path") or "")) for item in views]
|
|
if not paths or not all(path.is_file() for path in paths):
|
|
raise VisualReviewError("Candidate review render manifest references missing image files")
|
|
content: list[dict[str, Any]] = [{
|
|
"type": "text",
|
|
"text": json.dumps({
|
|
"node_id": node_id,
|
|
"source_requirements": source_requirements,
|
|
"frozen_requirements": requirements,
|
|
"completion_checklist": checklist,
|
|
"batch_goal": batch_goal,
|
|
"deterministic_report": deterministic_report,
|
|
"instruction": (
|
|
"Assess this staged batch, not overall task completion. Accept only if the batch goal is achieved "
|
|
"without visibly contradicting source requirements, frozen requirements, or deterministic evidence. "
|
|
"Source requirements are the original user contract; frozen requirements may add measurable assumptions "
|
|
"but may not remove, replace, or weaken the source contract. Reject when a batch visibly downgrades or "
|
|
"contradicts source intent. An incomplete but coherent "
|
|
"intermediate model may be accepted when its batch goal is achieved. Return one coverage row for every "
|
|
"completion checklist item, preserving exact item text. Use pending for future work and regressed when "
|
|
"this batch breaks previously achieved work. Reject on disconnected geometry when the requirements call "
|
|
"for one body, wrong orientation, visibly wrong geometry, or a failed batch goal."
|
|
),
|
|
"render_manifest": {
|
|
"renderer": manifest.get("renderer"),
|
|
"source": manifest.get("source"),
|
|
"views": [{"id": item.get("id"), "camera": item.get("camera")} for item in views],
|
|
},
|
|
}, ensure_ascii=False),
|
|
}]
|
|
content.extend(_image_part(path) for path in paths)
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD batch reviewer. You may only call review_candidate_batch."},
|
|
{"role": "user", "content": content},
|
|
],
|
|
"tools": [candidate_review_tool()],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_candidate_batch"}},
|
|
"temperature": 0,
|
|
}
|
|
payload.update(provider.chat_completion_options)
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
|
|
async def request_review(client: httpx.AsyncClient, request_payload: dict[str, Any]) -> httpx.Response:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if _thinking_tool_choice_rejected(response):
|
|
request_payload.pop("tool_choice", None)
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Candidate visual review request failed ({response.status_code}): {response.text[:500]}")
|
|
return response
|
|
|
|
def validate_response(response: httpx.Response) -> dict[str, Any]:
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_candidate_batch":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise _CandidateReviewFormatError("Candidate reviewer did not return a valid review tool call") from error
|
|
allowed = {"verdict", "confidence", "batch_goal_status", "coverage", "evidence"}
|
|
if not isinstance(result, dict) or set(result) != allowed or result.get("verdict") not in {"accept", "reject"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise _CandidateReviewFormatError("Candidate reviewer confidence is outside [0, 1]")
|
|
if result.get("batch_goal_status") not in {"achieved", "partial", "failed"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid batch goal status")
|
|
if result.get("verdict") == "accept" and result.get("batch_goal_status") != "achieved":
|
|
raise _CandidateReviewFormatError("Candidate reviewer may accept only an achieved batch goal")
|
|
if not isinstance(result.get("evidence"), list) or not all(isinstance(item, str) for item in result["evidence"]):
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned invalid evidence")
|
|
coverage = result.get("coverage")
|
|
if not isinstance(coverage, list) or len(coverage) != len(checklist):
|
|
raise _CandidateReviewFormatError("Candidate reviewer must return coverage for every checklist item")
|
|
expected = {item.casefold(): item for item in checklist}
|
|
seen: set[str] = set()
|
|
for item in coverage:
|
|
if not isinstance(item, dict) or set(item) != {"item", "status", "evidence"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid coverage row")
|
|
key = str(item.get("item") or "").casefold()
|
|
if key not in expected or key in seen:
|
|
raise _CandidateReviewFormatError("Candidate reviewer coverage does not match the frozen checklist")
|
|
if item.get("status") not in {"complete", "pending", "regressed", "uncertain"} or not str(item.get("evidence") or "").strip():
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an incomplete coverage row")
|
|
item["item"] = expected[key]
|
|
seen.add(key)
|
|
return result
|
|
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await request_review(client, payload)
|
|
try:
|
|
result = validate_response(response)
|
|
except _CandidateReviewFormatError as error:
|
|
retry_payload = copy.deepcopy(payload)
|
|
retry_payload["messages"].append({
|
|
"role": "user",
|
|
"content": (
|
|
"Your previous tool call was rejected by the local validator: "
|
|
f"{error}. Review the same rendered candidate again. Return only a complete "
|
|
"review_candidate_batch tool call with exactly one coverage row for every frozen checklist item."
|
|
),
|
|
})
|
|
result = validate_response(await request_review(client, retry_payload))
|
|
return {
|
|
"schema_version": "cad.candidate-review.v1",
|
|
"node_id": node_id,
|
|
"model": model.id,
|
|
"batch_goal": batch_goal,
|
|
**result,
|
|
}
|