592 lines
33 KiB
Python
592 lines
33 KiB
Python
"""Independent, structured visual review of generated checkpoint renders."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import copy
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.settings import ProviderConfig, ProviderModel, Settings
|
|
|
|
|
|
VISUAL_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_rendered_checkpoint",
|
|
"description": "Review fixed CAD render views against frozen requirements. Never author or modify CDSL.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["pass", "warning", "repair"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"affected_node_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
"requirement_ids": {"type": "array", "items": {"type": "string"}, "maxItems": 32},
|
|
"evidence": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
},
|
|
"required": ["verdict", "confidence", "affected_node_ids", "requirement_ids", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
CANDIDATE_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_candidate_batch",
|
|
"description": "Independently review one staged CAD batch. Never author, modify, or approve CDSL outside this verdict.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["accept", "reject"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"batch_goal_status": {"enum": ["achieved", "partial", "failed"]},
|
|
"coverage": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"item": {"type": "string"},
|
|
"status": {"enum": ["complete", "pending", "regressed", "uncertain"]},
|
|
"evidence": {"type": "string"},
|
|
},
|
|
"required": ["item", "status", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
"evidence": {"type": "array", "items": {"type": "string"}, "maxItems": 12},
|
|
},
|
|
"required": ["verdict", "confidence", "batch_goal_status", "coverage", "evidence"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
MODELING_PLAN_REVIEW_TOOL = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "review_modeling_plan",
|
|
"description": "Independently review a CAD modeling plan for requirement coverage, coherent step grouping, dependencies, and observable evidence. Never generate or modify CDSL.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": ["pass", "revise"]},
|
|
"confidence": {"type": "number", "minimum": 0, "maximum": 1},
|
|
"issues": {"type": "array", "items": {"type": "object", "properties": {"type": {"type": "string"}, "step_id": {"type": "string"}, "message": {"type": "string"}}, "required": ["type", "message"], "additionalProperties": False}, "maxItems": 20},
|
|
"coverage": {"type": "array", "items": {"type": "object", "properties": {"requirement": {"type": "string"}, "step_id": {"type": "string"}, "status": {"enum": ["covered", "missing"]}, "evidence": {"type": "string"}}, "required": ["requirement", "step_id", "status", "evidence"], "additionalProperties": False}},
|
|
"step_checks": {"type": "array", "items": {"type": "object", "properties": {"step_id": {"type": "string"}, "status": {"enum": ["pass", "fail"]}, "notes": {"type": "string"}}, "required": ["step_id", "status", "notes"], "additionalProperties": False}},
|
|
"action_checks": {"type": "array", "items": {"type": "object", "properties": {"step_id": {"type": "string"}, "status": {"enum": ["pass", "fail"]}, "notes": {"type": "string"}, "required_action_count": {"type": "integer", "minimum": 1}}, "required": ["step_id", "status", "notes", "required_action_count"], "additionalProperties": False}},
|
|
},
|
|
"required": ["verdict", "confidence", "issues", "coverage", "step_checks", "action_checks"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class VisualReviewError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class _CandidateReviewFormatError(VisualReviewError):
|
|
"""A locally detected invalid reviewer tool result, eligible for one retry."""
|
|
|
|
|
|
class _ModelingPlanReviewFormatError(VisualReviewError):
|
|
"""A locally detected invalid modeling-plan verdict, eligible for one retry."""
|
|
|
|
|
|
def _checklist_key(value: Any) -> str:
|
|
return " ".join(str(value or "").strip().split()).casefold()
|
|
|
|
|
|
def _looks_like_operation_id(value: str) -> bool:
|
|
token = str(value or "").strip().casefold()
|
|
if not token or "_" not in token:
|
|
return token in {"fillet", "chamfer"}
|
|
prefixes = ("extrude_", "revolve_", "hole_", "pattern_", "sphere_", "reference_", "cylinder_", "sweep_")
|
|
suffixes = ("_add", "_cut", "_blind", "_wizard", "_linear", "_circular", "_mirror")
|
|
return token.startswith(prefixes) or token.endswith(suffixes)
|
|
|
|
|
|
def _unsupported_operation_mentions(result: dict[str, Any], runtime_operations: list[dict[str, Any]]) -> list[str]:
|
|
"""Find capability-shaped names in reviewer prose that Runtime cannot execute."""
|
|
supported = {
|
|
str(item.get("atomic_id") or "").strip().casefold()
|
|
for item in runtime_operations
|
|
if isinstance(item, dict) and str(item.get("atomic_id") or "").strip()
|
|
}
|
|
mentions: set[str] = set()
|
|
for section in (result.get("issues"), result.get("coverage"), result.get("step_checks"), result.get("action_checks")):
|
|
if not isinstance(section, list):
|
|
continue
|
|
for row in section:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
for value in row.values():
|
|
if not isinstance(value, str):
|
|
continue
|
|
for token in re.findall(r"(?<![A-Za-z0-9_])([A-Za-z][A-Za-z0-9_]*)(?![A-Za-z0-9_])", value):
|
|
normalized = token.casefold()
|
|
if normalized in supported or not _looks_like_operation_id(normalized):
|
|
continue
|
|
# Runtime capabilities are exact identifiers. A prefix
|
|
# or human shorthand (for example extrude_cut for
|
|
# extrude_cut_blind) is still invalid and must be revised.
|
|
mentions.add(token)
|
|
return sorted(mentions, key=str.casefold)
|
|
|
|
|
|
def _thinking_tool_choice_rejected(response: httpx.Response) -> bool:
|
|
"""Recognize the only compatibility error for which retrying is sound.
|
|
|
|
Some OpenAI-compatible reasoning endpoints expose function tools but
|
|
reject a forced function selection while thinking is enabled. Retrying
|
|
without ``tool_choice`` retains the tool contract and the local output
|
|
validation below; it merely lets that endpoint choose the sole available
|
|
review tool itself.
|
|
"""
|
|
return (
|
|
response.status_code == 400
|
|
and "thinking mode does not support this tool_choice" in response.text.lower()
|
|
)
|
|
|
|
|
|
def visual_review_tool() -> dict[str, Any]:
|
|
"""Normal function-call schema; local validation remains authoritative."""
|
|
return json.loads(json.dumps(VISUAL_REVIEW_TOOL))
|
|
|
|
|
|
def candidate_review_tool() -> dict[str, Any]:
|
|
return json.loads(json.dumps(CANDIDATE_REVIEW_TOOL))
|
|
|
|
|
|
def modeling_plan_review_tool() -> dict[str, Any]:
|
|
return json.loads(json.dumps(MODELING_PLAN_REVIEW_TOOL))
|
|
|
|
|
|
def _image_part(path: Path) -> dict[str, Any]:
|
|
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
|
media = "image/jpeg" if path.suffix.lower() in {".jpg", ".jpeg"} else "image/png"
|
|
return {"type": "image_url", "image_url": {"url": f"data:{media};base64,{encoded}"}}
|
|
|
|
|
|
def _selected_review_views(manifest: dict[str, Any], *, final_checkpoint: bool) -> list[dict[str, Any]]:
|
|
"""Keep each reviewer call bounded while canonical evidence stays archived.
|
|
|
|
A contact sheet establishes global context. Up to two planned detail views
|
|
provide node-specific evidence. The final checkpoint adds full canonical
|
|
views because it is the only point where those extra image tokens pay off.
|
|
"""
|
|
views = [item for item in manifest.get("views") or () if isinstance(item, dict)]
|
|
by_id = {str(item.get("id") or ""): item for item in views}
|
|
selected: list[dict[str, Any]] = []
|
|
contact = Path(str(manifest.get("contact_sheet_path") or ""))
|
|
if contact.is_file():
|
|
selected.append({"id": "contact-sheet", "path": str(contact), "camera": {"projection": "mixed"}})
|
|
for view_id in ("detail-1", "detail-2"):
|
|
item = by_id.get(view_id)
|
|
if item is not None:
|
|
selected.append(item)
|
|
if final_checkpoint:
|
|
selected.extend(by_id[view_id] for view_id in ("top", "bottom", "front", "back", "left", "right", "isometric") if view_id in by_id)
|
|
else:
|
|
selected.extend(by_id[view_id] for view_id in ("top", "front", "right", "isometric") if view_id in by_id)
|
|
return selected or views[:1]
|
|
|
|
|
|
async def review_checkpoint(
|
|
settings: Settings,
|
|
*,
|
|
manifest: dict[str, Any],
|
|
requirements: str | list[dict[str, Any]],
|
|
source_requirements: str = "",
|
|
node_id: str = "final",
|
|
deterministic_report: dict[str, Any],
|
|
source_images: list[Path] | None = None,
|
|
final_checkpoint: bool = False,
|
|
) -> dict[str, Any]:
|
|
provider, model = settings.resolve_review_model()
|
|
views = _selected_review_views(manifest, final_checkpoint=final_checkpoint)
|
|
paths = [Path(str(item.get("path") or "")) for item in views]
|
|
if not paths or not all(path.is_file() for path in paths):
|
|
raise VisualReviewError("Review render manifest references missing image files")
|
|
content: list[dict[str, Any]] = [{
|
|
"type": "text",
|
|
"text": json.dumps({
|
|
"node_id": node_id,
|
|
"requirements": requirements,
|
|
"source_requirements": source_requirements,
|
|
"deterministic_report": deterministic_report,
|
|
"render_manifest": {
|
|
"renderer": manifest.get("renderer"),
|
|
"source": manifest.get("source"),
|
|
"views": [{"id": item.get("id"), "camera": item.get("camera"), "diagnostics": item.get("diagnostics")} for item in views],
|
|
},
|
|
"instruction": "Identify visible missing geometry, wrong silhouette, orientation, or proportion. First reconcile every claimed structural element with final_model_evidence: never claim a feature, mirrored copy, slot, cut, or chamfer exists when it is absent from that evidence. Do not infer hidden dimensions. Source requirements are the original user contract. Frozen requirements may add measurable assumptions but may never remove, replace, or weaken source requirements. Return pass only when both contracts are satisfied by the views and evidence; return warning or repair for any unsatisfied or downgraded source requirement.",
|
|
}, ensure_ascii=False),
|
|
}]
|
|
content.extend(_image_part(path) for path in paths)
|
|
# Reference images are only supplementary evidence. Keep this bounded so
|
|
# an attachment-heavy request does not dominate every checkpoint review.
|
|
for path in (source_images or [])[:2]:
|
|
if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"}:
|
|
content.append(_image_part(path))
|
|
tool = visual_review_tool()
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD visual reviewer. You may only call review_rendered_checkpoint."},
|
|
{"role": "user", "content": content},
|
|
],
|
|
"tools": [tool],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_rendered_checkpoint"}},
|
|
"temperature": 0,
|
|
}
|
|
payload.update(provider.chat_completion_options)
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=payload)
|
|
if _thinking_tool_choice_rejected(response):
|
|
# Do not remove the tool itself and do not accept prose as a
|
|
# fallback. This compatibility retry is valid only because the
|
|
# reviewer receives one callable tool and its response is still
|
|
# verified by the strict local validator below.
|
|
payload.pop("tool_choice", None)
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Visual review request failed ({response.status_code}): {response.text[:500]}")
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_rendered_checkpoint":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise VisualReviewError("Visual reviewer did not return a valid review tool call") from error
|
|
allowed_fields = {"verdict", "confidence", "affected_node_ids", "requirement_ids", "evidence"}
|
|
if not isinstance(result, dict) or set(result) != allowed_fields or result.get("verdict") not in {"pass", "warning", "repair"}:
|
|
raise VisualReviewError("Visual reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise VisualReviewError("Visual reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise VisualReviewError("Visual reviewer confidence is outside [0, 1]")
|
|
for key in ("affected_node_ids", "requirement_ids", "evidence"):
|
|
if not isinstance(result.get(key), list) or not all(isinstance(item, str) for item in result[key]):
|
|
raise VisualReviewError(f"Visual reviewer returned an invalid {key}")
|
|
return {"schema_version": "cad.visual-review.v1", "node_id": node_id, "model": model.id, **result}
|
|
|
|
|
|
async def review_candidate_batch(
|
|
settings: Settings,
|
|
*,
|
|
manifest: dict[str, Any],
|
|
requirements: str,
|
|
source_requirements: str = "",
|
|
checklist: list[str],
|
|
batch_goal: str,
|
|
deterministic_report: dict[str, Any],
|
|
node_id: str,
|
|
plan_step: dict[str, Any] | None = None,
|
|
plan_feature_ids: list[str] | None = None,
|
|
plan_action: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Review a staged batch against its local goal and the frozen checklist."""
|
|
provider, model = settings.resolve_review_model()
|
|
views = _selected_review_views(manifest, final_checkpoint=False)
|
|
paths = [Path(str(item.get("path") or "")) for item in views]
|
|
if not paths or not all(path.is_file() for path in paths):
|
|
raise VisualReviewError("Candidate review render manifest references missing image files")
|
|
content: list[dict[str, Any]] = [{
|
|
"type": "text",
|
|
"text": json.dumps({
|
|
"node_id": node_id,
|
|
"source_requirements": source_requirements,
|
|
"frozen_requirements": requirements,
|
|
"completion_checklist": checklist,
|
|
"batch_goal": batch_goal,
|
|
"plan_step": plan_step or None,
|
|
"plan_feature_ids": plan_feature_ids or [],
|
|
"plan_action": plan_action or None,
|
|
"deterministic_report": deterministic_report,
|
|
"instruction": (
|
|
"Assess this staged batch, not overall task completion. Accept only if the batch goal is achieved "
|
|
"without visibly contradicting source requirements, frozen requirements, or deterministic evidence. "
|
|
"Source requirements are the original user contract; frozen requirements may add measurable assumptions "
|
|
"but may not remove, replace, or weaken the source contract. Reject when a batch visibly downgrades or "
|
|
"contradicts source intent. An incomplete but coherent "
|
|
"intermediate model may be accepted when its batch goal is achieved. Return one coverage row for every "
|
|
"completion checklist item, preserving exact item text. Use pending for future work and regressed when "
|
|
"this batch breaks previously achieved work. Reject on disconnected geometry when the requirements call "
|
|
"for one body, wrong orientation, visibly wrong geometry, or a failed batch goal. "
|
|
"When plan_step is supplied, reject operations outside that step and confirm its planned feature goal. "
|
|
"A step may contain multiple related actions; when plan_action is supplied, assess only that action, confirm the candidate changes its target, and reject unrelated or multiple independent profiles."
|
|
),
|
|
"render_manifest": {
|
|
"renderer": manifest.get("renderer"),
|
|
"source": manifest.get("source"),
|
|
"views": [{"id": item.get("id"), "camera": item.get("camera")} for item in views],
|
|
},
|
|
}, ensure_ascii=False),
|
|
}]
|
|
content.extend(_image_part(path) for path in paths)
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD batch reviewer. You may only call review_candidate_batch."},
|
|
{"role": "user", "content": content},
|
|
],
|
|
"tools": [candidate_review_tool()],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_candidate_batch"}},
|
|
"temperature": 0,
|
|
}
|
|
payload.update(provider.chat_completion_options)
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
|
|
async def request_review(client: httpx.AsyncClient, request_payload: dict[str, Any]) -> httpx.Response:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if _thinking_tool_choice_rejected(response):
|
|
request_payload.pop("tool_choice", None)
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Candidate visual review request failed ({response.status_code}): {response.text[:500]}")
|
|
return response
|
|
|
|
def validate_response(response: httpx.Response) -> dict[str, Any]:
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_candidate_batch":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise _CandidateReviewFormatError("Candidate reviewer did not return a valid review tool call") from error
|
|
allowed = {"verdict", "confidence", "batch_goal_status", "coverage", "evidence"}
|
|
if not isinstance(result, dict) or set(result) != allowed or result.get("verdict") not in {"accept", "reject"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise _CandidateReviewFormatError("Candidate reviewer confidence is outside [0, 1]")
|
|
if result.get("batch_goal_status") not in {"achieved", "partial", "failed"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid batch goal status")
|
|
if result.get("verdict") == "accept" and result.get("batch_goal_status") != "achieved":
|
|
raise _CandidateReviewFormatError("Candidate reviewer may accept only an achieved batch goal")
|
|
if not isinstance(result.get("evidence"), list) or not all(isinstance(item, str) for item in result["evidence"]):
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned invalid evidence")
|
|
coverage = result.get("coverage")
|
|
if not isinstance(coverage, list) or len(coverage) != len(checklist):
|
|
raise _CandidateReviewFormatError("Candidate reviewer must return coverage for every checklist item")
|
|
expected = {item.casefold(): item for item in checklist}
|
|
seen: set[str] = set()
|
|
for item in coverage:
|
|
if not isinstance(item, dict) or set(item) != {"item", "status", "evidence"}:
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an invalid coverage row")
|
|
key = str(item.get("item") or "").casefold()
|
|
if key not in expected or key in seen:
|
|
raise _CandidateReviewFormatError("Candidate reviewer coverage does not match the frozen checklist")
|
|
if item.get("status") not in {"complete", "pending", "regressed", "uncertain"} or not str(item.get("evidence") or "").strip():
|
|
raise _CandidateReviewFormatError("Candidate reviewer returned an incomplete coverage row")
|
|
item["item"] = expected[key]
|
|
seen.add(key)
|
|
return result
|
|
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await request_review(client, payload)
|
|
try:
|
|
result = validate_response(response)
|
|
except _CandidateReviewFormatError as error:
|
|
retry_payload = copy.deepcopy(payload)
|
|
retry_payload["messages"].append({
|
|
"role": "user",
|
|
"content": (
|
|
"Your previous tool call was rejected by the local validator: "
|
|
f"{error}. Review the same rendered candidate again. Return only a complete "
|
|
"review_candidate_batch tool call with exactly one coverage row for every frozen checklist item."
|
|
),
|
|
})
|
|
result = validate_response(await request_review(client, retry_payload))
|
|
return {
|
|
"schema_version": "cad.candidate-review.v1",
|
|
"node_id": node_id,
|
|
"model": model.id,
|
|
"batch_goal": batch_goal,
|
|
**result,
|
|
}
|
|
|
|
|
|
async def review_modeling_plan(
|
|
settings: Settings,
|
|
*,
|
|
source_requirements: str,
|
|
requirements: str,
|
|
checklist: list[str],
|
|
plan: dict[str, Any],
|
|
runtime_operations: list[dict[str, Any]],
|
|
node_id: str = "modeling-plan",
|
|
) -> dict[str, Any]:
|
|
"""Ask an independent model to simulate and review a modeling plan."""
|
|
provider, model = settings.resolve_review_model()
|
|
payload_context = {
|
|
"node_id": node_id,
|
|
"source_requirements": source_requirements,
|
|
"frozen_requirements": requirements,
|
|
"completion_checklist": checklist,
|
|
"modeling_plan": plan,
|
|
"runtime_operations": runtime_operations,
|
|
"instruction": (
|
|
"Review the plan only; do not generate CDSL. Treat modeling_plan.plan_text as semantic "
|
|
"guidance, not a schema: missing structures/features arrays or machine IDs are not by "
|
|
"themselves failures. Infer intended structures, ordering, relationships, and evidence "
|
|
"from the prose. Check every checklist item, dependency order, step cohesion, topology "
|
|
"preconditions, Runtime feasibility, action granularity, and observable evidence. A semantic "
|
|
"step may contain related or repeated targets, but each independently located profile "
|
|
"or Runtime feature must be enumerated as a separate action within that same step. Each action "
|
|
"must be executable as one Runtime feature. A multi-position hole or supported pattern may remain "
|
|
"one action when all positions share the same host and operation contract. Treat any supplied "
|
|
"action records as execution units, and set required_action_count to the minimum number the step's "
|
|
"semantics require. Return fail when the plan contains fewer actions than that number. Return pass when the plan "
|
|
"is actionable enough for an LLM to generate CDSL in coherent batches. Do not require "
|
|
"colors or materials when runtime_operations does not provide them; accept a geometrically "
|
|
"equivalent ring, groove, or separated feature and mention the limitation only as guidance. "
|
|
"Only return revise for a real requirement omission, contradictory geometry, unsafe ordering, "
|
|
"an operation name that is not present in runtime_operations, or an operation that cannot plausibly be implemented. "
|
|
"Do not downgrade an unsupported operation to vague guidance. If a machine operation is mentioned, it must be an exact ID from runtime_operations; otherwise return revise and describe the semantic intent without inventing an operation name. "
|
|
"Return exactly one coverage row per checklist item."
|
|
),
|
|
}
|
|
payload = {
|
|
"model": model.id,
|
|
"messages": [
|
|
{"role": "system", "content": "You are an independent CAD modeling-plan reviewer. You may only call review_modeling_plan."},
|
|
{"role": "user", "content": json.dumps(payload_context, ensure_ascii=False)},
|
|
],
|
|
"tools": [modeling_plan_review_tool()],
|
|
"tool_choice": {"type": "function", "function": {"name": "review_modeling_plan"}},
|
|
"temperature": 0,
|
|
}
|
|
payload.update(provider.chat_completion_options)
|
|
headers = {"Authorization": f"Bearer {provider.api_key}", "Content-Type": "application/json"}
|
|
|
|
async def request_review(client: httpx.AsyncClient, request_payload: dict[str, Any]) -> httpx.Response:
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if _thinking_tool_choice_rejected(response):
|
|
request_payload.pop("tool_choice", None)
|
|
response = await client.post(f"{provider.base_url}/chat/completions", headers=headers, json=request_payload)
|
|
if response.status_code >= 400:
|
|
raise VisualReviewError(f"Modeling plan review request failed ({response.status_code}): {response.text[:500]}")
|
|
return response
|
|
|
|
def validate_response(response: httpx.Response) -> dict[str, Any]:
|
|
try:
|
|
call = response.json()["choices"][0]["message"]["tool_calls"][0]
|
|
if call["function"]["name"] != "review_modeling_plan":
|
|
raise KeyError("wrong tool")
|
|
result = json.loads(call["function"]["arguments"])
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer did not return a valid review tool call") from error
|
|
allowed = {"verdict", "confidence", "issues", "coverage", "step_checks", "action_checks"}
|
|
if not isinstance(result, dict) or set(result) != allowed or result.get("verdict") not in {"pass", "revise"}:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned an invalid verdict")
|
|
try:
|
|
confidence = float(result.get("confidence"))
|
|
except (TypeError, ValueError) as error:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned an invalid confidence") from error
|
|
if not 0 <= confidence <= 1:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer confidence is outside [0, 1]")
|
|
if not isinstance(result.get("issues"), list) or not all(isinstance(item, dict) for item in result["issues"]):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned invalid issues")
|
|
expected = {_checklist_key(item): item for item in checklist}
|
|
coverage = result.get("coverage")
|
|
if not isinstance(coverage, list) or len(coverage) != len(checklist):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer must return coverage for every checklist item")
|
|
seen: set[str] = set()
|
|
for item in coverage:
|
|
if not isinstance(item, dict) or set(item) != {"requirement", "step_id", "status", "evidence"}:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned an invalid coverage row")
|
|
key = _checklist_key(item.get("requirement"))
|
|
if key not in expected or key in seen or item.get("status") not in {"covered", "missing"} or not str(item.get("evidence") or "").strip():
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer coverage does not match the frozen checklist")
|
|
item["requirement"] = expected[key]
|
|
seen.add(key)
|
|
step_records = {
|
|
str(item.get("step_id") or ""): item
|
|
for item in plan.get("steps") or () if isinstance(item, dict)
|
|
}
|
|
steps = set(step_records)
|
|
checks = result.get("step_checks")
|
|
if not isinstance(checks, list) or len(checks) != len(steps):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer must return one step check per plan step")
|
|
checked: set[str] = set()
|
|
for item in checks:
|
|
if not isinstance(item, dict) or set(item) != {"step_id", "status", "notes"}:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned an invalid step check")
|
|
step_id = str(item.get("step_id") or "")
|
|
if step_id not in steps or step_id in checked or item.get("status") not in {"pass", "fail"}:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer step checks do not match the plan")
|
|
checked.add(step_id)
|
|
action_checks = result.get("action_checks")
|
|
if not isinstance(action_checks, list) or len(action_checks) != len(steps):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer must return one action check per plan step")
|
|
action_checked: set[str] = set()
|
|
for item in action_checks:
|
|
if not isinstance(item, dict) or set(item) != {"step_id", "status", "notes", "required_action_count"}:
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer returned an invalid action check")
|
|
step_id = str(item.get("step_id") or "")
|
|
required_count = item.get("required_action_count")
|
|
if (
|
|
step_id not in steps
|
|
or step_id in action_checked
|
|
or item.get("status") not in {"pass", "fail"}
|
|
or not isinstance(required_count, int)
|
|
or isinstance(required_count, bool)
|
|
or required_count < 1
|
|
):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer action checks do not match the plan")
|
|
planned_actions = [
|
|
action for action in (step_records[step_id].get("actions") or ())
|
|
if isinstance(action, dict)
|
|
]
|
|
if required_count > len(planned_actions) and item.get("status") != "fail":
|
|
raise _ModelingPlanReviewFormatError(
|
|
"Modeling plan reviewer must fail an action check when required_action_count exceeds the plan's action count"
|
|
)
|
|
action_checked.add(step_id)
|
|
if result["verdict"] == "pass" and any(item.get("status") != "covered" for item in coverage):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer may pass only when every checklist item is covered")
|
|
if result["verdict"] == "pass" and any(item.get("status") != "pass" for item in checks):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer may pass only when every step check passes")
|
|
if result["verdict"] == "pass" and any(item.get("status") != "pass" for item in action_checks):
|
|
raise _ModelingPlanReviewFormatError("Modeling plan reviewer may pass only when every action check passes")
|
|
unsupported = _unsupported_operation_mentions(result, runtime_operations)
|
|
if unsupported:
|
|
# These strings came from the reviewer's explanatory prose, not
|
|
# from the submitted plan. The plan parser enforces exact IDs for
|
|
# explicit operation declarations; do not reject an otherwise
|
|
# valid semantic plan because the reviewer hallucinated a
|
|
# shorthand while describing an alternative. Keep this warning in
|
|
# the raw audit record and scrub it before author exposure.
|
|
result["unsupported_operations"] = unsupported
|
|
result["review_warnings"] = [
|
|
"Reviewer mentioned operation name(s) absent from runtime_operations; those names were ignored as guidance."
|
|
]
|
|
return result
|
|
|
|
async with httpx.AsyncClient(timeout=settings.llm_timeout_s) as client:
|
|
response = await request_review(client, payload)
|
|
try:
|
|
result = validate_response(response)
|
|
except _ModelingPlanReviewFormatError as error:
|
|
retry_payload = copy.deepcopy(payload)
|
|
retry_payload["messages"].append({"role": "user", "content": f"Your previous review was rejected locally: {error}. Return only a complete review_modeling_plan tool call with one coverage row per checklist item and one step_checks and action_checks row per plan step."})
|
|
result = validate_response(await request_review(client, retry_payload))
|
|
return {"schema_version": "cad.modeling-plan-review.v1", "node_id": node_id, "model": model.id, **result}
|