650 lines
37 KiB
Python
650 lines
37 KiB
Python
#!/usr/bin/env python3
|
|
"""Direct Doubao -> makeCAD CLI A/B benchmark. Never invokes Codex or ACP."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import html
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from cad_knowledge import KnowledgeBase # noqa: E402
|
|
|
|
FAMILIES = ["mounting-plate", "stepped-shaft", "hex-head-bolt", "ribbed-bearing-housing",
|
|
"involute-spur-gear", "covered-thin-wall-enclosure"]
|
|
ALLOWED_TOOLS = {"list_commands", "describe_command", "execute", "batch", "inspect_design",
|
|
"measure", "body_topology", "set_parameter", "check_parameters", "undo", "redo"}
|
|
PRINT_LOCK = threading.Lock()
|
|
RUNNER_VERSION = "0.2.0"
|
|
|
|
|
|
def sha256_file(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def log(message):
|
|
with PRINT_LOCK:
|
|
print(f"[{datetime.now().astimezone().isoformat(timespec='seconds')}] {message}", flush=True)
|
|
|
|
|
|
def write_json(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
|
|
|
|
def env_file(path):
|
|
result = {}
|
|
if not path:
|
|
return result
|
|
for line in Path(path).expanduser().read_text(encoding="utf-8").splitlines():
|
|
match = re.match(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$", line)
|
|
if match:
|
|
key, value = match.groups()
|
|
if value.startswith(("'", '"')) and value.endswith(value[0]):
|
|
value = value[1:-1]
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def load_config(path):
|
|
config = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
values = {**env_file(config.get("env_file")), **os.environ}
|
|
key = values.get(config.get("api_key_env", "ARK_API_KEY"), "")
|
|
if not key:
|
|
raise ValueError("Doubao API key is missing; set api_key_env or env_file in local config")
|
|
if config.get("model_env"):
|
|
config["model"] = values.get(config["model_env"], config.get("model"))
|
|
if not config.get("model") or "doubao" not in config["model"].lower():
|
|
raise ValueError("This benchmark requires an explicit Doubao model ID")
|
|
config["cad_cli"] = str(Path(config["cad_cli"]).expanduser().resolve())
|
|
if not Path(config["cad_cli"]).is_file():
|
|
raise ValueError("cad_cli does not exist")
|
|
config.setdefault("max_model_turns", 48)
|
|
config.setdefault("max_tokens", 16000)
|
|
config.setdefault("workers", 2)
|
|
config.setdefault("thinking", "disabled")
|
|
config.setdefault("views", ["iso", "top", "front"])
|
|
config.setdefault("image_width", 1200)
|
|
config.setdefault("image_height", 900)
|
|
if config["thinking"] not in {"enabled", "disabled", "auto"}:
|
|
raise ValueError("thinking must be enabled, disabled or auto")
|
|
if not 1 <= config["workers"] <= 6 or not 1 <= config["max_model_turns"] <= 200:
|
|
raise ValueError("workers must be 1..6 and max_model_turns must be 1..200")
|
|
if not all(view in {"iso", "top", "front", "right", "back", "left", "bottom"} for view in config["views"]):
|
|
raise ValueError("views must be standard makeCAD view names")
|
|
config["api_key"] = key
|
|
return config
|
|
|
|
|
|
def public_config(config):
|
|
return {k: v for k, v in config.items() if k != "api_key"}
|
|
|
|
|
|
def redact(value, secret):
|
|
if isinstance(value, str):
|
|
return value.replace(secret, "[REDACTED]") if secret else value
|
|
if isinstance(value, list):
|
|
return [redact(item, secret) for item in value]
|
|
if isinstance(value, dict):
|
|
return {key: redact(item, secret) for key, item in value.items()}
|
|
return value
|
|
|
|
|
|
class MCP:
|
|
"""One private headless CAD process per condition; no UI chat backend."""
|
|
|
|
def __init__(self, executable, directory):
|
|
self.directory = Path(directory).resolve()
|
|
self.errors = (self.directory / "cad-stderr.log").open("w", encoding="utf-8")
|
|
self.events = (self.directory / "cad-events.jsonl").open("a", encoding="utf-8")
|
|
self.process = subprocess.Popen(
|
|
[executable, "mcp", "--headless", "--compact", "--allow-path", str(self.directory)],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.errors,
|
|
text=True, encoding="utf-8", cwd=self.directory)
|
|
self.counter = 0
|
|
self.initialization = self.rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
|
|
"clientInfo": {"name": "doubao-cad-ab", "version": "0.1.0"}})
|
|
self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
|
|
self.process.stdin.flush()
|
|
self.tools = self.rpc("tools/list", {})["tools"]
|
|
|
|
def rpc(self, method, params):
|
|
self.counter += 1
|
|
request = {"jsonrpc": "2.0", "id": self.counter, "method": method, "params": params}
|
|
started = time.monotonic()
|
|
self.process.stdin.write(json.dumps(request, ensure_ascii=False) + "\n")
|
|
self.process.stdin.flush()
|
|
while True:
|
|
line = self.process.stdout.readline()
|
|
if not line:
|
|
raise RuntimeError(f"CAD subprocess closed stdout (exit={self.process.poll()}); see cad-stderr.log")
|
|
reply = json.loads(line)
|
|
if reply.get("id") == self.counter:
|
|
break
|
|
# Image bytes stay in their PNGs, not duplicated into the tool log.
|
|
compact = json.loads(json.dumps(reply))
|
|
for item in compact.get("result", {}).get("content", []):
|
|
if item.get("type") == "image":
|
|
item["data"] = f"[base64 image, {len(item.get('data', ''))} chars]"
|
|
self.events.write(json.dumps({"request": request, "reply": compact,
|
|
"elapsed_s": time.monotonic() - started}, ensure_ascii=False) + "\n")
|
|
self.events.flush()
|
|
if "error" in reply:
|
|
raise RuntimeError(json.dumps(reply["error"], ensure_ascii=False))
|
|
return reply.get("result", {})
|
|
|
|
def call(self, name, arguments):
|
|
return self.rpc("tools/call", {"name": name, "arguments": arguments})
|
|
|
|
def close(self):
|
|
if self.process.poll() is None:
|
|
self.process.stdin.close()
|
|
try:
|
|
self.process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
self.process.terminate()
|
|
self.process.wait()
|
|
self.errors.close()
|
|
self.events.close()
|
|
|
|
|
|
def tool_value(result):
|
|
texts = [item["text"] for item in result.get("content", []) if item.get("type") == "text"]
|
|
if len(texts) == 1:
|
|
try:
|
|
return json.loads(texts[0])
|
|
except json.JSONDecodeError:
|
|
return texts[0]
|
|
return texts
|
|
|
|
|
|
def tool_text(result):
|
|
return "\n".join(item["text"] for item in result.get("content", []) if item.get("type") == "text")
|
|
|
|
|
|
def stream_completion(config, messages, tools, directory, turn):
|
|
"""SSE request made directly to the configured provider, with no deadline."""
|
|
directory = Path(directory)
|
|
payload = {"model": config["model"], "messages": messages, "tools": tools,
|
|
"tool_choice": "auto", "stream": True, "max_tokens": config["max_tokens"],
|
|
"thinking": {"type": config["thinking"]}}
|
|
if config.get("reasoning_effort"):
|
|
payload["reasoning_effort"] = config["reasoning_effort"]
|
|
write_json(directory / f"request-{turn:03}.json", redact(payload, config["api_key"]))
|
|
request = urllib.request.Request(
|
|
config["base_url"].rstrip("/") + "/chat/completions",
|
|
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), method="POST",
|
|
headers={"Content-Type": "application/json", "Authorization": "Bearer " + config["api_key"]})
|
|
try:
|
|
import certifi
|
|
context = ssl.create_default_context(cafile=certifi.where())
|
|
except ImportError:
|
|
context = ssl.create_default_context()
|
|
content, reasoning, calls, usage, finish = [], [], {}, None, None
|
|
with (directory / f"response-{turn:03}.jsonl").open("w", encoding="utf-8") as raw:
|
|
try:
|
|
# Explicitly no HTTP/request deadline. The provider may impose its own limits.
|
|
with urllib.request.urlopen(request, context=context, timeout=None) as response:
|
|
for line in response:
|
|
line = line.decode("utf-8").strip()
|
|
if not line.startswith("data:"):
|
|
continue
|
|
data = line[5:].strip()
|
|
if data == "[DONE]":
|
|
break
|
|
event = json.loads(data)
|
|
raw.write(json.dumps(redact(event, config["api_key"]), ensure_ascii=False) + "\n")
|
|
raw.flush()
|
|
if event.get("error"):
|
|
raise RuntimeError(json.dumps(event["error"], ensure_ascii=False))
|
|
if event.get("usage"):
|
|
usage = event["usage"]
|
|
for choice in event.get("choices", []):
|
|
if choice.get("finish_reason"):
|
|
finish = choice["finish_reason"]
|
|
delta = choice.get("delta", {})
|
|
if delta.get("content"):
|
|
content.append(delta["content"])
|
|
if delta.get("reasoning_content"):
|
|
reasoning.append(delta["reasoning_content"])
|
|
for item in delta.get("tool_calls", []):
|
|
index = item.get("index", 0)
|
|
current = calls.setdefault(index, {"id": "", "type": "function",
|
|
"function": {"name": "", "arguments": ""}})
|
|
if item.get("id"):
|
|
current["id"] = item["id"]
|
|
function = item.get("function", {})
|
|
current["function"]["name"] += function.get("name", "")
|
|
current["function"]["arguments"] += function.get("arguments", "")
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(redact(f"Doubao HTTP {exc.code}: {detail}", config["api_key"])) from None
|
|
if finish is None:
|
|
raise RuntimeError("Doubao stream ended without finish_reason; incomplete response retained")
|
|
if not content and not calls:
|
|
raise RuntimeError(f"Empty Doubao completion (finish_reason={finish})")
|
|
message = {"role": "assistant", "content": "".join(content) or None}
|
|
if calls:
|
|
message["tool_calls"] = [calls[index] for index in sorted(calls)]
|
|
result = {"message": message, "reasoning": "".join(reasoning), "usage": usage, "finish_reason": finish}
|
|
write_json(directory / f"completion-{turn:03}.json", redact(result, config["api_key"]))
|
|
return result
|
|
|
|
|
|
def extract_task(kb, family_id):
|
|
example = kb.documents[kb.families[family_id]["meta"]["example_id"]]
|
|
return example["body"].split("## 完整需求\n\n", 1)[1].split("\n\n## ", 1)[0].strip()
|
|
|
|
|
|
def base_prompt(kb, technical_reference):
|
|
return """You are building a CAD part in a private makeCAD headless session. Use only the provided
|
|
function tools to construct the requested geometry. This is an execution task, not a request for a plan.
|
|
Do not claim success until you actually inspect and measure the design. Read command contracts with
|
|
describe_command when needed; never invent commands or silently substitute requested structures.
|
|
Each independent call may use batch. Read a failed batch's index and fix the failure; avoid repeating
|
|
the entire build or retrying a command that is still running. Use named bodies and features.
|
|
Use engine geometry only; no external code execution, imported meshes or external CAD files.
|
|
The final response must state what was built, what is simplified or unsupported, and remaining errors.
|
|
The harness saves the design, measures it and captures screenshots after your final response; you do
|
|
not need to save files, export, render, choose a camera or change display appearance yourself.
|
|
Keep lengths in mm, Z up. The following common conventions and technical command reference are
|
|
identical for both conditions; any later family reference is additional knowledge, not a replacement
|
|
for the user's request. Do not load any complete worked example from a knowledge library.
|
|
""" + "\n\n" + kb.documents["common-conventions"]["body"] + "\n\n# CAD tool reference\n\n" + technical_reference
|
|
|
|
|
|
def run_agent(config, mcp, messages, directory, label):
|
|
schemas = [{"type": "function", "function": {"name": t["name"], "description": t["description"],
|
|
"parameters": t["inputSchema"]}} for t in mcp.tools if t["name"] in ALLOWED_TOOLS]
|
|
rounds, tool_errors, uses = 0, 0, []
|
|
history = Path(directory) / "conversation.json"
|
|
for turn in range(1, config["max_model_turns"] + 1):
|
|
rounds += 1
|
|
log(f"{label}: Doubao round {turn}")
|
|
completion = stream_completion(config, messages, schemas, directory, turn)
|
|
if completion.get("usage"):
|
|
uses.append(completion["usage"])
|
|
message = completion["message"]
|
|
messages.append(message)
|
|
calls = message.get("tool_calls", [])
|
|
if completion["finish_reason"] in {"length", "content_filter"}:
|
|
# Never execute a truncated tool-call prefix, even if it is valid JSON.
|
|
write_json(history, messages)
|
|
return {"ended": False, "stop_reason": completion["finish_reason"], "rounds": rounds,
|
|
"tool_errors": tool_errors, "usage": uses, "final_reply": message.get("content")}
|
|
if not calls:
|
|
write_json(history, messages)
|
|
return {"ended": completion["finish_reason"] == "stop",
|
|
"stop_reason": completion["finish_reason"], "rounds": rounds,
|
|
"tool_errors": tool_errors, "usage": uses, "final_reply": message.get("content")}
|
|
for call in calls:
|
|
name = call["function"]["name"]
|
|
log(f"{label}: {name}")
|
|
try:
|
|
arguments = json.loads(call["function"]["arguments"])
|
|
if name not in ALLOWED_TOOLS:
|
|
raise ValueError("Tool is outside this benchmark's advertised set")
|
|
result = mcp.call(name, arguments)
|
|
tool_errors += int(result.get("isError", False))
|
|
text = tool_text(result)
|
|
except (ValueError, RuntimeError) as exc:
|
|
tool_errors += 1
|
|
text = json.dumps({"error": str(exc)}, ensure_ascii=False)
|
|
messages.append({"role": "tool", "tool_call_id": call["id"], "content": text})
|
|
write_json(history, messages)
|
|
return {"ended": False, "stop_reason": "model_turn_budget", "rounds": rounds,
|
|
"tool_errors": tool_errors, "usage": uses, "final_reply": None}
|
|
|
|
|
|
def cylinder_matches(cylinder, radius, axis, origin, extent=None, tolerance=0.05):
|
|
if abs(cylinder.get("radius", -1) - radius) > tolerance:
|
|
return False
|
|
actual_axis = cylinder.get("axis", [])
|
|
if len(actual_axis) != 3 or abs(sum(a*b for a, b in zip(actual_axis, axis))) < 0.999:
|
|
return False
|
|
actual = cylinder.get("origin", [math.inf] * 3)
|
|
delta = [a-b for a, b in zip(actual, origin)]
|
|
parallel = sum(a*b for a, b in zip(delta, axis))
|
|
if math.sqrt(sum((a-parallel*b)**2 for a, b in zip(delta, axis))) > tolerance:
|
|
return False
|
|
if extent is not None:
|
|
ends = cylinder.get("extent", [])
|
|
if len(ends) != 2 or abs(abs(ends[1]-ends[0]) - extent) > tolerance:
|
|
return False
|
|
return True
|
|
|
|
|
|
def assess(family_id, params, derived, inspection, measurement, topologies):
|
|
"""Partial automatic geometry gates. Unmeasured properties remain explicit."""
|
|
checks = []
|
|
|
|
def check(name, passed, actual=None, expected=None):
|
|
checks.append({"name": name, "passed": bool(passed), "actual": actual, "expected": expected})
|
|
|
|
bodies = [b for b in inspection.get("bodies", []) if b.get("visible", True)] if isinstance(inspection, dict) else []
|
|
expected_count = 2 if family_id == "covered-thin-wall-enclosure" else 1
|
|
check("visible_body_count", len(bodies) == expected_count, len(bodies), expected_count)
|
|
measured = measurement.get("bodies", []) if isinstance(measurement, dict) else []
|
|
visible_names = {b["name"] for b in bodies}
|
|
measured = [b for b in measured if b.get("name") in visible_names]
|
|
check("closed_positive_solids", bool(measured) and len(measured) == len(bodies) and
|
|
all(b.get("closed") is True and b.get("volume_mm3", 0) > 0 for b in measured))
|
|
errors = [f for f in inspection.get("timeline", []) if f.get("error")] if isinstance(inspection, dict) else []
|
|
check("no_failed_timeline_features", not errors, errors)
|
|
if measured:
|
|
low = [min(b["bbox"]["min"][i] for b in measured) for i in range(3)]
|
|
high = [max(b["bbox"]["max"][i] for b in measured) for i in range(3)]
|
|
size = [h-l for h, l in zip(high, low)]
|
|
if family_id == "mounting-plate":
|
|
expected = [params["length"], params["width"], params["thickness"]]
|
|
elif family_id == "stepped-shaft":
|
|
expected = [derived["max_diameter"], derived["max_diameter"], derived["total_length"]]
|
|
elif family_id == "hex-head-bolt":
|
|
expected = None # Hexagon orientation is not fixed in the original request.
|
|
check("axial_total_length", abs(size[2] - derived["total_length"]) <= 0.05, size[2], derived["total_length"])
|
|
elif family_id == "ribbed-bearing-housing":
|
|
expected = [params["base_length"], params["base_width"], derived["total_height"]]
|
|
elif family_id == "involute-spur-gear":
|
|
expected = None # Angular orientation changes the XY bounding box slightly.
|
|
check("axial_total_length", abs(size[2] - derived["total_length"]) <= 0.05, size[2], derived["total_length"])
|
|
check("outside_envelope", all(abs(v-derived["addendum_diameter"]) < 0.5 for v in size[:2]), size[:2], derived["addendum_diameter"])
|
|
else:
|
|
expected = [params["outer_length"], params["outer_width"], derived["total_height"]]
|
|
if expected:
|
|
check("bounding_box_dimensions", all(abs(a-b) <= 0.05 for a, b in zip(size, expected)), size, expected)
|
|
if "volume" in derived:
|
|
volume = sum(b["volume_mm3"] for b in measured)
|
|
# 0.2% is an engine measurement gate, not a manufacturing tolerance.
|
|
check("analytic_volume", math.isclose(volume, derived["volume"], rel_tol=0.002, abs_tol=0.05), volume, derived["volume"])
|
|
cylinders = [c for topo in topologies.values() if isinstance(topo, dict) for c in topo.get("cylinders", [])]
|
|
|
|
def cyl(name, radius, axis, origin, extent=None):
|
|
check(name, any(cylinder_matches(c, radius, axis, origin, extent) for c in cylinders),
|
|
expected={"radius": radius, "axis": axis, "origin": origin, "extent_length": extent})
|
|
|
|
if family_id == "mounting-plate":
|
|
for i, h in enumerate(derived["holes"]):
|
|
cyl(f"hole_{i+1}_axis_radius_extent", h["diameter"]/2, [0, 0, 1], [h["x"], h["y"], 0], params["thickness"])
|
|
elif family_id == "stepped-shaft":
|
|
for i, segment in enumerate(params["segments"]):
|
|
cyl(f"shaft_segment_{i+1}", segment["diameter"]/2, [0, 0, 1], [0, 0, 0], segment["length"])
|
|
elif family_id == "hex-head-bolt":
|
|
cyl("nominal_shank", params["nominal_diameter"]/2, [0, 0, 1], [0, 0, 0], params["under_head_length"])
|
|
elif family_id == "ribbed-bearing-housing":
|
|
cyl("horizontal_bearing_bore", params["bore_diameter"]/2, [0, 1, 0], [0, 0, params["center_height"]])
|
|
if params.get("counterbore"):
|
|
cyl("counterbore_axis_radius", params["counterbore"]["diameter"]/2, [0, 1, 0], [0, 0, params["center_height"]], params["counterbore"]["depth"])
|
|
for i, h in enumerate(params["mount_holes"]):
|
|
cyl(f"vertical_mounting_hole_{i+1}", h["diameter"]/2, [0, 0, 1], [h["x"], h["y"], 0], params["base_thickness"])
|
|
elif family_id == "involute-spur-gear":
|
|
cyl("hub_outer_radius", params["hub_diameter"]/2, [0, 0, 1], [0, 0, 0])
|
|
cyl("axial_bore_radius", params["bore_diameter"]/2, [0, 0, 1], [0, 0, 0])
|
|
else:
|
|
for i, centre in enumerate(derived["fastener_centers"]):
|
|
for role, diameter, extent in [("boss_pilot", params["boss_pilot_diameter"], params["pilot_depth"]),
|
|
("lid_clearance", params["cover_hole_diameter"], params["lid_thickness"])]:
|
|
cyl(f"{role}_{i+1}", diameter/2, [0, 0, 1], [centre["x"], centre["y"], 0], extent)
|
|
manual = {
|
|
"mounting-plate": ["No unintended extra features; confirm exact outer origin and bore material side"],
|
|
"stepped-shaft": ["Check shoulder axial positions and native parameter-edit behaviour"],
|
|
"hex-head-bolt": ["Check regular hexagon across flats/head height and simplified-thread semantic marking"],
|
|
"ribbed-bearing-housing": ["Check both rib profiles, their connections, all local wall thicknesses and counterbore side"],
|
|
"involute-spur-gear": ["Verify 24 teeth, involute deviation, pitch-circle tooth thickness, keyway sizes and disclosed root simplification"],
|
|
"covered-thin-wall-enclosure": ["Check cavity dimensions, four rib connections, actual cover plane gap and shared interface dependencies"],
|
|
}[family_id]
|
|
return {"automatic_checks_passed": bool(checks) and all(c["passed"] for c in checks),
|
|
"checks": checks, "manual_review_required": manual, "full_geometry_certified": False,
|
|
"linear_check_tolerance_mm": 0.05, "volume_relative_tolerance": 0.002}
|
|
|
|
|
|
def capture(mcp, config, directory, family_id):
|
|
results = {}
|
|
for view in config["views"]:
|
|
path = Path(directory) / f"{view}.png"
|
|
result = mcp.call("screenshot", {"source": "model", "view": view, "path": str(path),
|
|
"grid": False, "axes": False, "width": config["image_width"], "height": config["image_height"]})
|
|
results[view] = {"path": path.name, "ok": path.exists() and not result.get("isError"), "info": tool_value(result)}
|
|
if family_id in {"ribbed-bearing-housing", "covered-thin-wall-enclosure"}:
|
|
plane = "YZ" if family_id == "ribbed-bearing-housing" else "XZ"
|
|
section = mcp.call("execute", {"command": "inspect.section", "params": {"plane": plane}})
|
|
if not section.get("isError"):
|
|
path = Path(directory) / "section.png"
|
|
result = mcp.call("screenshot", {"source": "model", "view": "iso", "path": str(path),
|
|
"grid": False, "axes": False, "width": config["image_width"], "height": config["image_height"]})
|
|
results["section"] = {"path": path.name, "ok": path.exists() and not result.get("isError"), "info": tool_value(result)}
|
|
mcp.call("execute", {"command": "inspect.section", "params": {"clear": True}})
|
|
return results
|
|
|
|
|
|
def run_case(config, kb, family_id, condition, directory, repeat):
|
|
directory = Path(directory).resolve()
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
started = time.monotonic()
|
|
label = f"{family_id}/{condition}/r{repeat}"
|
|
summary = {"family_id": family_id, "condition": condition, "repeat": repeat,
|
|
"model": config["model"], "thinking": config["thinking"], "status": "running",
|
|
"directory": str(directory), "provider_called_directly": True, "codex_used_for_generation": False}
|
|
write_json(directory / "result.json", summary)
|
|
mcp = None
|
|
try:
|
|
if config.get("cad_cli_sha256") and sha256_file(config["cad_cli"]) != config["cad_cli_sha256"]:
|
|
raise RuntimeError("CAD executable changed during this run; refusing to mix engine versions")
|
|
mcp = MCP(config["cad_cli"], directory)
|
|
# The runtime binary owns the technical contract, so its matching reference is used.
|
|
reference = None
|
|
for skill in ("makecad-modeling", "solidcraft-modeling"):
|
|
value = mcp.call("skill_reference", {"skill": skill, "file": "reference.md"})
|
|
if not value.get("isError"):
|
|
reference = tool_value(value)
|
|
if isinstance(reference, dict):
|
|
reference = json.dumps(reference, ensure_ascii=False)
|
|
if isinstance(reference, str) and reference.strip():
|
|
break
|
|
if not reference:
|
|
raise RuntimeError("No matching runtime command reference available")
|
|
system = base_prompt(kb, reference)
|
|
(directory / "common-prompt.md").write_text(system, encoding="utf-8")
|
|
messages = [{"role": "system", "content": system}]
|
|
if condition == "B":
|
|
context = kb.build_context([family_id], include_examples=False)
|
|
# common-conventions is already identical in both system prompts.
|
|
family_context = context["text"].split("\n\n---\n\n", 1)[1]
|
|
messages.append({"role": "system", "content": "Additional engineering reference; use only within the user's requested scope:\n\n" + family_context})
|
|
write_json(directory / "knowledge-manifest.json", context["manifest"])
|
|
(directory / "knowledge-context.md").write_text(family_context, encoding="utf-8")
|
|
else:
|
|
write_json(directory / "knowledge-manifest.json", [])
|
|
task = extract_task(kb, family_id)
|
|
(directory / "task.md").write_text(task + "\n", encoding="utf-8")
|
|
messages.append({"role": "user", "content": task})
|
|
summary["agent"] = run_agent(config, mcp, messages, directory, label)
|
|
summary["status"] = "completed" if summary["agent"]["ended"] else "budget_or_provider_stop"
|
|
except Exception as exc:
|
|
summary["status"] = "error"
|
|
summary["error"] = redact(f"{type(exc).__name__}: {exc}", config["api_key"])
|
|
log(f"{label}: {summary['error']}")
|
|
finally:
|
|
if mcp is not None:
|
|
for stage in ("inspect", "measure", "save", "export", "screenshots", "assessment"):
|
|
try:
|
|
if stage == "inspect":
|
|
result = mcp.call("inspect_design", {"measure": True, "sketches": False})
|
|
inspection = tool_value(result)
|
|
write_json(directory / "inspection.json", inspection)
|
|
elif stage == "measure":
|
|
measurement = tool_value(mcp.call("measure", {}))
|
|
write_json(directory / "measurement.json", measurement)
|
|
topologies = {}
|
|
for body in inspection.get("bodies", []) if isinstance(inspection, dict) else []:
|
|
topologies[body["name"]] = tool_value(mcp.call("body_topology", {"body": body["name"], "faces": "cylinders"}))
|
|
write_json(directory / "cylinders.json", topologies)
|
|
elif stage == "save":
|
|
result = mcp.call("save", {"path": str(directory / "design.sc")})
|
|
summary["native_saved"] = (directory / "design.sc").exists() and not result.get("isError")
|
|
elif stage == "export":
|
|
result = mcp.call("export", {"path": str(directory / "design.step")})
|
|
summary["step_exported"] = (directory / "design.step").exists() and not result.get("isError")
|
|
if result.get("isError"):
|
|
summary["export_error"] = tool_value(result)
|
|
elif stage == "screenshots":
|
|
summary["screenshots"] = capture(mcp, config, directory, family_id)
|
|
else:
|
|
example = kb.documents[kb.families[family_id]["meta"]["example_id"]]["meta"]
|
|
derived = kb.validate_params(family_id, example["params"])["derived"]
|
|
summary["assessment"] = assess(family_id, example["params"], derived, inspection, measurement, topologies)
|
|
write_json(directory / "assessment.json", summary["assessment"])
|
|
except Exception as exc:
|
|
summary.setdefault("artifact_errors", {})[stage] = redact(str(exc), config["api_key"])
|
|
mcp.close()
|
|
summary["elapsed_s"] = round(time.monotonic() - started, 3)
|
|
write_json(directory / "result.json", redact(summary, config["api_key"]))
|
|
log(f"{label}: {summary['status']}, {summary['elapsed_s']} s")
|
|
return summary
|
|
|
|
|
|
def report(directory):
|
|
directory = Path(directory).resolve()
|
|
results = [json.loads(p.read_text()) for p in sorted(directory.glob("r*/*/[AB]/result.json"))]
|
|
write_json(directory / "results.json", results)
|
|
rows = ["# 六类零件:豆包直接调用 A/B 生成记录", "",
|
|
"A:共同技术说明与通用流程。B:额外加载对应零件族与共享特征,不加载完整示例。每个条件独立进程、独立会话。",
|
|
"请求由 Python 直接发送给豆包,使用 makeCAD 命令行 MCP;生成链路不调用 Codex、ACP 或 makeCAD 内置聊天。",
|
|
"自动几何检查为部分检查,完整结构与齿形等仍需人工核对。不能据此报告全面建模成功率。", "",
|
|
"| 零件 | 组 | 状态 | 自动项通过/总数 | 耗时秒 | 截图 |", "| --- | --- | --- | --- | ---: | --- |"]
|
|
cards = []
|
|
for result in results:
|
|
path = Path(result["directory"]).relative_to(directory).as_posix()
|
|
checks = result.get("assessment", {}).get("checks", [])
|
|
count = f"{sum(c['passed'] for c in checks)}/{len(checks)}" if checks else "未测量"
|
|
pictures = result.get("screenshots", {})
|
|
links = " ".join(f"[{view}]({path}/{value['path']})" for view, value in pictures.items() if value["ok"])
|
|
rows.append(f"| {result['family_id']} | {result['condition']} | {result['status']} | {count} | {result.get('elapsed_s', 0)} | {links} |")
|
|
failed = [c["name"] for c in checks if not c["passed"]]
|
|
rows += []
|
|
image = f'<img src="{html.escape(path)}/iso.png" loading="lazy">' if pictures.get("iso", {}).get("ok") else '<p>未取得截图</p>'
|
|
details = html.escape(json.dumps({"failed_checks": failed, "error": result.get("error"),
|
|
"manual_review": result.get("assessment", {}).get("manual_review_required", [])}, ensure_ascii=False, indent=2))
|
|
cards.append(f'<article><h2>{html.escape(result["family_id"])} · {result["condition"]} · r{result["repeat"]}</h2>{image}<p>{html.escape(result["status"])} · {count} 自动检查 · {result.get("elapsed_s",0)} s</p><p><a href="{path}/result.json">结果</a> · <a href="{path}/design.sc">设计</a> · <a href="{path}/measurement.json">测量</a> · <a href="{path}/cad-events.jsonl">工具日志</a></p><details><summary>失败项和待核对项</summary><pre>{details}</pre></details></article>')
|
|
rows += ["", "## 查看记录", "", "[并排截图页面](index.html) · [全部结果 JSON](results.json) · [运行配置与来源快照](run-manifest.json)",
|
|
"", "每个案例目录保留 task.md、common-prompt.md、B 组知识上下文、逐轮请求/SSE 响应、conversation.json、cad-events.jsonl、measurement.json、assessment.json、design.sc 和实际截图。失败案例同样记录残留模型。"]
|
|
(directory / "report.md").write_text("\n".join(rows) + "\n", encoding="utf-8")
|
|
(directory / "index.html").write_text('<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>豆包 CAD A/B 对照</title><style>body{font:15px system-ui;margin:24px;background:#f3f5f7;color:#18212b}main{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:20px}article{background:white;padding:16px;border-radius:10px}h2{font-size:16px}img{width:100%;height:auto}pre{white-space:pre-wrap}a{color:#1756a9}</style><h1>六类零件:豆包 A/B 生成记录</h1><p>A:共同说明;B:增加零件知识。截图来自实际 makeCAD 渲染。自动检查只覆盖已实现的部分几何要求。</p><main>' + "\n".join(cards) + '</main></html>', encoding="utf-8")
|
|
try:
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
font = ImageFont.truetype("/System/Library/Fonts/Supplemental/Arial.ttf", 18)
|
|
cell_w, cell_h = 600, 490
|
|
sheet = Image.new("RGB", (cell_w * 2, cell_h * max(1, (len(results)+1)//2)), "#edf0f3")
|
|
draw = ImageDraw.Draw(sheet)
|
|
for index, result in enumerate(results):
|
|
x, y = (index % 2) * cell_w, (index // 2) * cell_h
|
|
draw.text((x+12, y+10), f"{result['family_id']} / {result['condition']}", fill="#18212b", font=font)
|
|
path = Path(result["directory"]) / "iso.png"
|
|
if path.exists():
|
|
with Image.open(path) as im:
|
|
im = im.convert("RGB")
|
|
im.thumbnail((cell_w-20, cell_h-45))
|
|
sheet.paste(im, (x+(cell_w-im.width)//2, y+38))
|
|
sheet.save(directory / "comparison.png")
|
|
except (ImportError, OSError) as exc:
|
|
log(f"Contact sheet unavailable; individual PNGs and HTML are intact: {exc}")
|
|
return results
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--config", type=Path, default=ROOT / "benchmark/config.local.json")
|
|
parser.add_argument("--out", type=Path)
|
|
parser.add_argument("--families", nargs="+", choices=FAMILIES, default=FAMILIES)
|
|
parser.add_argument("--conditions", nargs="+", choices=["A", "B"], default=["A", "B"])
|
|
parser.add_argument("--repeat", type=int, default=1)
|
|
parser.add_argument("--workers", type=int)
|
|
parser.add_argument("--resume", action="store_true", help="Skip all recorded cases, including failures; never silently rerun a failed trial")
|
|
parser.add_argument("--report-only", action="store_true")
|
|
args = parser.parse_args()
|
|
output = (args.out or ROOT / "runs" / datetime.now().astimezone().strftime("doubao-ab-%Y%m%d-%H%M%S")).resolve()
|
|
if args.report_only:
|
|
report(output)
|
|
return 0
|
|
config = load_config(args.config)
|
|
config["cad_cli_sha256"] = sha256_file(config["cad_cli"])
|
|
if args.workers:
|
|
if not 1 <= args.workers <= 6:
|
|
parser.error("--workers must be 1..6")
|
|
config["workers"] = args.workers
|
|
if not 1 <= args.repeat <= 100:
|
|
parser.error("--repeat must be 1..100")
|
|
if output.exists() and any(output.iterdir()) and not args.resume:
|
|
parser.error("Output is nonempty; choose a new directory or --resume")
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
kb = KnowledgeBase(ROOT)
|
|
manifest_path = output / "run-manifest.json"
|
|
source_files = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for folder in ["cad_knowledge", "references", "schemas", "benchmark"]
|
|
for p in sorted((ROOT/folder).rglob('*')) if p.is_file() and '__pycache__' not in p.parts and p.name != "config.local.json"}
|
|
manifest = {"created_at": datetime.now(timezone.utc).isoformat(), "runner_version": RUNNER_VERSION,
|
|
"config": public_config(config),
|
|
"families": args.families, "conditions": args.conditions, "repeat": args.repeat,
|
|
"source_hashes": source_files, "generation_provider": "Doubao direct HTTP SSE",
|
|
"http_timeout": None, "cad_operation_timeout": None, "full_examples_loaded": False,
|
|
"codex_used_for_generation": False}
|
|
if manifest_path.exists():
|
|
old = json.loads(manifest_path.read_text())
|
|
for key in ["config", "families", "conditions", "repeat", "source_hashes"]:
|
|
if old[key] != manifest[key]:
|
|
raise ValueError(f"Cannot resume with changed {key}; start a new run")
|
|
else:
|
|
write_json(manifest_path, manifest)
|
|
for name in source_files:
|
|
snapshot = output / "source-snapshot" / name
|
|
snapshot.parent.mkdir(parents=True, exist_ok=True)
|
|
snapshot.write_bytes((ROOT / name).read_bytes())
|
|
jobs = []
|
|
for repeat in range(1, args.repeat+1):
|
|
for family in args.families:
|
|
for condition in args.conditions:
|
|
directory = output / f"r{repeat:02}" / family / condition
|
|
if args.resume and (directory / "result.json").exists():
|
|
prior = json.loads((directory / "result.json").read_text())
|
|
if prior.get("status") != "running":
|
|
continue
|
|
raise ValueError(f"Interrupted trial exists at {directory}; preserve it and start a new run")
|
|
jobs.append((family, condition, directory, repeat))
|
|
log(f"Starting {len(jobs)} isolated trials; model={config['model']}, thinking={config['thinking']}, workers={config['workers']}; out={output}")
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=config["workers"]) as pool:
|
|
futures = [pool.submit(run_case, config, kb, *job) for job in jobs]
|
|
for future in as_completed(futures):
|
|
future.result()
|
|
report(output)
|
|
finally:
|
|
results = report(output)
|
|
log(f"Recorded {len(results)} trials: {output / 'report.md'}")
|
|
return 0 if all(r["status"] == "completed" for r in results) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|