Files
2026-11-12 20:40:21 +08:00

650 lines
37 KiB
Python

#!/usr/bin/env python3
"""Direct Doubao -> makeCAD CLI A/B benchmark. Never invokes Codex or ACP."""
from __future__ import annotations
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
import hashlib
import html
import json
import math
import os
from pathlib import Path
import re
import ssl
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from cad_knowledge import KnowledgeBase # noqa: E402
FAMILIES = ["mounting-plate", "stepped-shaft", "hex-head-bolt", "ribbed-bearing-housing",
"involute-spur-gear", "covered-thin-wall-enclosure"]
ALLOWED_TOOLS = {"list_commands", "describe_command", "execute", "batch", "inspect_design",
"measure", "body_topology", "set_parameter", "check_parameters", "undo", "redo"}
PRINT_LOCK = threading.Lock()
RUNNER_VERSION = "0.2.0"
def sha256_file(path):
digest = hashlib.sha256()
with Path(path).open("rb") as source:
for block in iter(lambda: source.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def log(message):
with PRINT_LOCK:
print(f"[{datetime.now().astimezone().isoformat(timespec='seconds')}] {message}", flush=True)
def write_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
temporary.replace(path)
def env_file(path):
result = {}
if not path:
return result
for line in Path(path).expanduser().read_text(encoding="utf-8").splitlines():
match = re.match(r"\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$", line)
if match:
key, value = match.groups()
if value.startswith(("'", '"')) and value.endswith(value[0]):
value = value[1:-1]
result[key] = value
return result
def load_config(path):
config = json.loads(Path(path).read_text(encoding="utf-8"))
values = {**env_file(config.get("env_file")), **os.environ}
key = values.get(config.get("api_key_env", "ARK_API_KEY"), "")
if not key:
raise ValueError("Doubao API key is missing; set api_key_env or env_file in local config")
if config.get("model_env"):
config["model"] = values.get(config["model_env"], config.get("model"))
if not config.get("model") or "doubao" not in config["model"].lower():
raise ValueError("This benchmark requires an explicit Doubao model ID")
config["cad_cli"] = str(Path(config["cad_cli"]).expanduser().resolve())
if not Path(config["cad_cli"]).is_file():
raise ValueError("cad_cli does not exist")
config.setdefault("max_model_turns", 48)
config.setdefault("max_tokens", 16000)
config.setdefault("workers", 2)
config.setdefault("thinking", "disabled")
config.setdefault("views", ["iso", "top", "front"])
config.setdefault("image_width", 1200)
config.setdefault("image_height", 900)
if config["thinking"] not in {"enabled", "disabled", "auto"}:
raise ValueError("thinking must be enabled, disabled or auto")
if not 1 <= config["workers"] <= 6 or not 1 <= config["max_model_turns"] <= 200:
raise ValueError("workers must be 1..6 and max_model_turns must be 1..200")
if not all(view in {"iso", "top", "front", "right", "back", "left", "bottom"} for view in config["views"]):
raise ValueError("views must be standard makeCAD view names")
config["api_key"] = key
return config
def public_config(config):
return {k: v for k, v in config.items() if k != "api_key"}
def redact(value, secret):
if isinstance(value, str):
return value.replace(secret, "[REDACTED]") if secret else value
if isinstance(value, list):
return [redact(item, secret) for item in value]
if isinstance(value, dict):
return {key: redact(item, secret) for key, item in value.items()}
return value
class MCP:
"""One private headless CAD process per condition; no UI chat backend."""
def __init__(self, executable, directory):
self.directory = Path(directory).resolve()
self.errors = (self.directory / "cad-stderr.log").open("w", encoding="utf-8")
self.events = (self.directory / "cad-events.jsonl").open("a", encoding="utf-8")
self.process = subprocess.Popen(
[executable, "mcp", "--headless", "--compact", "--allow-path", str(self.directory)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.errors,
text=True, encoding="utf-8", cwd=self.directory)
self.counter = 0
self.initialization = self.rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
"clientInfo": {"name": "doubao-cad-ab", "version": "0.1.0"}})
self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
self.process.stdin.flush()
self.tools = self.rpc("tools/list", {})["tools"]
def rpc(self, method, params):
self.counter += 1
request = {"jsonrpc": "2.0", "id": self.counter, "method": method, "params": params}
started = time.monotonic()
self.process.stdin.write(json.dumps(request, ensure_ascii=False) + "\n")
self.process.stdin.flush()
while True:
line = self.process.stdout.readline()
if not line:
raise RuntimeError(f"CAD subprocess closed stdout (exit={self.process.poll()}); see cad-stderr.log")
reply = json.loads(line)
if reply.get("id") == self.counter:
break
# Image bytes stay in their PNGs, not duplicated into the tool log.
compact = json.loads(json.dumps(reply))
for item in compact.get("result", {}).get("content", []):
if item.get("type") == "image":
item["data"] = f"[base64 image, {len(item.get('data', ''))} chars]"
self.events.write(json.dumps({"request": request, "reply": compact,
"elapsed_s": time.monotonic() - started}, ensure_ascii=False) + "\n")
self.events.flush()
if "error" in reply:
raise RuntimeError(json.dumps(reply["error"], ensure_ascii=False))
return reply.get("result", {})
def call(self, name, arguments):
return self.rpc("tools/call", {"name": name, "arguments": arguments})
def close(self):
if self.process.poll() is None:
self.process.stdin.close()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
self.process.terminate()
self.process.wait()
self.errors.close()
self.events.close()
def tool_value(result):
texts = [item["text"] for item in result.get("content", []) if item.get("type") == "text"]
if len(texts) == 1:
try:
return json.loads(texts[0])
except json.JSONDecodeError:
return texts[0]
return texts
def tool_text(result):
return "\n".join(item["text"] for item in result.get("content", []) if item.get("type") == "text")
def stream_completion(config, messages, tools, directory, turn):
"""SSE request made directly to the configured provider, with no deadline."""
directory = Path(directory)
payload = {"model": config["model"], "messages": messages, "tools": tools,
"tool_choice": "auto", "stream": True, "max_tokens": config["max_tokens"],
"thinking": {"type": config["thinking"]}}
if config.get("reasoning_effort"):
payload["reasoning_effort"] = config["reasoning_effort"]
write_json(directory / f"request-{turn:03}.json", redact(payload, config["api_key"]))
request = urllib.request.Request(
config["base_url"].rstrip("/") + "/chat/completions",
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), method="POST",
headers={"Content-Type": "application/json", "Authorization": "Bearer " + config["api_key"]})
try:
import certifi
context = ssl.create_default_context(cafile=certifi.where())
except ImportError:
context = ssl.create_default_context()
content, reasoning, calls, usage, finish = [], [], {}, None, None
with (directory / f"response-{turn:03}.jsonl").open("w", encoding="utf-8") as raw:
try:
# Explicitly no HTTP/request deadline. The provider may impose its own limits.
with urllib.request.urlopen(request, context=context, timeout=None) as response:
for line in response:
line = line.decode("utf-8").strip()
if not line.startswith("data:"):
continue
data = line[5:].strip()
if data == "[DONE]":
break
event = json.loads(data)
raw.write(json.dumps(redact(event, config["api_key"]), ensure_ascii=False) + "\n")
raw.flush()
if event.get("error"):
raise RuntimeError(json.dumps(event["error"], ensure_ascii=False))
if event.get("usage"):
usage = event["usage"]
for choice in event.get("choices", []):
if choice.get("finish_reason"):
finish = choice["finish_reason"]
delta = choice.get("delta", {})
if delta.get("content"):
content.append(delta["content"])
if delta.get("reasoning_content"):
reasoning.append(delta["reasoning_content"])
for item in delta.get("tool_calls", []):
index = item.get("index", 0)
current = calls.setdefault(index, {"id": "", "type": "function",
"function": {"name": "", "arguments": ""}})
if item.get("id"):
current["id"] = item["id"]
function = item.get("function", {})
current["function"]["name"] += function.get("name", "")
current["function"]["arguments"] += function.get("arguments", "")
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(redact(f"Doubao HTTP {exc.code}: {detail}", config["api_key"])) from None
if finish is None:
raise RuntimeError("Doubao stream ended without finish_reason; incomplete response retained")
if not content and not calls:
raise RuntimeError(f"Empty Doubao completion (finish_reason={finish})")
message = {"role": "assistant", "content": "".join(content) or None}
if calls:
message["tool_calls"] = [calls[index] for index in sorted(calls)]
result = {"message": message, "reasoning": "".join(reasoning), "usage": usage, "finish_reason": finish}
write_json(directory / f"completion-{turn:03}.json", redact(result, config["api_key"]))
return result
def extract_task(kb, family_id):
example = kb.documents[kb.families[family_id]["meta"]["example_id"]]
return example["body"].split("## 完整需求\n\n", 1)[1].split("\n\n## ", 1)[0].strip()
def base_prompt(kb, technical_reference):
return """You are building a CAD part in a private makeCAD headless session. Use only the provided
function tools to construct the requested geometry. This is an execution task, not a request for a plan.
Do not claim success until you actually inspect and measure the design. Read command contracts with
describe_command when needed; never invent commands or silently substitute requested structures.
Each independent call may use batch. Read a failed batch's index and fix the failure; avoid repeating
the entire build or retrying a command that is still running. Use named bodies and features.
Use engine geometry only; no external code execution, imported meshes or external CAD files.
The final response must state what was built, what is simplified or unsupported, and remaining errors.
The harness saves the design, measures it and captures screenshots after your final response; you do
not need to save files, export, render, choose a camera or change display appearance yourself.
Keep lengths in mm, Z up. The following common conventions and technical command reference are
identical for both conditions; any later family reference is additional knowledge, not a replacement
for the user's request. Do not load any complete worked example from a knowledge library.
""" + "\n\n" + kb.documents["common-conventions"]["body"] + "\n\n# CAD tool reference\n\n" + technical_reference
def run_agent(config, mcp, messages, directory, label):
schemas = [{"type": "function", "function": {"name": t["name"], "description": t["description"],
"parameters": t["inputSchema"]}} for t in mcp.tools if t["name"] in ALLOWED_TOOLS]
rounds, tool_errors, uses = 0, 0, []
history = Path(directory) / "conversation.json"
for turn in range(1, config["max_model_turns"] + 1):
rounds += 1
log(f"{label}: Doubao round {turn}")
completion = stream_completion(config, messages, schemas, directory, turn)
if completion.get("usage"):
uses.append(completion["usage"])
message = completion["message"]
messages.append(message)
calls = message.get("tool_calls", [])
if completion["finish_reason"] in {"length", "content_filter"}:
# Never execute a truncated tool-call prefix, even if it is valid JSON.
write_json(history, messages)
return {"ended": False, "stop_reason": completion["finish_reason"], "rounds": rounds,
"tool_errors": tool_errors, "usage": uses, "final_reply": message.get("content")}
if not calls:
write_json(history, messages)
return {"ended": completion["finish_reason"] == "stop",
"stop_reason": completion["finish_reason"], "rounds": rounds,
"tool_errors": tool_errors, "usage": uses, "final_reply": message.get("content")}
for call in calls:
name = call["function"]["name"]
log(f"{label}: {name}")
try:
arguments = json.loads(call["function"]["arguments"])
if name not in ALLOWED_TOOLS:
raise ValueError("Tool is outside this benchmark's advertised set")
result = mcp.call(name, arguments)
tool_errors += int(result.get("isError", False))
text = tool_text(result)
except (ValueError, RuntimeError) as exc:
tool_errors += 1
text = json.dumps({"error": str(exc)}, ensure_ascii=False)
messages.append({"role": "tool", "tool_call_id": call["id"], "content": text})
write_json(history, messages)
return {"ended": False, "stop_reason": "model_turn_budget", "rounds": rounds,
"tool_errors": tool_errors, "usage": uses, "final_reply": None}
def cylinder_matches(cylinder, radius, axis, origin, extent=None, tolerance=0.05):
if abs(cylinder.get("radius", -1) - radius) > tolerance:
return False
actual_axis = cylinder.get("axis", [])
if len(actual_axis) != 3 or abs(sum(a*b for a, b in zip(actual_axis, axis))) < 0.999:
return False
actual = cylinder.get("origin", [math.inf] * 3)
delta = [a-b for a, b in zip(actual, origin)]
parallel = sum(a*b for a, b in zip(delta, axis))
if math.sqrt(sum((a-parallel*b)**2 for a, b in zip(delta, axis))) > tolerance:
return False
if extent is not None:
ends = cylinder.get("extent", [])
if len(ends) != 2 or abs(abs(ends[1]-ends[0]) - extent) > tolerance:
return False
return True
def assess(family_id, params, derived, inspection, measurement, topologies):
"""Partial automatic geometry gates. Unmeasured properties remain explicit."""
checks = []
def check(name, passed, actual=None, expected=None):
checks.append({"name": name, "passed": bool(passed), "actual": actual, "expected": expected})
bodies = [b for b in inspection.get("bodies", []) if b.get("visible", True)] if isinstance(inspection, dict) else []
expected_count = 2 if family_id == "covered-thin-wall-enclosure" else 1
check("visible_body_count", len(bodies) == expected_count, len(bodies), expected_count)
measured = measurement.get("bodies", []) if isinstance(measurement, dict) else []
visible_names = {b["name"] for b in bodies}
measured = [b for b in measured if b.get("name") in visible_names]
check("closed_positive_solids", bool(measured) and len(measured) == len(bodies) and
all(b.get("closed") is True and b.get("volume_mm3", 0) > 0 for b in measured))
errors = [f for f in inspection.get("timeline", []) if f.get("error")] if isinstance(inspection, dict) else []
check("no_failed_timeline_features", not errors, errors)
if measured:
low = [min(b["bbox"]["min"][i] for b in measured) for i in range(3)]
high = [max(b["bbox"]["max"][i] for b in measured) for i in range(3)]
size = [h-l for h, l in zip(high, low)]
if family_id == "mounting-plate":
expected = [params["length"], params["width"], params["thickness"]]
elif family_id == "stepped-shaft":
expected = [derived["max_diameter"], derived["max_diameter"], derived["total_length"]]
elif family_id == "hex-head-bolt":
expected = None # Hexagon orientation is not fixed in the original request.
check("axial_total_length", abs(size[2] - derived["total_length"]) <= 0.05, size[2], derived["total_length"])
elif family_id == "ribbed-bearing-housing":
expected = [params["base_length"], params["base_width"], derived["total_height"]]
elif family_id == "involute-spur-gear":
expected = None # Angular orientation changes the XY bounding box slightly.
check("axial_total_length", abs(size[2] - derived["total_length"]) <= 0.05, size[2], derived["total_length"])
check("outside_envelope", all(abs(v-derived["addendum_diameter"]) < 0.5 for v in size[:2]), size[:2], derived["addendum_diameter"])
else:
expected = [params["outer_length"], params["outer_width"], derived["total_height"]]
if expected:
check("bounding_box_dimensions", all(abs(a-b) <= 0.05 for a, b in zip(size, expected)), size, expected)
if "volume" in derived:
volume = sum(b["volume_mm3"] for b in measured)
# 0.2% is an engine measurement gate, not a manufacturing tolerance.
check("analytic_volume", math.isclose(volume, derived["volume"], rel_tol=0.002, abs_tol=0.05), volume, derived["volume"])
cylinders = [c for topo in topologies.values() if isinstance(topo, dict) for c in topo.get("cylinders", [])]
def cyl(name, radius, axis, origin, extent=None):
check(name, any(cylinder_matches(c, radius, axis, origin, extent) for c in cylinders),
expected={"radius": radius, "axis": axis, "origin": origin, "extent_length": extent})
if family_id == "mounting-plate":
for i, h in enumerate(derived["holes"]):
cyl(f"hole_{i+1}_axis_radius_extent", h["diameter"]/2, [0, 0, 1], [h["x"], h["y"], 0], params["thickness"])
elif family_id == "stepped-shaft":
for i, segment in enumerate(params["segments"]):
cyl(f"shaft_segment_{i+1}", segment["diameter"]/2, [0, 0, 1], [0, 0, 0], segment["length"])
elif family_id == "hex-head-bolt":
cyl("nominal_shank", params["nominal_diameter"]/2, [0, 0, 1], [0, 0, 0], params["under_head_length"])
elif family_id == "ribbed-bearing-housing":
cyl("horizontal_bearing_bore", params["bore_diameter"]/2, [0, 1, 0], [0, 0, params["center_height"]])
if params.get("counterbore"):
cyl("counterbore_axis_radius", params["counterbore"]["diameter"]/2, [0, 1, 0], [0, 0, params["center_height"]], params["counterbore"]["depth"])
for i, h in enumerate(params["mount_holes"]):
cyl(f"vertical_mounting_hole_{i+1}", h["diameter"]/2, [0, 0, 1], [h["x"], h["y"], 0], params["base_thickness"])
elif family_id == "involute-spur-gear":
cyl("hub_outer_radius", params["hub_diameter"]/2, [0, 0, 1], [0, 0, 0])
cyl("axial_bore_radius", params["bore_diameter"]/2, [0, 0, 1], [0, 0, 0])
else:
for i, centre in enumerate(derived["fastener_centers"]):
for role, diameter, extent in [("boss_pilot", params["boss_pilot_diameter"], params["pilot_depth"]),
("lid_clearance", params["cover_hole_diameter"], params["lid_thickness"])]:
cyl(f"{role}_{i+1}", diameter/2, [0, 0, 1], [centre["x"], centre["y"], 0], extent)
manual = {
"mounting-plate": ["No unintended extra features; confirm exact outer origin and bore material side"],
"stepped-shaft": ["Check shoulder axial positions and native parameter-edit behaviour"],
"hex-head-bolt": ["Check regular hexagon across flats/head height and simplified-thread semantic marking"],
"ribbed-bearing-housing": ["Check both rib profiles, their connections, all local wall thicknesses and counterbore side"],
"involute-spur-gear": ["Verify 24 teeth, involute deviation, pitch-circle tooth thickness, keyway sizes and disclosed root simplification"],
"covered-thin-wall-enclosure": ["Check cavity dimensions, four rib connections, actual cover plane gap and shared interface dependencies"],
}[family_id]
return {"automatic_checks_passed": bool(checks) and all(c["passed"] for c in checks),
"checks": checks, "manual_review_required": manual, "full_geometry_certified": False,
"linear_check_tolerance_mm": 0.05, "volume_relative_tolerance": 0.002}
def capture(mcp, config, directory, family_id):
results = {}
for view in config["views"]:
path = Path(directory) / f"{view}.png"
result = mcp.call("screenshot", {"source": "model", "view": view, "path": str(path),
"grid": False, "axes": False, "width": config["image_width"], "height": config["image_height"]})
results[view] = {"path": path.name, "ok": path.exists() and not result.get("isError"), "info": tool_value(result)}
if family_id in {"ribbed-bearing-housing", "covered-thin-wall-enclosure"}:
plane = "YZ" if family_id == "ribbed-bearing-housing" else "XZ"
section = mcp.call("execute", {"command": "inspect.section", "params": {"plane": plane}})
if not section.get("isError"):
path = Path(directory) / "section.png"
result = mcp.call("screenshot", {"source": "model", "view": "iso", "path": str(path),
"grid": False, "axes": False, "width": config["image_width"], "height": config["image_height"]})
results["section"] = {"path": path.name, "ok": path.exists() and not result.get("isError"), "info": tool_value(result)}
mcp.call("execute", {"command": "inspect.section", "params": {"clear": True}})
return results
def run_case(config, kb, family_id, condition, directory, repeat):
directory = Path(directory).resolve()
directory.mkdir(parents=True, exist_ok=True)
started = time.monotonic()
label = f"{family_id}/{condition}/r{repeat}"
summary = {"family_id": family_id, "condition": condition, "repeat": repeat,
"model": config["model"], "thinking": config["thinking"], "status": "running",
"directory": str(directory), "provider_called_directly": True, "codex_used_for_generation": False}
write_json(directory / "result.json", summary)
mcp = None
try:
if config.get("cad_cli_sha256") and sha256_file(config["cad_cli"]) != config["cad_cli_sha256"]:
raise RuntimeError("CAD executable changed during this run; refusing to mix engine versions")
mcp = MCP(config["cad_cli"], directory)
# The runtime binary owns the technical contract, so its matching reference is used.
reference = None
for skill in ("makecad-modeling", "solidcraft-modeling"):
value = mcp.call("skill_reference", {"skill": skill, "file": "reference.md"})
if not value.get("isError"):
reference = tool_value(value)
if isinstance(reference, dict):
reference = json.dumps(reference, ensure_ascii=False)
if isinstance(reference, str) and reference.strip():
break
if not reference:
raise RuntimeError("No matching runtime command reference available")
system = base_prompt(kb, reference)
(directory / "common-prompt.md").write_text(system, encoding="utf-8")
messages = [{"role": "system", "content": system}]
if condition == "B":
context = kb.build_context([family_id], include_examples=False)
# common-conventions is already identical in both system prompts.
family_context = context["text"].split("\n\n---\n\n", 1)[1]
messages.append({"role": "system", "content": "Additional engineering reference; use only within the user's requested scope:\n\n" + family_context})
write_json(directory / "knowledge-manifest.json", context["manifest"])
(directory / "knowledge-context.md").write_text(family_context, encoding="utf-8")
else:
write_json(directory / "knowledge-manifest.json", [])
task = extract_task(kb, family_id)
(directory / "task.md").write_text(task + "\n", encoding="utf-8")
messages.append({"role": "user", "content": task})
summary["agent"] = run_agent(config, mcp, messages, directory, label)
summary["status"] = "completed" if summary["agent"]["ended"] else "budget_or_provider_stop"
except Exception as exc:
summary["status"] = "error"
summary["error"] = redact(f"{type(exc).__name__}: {exc}", config["api_key"])
log(f"{label}: {summary['error']}")
finally:
if mcp is not None:
for stage in ("inspect", "measure", "save", "export", "screenshots", "assessment"):
try:
if stage == "inspect":
result = mcp.call("inspect_design", {"measure": True, "sketches": False})
inspection = tool_value(result)
write_json(directory / "inspection.json", inspection)
elif stage == "measure":
measurement = tool_value(mcp.call("measure", {}))
write_json(directory / "measurement.json", measurement)
topologies = {}
for body in inspection.get("bodies", []) if isinstance(inspection, dict) else []:
topologies[body["name"]] = tool_value(mcp.call("body_topology", {"body": body["name"], "faces": "cylinders"}))
write_json(directory / "cylinders.json", topologies)
elif stage == "save":
result = mcp.call("save", {"path": str(directory / "design.sc")})
summary["native_saved"] = (directory / "design.sc").exists() and not result.get("isError")
elif stage == "export":
result = mcp.call("export", {"path": str(directory / "design.step")})
summary["step_exported"] = (directory / "design.step").exists() and not result.get("isError")
if result.get("isError"):
summary["export_error"] = tool_value(result)
elif stage == "screenshots":
summary["screenshots"] = capture(mcp, config, directory, family_id)
else:
example = kb.documents[kb.families[family_id]["meta"]["example_id"]]["meta"]
derived = kb.validate_params(family_id, example["params"])["derived"]
summary["assessment"] = assess(family_id, example["params"], derived, inspection, measurement, topologies)
write_json(directory / "assessment.json", summary["assessment"])
except Exception as exc:
summary.setdefault("artifact_errors", {})[stage] = redact(str(exc), config["api_key"])
mcp.close()
summary["elapsed_s"] = round(time.monotonic() - started, 3)
write_json(directory / "result.json", redact(summary, config["api_key"]))
log(f"{label}: {summary['status']}, {summary['elapsed_s']} s")
return summary
def report(directory):
directory = Path(directory).resolve()
results = [json.loads(p.read_text()) for p in sorted(directory.glob("r*/*/[AB]/result.json"))]
write_json(directory / "results.json", results)
rows = ["# 六类零件:豆包直接调用 A/B 生成记录", "",
"A:共同技术说明与通用流程。B:额外加载对应零件族与共享特征,不加载完整示例。每个条件独立进程、独立会话。",
"请求由 Python 直接发送给豆包,使用 makeCAD 命令行 MCP;生成链路不调用 Codex、ACP 或 makeCAD 内置聊天。",
"自动几何检查为部分检查,完整结构与齿形等仍需人工核对。不能据此报告全面建模成功率。", "",
"| 零件 | 组 | 状态 | 自动项通过/总数 | 耗时秒 | 截图 |", "| --- | --- | --- | --- | ---: | --- |"]
cards = []
for result in results:
path = Path(result["directory"]).relative_to(directory).as_posix()
checks = result.get("assessment", {}).get("checks", [])
count = f"{sum(c['passed'] for c in checks)}/{len(checks)}" if checks else "未测量"
pictures = result.get("screenshots", {})
links = " ".join(f"[{view}]({path}/{value['path']})" for view, value in pictures.items() if value["ok"])
rows.append(f"| {result['family_id']} | {result['condition']} | {result['status']} | {count} | {result.get('elapsed_s', 0)} | {links} |")
failed = [c["name"] for c in checks if not c["passed"]]
rows += []
image = f'<img src="{html.escape(path)}/iso.png" loading="lazy">' if pictures.get("iso", {}).get("ok") else '<p>未取得截图</p>'
details = html.escape(json.dumps({"failed_checks": failed, "error": result.get("error"),
"manual_review": result.get("assessment", {}).get("manual_review_required", [])}, ensure_ascii=False, indent=2))
cards.append(f'<article><h2>{html.escape(result["family_id"])} · {result["condition"]} · r{result["repeat"]}</h2>{image}<p>{html.escape(result["status"])} · {count} 自动检查 · {result.get("elapsed_s",0)} s</p><p><a href="{path}/result.json">结果</a> · <a href="{path}/design.sc">设计</a> · <a href="{path}/measurement.json">测量</a> · <a href="{path}/cad-events.jsonl">工具日志</a></p><details><summary>失败项和待核对项</summary><pre>{details}</pre></details></article>')
rows += ["", "## 查看记录", "", "[并排截图页面](index.html) · [全部结果 JSON](results.json) · [运行配置与来源快照](run-manifest.json)",
"", "每个案例目录保留 task.md、common-prompt.md、B 组知识上下文、逐轮请求/SSE 响应、conversation.json、cad-events.jsonl、measurement.json、assessment.json、design.sc 和实际截图。失败案例同样记录残留模型。"]
(directory / "report.md").write_text("\n".join(rows) + "\n", encoding="utf-8")
(directory / "index.html").write_text('<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>豆包 CAD A/B 对照</title><style>body{font:15px system-ui;margin:24px;background:#f3f5f7;color:#18212b}main{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:20px}article{background:white;padding:16px;border-radius:10px}h2{font-size:16px}img{width:100%;height:auto}pre{white-space:pre-wrap}a{color:#1756a9}</style><h1>六类零件:豆包 A/B 生成记录</h1><p>A:共同说明;B:增加零件知识。截图来自实际 makeCAD 渲染。自动检查只覆盖已实现的部分几何要求。</p><main>' + "\n".join(cards) + '</main></html>', encoding="utf-8")
try:
from PIL import Image, ImageDraw, ImageFont
font = ImageFont.truetype("/System/Library/Fonts/Supplemental/Arial.ttf", 18)
cell_w, cell_h = 600, 490
sheet = Image.new("RGB", (cell_w * 2, cell_h * max(1, (len(results)+1)//2)), "#edf0f3")
draw = ImageDraw.Draw(sheet)
for index, result in enumerate(results):
x, y = (index % 2) * cell_w, (index // 2) * cell_h
draw.text((x+12, y+10), f"{result['family_id']} / {result['condition']}", fill="#18212b", font=font)
path = Path(result["directory"]) / "iso.png"
if path.exists():
with Image.open(path) as im:
im = im.convert("RGB")
im.thumbnail((cell_w-20, cell_h-45))
sheet.paste(im, (x+(cell_w-im.width)//2, y+38))
sheet.save(directory / "comparison.png")
except (ImportError, OSError) as exc:
log(f"Contact sheet unavailable; individual PNGs and HTML are intact: {exc}")
return results
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=ROOT / "benchmark/config.local.json")
parser.add_argument("--out", type=Path)
parser.add_argument("--families", nargs="+", choices=FAMILIES, default=FAMILIES)
parser.add_argument("--conditions", nargs="+", choices=["A", "B"], default=["A", "B"])
parser.add_argument("--repeat", type=int, default=1)
parser.add_argument("--workers", type=int)
parser.add_argument("--resume", action="store_true", help="Skip all recorded cases, including failures; never silently rerun a failed trial")
parser.add_argument("--report-only", action="store_true")
args = parser.parse_args()
output = (args.out or ROOT / "runs" / datetime.now().astimezone().strftime("doubao-ab-%Y%m%d-%H%M%S")).resolve()
if args.report_only:
report(output)
return 0
config = load_config(args.config)
config["cad_cli_sha256"] = sha256_file(config["cad_cli"])
if args.workers:
if not 1 <= args.workers <= 6:
parser.error("--workers must be 1..6")
config["workers"] = args.workers
if not 1 <= args.repeat <= 100:
parser.error("--repeat must be 1..100")
if output.exists() and any(output.iterdir()) and not args.resume:
parser.error("Output is nonempty; choose a new directory or --resume")
output.mkdir(parents=True, exist_ok=True)
kb = KnowledgeBase(ROOT)
manifest_path = output / "run-manifest.json"
source_files = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest()
for folder in ["cad_knowledge", "references", "schemas", "benchmark"]
for p in sorted((ROOT/folder).rglob('*')) if p.is_file() and '__pycache__' not in p.parts and p.name != "config.local.json"}
manifest = {"created_at": datetime.now(timezone.utc).isoformat(), "runner_version": RUNNER_VERSION,
"config": public_config(config),
"families": args.families, "conditions": args.conditions, "repeat": args.repeat,
"source_hashes": source_files, "generation_provider": "Doubao direct HTTP SSE",
"http_timeout": None, "cad_operation_timeout": None, "full_examples_loaded": False,
"codex_used_for_generation": False}
if manifest_path.exists():
old = json.loads(manifest_path.read_text())
for key in ["config", "families", "conditions", "repeat", "source_hashes"]:
if old[key] != manifest[key]:
raise ValueError(f"Cannot resume with changed {key}; start a new run")
else:
write_json(manifest_path, manifest)
for name in source_files:
snapshot = output / "source-snapshot" / name
snapshot.parent.mkdir(parents=True, exist_ok=True)
snapshot.write_bytes((ROOT / name).read_bytes())
jobs = []
for repeat in range(1, args.repeat+1):
for family in args.families:
for condition in args.conditions:
directory = output / f"r{repeat:02}" / family / condition
if args.resume and (directory / "result.json").exists():
prior = json.loads((directory / "result.json").read_text())
if prior.get("status") != "running":
continue
raise ValueError(f"Interrupted trial exists at {directory}; preserve it and start a new run")
jobs.append((family, condition, directory, repeat))
log(f"Starting {len(jobs)} isolated trials; model={config['model']}, thinking={config['thinking']}, workers={config['workers']}; out={output}")
try:
with ThreadPoolExecutor(max_workers=config["workers"]) as pool:
futures = [pool.submit(run_case, config, kb, *job) for job in jobs]
for future in as_completed(futures):
future.result()
report(output)
finally:
results = report(output)
log(f"Recorded {len(results)} trials: {output / 'report.md'}")
return 0 if all(r["status"] == "completed" for r in results) else 1
if __name__ == "__main__":
raise SystemExit(main())