Files
cadSet/cad-experience-plugin/benchmarks/ab50/run_case.py
T

89 lines
3.0 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import subprocess
from pathlib import Path
HERE = Path(__file__).resolve().parent
WORKSPACE = HERE.parents[2]
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("case_id")
parser.add_argument("variant", choices=("baseline", "experience"))
args = parser.parse_args()
manifest = json.loads((HERE / "manifest.json").read_text())
case = next(item for item in manifest["cases"] if item["id"] == args.case_id)
output = Path(case[f"{args.variant}_step"])
output.parent.mkdir(parents=True, exist_ok=True)
if output.is_file():
print(json.dumps({"status": "skipped", "output": str(output)}))
return
description = Path(case["description"]).read_text()
library = (
HERE / "empty-library.json"
if args.variant == "baseline"
else HERE / "train-library.json"
)
prompt = f"""
This is a controlled CAD reconstruction benchmark.
Use $cad-router and $cad to reconstruct the single mechanical part shown in
the attached multi-view image and described below. Generate a new parametric
STEP from the image and description only.
Hard isolation rules:
- Do not search for, inspect, import, copy, or reference any teacher STEP,
SLDPRT, parser/input, parser/output, case JSON, or SLDPRT-NET dataset file.
- The attached image and the description in this prompt are the only
part-specific evidence.
- Run cad-router with the explicit generalized experience library
{library.resolve()}; do not use any other experience library.
- Use the router-selected backend and actually apply returned generalized
methods when present.
- Do not preserve an exact STEP base: this must be a blind reconstruction.
Write the primary output exactly to:
{output.resolve()}
Keep the editable generator and model-spec.json in the same directory.
Also write the router result to route.json and record in cad-task.json the
experience methods returned and the subset actually applied. Record an empty
list when nothing matches; never claim an experience method was used unless
the router returned it.
Validate that the STEP opens and contains positive-volume geometry. For this
batch benchmark, do not open CAD Viewer or create browser tabs.
Description:
{description}
"""
command = [
"codex",
"exec",
"--ephemeral",
"--json",
"--image",
case["image"],
"--cd",
str(WORKSPACE),
"--sandbox",
"danger-full-access",
"--output-last-message",
case[f"{args.variant}_message"],
prompt,
]
with Path(case[f"{args.variant}_log"]).open("w") as stream:
completed = subprocess.run(command, stdout=stream, stderr=subprocess.STDOUT)
print(json.dumps({
"status": "ok" if completed.returncode == 0 and output.is_file() else "failed",
"returncode": completed.returncode,
"output": str(output),
}))
raise SystemExit(0 if completed.returncode == 0 and output.is_file() else 1)
if __name__ == "__main__":
main()