Files
2026-09-24 10:40:57 +08:00

98 lines
3.7 KiB
Python

"""回归测试跑批:把一份测试用例json里的每条描述都通过真实的②理解层->④生成层
管线跑一遍,记录成功/失败、体积、实体数,落一份结果json。
这是"第二步:回归测试集"的最小起步版本——先只做"能不能成功生成、生成的基本形态是什么",
后续会在这个骨架上接入claims自动核验,替换掉现在的人工肉眼核对。
用法:
python scripts/run_canary_tests.py [输入文件名] [输出文件名]
(默认: tests/canary_parts.json -> tests/canary_results.json)
"""
import json
import sys
import time
from pathlib import Path
from dotenv import load_dotenv
BASE_DIR = Path(__file__).resolve().parent.parent
load_dotenv(BASE_DIR / ".env")
sys.path.insert(0, str(BASE_DIR))
from engine.pipeline import PipelineError, generate_from_description_best as generate_from_description # noqa: E402
def _solid_count(step_path: str) -> int:
from build123d import import_step
try:
return len(import_step(step_path).solids())
except Exception: # noqa: BLE001
return -1
def main():
in_name = sys.argv[1] if len(sys.argv) > 1 else "canary_parts.json"
out_name = sys.argv[2] if len(sys.argv) > 2 else in_name.replace("_parts.json", "_results.json")
cases = json.loads((BASE_DIR / "tests" / in_name).read_text(encoding="utf-8"))
results = []
for i, case in enumerate(cases, 1):
label = case.get("name", case["id"])
print(f"\n===== [{i}/{len(cases)}] {case['id']} ({label}) =====")
print(f"描述: {case['description']}")
if case.get("stresses"):
print(f"侧重点: {case['stresses']}")
t0 = time.time()
try:
report = generate_from_description(case["description"], BASE_DIR)
elapsed = time.time() - t0
solids = _solid_count(report["step_file"])
result = {
"id": case["id"],
"name": label,
"description": case["description"],
"status": "ok",
"part_name": report["part_name"],
"volume_mm3": round(report["volume_mm3"], 2),
"solid_count": solids,
"elapsed_s": round(elapsed, 1),
"degraded_fallback": bool(report.get("degraded_fallback")),
"placeholder": bool(report.get("is_fallback_placeholder")),
"assumptions": report.get("llm_assumptions", []),
"step_file": report["step_file"],
}
print(f"[成功] part={report['part_name']} volume={report['volume_mm3']:.1f} "
f"solids={solids} degraded={result['degraded_fallback']} "
f"placeholder={result['placeholder']} 用时{elapsed:.0f}s")
except PipelineError as e:
elapsed = time.time() - t0
result = {
"id": case["id"],
"name": label,
"description": case["description"],
"status": "fail",
"error": str(e),
"elapsed_s": round(elapsed, 1),
}
print(f"[失败] {e}")
results.append(result)
out_path = BASE_DIR / "tests" / out_name
out_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print("\n\n===== 汇总 =====")
ok = sum(1 for r in results if r["status"] == "ok")
degraded = sum(1 for r in results if r.get("degraded_fallback"))
placeholder = sum(1 for r in results if r.get("placeholder"))
print(f"总数 {len(results)}, 成功 {ok}, 失败 {len(results) - ok}, "
f"其中降级 {degraded}, 占位兜底 {placeholder}")
print(f"详细结果已写入 {out_path}")
if __name__ == "__main__":
main()