98 lines
3.7 KiB
Python
98 lines
3.7 KiB
Python
"""回归测试跑批:把一份测试用例json里的每条描述都通过真实的②理解层->④生成层
|
|
管线跑一遍,记录成功/失败、体积、实体数,落一份结果json。
|
|
|
|
这是"第二步:回归测试集"的最小起步版本——先只做"能不能成功生成、生成的基本形态是什么",
|
|
后续会在这个骨架上接入claims自动核验,替换掉现在的人工肉眼核对。
|
|
|
|
用法:
|
|
python scripts/run_canary_tests.py [输入文件名] [输出文件名]
|
|
(默认: tests/canary_parts.json -> tests/canary_results.json)
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
load_dotenv(BASE_DIR / ".env")
|
|
sys.path.insert(0, str(BASE_DIR))
|
|
|
|
from engine.pipeline import PipelineError, generate_from_description_best as generate_from_description # noqa: E402
|
|
|
|
|
|
def _solid_count(step_path: str) -> int:
|
|
from build123d import import_step
|
|
|
|
try:
|
|
return len(import_step(step_path).solids())
|
|
except Exception: # noqa: BLE001
|
|
return -1
|
|
|
|
|
|
def main():
|
|
in_name = sys.argv[1] if len(sys.argv) > 1 else "canary_parts.json"
|
|
out_name = sys.argv[2] if len(sys.argv) > 2 else in_name.replace("_parts.json", "_results.json")
|
|
|
|
cases = json.loads((BASE_DIR / "tests" / in_name).read_text(encoding="utf-8"))
|
|
results = []
|
|
|
|
for i, case in enumerate(cases, 1):
|
|
label = case.get("name", case["id"])
|
|
print(f"\n===== [{i}/{len(cases)}] {case['id']} ({label}) =====")
|
|
print(f"描述: {case['description']}")
|
|
if case.get("stresses"):
|
|
print(f"侧重点: {case['stresses']}")
|
|
t0 = time.time()
|
|
try:
|
|
report = generate_from_description(case["description"], BASE_DIR)
|
|
elapsed = time.time() - t0
|
|
solids = _solid_count(report["step_file"])
|
|
result = {
|
|
"id": case["id"],
|
|
"name": label,
|
|
"description": case["description"],
|
|
"status": "ok",
|
|
"part_name": report["part_name"],
|
|
"volume_mm3": round(report["volume_mm3"], 2),
|
|
"solid_count": solids,
|
|
"elapsed_s": round(elapsed, 1),
|
|
"degraded_fallback": bool(report.get("degraded_fallback")),
|
|
"placeholder": bool(report.get("is_fallback_placeholder")),
|
|
"assumptions": report.get("llm_assumptions", []),
|
|
"step_file": report["step_file"],
|
|
}
|
|
print(f"[成功] part={report['part_name']} volume={report['volume_mm3']:.1f} "
|
|
f"solids={solids} degraded={result['degraded_fallback']} "
|
|
f"placeholder={result['placeholder']} 用时{elapsed:.0f}s")
|
|
except PipelineError as e:
|
|
elapsed = time.time() - t0
|
|
result = {
|
|
"id": case["id"],
|
|
"name": label,
|
|
"description": case["description"],
|
|
"status": "fail",
|
|
"error": str(e),
|
|
"elapsed_s": round(elapsed, 1),
|
|
}
|
|
print(f"[失败] {e}")
|
|
|
|
results.append(result)
|
|
|
|
out_path = BASE_DIR / "tests" / out_name
|
|
out_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
print("\n\n===== 汇总 =====")
|
|
ok = sum(1 for r in results if r["status"] == "ok")
|
|
degraded = sum(1 for r in results if r.get("degraded_fallback"))
|
|
placeholder = sum(1 for r in results if r.get("placeholder"))
|
|
print(f"总数 {len(results)}, 成功 {ok}, 失败 {len(results) - ok}, "
|
|
f"其中降级 {degraded}, 占位兜底 {placeholder}")
|
|
print(f"详细结果已写入 {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|