refactor: add DesignIR distillation pipeline
This commit is contained in:
Jerry
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate blind baseline/experience STEP reconstructions against held-out STEP."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import importlib.util
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from build123d import import_step
|
||||
from scipy.spatial import cKDTree
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
EXTRACTOR = (
|
||||
HERE.parents[1]
|
||||
/ "skills/cad-experience-builder/scripts/step_to_case.py"
|
||||
)
|
||||
|
||||
|
||||
def load_extractor():
|
||||
spec = importlib.util.spec_from_file_location("ab50_step_to_case", EXTRACTOR)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
EXTRACT = load_extractor()
|
||||
|
||||
|
||||
def rel_error(actual: float, expected: float) -> float:
|
||||
return abs(actual - expected) / max(abs(expected), 1e-9)
|
||||
|
||||
|
||||
def feature_names(payload: dict[str, Any]) -> set[str]:
|
||||
result: set[str] = set()
|
||||
for item in payload.get("design_ir", {}).get("features", []):
|
||||
if isinstance(item, str):
|
||||
result.add(item)
|
||||
elif isinstance(item, dict):
|
||||
name = item.get("kind") or item.get("type") or item.get("name")
|
||||
if name:
|
||||
result.add(str(name))
|
||||
return result
|
||||
|
||||
|
||||
def f1_score(left: set[str], right: set[str]) -> float:
|
||||
if not left and not right:
|
||||
return 1.0
|
||||
if not left or not right:
|
||||
return 0.0
|
||||
overlap = len(left & right)
|
||||
precision = overlap / len(right)
|
||||
recall = overlap / len(left)
|
||||
return 2 * precision * recall / max(precision + recall, 1e-9)
|
||||
|
||||
|
||||
def mesh(shape, tolerance: float = 0.35) -> tuple[np.ndarray, np.ndarray]:
|
||||
vertices, triangles = shape.tessellate(tolerance)
|
||||
points = np.asarray([[v.X, v.Y, v.Z] for v in vertices], dtype=float)
|
||||
faces = np.asarray(triangles, dtype=int)
|
||||
return points, faces
|
||||
|
||||
|
||||
def sample_surface(
|
||||
vertices: np.ndarray, faces: np.ndarray, count: int, seed: int
|
||||
) -> np.ndarray:
|
||||
tri = vertices[faces]
|
||||
cross = np.cross(tri[:, 1] - tri[:, 0], tri[:, 2] - tri[:, 0])
|
||||
areas = np.linalg.norm(cross, axis=1) / 2.0
|
||||
total = float(areas.sum())
|
||||
if total <= 1e-12:
|
||||
return vertices.copy()
|
||||
rng = np.random.default_rng(seed)
|
||||
chosen = rng.choice(len(faces), size=count, p=areas / total)
|
||||
selected = tri[chosen]
|
||||
u = rng.random(count)
|
||||
v = rng.random(count)
|
||||
reflected = u + v > 1.0
|
||||
u[reflected] = 1.0 - u[reflected]
|
||||
v[reflected] = 1.0 - v[reflected]
|
||||
return (
|
||||
selected[:, 0]
|
||||
+ u[:, None] * (selected[:, 1] - selected[:, 0])
|
||||
+ v[:, None] * (selected[:, 2] - selected[:, 0])
|
||||
)
|
||||
|
||||
|
||||
def cube_transforms() -> list[np.ndarray]:
|
||||
transforms = []
|
||||
for permutation in itertools.permutations(range(3)):
|
||||
base = np.eye(3)[:, permutation]
|
||||
for signs in itertools.product((-1.0, 1.0), repeat=3):
|
||||
transforms.append(base @ np.diag(signs))
|
||||
return transforms
|
||||
|
||||
|
||||
TRANSFORMS = cube_transforms()
|
||||
|
||||
|
||||
def surface_distance(
|
||||
reference, generated, seed: int, samples: int
|
||||
) -> tuple[float, float]:
|
||||
ref_v, ref_f = mesh(reference)
|
||||
gen_v, gen_f = mesh(generated)
|
||||
ref = sample_surface(ref_v, ref_f, samples, seed)
|
||||
gen = sample_surface(gen_v, gen_f, samples, seed + 1)
|
||||
ref -= (ref.min(axis=0) + ref.max(axis=0)) / 2.0
|
||||
gen -= (gen.min(axis=0) + gen.max(axis=0)) / 2.0
|
||||
ref_tree = cKDTree(ref)
|
||||
best_chamfer = math.inf
|
||||
best_hausdorff = math.inf
|
||||
for transform in TRANSFORMS:
|
||||
candidate = gen @ transform
|
||||
gen_tree = cKDTree(candidate)
|
||||
d_gr = ref_tree.query(candidate, workers=-1)[0]
|
||||
d_rg = gen_tree.query(ref, workers=-1)[0]
|
||||
chamfer = float((d_gr.mean() + d_rg.mean()) / 2.0)
|
||||
hausdorff = float(max(np.quantile(d_gr, 0.95), np.quantile(d_rg, 0.95)))
|
||||
if chamfer < best_chamfer:
|
||||
best_chamfer = chamfer
|
||||
best_hausdorff = hausdorff
|
||||
diagonal = max(float(np.linalg.norm(np.ptp(ref, axis=0))), 1e-9)
|
||||
return best_chamfer / diagonal, best_hausdorff / diagonal
|
||||
|
||||
|
||||
def task_experience(step_path: Path) -> tuple[int, int]:
|
||||
task = step_path.parent / "cad-task.json"
|
||||
if not task.is_file():
|
||||
return 0, 0
|
||||
payload = json.loads(task.read_text())
|
||||
exp = payload.get("experience", {})
|
||||
returned = exp.get("methods_returned")
|
||||
if returned is None:
|
||||
returned = len(exp.get("methods", exp.get("methods_applied", [])))
|
||||
if isinstance(returned, list):
|
||||
returned = len(returned)
|
||||
return int(returned or 0), len(exp.get("methods_applied", []))
|
||||
|
||||
|
||||
def evaluate_one(
|
||||
case: dict[str, Any], variant: str, samples: int
|
||||
) -> dict[str, Any]:
|
||||
generated_path = Path(case[f"{variant}_step"])
|
||||
row: dict[str, Any] = {
|
||||
"id": case["id"],
|
||||
"family": case["family"],
|
||||
"variant": variant,
|
||||
"generated": generated_path.is_file(),
|
||||
}
|
||||
if not generated_path.is_file():
|
||||
row["error"] = "missing STEP"
|
||||
return row
|
||||
try:
|
||||
reference = import_step(case["reference_step"])
|
||||
generated = import_step(str(generated_path))
|
||||
ref_case = json.loads(Path(case["case_json"]).read_text())
|
||||
gen_case = EXTRACT.extract_step_case(generated_path)
|
||||
ref_bbox = sorted(float(v) for v in reference.bounding_box().size)
|
||||
gen_bbox = sorted(float(v) for v in generated.bounding_box().size)
|
||||
bbox_error = float(
|
||||
np.mean([rel_error(a, b) for a, b in zip(gen_bbox, ref_bbox)])
|
||||
)
|
||||
volume_error = rel_error(float(generated.volume), float(reference.volume))
|
||||
area_error = rel_error(float(generated.area), float(reference.area))
|
||||
face_error = rel_error(len(generated.faces()), len(reference.faces()))
|
||||
edge_error = rel_error(len(generated.edges()), len(reference.edges()))
|
||||
chamfer, hausdorff95 = surface_distance(
|
||||
reference, generated, int(case["id"]), samples
|
||||
)
|
||||
feature_f1 = f1_score(feature_names(ref_case), feature_names(gen_case))
|
||||
family_match = float(
|
||||
ref_case["design_ir"]["part_family"]
|
||||
== gen_case["design_ir"]["part_family"]
|
||||
)
|
||||
returned, applied = task_experience(generated_path)
|
||||
|
||||
# Transparent 0–100 similarity index. It is a benchmark summary, not a
|
||||
# claim of manufacturing interchangeability or exact B-rep identity.
|
||||
score = 100.0 * (
|
||||
0.18 * math.exp(-volume_error)
|
||||
+ 0.12 * math.exp(-area_error)
|
||||
+ 0.16 * math.exp(-bbox_error)
|
||||
+ 0.24 * math.exp(-8.0 * chamfer)
|
||||
+ 0.10 * math.exp(-(face_error + edge_error) / 2.0)
|
||||
+ 0.15 * feature_f1
|
||||
+ 0.05 * family_match
|
||||
)
|
||||
row.update(
|
||||
{
|
||||
"valid": bool(generated.is_valid),
|
||||
"solid_count": len(generated.solids()),
|
||||
"reference_volume": float(reference.volume),
|
||||
"generated_volume": float(generated.volume),
|
||||
"volume_rel_error": volume_error,
|
||||
"surface_area_rel_error": area_error,
|
||||
"bbox_mape": bbox_error,
|
||||
"face_rel_error": face_error,
|
||||
"edge_rel_error": edge_error,
|
||||
"surface_chamfer_norm": chamfer,
|
||||
"surface_hausdorff95_norm": hausdorff95,
|
||||
"feature_f1": feature_f1,
|
||||
"family_match": family_match,
|
||||
"experience_methods_returned": returned,
|
||||
"experience_methods_applied": applied,
|
||||
"similarity_score": score,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
row["error"] = f"{type(exc).__name__}: {exc}"
|
||||
return row
|
||||
|
||||
|
||||
def mean(rows: list[dict[str, Any]], key: str) -> float:
|
||||
values = [float(row[key]) for row in rows if key in row]
|
||||
return sum(values) / len(values) if values else math.nan
|
||||
|
||||
|
||||
def write_report(rows: list[dict[str, Any]]) -> None:
|
||||
complete = [row for row in rows if "similarity_score" in row]
|
||||
baseline = [row for row in complete if row["variant"] == "baseline"]
|
||||
experience = [row for row in complete if row["variant"] == "experience"]
|
||||
by_id = {}
|
||||
for row in complete:
|
||||
by_id.setdefault(row["id"], {})[row["variant"]] = row
|
||||
pairs = [
|
||||
item for item in by_id.values() if {"baseline", "experience"} <= item.keys()
|
||||
]
|
||||
paired_baseline = [item["baseline"] for item in pairs]
|
||||
paired_experience = [item["experience"] for item in pairs]
|
||||
deltas = [
|
||||
item["experience"]["similarity_score"]
|
||||
- item["baseline"]["similarity_score"]
|
||||
for item in pairs
|
||||
]
|
||||
wins = sum(delta > 0.25 for delta in deltas)
|
||||
ties = sum(abs(delta) <= 0.25 for delta in deltas)
|
||||
losses = sum(delta < -0.25 for delta in deltas)
|
||||
methods_returned = sum(
|
||||
row.get("experience_methods_returned", 0) for row in experience
|
||||
)
|
||||
methods_applied = sum(
|
||||
row.get("experience_methods_applied", 0) for row in experience
|
||||
)
|
||||
report = f"""# CAD 经验库 50 件 A/B 盲重建报告
|
||||
|
||||
## 口径
|
||||
|
||||
- 测试集：固定分层抽取 50 件；这 50 件不进入经验库。
|
||||
- 训练经验：其余 950 件生成的通用经验库。
|
||||
- 两组输入完全相同：仅多视图图片和文字描述；生成过程禁止读取参考 STEP 和解析 JSON。
|
||||
- Baseline：空经验库；Experience：950 件经验库。
|
||||
- 参考 STEP 只在生成完成后用于评分。
|
||||
- “相似度”是体积、面积、包围盒、表面距离、拓扑和语义特征的综合诊断分，不代表制造级 1:1 或可互换性。
|
||||
|
||||
## 当前完成度
|
||||
|
||||
- 有效 Baseline：{len(baseline)}/50
|
||||
- 有效 Experience：{len(experience)}/50
|
||||
- 完整配对：{len(pairs)}/50
|
||||
- 经验方法返回/实际采用：{methods_returned}/{methods_applied}
|
||||
|
||||
## 汇总
|
||||
|
||||
| 指标 | 无经验 | 有经验 | 差值 |
|
||||
|---|---:|---:|---:|
|
||||
| 综合相似度（0–100，越高越好） | {mean(paired_baseline, "similarity_score"):.2f} | {mean(paired_experience, "similarity_score"):.2f} | {mean(paired_experience, "similarity_score") - mean(paired_baseline, "similarity_score"):+.2f} |
|
||||
| 体积相对误差（越低越好） | {mean(paired_baseline, "volume_rel_error"):.3f} | {mean(paired_experience, "volume_rel_error"):.3f} | {mean(paired_experience, "volume_rel_error") - mean(paired_baseline, "volume_rel_error"):+.3f} |
|
||||
| 表面积相对误差 | {mean(paired_baseline, "surface_area_rel_error"):.3f} | {mean(paired_experience, "surface_area_rel_error"):.3f} | {mean(paired_experience, "surface_area_rel_error") - mean(paired_baseline, "surface_area_rel_error"):+.3f} |
|
||||
| 包围盒尺寸 MAPE | {mean(paired_baseline, "bbox_mape"):.3f} | {mean(paired_experience, "bbox_mape"):.3f} | {mean(paired_experience, "bbox_mape") - mean(paired_baseline, "bbox_mape"):+.3f} |
|
||||
| 归一化表面 Chamfer | {mean(paired_baseline, "surface_chamfer_norm"):.4f} | {mean(paired_experience, "surface_chamfer_norm"):.4f} | {mean(paired_experience, "surface_chamfer_norm") - mean(paired_baseline, "surface_chamfer_norm"):+.4f} |
|
||||
| 特征 F1 | {mean(paired_baseline, "feature_f1"):.3f} | {mean(paired_experience, "feature_f1"):.3f} | {mean(paired_experience, "feature_f1") - mean(paired_baseline, "feature_f1"):+.3f} |
|
||||
|
||||
经验组胜/平/负（±0.25 分视为平）：**{wins}/{ties}/{losses}**。
|
||||
"""
|
||||
(HERE / "report.md").write_text(report)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--samples", type=int, default=1600)
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((HERE / "manifest.json").read_text())
|
||||
rows = [
|
||||
evaluate_one(case, variant, args.samples)
|
||||
for case in manifest["cases"]
|
||||
for variant in ("baseline", "experience")
|
||||
]
|
||||
(HERE / "metrics.json").write_text(
|
||||
json.dumps(rows, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
keys = sorted({key for row in rows for key in row})
|
||||
with (HERE / "metrics.csv").open("w", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=keys)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
write_report(rows)
|
||||
completed = sum("similarity_score" in row for row in rows)
|
||||
print(json.dumps({"evaluated": completed, "expected": 100}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
bbox_mape,edge_rel_error,error,experience_methods_applied,experience_methods_returned,face_rel_error,family,family_match,feature_f1,generated,generated_volume,id,reference_volume,similarity_score,solid_count,surface_area_rel_error,surface_chamfer_norm,surface_hausdorff95_norm,valid,variant,volume_rel_error
|
||||
0.0,0.5,,0,0,0.25,general_mechanical_part,1.0,0.8571428571428571,True,125663.70614359173,000166,125663.70614359173,93.81711632834559,1,6.425513440379324e-14,0.004847580881730266,0.00977766538694562,True,baseline,0.0
|
||||
0.0,0.5,,0,0,0.25,general_mechanical_part,1.0,0.8571428571428571,True,125663.70614359173,000166,125663.70614359173,93.81711632834559,1,6.425513440379324e-14,0.004847580881730266,0.00977766538694562,True,experience,0.0
|
||||
0.45066264681866697,0.125,,0,0,0.1111111111111111,patterned_plate,0.0,0.8571428571428571,True,106129.48703937844,004764,36749.40939347248,54.33835930595081,1,1.0309687869495758,0.05550481239248205,0.1775660613682025,True,baseline,1.8879236099568286
|
||||
0.3413730293323282,0.125,,3,18,0.1111111111111111,patterned_plate,0.0,0.8571428571428571,True,88774.08881569223,004764,36749.40939347248,59.488074232257325,1,0.7876989178243013,0.0465090322155671,0.15562347480527094,True,experience,1.4156602862700836
|
||||
0.5696649029982362,0.0,,0,0,0.0,plate_or_bracket,1.0,1.0,True,19016.0,011680,898496.0,60.445468751589296,1,0.9780420050604243,0.10796723414266624,0.36481967349123434,True,baseline,0.9788357432865589
|
||||
0.583774250440917,0.0,,4,10,0.0,plate_or_bracket,1.0,1.0,True,13547.999999999998,011680,898496.0,59.90070139134242,1,0.9842171382062035,0.11235674854244815,0.374964117147598,True,experience,0.9849214687655816
|
||||
0.18571427761876255,0.2631578947368421,,0,0,0.26666666666666666,patterned_plate,1.0,1.0,True,23882.59512641282,016033,44618.37587532573,81.50778192907364,1,0.34116223818430896,0.018452946285241585,0.08713798240651745,True,baseline,0.4647363410728704
|
||||
0.3238095238095235,0.02631578947368421,,3,10,0.06666666666666667,patterned_plate,1.0,0.9411764705882353,True,13866.369040708962,016033,44618.37587532573,75.91174289423121,1,0.5000404499582691,0.026851094204940862,0.12169100582216402,True,experience,0.6892229094251465
|
||||
0.5959595959595959,0.2857142857142857,,0,0,0.28125,flanged_rotational_part,1.0,1.0,True,118756.98485692723,020543,24145.808587596228,51.56160371886595,1,1.5105685440071281,0.08451218347517844,0.22747541674825858,True,baseline,3.9183271053483395
|
||||
0.23232323232323235,0.05194805194805195,,0,0,0.1875,flanged_rotational_part,1.0,0.888888888888889,True,55638.90609255368,020543,24145.808587596228,69.33575293745653,1,0.4640684949891876,0.042969081471212464,0.11104259063963073,True,experience,1.3042883774509648
|
||||
0.06666666666666685,0.2,,0,0,0.19480519480519481,patterned_prismatic_part,1.0,1.0,True,569438.4483554161,024675,580617.6091315141,91.92371160771847,1,0.2739018704584683,0.011073315847120254,0.07352156562273186,True,baseline,0.019253912730652068
|
||||
0.5777777777777777,0.8933333333333333,,0,0,1.4025974025974026,patterned_prismatic_part,1.0,0.9333333333333333,True,38876.03946418978,024675,580617.6091315141,56.84188202691976,1,0.8746474563374864,0.07094038930770122,0.33837892456299423,True,experience,0.9330436437807312
|
||||
0.0980952380952385,0.16666666666666666,,0,0,0.14285714285714285,patterned_plate,1.0,0.9333333333333333,True,73807.58425433026,029190,131254.75510206894,83.74861954252185,1,0.1560183040026604,0.024101504873521428,0.04232654923555514,True,baseline,0.4376768735202429
|
||||
0.14047619047618823,1.5277777777777777,,0,0,1.6428571428571428,patterned_plate,1.0,0.7999999999999999,True,71846.37021205797,029190,131254.75510206894,73.91053356207678,1,0.20263501345527155,0.02460078307658505,0.04143094591291666,True,experience,0.45261891535901033
|
||||
0.31241830065359194,0.9285714285714286,,0,0,0.5625,patterned_plate,1.0,0.923076923076923,True,34352.594678913054,030352,60881.15044407846,73.94372815488485,1,0.48742381144539443,0.02510282770499014,0.10902462973941958,True,baseline,0.4357433388111292
|
||||
0.4306604575163398,2.642857142857143,,3,10,2.0625,patterned_plate,1.0,0.7692307692307692,True,11173.345159683424,030352,60881.15044407846,59.45331217188532,1,0.6732862799472543,0.03957952772551367,0.16850566658178062,True,experience,0.8164728314398964
|
||||
0.21212121212121218,0.5,,0,0,0.43243243243243246,flanged_rotational_part,1.0,0.9600000000000001,True,73578.89519968569,046014,162271.3445381513,75.92217396437314,1,0.333964798541362,0.033942930350360764,0.05126001418573268,True,baseline,0.5465687709120651
|
||||
0.13636363636363644,0.5,,4,10,0.43243243243243246,flanged_rotational_part,1.0,0.9600000000000001,True,111532.3028776836,046014,162271.3445381513,81.61765078194381,1,0.24200901291799348,0.02662187229047825,0.043202965297600246,True,experience,0.31268023202050027
|
||||
0.6851851851851847,0.3709677419354839,,0,0,0.34782608695652173,patterned_prismatic_part,1.0,0.823529411764706,True,167251.53322574112,047048,33461.296317388085,44.79000279500974,1,1.6260155362793598,0.11322328439048983,0.2789388077989153,True,baseline,3.9983578531842254
|
||||
0.7175925925925921,0.1693548387096774,,3,18,0.2391304347826087,patterned_prismatic_part,1.0,0.823529411764706,True,169277.67589913897,047048,33461.296317388085,44.38628038601891,1,1.587008333438104,0.13259494837120114,0.321530970786483,True,experience,4.058909681606514
|
||||
1.4575235605335388e-16,0.5,,0,0,0.3125,hollow_or_stepped_shaft,1.0,0.9166666666666666,True,8278002.394429736,048346,8341713.8934443,93.45850460439323,1,0.01545893719801078,0.008802226092164953,0.016975575422477428,True,baseline,0.007637698898380382
|
||||
1.4575235605335388e-16,0.5,,0,0,0.3125,hollow_or_stepped_shaft,1.0,0.9166666666666666,True,8292610.800268921,048346,8341713.8934443,93.51807327218812,1,0.012238325281746575,0.008857323306102671,0.017730676062993916,True,experience,0.005886451369899948
|
||||
0.02857142857142833,0.2222222222222222,,0,0,0.21212121212121213,patterned_plate,0.0,0.8571428571428571,True,315117.2806010815,053664,349435.5647059466,85.57701767939263,1,0.07576465816363272,0.012696566244914555,0.02434484580541925,True,baseline,0.0982106218459597
|
||||
0.07936507936507906,0.0,,0,0,0.030303030303030304,patterned_plate,0.0,0.8571428571428571,True,282776.14120936824,053664,349435.5647059466,83.57268406993335,1,0.167398697885095,0.016320456106002173,0.05162627100789126,True,experience,0.19076313412080106
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,063487,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,063487,,,,,,,,experience,
|
||||
0.05572210294349599,0.420863309352518,,0,0,0.4396551724137931,flanged_rotational_part,1.0,0.923076923076923,True,68680.44493144708,065487,81354.16556588435,87.21779338748289,1,0.060356551334099576,0.022572421105818447,0.053770414665160235,True,baseline,0.15578453231351141
|
||||
,,missing STEP,,,,flanged_rotational_part,,,False,,065487,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,067186,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,067186,,,,,,,,experience,
|
||||
,,missing STEP,,,,rotational_part,,,False,,073706,,,,,,,,baseline,
|
||||
,,missing STEP,,,,rotational_part,,,False,,073706,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,077955,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,077955,,,,,,,,experience,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,082187,,,,,,,,baseline,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,082187,,,,,,,,experience,
|
||||
,,missing STEP,,,,rotational_part,,,False,,089970,,,,,,,,baseline,
|
||||
,,missing STEP,,,,rotational_part,,,False,,089970,,,,,,,,experience,
|
||||
,,missing STEP,,,,shaft,,,False,,092092,,,,,,,,baseline,
|
||||
,,missing STEP,,,,shaft,,,False,,092092,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,095520,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,095520,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,102128,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,102128,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,104465,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,104465,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,110143,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,110143,,,,,,,,experience,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,112069,,,,,,,,baseline,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,112069,,,,,,,,experience,
|
||||
0.22361111004444456,0.6538461538461539,,0,0,0.6153846153846154,rotational_part,1.0,0.761904761904762,True,67519.74039233122,116166,147536.10631008807,71.3680012563105,1,0.38925591734814796,0.03424668647245051,0.07518718288900171,True,baseline,0.5423510754010293
|
||||
0.18888888915554566,0.7307692307692307,,3,16,0.6923076923076923,rotational_part,1.0,0.8571428571428572,True,69565.58402815007,116166,147536.10631008807,73.4520592182128,1,0.365981954180488,0.032489227662340435,0.07289222461786297,True,experience,0.528484343473599
|
||||
0.6746686351889055,0.45217391304347826,,0,0,0.45,multi_axis_manifold,0.0,0.7692307692307692,True,34234.94531047202,134828,349556.2389887021,48.526262227273214,1,0.9239731944207157,0.10449739414590258,0.3144594900968871,True,baseline,0.9020616956815967
|
||||
0.7062506154470962,0.5043478260869565,,5,14,0.5,multi_axis_manifold,0.0,0.7692307692307692,True,38190.323227729335,134828,349556.2389887021,48.60371993717729,1,0.9152269986233839,0.09836507300701838,0.29918959165673675,True,experience,0.8907462692177447
|
||||
7.3809523809523805,0.5,,0,0,0.3333333333333333,rotational_part,1.0,0.9411764705882353,True,25446.900494077883,136930,38.877209088173686,25.720060532485707,1,72.6363636363658,1.666072751413829,2.6857505921882785,True,baseline,653.545454545469
|
||||
6.936507936507937,0.5,,0,0,0.3333333333333333,rotational_part,1.0,0.9411764705882353,True,21205.750411731566,136930,38.877209088173686,25.725642575854934,1,67.18181818181998,1.6549008981976585,2.6698915132197145,True,experience,544.4545454545574
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,138022,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,138022,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,141680,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,141680,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,145574,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,145574,,,,,,,,experience,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,150196,,,,,,,,baseline,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,150196,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,150587,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,150587,,,,,,,,experience,
|
||||
,,missing STEP,,,,multi_axis_manifold,,,False,,152568,,,,,,,,baseline,
|
||||
,,missing STEP,,,,multi_axis_manifold,,,False,,152568,,,,,,,,experience,
|
||||
,,missing STEP,,,,plate_or_bracket,,,False,,153245,,,,,,,,baseline,
|
||||
,,missing STEP,,,,plate_or_bracket,,,False,,153245,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,158256,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,158256,,,,,,,,experience,
|
||||
,,missing STEP,,,,hollow_or_stepped_shaft,,,False,,162558,,,,,,,,baseline,
|
||||
,,missing STEP,,,,hollow_or_stepped_shaft,,,False,,162558,,,,,,,,experience,
|
||||
,,missing STEP,,,,multi_axis_manifold,,,False,,167962,,,,,,,,baseline,
|
||||
,,missing STEP,,,,multi_axis_manifold,,,False,,167962,,,,,,,,experience,
|
||||
,,missing STEP,,,,rotational_part,,,False,,168239,,,,,,,,baseline,
|
||||
,,missing STEP,,,,rotational_part,,,False,,168239,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,174675,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,174675,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,183151,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,183151,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,189669,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,189669,,,,,,,,experience,
|
||||
,,missing STEP,,,,rotational_part,,,False,,194029,,,,,,,,baseline,
|
||||
,,missing STEP,,,,rotational_part,,,False,,194029,,,,,,,,experience,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,201547,,,,,,,,baseline,
|
||||
,,missing STEP,,,,prismatic_block,,,False,,201547,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,204730,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,204730,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,211326,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,211326,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,214404,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_plate,,,False,,214404,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,229115,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,229115,,,,,,,,experience,
|
||||
,,missing STEP,,,,hybrid_freeform_part,,,False,,234097,,,,,,,,baseline,
|
||||
,,missing STEP,,,,hybrid_freeform_part,,,False,,234097,,,,,,,,experience,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,242307,,,,,,,,baseline,
|
||||
,,missing STEP,,,,patterned_prismatic_part,,,False,,242307,,,,,,,,experience,
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,39 @@
|
||||
# CAD 经验库 50 件 A/B 盲重建报告
|
||||
|
||||
> 运行状态：2026-07-23 因 Codex 账户用量上限中断；当前为 15/50
|
||||
> 完整配对的阶段结果。系统提示可在 2026-07-29 10:24 后继续。
|
||||
|
||||
## 口径
|
||||
|
||||
- 测试集：固定分层抽取 50 件；这 50 件不进入经验库。
|
||||
- 训练经验：其余 950 件生成的通用经验库。
|
||||
- 两组输入完全相同：仅多视图图片和文字描述；生成过程禁止读取参考 STEP 和解析 JSON。
|
||||
- Baseline：空经验库；Experience：950 件经验库。
|
||||
- 参考 STEP 只在生成完成后用于评分。
|
||||
- “相似度”是体积、面积、包围盒、表面距离、拓扑和语义特征的综合诊断分，不代表制造级 1:1 或可互换性。
|
||||
|
||||
## 当前完成度
|
||||
|
||||
- 有效 Baseline：16/50
|
||||
- 有效 Experience：15/50
|
||||
- 完整配对：15/50
|
||||
- 经验方法返回/实际采用：106/28
|
||||
|
||||
## 汇总
|
||||
|
||||
| 指标 | 无经验 | 有经验 | 差值 |
|
||||
|---|---:|---:|---:|
|
||||
| 综合相似度（0–100，越高越好） | 69.11 | 66.64 | -2.47 |
|
||||
| 体积相对误差（越低越好） | 44.519 | 37.136 | -7.383 |
|
||||
| 表面积相对误差 | 5.385 | 4.997 | -0.388 |
|
||||
| 包围盒尺寸 MAPE | 0.766 | 0.760 | -0.006 |
|
||||
| 归一化表面 Chamfer | 0.1537 | 0.1559 | +0.0023 |
|
||||
| 特征 F1 | 0.907 | 0.878 | -0.029 |
|
||||
|
||||
经验组胜/平/负（±0.25 分视为平）：**4/4/7**。
|
||||
|
||||
## 阶段解释
|
||||
|
||||
- 经验组的体积、表面积和包围盒平均误差较低，说明通用构造经验对尺度/主体关系有一定帮助。
|
||||
- 经验组的表面距离、特征 F1 和综合分暂时较差，说明部分经验会诱导多余或错误特征，当前不能宣称整体提升。
|
||||
- 每组由独立的大模型运行生成；即使经验命中为 0，两次结果也可能因模型随机性不同。因此最终应完成 50 对，并对若干样本做重复运行，才能把经验效果与随机波动分开。
|
||||
Reference in New Issue
Block a user