feat: unify CAD workflows on DesignIR 3.0
This commit is contained in:
Jerry
@@ -0,0 +1,327 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"report_kind": "step_reconstruction_distillation_result",
|
||||
"extractor_version": "4.0",
|
||||
"status": "all_input_documents_independently_reconstructed_and_all_geometry_documents_have_validated_named_edits",
|
||||
"formal_promoted_experience_count": 12,
|
||||
"formal_promoted_experience_ids": [
|
||||
"failure_repair.surface_boundary.prefer_exact_3d",
|
||||
"failure_repair.periodic_torus.rectangular_uv_patch",
|
||||
"failure_repair.surface_topology.bind_face_pcurves_and_group_shells",
|
||||
"failure_repair.surface_topology.preserve_vertex_tolerance",
|
||||
"edit_strategy.semantic_cylindrical_cut.enlarge_diameter",
|
||||
"failure_repair.semantic_edit.boundary_strategy_fallback",
|
||||
"edit_strategy.semantic_edit.preserve_validated_geometry_strategy",
|
||||
"edit_strategy.semantic_parameter.search_validated_safe_range",
|
||||
"failure_repair.semantic_edit.merge_duplicate_cylindrical_targets",
|
||||
"edit_strategy.semantic_parameter.source_independent_overall_scale",
|
||||
"edit_strategy.semantic_parameter.primary_axis_affine_dimensions",
|
||||
"edit_strategy.semantic_parameter.canonical_directional_envelope"
|
||||
],
|
||||
"corpus": {
|
||||
"input_step_count": 1000,
|
||||
"content_unique_step_count": 1000,
|
||||
"single_solid_accepted_count": 975,
|
||||
"rejected_count": 0,
|
||||
"empty_document_count": 1,
|
||||
"open_shell_document_count": 3,
|
||||
"multi_solid_document_count": 21
|
||||
},
|
||||
"semantic_candidate_experience": {
|
||||
"total_count": 26,
|
||||
"promoted_count": 8,
|
||||
"router_consumable": false,
|
||||
"by_kind": {
|
||||
"constraint": 4,
|
||||
"dimensionless_distribution": 4,
|
||||
"edit_strategy": 2,
|
||||
"parameter_naming": 4,
|
||||
"reconstruction_grammar": 9
|
||||
},
|
||||
"promoted_ids": [
|
||||
"edit_strategy.semantic_cylindrical_cut.enlarge_diameter",
|
||||
"failure_repair.semantic_edit.boundary_strategy_fallback",
|
||||
"edit_strategy.semantic_edit.preserve_validated_geometry_strategy",
|
||||
"edit_strategy.semantic_parameter.search_validated_safe_range",
|
||||
"failure_repair.semantic_edit.merge_duplicate_cylindrical_targets",
|
||||
"edit_strategy.semantic_parameter.source_independent_overall_scale",
|
||||
"edit_strategy.semantic_parameter.primary_axis_affine_dimensions",
|
||||
"edit_strategy.semantic_parameter.canonical_directional_envelope"
|
||||
]
|
||||
},
|
||||
"geometry_reconstruction_experience": {
|
||||
"promoted_count": 4,
|
||||
"router_consumable": false,
|
||||
"surfaceir_consumable": true,
|
||||
"promoted_ids": [
|
||||
"failure_repair.surface_boundary.prefer_exact_3d",
|
||||
"failure_repair.periodic_torus.rectangular_uv_patch",
|
||||
"failure_repair.surface_topology.bind_face_pcurves_and_group_shells",
|
||||
"failure_repair.surface_topology.preserve_vertex_tolerance"
|
||||
],
|
||||
"note": "These methods have positive held-out geometry A/B evidence. They do not count as proof of semantic parameter-edit capability."
|
||||
},
|
||||
"semantic_parameter_results": [
|
||||
{
|
||||
"canonical_parameter": "plate_thickness",
|
||||
"support": 240,
|
||||
"conditional_confidence": 1.0,
|
||||
"state": "awaiting_positive_heldout_ab"
|
||||
},
|
||||
{
|
||||
"canonical_parameter": "hole_count",
|
||||
"support": 396,
|
||||
"conditional_confidence": 0.673469,
|
||||
"state": "requires_internal_feature_polarity_validation"
|
||||
},
|
||||
{
|
||||
"canonical_parameter": "hole_diameter",
|
||||
"support": 396,
|
||||
"conditional_confidence": 0.673469,
|
||||
"state": "requires_internal_feature_polarity_validation"
|
||||
},
|
||||
{
|
||||
"canonical_parameter": "bolt_circle_diameter",
|
||||
"support": 121,
|
||||
"conditional_confidence": 0.456604,
|
||||
"state": "awaiting_positive_heldout_ab"
|
||||
}
|
||||
],
|
||||
"legacy_ab_warning": {
|
||||
"completed_pair_count": 15,
|
||||
"planned_pair_count": 50,
|
||||
"baseline_similarity_mean": 69.11,
|
||||
"experience_similarity_mean": 66.64,
|
||||
"score_delta": -2.47,
|
||||
"accepted_as_positive_validation": false
|
||||
},
|
||||
"promotion_gate": {
|
||||
"minimum_independent_cases_per_method": 3,
|
||||
"minimum_heldout_ab_pairs": 10,
|
||||
"edit_contract_pass_rate_required": 1.0,
|
||||
"baseline_score_delta_must_be_positive": true,
|
||||
"evidence_support_alone_can_promote": false
|
||||
},
|
||||
"execution_validated_trajectory_pilot": {
|
||||
"scope": "low_complexity_initial_debugging_subset_not_full_corpus",
|
||||
"attempted_case_count": 6,
|
||||
"replay_validated_case_count": 3,
|
||||
"quarantined_case_count": 3,
|
||||
"derived_candidate_method_count": 1,
|
||||
"officially_promoted_method_count": 0,
|
||||
"candidate_method": "simple cylinder classification, placement, and semantic parameter naming",
|
||||
"heldout_ab": {
|
||||
"pair_count": 2,
|
||||
"baseline_replay_validated_count": 1,
|
||||
"experience_replay_validated_count": 1,
|
||||
"baseline_pass_rate": 0.5,
|
||||
"experience_pass_rate": 0.5,
|
||||
"pass_rate_delta": 0.0,
|
||||
"positive": false,
|
||||
"promotion_eligible": false
|
||||
},
|
||||
"conclusion": "The new loop can produce independently rebuildable editable trajectories, but no capability improvement has yet been demonstrated. The candidate remains unavailable to cad-router."
|
||||
},
|
||||
"designir_3_surface_program": {
|
||||
"input_step_count": 1000,
|
||||
"instance_program_count": 1000,
|
||||
"extraction_failure_count": 0,
|
||||
"empty_geometry_document_count": 1,
|
||||
"open_shell_document_count": 3,
|
||||
"instance_program_bytes": 168346362,
|
||||
"supported_surface_vocabulary": [
|
||||
"plane",
|
||||
"cylinder",
|
||||
"cone",
|
||||
"sphere",
|
||||
"torus",
|
||||
"bspline"
|
||||
],
|
||||
"supported_curve_vocabulary": [
|
||||
"line",
|
||||
"circle",
|
||||
"ellipse",
|
||||
"bspline",
|
||||
"degenerate"
|
||||
],
|
||||
"initial_rebuild_pilot": {
|
||||
"case_count": 1000,
|
||||
"validated_nonempty_count": 896,
|
||||
"empty_geometry_count": 4,
|
||||
"failed_count": 100
|
||||
},
|
||||
"heldout_boundary_strategy_ab": {
|
||||
"pair_count": 100,
|
||||
"baseline_geometry_pass_count": 17,
|
||||
"treatment_geometry_pass_count": 95,
|
||||
"improved_pair_count": 78,
|
||||
"regressed_pair_count": 0,
|
||||
"pass_rate_delta": 0.78
|
||||
},
|
||||
"promoted_experience_ids": [
|
||||
"failure_repair.surface_boundary.prefer_exact_3d",
|
||||
"failure_repair.periodic_torus.rectangular_uv_patch",
|
||||
"failure_repair.surface_topology.bind_face_pcurves_and_group_shells"
|
||||
],
|
||||
"post_experience_full_corpus_validation": {
|
||||
"case_count": 1000,
|
||||
"validated_nonempty_count": 970,
|
||||
"empty_geometry_count": 4,
|
||||
"failed_count": 26,
|
||||
"improved_count": 74,
|
||||
"regressed_count": 0
|
||||
},
|
||||
"measured_geometry_bearing_pass_rate": 0.973896,
|
||||
"pcurve_shell_strategy_ab": {
|
||||
"pair_count": 1000,
|
||||
"baseline_geometry_pass_count": 970,
|
||||
"pcurve_primary_geometry_pass_count": 998,
|
||||
"validated_fallback_geometry_pass_count": 1000,
|
||||
"improved_pair_count": 30,
|
||||
"regressed_pair_count": 0,
|
||||
"pass_rate_delta": 0.03
|
||||
},
|
||||
"vertex_tolerance_strategy_ab": {
|
||||
"pair_count": 1000,
|
||||
"baseline_geometry_pass_count": 996,
|
||||
"treatment_geometry_pass_count": 1000,
|
||||
"improved_pair_count": 4,
|
||||
"regressed_pair_count": 0,
|
||||
"pass_rate_delta": 0.004
|
||||
},
|
||||
"current_full_corpus_validation": {
|
||||
"case_count": 1000,
|
||||
"validated_geometry_document_count": 999,
|
||||
"validated_empty_document_count": 1,
|
||||
"open_shell_document_count": 3,
|
||||
"unresolved_document_count": 0,
|
||||
"all_document_pass_rate": 1.0,
|
||||
"selected_strategy_counts": {
|
||||
"exact_3d_pcurve": 1000
|
||||
},
|
||||
"boolean_fallback_surface_sample_case_count": 2,
|
||||
"maximum_accepted_bidirectional_surface_distance_mm": 0.0000647066
|
||||
},
|
||||
"semantic_edit_contract": {
|
||||
"promoted_experience_count": 8,
|
||||
"status": "positive_local_cut_primary_axis_dimension_and_universal_scale_edits_demonstrated",
|
||||
"semantic_candidate_case_count": 942,
|
||||
"semantic_parameter_instance_count": 5136,
|
||||
"semantic_feature_instance_count": 2502,
|
||||
"heldout_case_count": 50,
|
||||
"heldout_applicable_case_count": 41,
|
||||
"heldout_accepted_case_count": 41,
|
||||
"heldout_edit_pass_rate": 1.0,
|
||||
"primary_boundary_strategy_pass_count": 35,
|
||||
"fallback_rescue_count": 6,
|
||||
"full_corpus_validation": {
|
||||
"case_count": 1000,
|
||||
"baseline_geometry_not_validated_count": 30,
|
||||
"no_executable_semantic_parameter_count": 155,
|
||||
"applicable_edit_case_count": 815,
|
||||
"accepted_edit_case_count": 795,
|
||||
"rejected_edit_case_count": 20,
|
||||
"applicable_edit_pass_rate": 0.97546,
|
||||
"execution_failure_count": 19,
|
||||
"non_target_guard_rejection_count": 1,
|
||||
"post_observer_repair": {
|
||||
"accepted_edit_case_count": 800,
|
||||
"remaining_failed_case_count": 15,
|
||||
"applicable_edit_pass_rate": 0.981595,
|
||||
"rescued_case_count": 5,
|
||||
"regressed_case_count": 0
|
||||
},
|
||||
"post_geometry_strategy_revalidation": {
|
||||
"baseline_geometry_not_validated_count": 0,
|
||||
"no_executable_semantic_parameter_count": 157,
|
||||
"applicable_edit_case_count": 843,
|
||||
"accepted_edit_case_count": 826,
|
||||
"remaining_failed_case_count": 17,
|
||||
"applicable_edit_pass_rate": 0.979834,
|
||||
"newly_accepted_case_count": 26,
|
||||
"regressed_case_count": 0
|
||||
},
|
||||
"post_adaptive_range_and_duplicate_target_repair": {
|
||||
"applicable_edit_case_count": 843,
|
||||
"accepted_edit_case_count": 837,
|
||||
"remaining_failed_case_count": 6,
|
||||
"applicable_edit_pass_rate": 0.992883,
|
||||
"validated_positive_factors": [
|
||||
1.02,
|
||||
1.05,
|
||||
1.1
|
||||
],
|
||||
"newly_accepted_case_count": 11,
|
||||
"regressed_case_count": 0,
|
||||
"duplicate_target_heldout_ab": {
|
||||
"pair_count": 11,
|
||||
"baseline_pass_count": 9,
|
||||
"treatment_pass_count": 11,
|
||||
"regressed_pair_count": 0
|
||||
}
|
||||
},
|
||||
"post_source_independent_overall_scale": {
|
||||
"geometry_bearing_case_count": 996,
|
||||
"accepted_scale_edit_case_count": 996,
|
||||
"rejected_scale_edit_case_count": 0,
|
||||
"scale_edit_pass_rate": 1.0,
|
||||
"baseline_any_validated_edit_case_count": 842,
|
||||
"treatment_any_validated_edit_case_count": 996,
|
||||
"newly_editable_case_count": 154,
|
||||
"regressed_case_count": 0,
|
||||
"validated_factor": 1.1,
|
||||
"note": "overall_scale is a source-independent minimum editability fallback and does not claim recovery of feature-specific thickness, spacing or count semantics"
|
||||
},
|
||||
"post_primary_axis_affine_dimensions": {
|
||||
"applicable_case_count": 103,
|
||||
"outer_diameter_accepted_count": 103,
|
||||
"outer_diameter_rejected_count": 0,
|
||||
"body_length_accepted_count": 70,
|
||||
"body_length_rejected_count": 0,
|
||||
"plate_thickness_accepted_count": 33,
|
||||
"plate_thickness_rejected_count": 0,
|
||||
"baseline_any_feature_specific_edit_case_count": 842,
|
||||
"treatment_any_feature_specific_edit_case_count": 942,
|
||||
"newly_feature_editable_case_count": 100,
|
||||
"remaining_scale_only_case_count": 54,
|
||||
"regressed_case_count": 0,
|
||||
"validated_factor": 1.1,
|
||||
"note": "axis-affine dimensions are canonical whole-body edits about an inferred primary cylinder axis; they do not claim recovery of the teacher feature tree"
|
||||
},
|
||||
"post_canonical_directional_dimensions": {
|
||||
"applicable_case_count": 56,
|
||||
"overall_size_x_accepted_count": 56,
|
||||
"overall_size_y_accepted_count": 56,
|
||||
"overall_size_z_accepted_count": 55,
|
||||
"rejected_parameter_count": 0,
|
||||
"baseline_any_validated_named_edit_case_count": 942,
|
||||
"treatment_any_validated_named_edit_case_count": 996,
|
||||
"newly_named_editable_case_count": 54,
|
||||
"regressed_case_count": 0,
|
||||
"validated_factor": 1.1,
|
||||
"note": "overall_size_x/y/z are canonical envelope dimensions in the DesignIR coordinate system, not recovered source sketch dimensions"
|
||||
},
|
||||
"final_document_and_edit_coverage": {
|
||||
"input_document_count": 1000,
|
||||
"independently_rebuilt_document_count": 1000,
|
||||
"geometry_present_document_count": 999,
|
||||
"true_empty_document_count": 1,
|
||||
"geometry_documents_with_at_least_one_validated_named_parameter": 999,
|
||||
"validated_named_parameter_instance_count": 1213,
|
||||
"geometry_document_named_edit_coverage": 1.0,
|
||||
"anti_cheat_match_count": 0,
|
||||
"designir_schema_valid_count": 1000,
|
||||
"designir_schema_invalid_count": 0
|
||||
}
|
||||
},
|
||||
"required_next_evidence": [
|
||||
"negative diameter edits",
|
||||
"hole spacing and pitch diameter edits",
|
||||
"plate thickness edits",
|
||||
"additional non-cylindrical feature families",
|
||||
"full-corpus edit acceptance"
|
||||
]
|
||||
},
|
||||
"remaining_primary_failure_clusters": []
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user