1267 lines
43 KiB
JSON
1267 lines
43 KiB
JSON
{
|
|
"status": "failed",
|
|
"author": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.4-mini"
|
|
},
|
|
"reviewer": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.5"
|
|
},
|
|
"author_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 1288
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"reviewer_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 533
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"results": [
|
|
{
|
|
"scenario": "rectangular_plate",
|
|
"repetition": 1,
|
|
"success": true,
|
|
"started_at": "2026-08-31T10:19:19.704593+00:00",
|
|
"finished_at": "2026-08-31T10:20:17.376308+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_2039eb170f23",
|
|
"lifecycle": "completed",
|
|
"revisionId": "rev_001"
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_2039eb170f23",
|
|
"phase": "COMPLETED",
|
|
"lifecycle": "completed",
|
|
"state_version": 9,
|
|
"active_revision": "rev_001",
|
|
"current_revision": "rev_001",
|
|
"published_revision": "rev_001",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 1,
|
|
"at": "2026-08-31 10:19:27",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 2,
|
|
"at": "2026-08-31 10:19:31",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 3,
|
|
"at": "2026-08-31 10:19:37",
|
|
"contract_hash": "a65da6add55ca1a2db47b72c8afb329ba61239589f3ec7e5d3875c87f90730ae",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 4,
|
|
"at": "2026-08-31 10:19:45",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create one connected solid rectangular plate with overall dimensions 80 mm by 50 mm by 8 mm in the current model.",
|
|
"intent": "Form the required base plate as a single connected part matching the specified millimetre dimensions.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_2039eb170f23:root:v3"
|
|
},
|
|
{
|
|
"sequence": 5,
|
|
"at": "2026-08-31 10:19:57",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_86a9ab7071766aa8",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb"
|
|
},
|
|
{
|
|
"sequence": 6,
|
|
"at": "2026-08-31 10:19:58",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_86a9ab7071766aa8",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 7,
|
|
"at": "2026-08-31 10:20:06",
|
|
"action_id": "act_004",
|
|
"actual_atomic_id": "extrude_add_blind",
|
|
"candidate_id": "candidate_86a9ab7071766aa8",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb",
|
|
"review_path": "revisions/rev_001/candidate-review.json",
|
|
"revision_id": "rev_001",
|
|
"selector_snapshot_id": "",
|
|
"working_head_before": "cad_2039eb170f23:root:v3"
|
|
},
|
|
{
|
|
"sequence": 8,
|
|
"at": "2026-08-31 10:20:09",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "final_validation_requested",
|
|
"revision_id": "rev_001"
|
|
},
|
|
{
|
|
"sequence": 9,
|
|
"at": "2026-08-31 10:20:17",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "completed",
|
|
"final_review_path": "revisions/rev_001/final-review.json",
|
|
"revision_id": "rev_001"
|
|
}
|
|
],
|
|
"revisions": [
|
|
{
|
|
"revision_id": "rev_001",
|
|
"status": "success",
|
|
"visibility": "final",
|
|
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
|
"step_path": "revisions/rev_001/model.step",
|
|
"glb_path": "revisions/rev_001/model.glb",
|
|
"report_path": "revisions/rev_001/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_001/candidate-review.json"
|
|
}
|
|
]
|
|
},
|
|
"usage": {
|
|
"calls": 9,
|
|
"prompt_tokens": 72947,
|
|
"completion_tokens": 1889,
|
|
"context_chars": 51372,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 334,
|
|
"context_chars": 878,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5234,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "7eb460d7a0240064a62a1238b05a1654d3c3e6b0cc99c08ced5a29f3f8a6dc28",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5568
|
|
},
|
|
{
|
|
"completion_tokens": 79,
|
|
"context_chars": 1684,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5442,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5521
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1172,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5005,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5076
|
|
},
|
|
{
|
|
"completion_tokens": 402,
|
|
"context_chars": 1282,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4961,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "fbc8fc3885260b3df2ea39c080a46cc7469e153ba405c80d6c5ae926eedbd4c5",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5363
|
|
},
|
|
{
|
|
"completion_tokens": 213,
|
|
"context_chars": 2350,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5142,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "267689e4fda03af2c5ca2311999445bcaae217f3ea424045ef748e2400285d14",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 5355
|
|
},
|
|
{
|
|
"completion_tokens": 262,
|
|
"context_chars": 7172,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6771,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "fdbe1ee1757bcdca738ee8d347cb75ecb0fd5efc9e9066ab47678185ab3b5ab3",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 7033
|
|
},
|
|
{
|
|
"completion_tokens": 239,
|
|
"context_chars": 21930,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19246,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "e8b0a855c46120aef598aa14e37a7d6ce99d38c654a40ac10767f2810e69556d",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19485
|
|
},
|
|
{
|
|
"completion_tokens": 50,
|
|
"context_chars": 4102,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6075,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "complete_task",
|
|
"total_tokens": 6125
|
|
},
|
|
{
|
|
"completion_tokens": 239,
|
|
"context_chars": 10802,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 15071,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "ceba073ff2dae05b9c08ee83af5d12d9b538505e6f778cbec4cb681b410a6396",
|
|
"role": "reviewer",
|
|
"tool": "review_final",
|
|
"total_tokens": 15310
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": true,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": true,
|
|
"required_operations": true,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": true,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": true
|
|
},
|
|
"rejection_codes": [],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T101800Z/artifacts/cad_2039eb170f23"
|
|
},
|
|
{
|
|
"scenario": "simple_flange",
|
|
"repetition": 1,
|
|
"success": false,
|
|
"started_at": "2026-08-31T10:20:17.378094+00:00",
|
|
"finished_at": "2026-08-31T10:22:33.804876+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_ef8202e48077",
|
|
"lifecycle": "completed",
|
|
"revisionId": "rev_002"
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_ef8202e48077",
|
|
"phase": "COMPLETED",
|
|
"lifecycle": "completed",
|
|
"state_version": 13,
|
|
"active_revision": "rev_002",
|
|
"current_revision": "rev_002",
|
|
"published_revision": "rev_002",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 11,
|
|
"at": "2026-08-31 10:20:34",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 12,
|
|
"at": "2026-08-31 10:20:41",
|
|
"contract_hash": "5b6ba3dbd137e308cf1b0ac6a37726d3a7ea5a94a37e47e9fef0395115aa4912",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 13,
|
|
"at": "2026-08-31 10:20:48",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create a 120 mm diameter cylindrical flange body 12 mm thick from the root sketch/profile, establishing the main connected solid in millimetres.",
|
|
"intent": "Build the base flange solid before adding the centered through bore.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_ef8202e48077:root:v3"
|
|
},
|
|
{
|
|
"sequence": 14,
|
|
"at": "2026-08-31 10:21:07",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_644388a67554ac7d",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078"
|
|
},
|
|
{
|
|
"sequence": 15,
|
|
"at": "2026-08-31 10:21:09",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_644388a67554ac7d",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"reason": "the target bore has not been introduced at this checkpoint"
|
|
},
|
|
"status": "pending"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 16,
|
|
"at": "2026-08-31 10:21:19",
|
|
"action_id": "act_004",
|
|
"actual_atomic_id": "extrude_add_blind",
|
|
"candidate_id": "candidate_644388a67554ac7d",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"reason": "the target bore has not been introduced at this checkpoint"
|
|
},
|
|
"status": "pending"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078",
|
|
"review_path": "revisions/rev_001/candidate-review.json",
|
|
"revision_id": "rev_001",
|
|
"selector_snapshot_id": "",
|
|
"working_head_before": "cad_ef8202e48077:root:v3"
|
|
},
|
|
{
|
|
"sequence": 17,
|
|
"at": "2026-08-31 10:21:29",
|
|
"action_id": "act_008",
|
|
"atomic_id": "hole_blind",
|
|
"contract_hash": "f62e12603feeca6d7e83acdab651fcb023e08a46c39fd1509e9dcb81d4f0c721",
|
|
"event": "proposed",
|
|
"expected_change": "Add one centered 40 mm diameter cylindrical hole through the 12 mm flange thickness, removing material from the existing solid so the bore passes fully through the part.",
|
|
"intent": "Create the required centered through bore in the finished flange body.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_ef8202e48077:rev_001:v7"
|
|
},
|
|
{
|
|
"sequence": 18,
|
|
"at": "2026-08-31 10:22:00",
|
|
"action_id": "act_008",
|
|
"candidate_id": "candidate_a6ed08c266993269",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "3eab7f4f6742d6ebc32b5982299b1f4d0c009e306cc43df00e69a10bfafa0998"
|
|
},
|
|
{
|
|
"sequence": 19,
|
|
"at": "2026-08-31 10:22:02",
|
|
"action_id": "act_008",
|
|
"candidate_id": "candidate_a6ed08c266993269",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 20,
|
|
"at": "2026-08-31 10:22:12",
|
|
"action_id": "act_008",
|
|
"actual_atomic_id": "hole_blind",
|
|
"candidate_id": "candidate_a6ed08c266993269",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "3eab7f4f6742d6ebc32b5982299b1f4d0c009e306cc43df00e69a10bfafa0998",
|
|
"review_path": "revisions/rev_002/candidate-review.json",
|
|
"revision_id": "rev_002",
|
|
"selector_snapshot_id": "cad_ef8202e48077/candidate_644388a67554ac7d",
|
|
"working_head_before": "cad_ef8202e48077:rev_001:v7"
|
|
},
|
|
{
|
|
"sequence": 21,
|
|
"at": "2026-08-31 10:22:22",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "final_validation_requested",
|
|
"revision_id": "rev_002"
|
|
},
|
|
{
|
|
"sequence": 22,
|
|
"at": "2026-08-31 10:22:33",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.01
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_007",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "completed",
|
|
"final_review_path": "revisions/rev_002/final-review.json",
|
|
"revision_id": "rev_002"
|
|
}
|
|
],
|
|
"revisions": [
|
|
{
|
|
"revision_id": "rev_001",
|
|
"status": "success",
|
|
"visibility": "checkpoint",
|
|
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
|
"step_path": "revisions/rev_001/model.step",
|
|
"glb_path": "revisions/rev_001/model.glb",
|
|
"report_path": "revisions/rev_001/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_001/candidate-review.json"
|
|
},
|
|
{
|
|
"revision_id": "rev_002",
|
|
"status": "success",
|
|
"visibility": "final",
|
|
"cdsl_path": "revisions/rev_002/model.cdsl.json",
|
|
"step_path": "revisions/rev_002/model.step",
|
|
"glb_path": "revisions/rev_002/model.glb",
|
|
"report_path": "revisions/rev_002/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_002/candidate-review.json"
|
|
}
|
|
]
|
|
},
|
|
"usage": {
|
|
"calls": 16,
|
|
"prompt_tokens": 133213,
|
|
"completion_tokens": 4905,
|
|
"context_chars": 118349,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 736,
|
|
"context_chars": 934,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5242,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "ddbd5623b25d8561fa8c0804534deb5f39b1d5abe0290f8478357398e88e2903",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5978
|
|
},
|
|
{
|
|
"completion_tokens": 68,
|
|
"context_chars": 2073,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5541,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5609
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1509,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5100,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5171
|
|
},
|
|
{
|
|
"completion_tokens": 288,
|
|
"context_chars": 1355,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4985,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "ae240facd59e8e4741fb47fb699b532def00adcc8b14f5a432721a44e801cc99",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5273
|
|
},
|
|
{
|
|
"completion_tokens": 206,
|
|
"context_chars": 2420,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5162,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "0104a85097dfd4dee5706a118dcf463106e3d43d89481d0861c4e5b9a2f6c281",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 5368
|
|
},
|
|
{
|
|
"completion_tokens": 144,
|
|
"context_chars": 2786,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5259,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "0e4c1562f9031cfb60f2c36487c44eee29c82c778133d4e138041a0d4d8cf6de",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 5403
|
|
},
|
|
{
|
|
"completion_tokens": 283,
|
|
"context_chars": 7608,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6888,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "31c65cdd0f45421d4fc38984c788e521d6ed99c8fb8fa58ab2ba62bc87526b1e",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 7171
|
|
},
|
|
{
|
|
"completion_tokens": 305,
|
|
"context_chars": 22656,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19473,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "8c74bce21c96269b82130cb7cb35f2d318e9714599599ba9e583ed3cf306aa08",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19778
|
|
},
|
|
{
|
|
"completion_tokens": 456,
|
|
"context_chars": 4980,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6281,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "cd87015d21863d5a301c5ed21a12322360c851cad3204d7fa6047627e6327624",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 6737
|
|
},
|
|
{
|
|
"completion_tokens": 376,
|
|
"context_chars": 5578,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6066,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "89abd3374b1f6eb20bc93717469a243f0596f4619f3540fe65d20025f657f1b6",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 6442
|
|
},
|
|
{
|
|
"completion_tokens": 283,
|
|
"context_chars": 5950,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6167,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f63a33f3d06df5403ee42eae6e3d3be1797c73ade7175743eac89dc1253982f5",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 6450
|
|
},
|
|
{
|
|
"completion_tokens": 194,
|
|
"context_chars": 9347,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 7246,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "4c513ec5bbdbc82a26d93652e12cbb9f7e5b2fa11c90f3963e5a8899d28cf3e7",
|
|
"tool": "inspect_topology",
|
|
"total_tokens": 7440
|
|
},
|
|
{
|
|
"completion_tokens": 321,
|
|
"context_chars": 11038,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 8406,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "6d796108b832bc3d33155fc09a27913120112d7d863343548f45231e2ee31ce8",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 8727
|
|
},
|
|
{
|
|
"completion_tokens": 336,
|
|
"context_chars": 23291,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19661,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "7d94d745a72edef2f342e0470e32e25283b54939634b6a7bdc779186a8955bf3",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19997
|
|
},
|
|
{
|
|
"completion_tokens": 515,
|
|
"context_chars": 5037,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6331,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "complete_task",
|
|
"total_tokens": 6846
|
|
},
|
|
{
|
|
"completion_tokens": 323,
|
|
"context_chars": 11787,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 15405,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f7dbfe4000ef981bc0266908c05d1b3c3e9a7785d9e35fd6376ac90b28b092b6",
|
|
"role": "reviewer",
|
|
"tool": "review_final",
|
|
"total_tokens": 15728
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": true,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": true,
|
|
"required_operations": true,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": true,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": false
|
|
},
|
|
"rejection_codes": [
|
|
"STALE_WORKING_HEAD",
|
|
"STALE_WORKING_HEAD"
|
|
],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T101800Z/artifacts/cad_ef8202e48077"
|
|
}
|
|
],
|
|
"suite": "smoke",
|
|
"require_live": true,
|
|
"report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T101800Z"
|
|
} |