1190 lines
40 KiB
JSON
1190 lines
40 KiB
JSON
{
|
|
"status": "passed",
|
|
"author": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.4-mini"
|
|
},
|
|
"reviewer": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.5"
|
|
},
|
|
"author_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 1372
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"reviewer_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 543
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"results": [
|
|
{
|
|
"scenario": "rectangular_plate",
|
|
"repetition": 1,
|
|
"success": true,
|
|
"started_at": "2026-08-31T10:28:37.832406+00:00",
|
|
"finished_at": "2026-08-31T10:29:32.753023+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_212841a7e248",
|
|
"lifecycle": "completed",
|
|
"revisionId": "rev_001"
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_212841a7e248",
|
|
"phase": "COMPLETED",
|
|
"lifecycle": "completed",
|
|
"state_version": 9,
|
|
"active_revision": "rev_001",
|
|
"current_revision": "rev_001",
|
|
"published_revision": "rev_001",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 1,
|
|
"at": "2026-08-31 10:28:45",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 2,
|
|
"at": "2026-08-31 10:28:48",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 3,
|
|
"at": "2026-08-31 10:28:52",
|
|
"contract_hash": "891abb525827ad48707b2e7aced262443318e11d71106959adca8e302827910d",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 4,
|
|
"at": "2026-08-31 10:28:59",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create a single connected solid rectangular plate measuring 80 mm by 50 mm by 8 mm.",
|
|
"intent": "Satisfy req_001 by adding the plate as one connected prismatic body in millimetres.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_212841a7e248:root:v3"
|
|
},
|
|
{
|
|
"sequence": 5,
|
|
"at": "2026-08-31 10:29:10",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_c364b4c5678722fb",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb"
|
|
},
|
|
{
|
|
"sequence": 6,
|
|
"at": "2026-08-31 10:29:11",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_c364b4c5678722fb",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 7,
|
|
"at": "2026-08-31 10:29:20",
|
|
"action_id": "act_004",
|
|
"actual_atomic_id": "extrude_add_blind",
|
|
"candidate_id": "candidate_c364b4c5678722fb",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb",
|
|
"review_path": "revisions/rev_001/candidate-review.json",
|
|
"revision_id": "rev_001",
|
|
"selector_snapshot_id": "",
|
|
"working_head_before": "cad_212841a7e248:root:v3"
|
|
},
|
|
{
|
|
"sequence": 8,
|
|
"at": "2026-08-31 10:29:24",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "final_validation_requested",
|
|
"revision_id": "rev_001"
|
|
},
|
|
{
|
|
"sequence": 9,
|
|
"at": "2026-08-31 10:29:32",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "completed",
|
|
"final_review_path": "revisions/rev_001/final-review.json",
|
|
"revision_id": "rev_001"
|
|
}
|
|
],
|
|
"revisions": [
|
|
{
|
|
"revision_id": "rev_001",
|
|
"status": "success",
|
|
"visibility": "final",
|
|
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
|
"step_path": "revisions/rev_001/model.step",
|
|
"glb_path": "revisions/rev_001/model.glb",
|
|
"report_path": "revisions/rev_001/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_001/candidate-review.json"
|
|
}
|
|
]
|
|
},
|
|
"usage": {
|
|
"calls": 9,
|
|
"prompt_tokens": 72883,
|
|
"completion_tokens": 1811,
|
|
"context_chars": 51142,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 334,
|
|
"context_chars": 878,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5234,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f0d6d9b69102c3fc061fb4cbbaf5b5cfe67da64ee41a8fd30eb6e28baa2b4a23",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5568
|
|
},
|
|
{
|
|
"completion_tokens": 82,
|
|
"context_chars": 1700,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5447,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5529
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1186,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5010,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5081
|
|
},
|
|
{
|
|
"completion_tokens": 302,
|
|
"context_chars": 1282,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4953,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "380ec9dea594d2498f7028273894bfdf09c4f6c30024c87aa1b3da4bd5012d33",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5255
|
|
},
|
|
{
|
|
"completion_tokens": 135,
|
|
"context_chars": 2254,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5104,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f4def2da351315b270f551588444f6f615a5d4b182249e4dbad7e8ee3e6f2f34",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 5239
|
|
},
|
|
{
|
|
"completion_tokens": 248,
|
|
"context_chars": 7076,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6759,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "fdbe1ee1757bcdca738ee8d347cb75ecb0fd5efc9e9066ab47678185ab3b5ab3",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 7007
|
|
},
|
|
{
|
|
"completion_tokens": 272,
|
|
"context_chars": 21896,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19238,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "fbd289b4eb30ffa7e584f08d8c51d013513cae871cb041a0141b2be0597e42c6",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19510
|
|
},
|
|
{
|
|
"completion_tokens": 122,
|
|
"context_chars": 4054,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6070,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "complete_task",
|
|
"total_tokens": 6192
|
|
},
|
|
{
|
|
"completion_tokens": 245,
|
|
"context_chars": 10816,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 15068,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "e10e72c02bb0495ebd65b1b2557730b3326c5c30ad1a73b6ed631cf476d8f190",
|
|
"role": "reviewer",
|
|
"tool": "review_final",
|
|
"total_tokens": 15313
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": true,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": true,
|
|
"required_operations": true,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": true,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": true
|
|
},
|
|
"rejection_codes": [],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T102709Z/artifacts/cad_212841a7e248"
|
|
},
|
|
{
|
|
"scenario": "simple_flange",
|
|
"repetition": 1,
|
|
"success": true,
|
|
"started_at": "2026-08-31T10:29:32.755510+00:00",
|
|
"finished_at": "2026-08-31T10:31:20.753138+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_71d72d14b433",
|
|
"lifecycle": "completed",
|
|
"revisionId": "rev_002"
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_71d72d14b433",
|
|
"phase": "COMPLETED",
|
|
"lifecycle": "completed",
|
|
"state_version": 13,
|
|
"active_revision": "rev_002",
|
|
"current_revision": "rev_002",
|
|
"published_revision": "rev_002",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 11,
|
|
"at": "2026-08-31 10:29:46",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 12,
|
|
"at": "2026-08-31 10:29:53",
|
|
"contract_hash": "af85f72e5d5efe317ca1f398b9676e21a4be90a83af01f5a6bad893c16a339b6",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 13,
|
|
"at": "2026-08-31 10:30:02",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create a solid round flange body in millimetres with outer diameter 120 mm and thickness 12 mm.",
|
|
"intent": "Establish the main flange geometry as a single connected base solid before adding the centered bore.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_71d72d14b433:root:v3"
|
|
},
|
|
{
|
|
"sequence": 14,
|
|
"at": "2026-08-31 10:30:12",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_241334e701818a16",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078"
|
|
},
|
|
{
|
|
"sequence": 15,
|
|
"at": "2026-08-31 10:30:14",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_241334e701818a16",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"reason": "the target bore has not been introduced at this checkpoint"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 16,
|
|
"at": "2026-08-31 10:30:25",
|
|
"action_id": "act_004",
|
|
"actual_atomic_id": "extrude_add_blind",
|
|
"candidate_id": "candidate_241334e701818a16",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"reason": "the target bore has not been introduced at this checkpoint"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078",
|
|
"review_path": "revisions/rev_001/candidate-review.json",
|
|
"revision_id": "rev_001",
|
|
"selector_snapshot_id": "",
|
|
"working_head_before": "cad_71d72d14b433:root:v3"
|
|
},
|
|
{
|
|
"sequence": 17,
|
|
"at": "2026-08-31 10:30:32",
|
|
"action_id": "act_008",
|
|
"atomic_id": "hole_blind",
|
|
"contract_hash": "f62e12603feeca6d7e83acdab651fcb023e08a46c39fd1509e9dcb81d4f0c721",
|
|
"event": "proposed",
|
|
"expected_change": "Cut a centered cylindrical hole through the 12 mm flange with 40 mm diameter by extending the bore to the full material thickness.",
|
|
"intent": "Introduce the required centered through bore while preserving the single connected flange body.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_71d72d14b433:rev_001:v7"
|
|
},
|
|
{
|
|
"sequence": 18,
|
|
"at": "2026-08-31 10:30:53",
|
|
"action_id": "act_008",
|
|
"candidate_id": "candidate_a9836999e50ab5f3",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "1e77ee8e187829845c362464afb50cf9a5cb2689ec1bcea9cebc073f8fe5a2a1"
|
|
},
|
|
{
|
|
"sequence": 19,
|
|
"at": "2026-08-31 10:30:55",
|
|
"action_id": "act_008",
|
|
"candidate_id": "candidate_a9836999e50ab5f3",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 20,
|
|
"at": "2026-08-31 10:31:07",
|
|
"action_id": "act_008",
|
|
"actual_atomic_id": "hole_blind",
|
|
"candidate_id": "candidate_a9836999e50ab5f3",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "1e77ee8e187829845c362464afb50cf9a5cb2689ec1bcea9cebc073f8fe5a2a1",
|
|
"review_path": "revisions/rev_002/candidate-review.json",
|
|
"revision_id": "rev_002",
|
|
"selector_snapshot_id": "cad_71d72d14b433/candidate_241334e701818a16",
|
|
"working_head_before": "cad_71d72d14b433:rev_001:v7"
|
|
},
|
|
{
|
|
"sequence": 21,
|
|
"at": "2026-08-31 10:31:11",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "final_validation_requested",
|
|
"revision_id": "rev_002"
|
|
},
|
|
{
|
|
"sequence": 22,
|
|
"at": "2026-08-31 10:31:20",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000014,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.00000000000009,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"through_bores": [
|
|
"body:feature_002:face:2"
|
|
]
|
|
},
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"event": "completed",
|
|
"final_review_path": "revisions/rev_002/final-review.json",
|
|
"revision_id": "rev_002"
|
|
}
|
|
],
|
|
"revisions": [
|
|
{
|
|
"revision_id": "rev_001",
|
|
"status": "success",
|
|
"visibility": "checkpoint",
|
|
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
|
"step_path": "revisions/rev_001/model.step",
|
|
"glb_path": "revisions/rev_001/model.glb",
|
|
"report_path": "revisions/rev_001/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_001/candidate-review.json"
|
|
},
|
|
{
|
|
"revision_id": "rev_002",
|
|
"status": "success",
|
|
"visibility": "final",
|
|
"cdsl_path": "revisions/rev_002/model.cdsl.json",
|
|
"step_path": "revisions/rev_002/model.step",
|
|
"glb_path": "revisions/rev_002/model.glb",
|
|
"report_path": "revisions/rev_002/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_002/candidate-review.json"
|
|
}
|
|
]
|
|
},
|
|
"usage": {
|
|
"calls": 14,
|
|
"prompt_tokens": 120729,
|
|
"completion_tokens": 3514,
|
|
"context_chars": 105477,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 516,
|
|
"context_chars": 934,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5242,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f106c56ab0ea27466f7e159e2f0f8941b1ef392709960088d9349e5c818cfa11",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5758
|
|
},
|
|
{
|
|
"completion_tokens": 117,
|
|
"context_chars": 1905,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5507,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5624
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1353,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5066,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5137
|
|
},
|
|
{
|
|
"completion_tokens": 434,
|
|
"context_chars": 1352,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4982,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "32c39ab63762970c2aa6be8196f469052916f43b2793550b9033a97fcd959297",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5416
|
|
},
|
|
{
|
|
"completion_tokens": 145,
|
|
"context_chars": 2383,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5133,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "7fb08e885b4c06efee8b4c34fdfd5b8cf4b40f5bb5fad90974318c47d08c74e1",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 5278
|
|
},
|
|
{
|
|
"completion_tokens": 204,
|
|
"context_chars": 7205,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6788,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "af5dd7a546cf394eab4affaf0e3486f75fedc950354cbac3345229497aaa0a90",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 6992
|
|
},
|
|
{
|
|
"completion_tokens": 319,
|
|
"context_chars": 22297,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19257,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "748f9f858ad36266bab0b65a10b649758be386669461c6b8022a20756c021c0a",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19576
|
|
},
|
|
{
|
|
"completion_tokens": 311,
|
|
"context_chars": 4598,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6194,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "ea671a4ef923dd91d9a66b03e9ce6e6c8ac6fbc78c5f9e643397f31dd6fb9cf7",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 6505
|
|
},
|
|
{
|
|
"completion_tokens": 203,
|
|
"context_chars": 5185,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5947,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "1375c42503256d751af6b845ffc59d8cdf37fe498c467e8aa75754de5c06df14",
|
|
"tool": "get_cdsl_operation_contract",
|
|
"total_tokens": 6150
|
|
},
|
|
{
|
|
"completion_tokens": 243,
|
|
"context_chars": 8582,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 7041,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "9e3c866c98c64757af25324c376141c17bffd3fdc676360e83f429c5ed61fde4",
|
|
"tool": "inspect_topology",
|
|
"total_tokens": 7284
|
|
},
|
|
{
|
|
"completion_tokens": 260,
|
|
"context_chars": 10645,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 8291,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "6ed475989c3ce0338e53ec1302d94dc0f0473493ea3e3d9fdabe5886b14408ff",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 8551
|
|
},
|
|
{
|
|
"completion_tokens": 317,
|
|
"context_chars": 22935,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19692,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "d46b81f9b006e8049f65319a80368020a4de5cba0664896341afd30de5722d6f",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 20009
|
|
},
|
|
{
|
|
"completion_tokens": 70,
|
|
"context_chars": 4658,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6244,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "complete_task",
|
|
"total_tokens": 6314
|
|
},
|
|
{
|
|
"completion_tokens": 304,
|
|
"context_chars": 11445,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 15345,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "5c317600b3e1d40d78df0e3362d74e8d73fad3d281e4e96fbc333d6bb9bf9255",
|
|
"role": "reviewer",
|
|
"tool": "review_final",
|
|
"total_tokens": 15649
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": true,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": true,
|
|
"required_operations": true,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": true,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": true
|
|
},
|
|
"rejection_codes": [],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T102709Z/artifacts/cad_71d72d14b433"
|
|
}
|
|
],
|
|
"suite": "smoke",
|
|
"require_live": true,
|
|
"report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T102709Z"
|
|
} |