{
  "status": "failed",
  "author": {
    "provider": "openai",
    "model": "gpt-5.4-mini"
  },
  "reviewer": {
    "provider": "openai",
    "model": "gpt-5.5"
  },
  "author_capability": {
    "schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
    "tool_count": 9,
    "supported": true,
    "failures": [],
    "usage": {
      "prompt_tokens": 41789,
      "completion_tokens": 1156
    },
    "mode": "openai_compatible_tool_calling"
  },
  "reviewer_capability": {
    "schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
    "tool_count": 9,
    "supported": true,
    "failures": [],
    "usage": {
      "prompt_tokens": 41789,
      "completion_tokens": 541
    },
    "mode": "openai_compatible_tool_calling"
  },
  "results": [
    {
      "scenario": "rectangular_plate",
      "repetition": 1,
      "success": true,
      "started_at": "2026-08-31T09:49:06.244495+00:00",
      "finished_at": "2026-08-31T09:50:11.574252+00:00",
      "terminal": {
        "taskId": "cad_52c2f993c7e5",
        "lifecycle": "completed",
        "revisionId": "rev_001"
      },
      "projection": {
        "schema_version": "3.0",
        "task_id": "cad_52c2f993c7e5",
        "phase": "COMPLETED",
        "lifecycle": "completed",
        "state_version": 9,
        "active_revision": "rev_001",
        "current_revision": "rev_001",
        "published_revision": "rev_001",
        "pending_action": null,
        "active_candidate_id": "",
        "repair_required": false,
        "last_error": "",
        "action_ledger_summary": [
          {
            "sequence": 1,
            "at": "2026-08-31 09:49:13",
            "draft_revision": 1,
            "event": "requirements_draft_updated",
            "item_count": 1
          },
          {
            "sequence": 2,
            "at": "2026-08-31 09:49:18",
            "draft_revision": 1,
            "event": "requirements_review_requested"
          },
          {
            "sequence": 3,
            "at": "2026-08-31 09:49:22",
            "contract_hash": "42badf61b3707fe34319e2f2c145d0e64a781a656c8f37ad9552d96733095503",
            "event": "requirements_contract_frozen",
            "requirement_count": 1
          },
          {
            "sequence": 4,
            "at": "2026-08-31 09:49:31",
            "action_id": "act_004",
            "atomic_id": "extrude_add_blind",
            "contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
            "event": "proposed",
            "expected_change": "Create a single solid rectangular plate with footprint 80 mm by 50 mm and thickness 8 mm.",
            "intent": "Build the required base solid for the plate in one verifiable step.",
            "requirement_ids": [
              "req_001"
            ],
            "working_head": "cad_52c2f993c7e5:root:v3"
          },
          {
            "sequence": 5,
            "at": "2026-08-31 09:49:39",
            "action_id": "act_004",
            "candidate_id": "candidate_4642dbd9452f51e3",
            "event": "candidate_building",
            "fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb"
          },
          {
            "sequence": 6,
            "at": "2026-08-31 09:49:40",
            "action_id": "act_004",
            "candidate_id": "candidate_4642dbd9452f51e3",
            "claim_results": [
              {
                "claim_id": "claim_001",
                "claim_kind": "solid_count_equals",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_002",
                "claim_kind": "single_connected_body",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_003",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 80.0,
                  "axis": "x",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_004",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 50.0,
                  "axis": "y",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_005",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 8.0,
                  "axis": "z",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_006",
                "claim_kind": "visual",
                "deterministic": false,
                "evidence": {
                  "reason": "requires independent visual review"
                },
                "status": "pending"
              }
            ],
            "event": "candidate_built"
          },
          {
            "sequence": 7,
            "at": "2026-08-31 09:49:49",
            "action_id": "act_004",
            "actual_atomic_id": "extrude_add_blind",
            "candidate_id": "candidate_4642dbd9452f51e3",
            "coverage": [
              {
                "claim_id": "claim_001",
                "claim_kind": "solid_count_equals",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_002",
                "claim_kind": "single_connected_body",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_003",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 80.0,
                  "axis": "x",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_004",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 50.0,
                  "axis": "y",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_005",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 8.0,
                  "axis": "z",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_006",
                "claim_kind": "visual",
                "deterministic": false,
                "evidence": {
                  "reason": "requires independent visual review"
                },
                "status": "pending"
              }
            ],
            "event": "accepted",
            "fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb",
            "review_path": "revisions/rev_001/candidate-review.json",
            "revision_id": "rev_001",
            "selector_snapshot_id": "",
            "working_head_before": "cad_52c2f993c7e5:root:v3"
          },
          {
            "sequence": 8,
            "at": "2026-08-31 09:50:00",
            "claim_results": [
              {
                "claim_id": "claim_001",
                "claim_kind": "solid_count_equals",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_002",
                "claim_kind": "single_connected_body",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_003",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 80.0,
                  "axis": "x",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_004",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 50.0,
                  "axis": "y",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_005",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 8.0,
                  "axis": "z",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_006",
                "claim_kind": "visual",
                "deterministic": false,
                "evidence": {
                  "reason": "requires independent visual review"
                },
                "status": "pending"
              }
            ],
            "event": "final_validation_requested",
            "revision_id": "rev_001"
          },
          {
            "sequence": 9,
            "at": "2026-08-31 09:50:11",
            "claim_results": [
              {
                "claim_id": "claim_001",
                "claim_kind": "solid_count_equals",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_002",
                "claim_kind": "single_connected_body",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_003",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 80.0,
                  "axis": "x",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_004",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 50.0,
                  "axis": "y",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_005",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 8.0,
                  "axis": "z",
                  "tolerance_mm": 0.0
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_006",
                "claim_kind": "visual",
                "deterministic": false,
                "evidence": {
                  "reason": "requires independent visual review"
                },
                "status": "pending"
              }
            ],
            "event": "completed",
            "final_review_path": "revisions/rev_001/final-review.json",
            "revision_id": "rev_001"
          }
        ],
        "revisions": [
          {
            "revision_id": "rev_001",
            "status": "success",
            "visibility": "final",
            "cdsl_path": "revisions/rev_001/model.cdsl.json",
            "step_path": "revisions/rev_001/model.step",
            "glb_path": "revisions/rev_001/model.glb",
            "report_path": "revisions/rev_001/rebuild-report.json",
            "candidate_review_path": "revisions/rev_001/candidate-review.json"
          }
        ]
      },
      "usage": {
        "calls": 8,
        "prompt_tokens": 67140,
        "completion_tokens": 2343,
        "context_chars": 44387,
        "records": [
          {
            "completion_tokens": 256,
            "context_chars": 878,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5234,
            "provider_id": "openai",
            "raw_arguments_hash": "2e18aabb4b24b0d6a4d9f1e99aac7c625e266fb9de1b6cecc9ae0ff110b32f3a",
            "tool": "submit_requirements_draft_batch",
            "total_tokens": 5490
          },
          {
            "completion_tokens": 84,
            "context_chars": 1715,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5450,
            "provider_id": "openai",
            "raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
            "tool": "finalize_requirements_draft",
            "total_tokens": 5534
          },
          {
            "completion_tokens": 71,
            "context_chars": 1195,
            "model_id": "gpt-5.5",
            "prompt_tokens": 5013,
            "provider_id": "openai",
            "raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
            "role": "reviewer",
            "tool": "review_requirements",
            "total_tokens": 5084
          },
          {
            "completion_tokens": 412,
            "context_chars": 1301,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 4972,
            "provider_id": "openai",
            "raw_arguments_hash": "14880afd2112d4805a5e96df077239ec12139c82dbfce41f563fce53471cf537",
            "tool": "propose_next_action",
            "total_tokens": 5384
          },
          {
            "completion_tokens": 366,
            "context_chars": 2253,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5651,
            "provider_id": "openai",
            "raw_arguments_hash": "d52011a37183174b1d6e1937fdbede565bda754adb6894e35e85e2fe0883774a",
            "tool": "submit_cdsl_fragment",
            "total_tokens": 6017
          },
          {
            "completion_tokens": 258,
            "context_chars": 21618,
            "model_id": "gpt-5.5",
            "prompt_tokens": 19414,
            "provider_id": "openai",
            "raw_arguments_hash": "077328095f2b06d7dc28fa29ad1fcf5b876c88bc7f32b7298db29d24f048e722",
            "role": "reviewer",
            "tool": "review_candidate",
            "total_tokens": 19672
          },
          {
            "completion_tokens": 612,
            "context_chars": 4419,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 6172,
            "provider_id": "openai",
            "raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
            "tool": "complete_task",
            "total_tokens": 6784
          },
          {
            "completion_tokens": 284,
            "context_chars": 11008,
            "model_id": "gpt-5.5",
            "prompt_tokens": 15234,
            "provider_id": "openai",
            "raw_arguments_hash": "f67fcf857c12765312b6a70ca5ccac33ddd5a9c84d1075142a92f3c0eb572870",
            "role": "reviewer",
            "tool": "review_final",
            "total_tokens": 15518
          }
        ]
      },
      "checks": {
        "expected_terminal_phase": true,
        "token_budget": true,
        "call_budget": true,
        "author_turn_budget": true,
        "reviewer_turn_budget": true,
        "required_claims": true,
        "required_operations": true,
        "required_operation_alternative": true,
        "immutable_artifacts": true,
        "unique_revisions": true,
        "raw_argument_audit": true,
        "no_schema_or_decision_rejections": true
      },
      "rejection_codes": [],
      "artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z/artifacts/cad_52c2f993c7e5"
    },
    {
      "scenario": "simple_flange",
      "repetition": 1,
      "success": false,
      "started_at": "2026-08-31T09:50:11.576785+00:00",
      "finished_at": "2026-08-31T09:52:12.813990+00:00",
      "terminal": {
        "taskId": "cad_e7a33b780edd",
        "lifecycle": "failed",
        "code": "FAILED_AUTHOR_FORMAT",
        "message": "Author repeatedly failed the same canonical schema or state contract."
      },
      "projection": {
        "schema_version": "3.0",
        "task_id": "cad_e7a33b780edd",
        "phase": "FAILED",
        "lifecycle": "failed",
        "state_version": 9,
        "active_revision": "",
        "current_revision": "",
        "published_revision": "",
        "pending_action": null,
        "active_candidate_id": "",
        "repair_required": true,
        "last_error": "FAILED_AUTHOR_FORMAT",
        "action_ledger_summary": [
          {
            "sequence": 10,
            "at": "2026-08-31 09:50:25",
            "draft_revision": 1,
            "event": "requirements_draft_updated",
            "item_count": 1
          },
          {
            "sequence": 11,
            "at": "2026-08-31 09:50:28",
            "draft_revision": 1,
            "event": "requirements_review_requested"
          },
          {
            "sequence": 12,
            "at": "2026-08-31 09:50:34",
            "contract_hash": "5e2b5bbbc2e75d41288e49ae3eb844a50e370d781cb9ce70cac8122ce7576ccb",
            "event": "requirements_contract_frozen",
            "requirement_count": 1
          },
          {
            "sequence": 13,
            "at": "2026-08-31 09:50:45",
            "action_id": "act_004",
            "atomic_id": "extrude_add_blind",
            "contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
            "event": "proposed",
            "expected_change": "Create the base flange solid as a centered 120 mm diameter, 12 mm thick round blank in millimetres.",
            "intent": "Establish the primary connected body that will later receive the centered 40 mm through bore.",
            "requirement_ids": [
              "req_001"
            ],
            "working_head": "cad_e7a33b780edd:root:v3"
          },
          {
            "sequence": 14,
            "at": "2026-08-31 09:50:53",
            "action_id": "act_004",
            "candidate_id": "candidate_b92d6c008fb7c644",
            "event": "candidate_building",
            "fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078"
          },
          {
            "sequence": 15,
            "at": "2026-08-31 09:50:55",
            "action_id": "act_004",
            "candidate_id": "candidate_b92d6c008fb7c644",
            "claim_results": [
              {
                "claim_id": "claim_001",
                "claim_kind": "single_connected_body",
                "deterministic": true,
                "evidence": {
                  "solid_count": 1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_002",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 120.0,
                  "axis": "x",
                  "tolerance_mm": 0.1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_003",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 120.0,
                  "axis": "y",
                  "tolerance_mm": 0.1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_004",
                "claim_kind": "bbox_dimension_mm",
                "deterministic": true,
                "evidence": {
                  "actual_mm": 12.0,
                  "axis": "z",
                  "tolerance_mm": 0.1
                },
                "status": "pass"
              },
              {
                "claim_id": "claim_005",
                "claim_kind": "through_cylindrical_bore",
                "deterministic": true,
                "evidence": {
                  "reason": "the target bore has not been introduced at this checkpoint"
                },
                "status": "pending"
              }
            ],
            "event": "candidate_built"
          },
          {
            "sequence": 16,
            "at": "2026-08-31 09:51:16",
            "candidate_id": "candidate_b92d6c008fb7c644",
            "deterministic_fail": [],
            "event": "candidate_rejected",
            "review_path": "candidate-review.json"
          },
          {
            "sequence": 17,
            "at": "2026-08-31 09:51:22",
            "action_id": "act_008",
            "atomic_id": "extrude_cut_blind",
            "contract_hash": "fe00ac225362469afe00cd4aee9b48259e8ceb74aa11d9a54ff038c6d12f0d50",
            "event": "proposed",
            "expected_change": "Cut a centered 40 mm diameter cylindrical void through the full 12 mm thickness of the flange, leaving the outer 120 mm round body intact.",
            "intent": "Introduce the centered through bore required by the specification while preserving the single connected flange body.",
            "requirement_ids": [
              "req_001"
            ],
            "working_head": "cad_e7a33b780edd:root:v7"
          },
          {
            "sequence": 18,
            "at": "2026-08-31 09:52:12",
            "event": "failed_author_format",
            "field_errors": [],
            "tool": "submit_cdsl_fragment"
          }
        ],
        "revisions": []
      },
      "usage": {
        "calls": 10,
        "prompt_tokens": 70265,
        "completion_tokens": 5557,
        "context_chars": 44063,
        "records": [
          {
            "completion_tokens": 745,
            "context_chars": 934,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5242,
            "provider_id": "openai",
            "raw_arguments_hash": "bea9adcee320f20c42e965058dc3d445a4772c88cf97d8881b39c0703c1f640d",
            "tool": "submit_requirements_draft_batch",
            "total_tokens": 5987
          },
          {
            "completion_tokens": 73,
            "context_chars": 1877,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5499,
            "provider_id": "openai",
            "raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
            "tool": "finalize_requirements_draft",
            "total_tokens": 5572
          },
          {
            "completion_tokens": 71,
            "context_chars": 1331,
            "model_id": "gpt-5.5",
            "prompt_tokens": 5058,
            "provider_id": "openai",
            "raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
            "role": "reviewer",
            "tool": "review_requirements",
            "total_tokens": 5129
          },
          {
            "completion_tokens": 544,
            "context_chars": 1329,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 4970,
            "provider_id": "openai",
            "raw_arguments_hash": "0c61ad302145316bcf37a93f2320442bf7f98bcfaf97f0007d02aa46f2bd4b38",
            "tool": "propose_next_action",
            "total_tokens": 5514
          },
          {
            "completion_tokens": 393,
            "context_chars": 2354,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5660,
            "provider_id": "openai",
            "raw_arguments_hash": "502aa2808018698f3ef18837ee58198e069831fe477e7ce8e02bb9b579b30696",
            "tool": "submit_cdsl_fragment",
            "total_tokens": 6053
          },
          {
            "completion_tokens": 396,
            "context_chars": 21651,
            "model_id": "gpt-5.5",
            "prompt_tokens": 19233,
            "provider_id": "openai",
            "raw_arguments_hash": "cb2915023d05ca8ee84de886fecf34aab773d93ec642f57e4c5ab0579411a283",
            "role": "reviewer",
            "tool": "review_candidate",
            "total_tokens": 19629
          },
          {
            "completion_tokens": 265,
            "context_chars": 3019,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 5719,
            "provider_id": "openai",
            "raw_arguments_hash": "00b71ef952389c68fcfbbd7871941cfcb755e180486f7420f01272158acd42d0",
            "tool": "propose_next_action",
            "total_tokens": 5984
          },
          {
            "completion_tokens": 1167,
            "context_chars": 3614,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 6244,
            "provider_id": "openai",
            "raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
            "tool": "submit_cdsl_fragment",
            "total_tokens": 7411
          },
          {
            "completion_tokens": 1271,
            "context_chars": 3977,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 6320,
            "provider_id": "openai",
            "raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
            "tool": "submit_cdsl_fragment",
            "total_tokens": 7591
          },
          {
            "completion_tokens": 632,
            "context_chars": 3977,
            "model_id": "gpt-5.4-mini",
            "prompt_tokens": 6320,
            "provider_id": "openai",
            "raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
            "tool": "submit_cdsl_fragment",
            "total_tokens": 6952
          }
        ]
      },
      "checks": {
        "expected_terminal_phase": false,
        "token_budget": true,
        "call_budget": true,
        "author_turn_budget": true,
        "reviewer_turn_budget": true,
        "required_claims": false,
        "required_operations": false,
        "required_operation_alternative": false,
        "immutable_artifacts": false,
        "unique_revisions": true,
        "raw_argument_audit": true,
        "no_schema_or_decision_rejections": true
      },
      "rejection_codes": [],
      "artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z/artifacts/cad_e7a33b780edd"
    }
  ],
  "suite": "smoke",
  "require_live": true,
  "report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z"
}