cdsl-cad/backend/live-evals/20260831T174752Z/report.json

{
  "status": "LIVE_EVAL_BLOCKED",
  "error": "MODEL_STRUCTURED_OUTPUT_UNSUPPORTED",
  "author_capability": {
    "schema_hash": "3935c6f40d3b8b9bb7c5672cdea37049a3cd9eb57c9f722b738397a45d77490c",
    "tool_count": 28,
    "supported": false,
    "failures": [
      {
        "tool": "submit_requirements_draft_batch",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "patch_requirements_draft",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "finalize_requirements_draft",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_requirements",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "propose_next_action",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "inspect_topology",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "get_cdsl_operation_contract",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_candidate",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "complete_task",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_final",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "record_geometry_conclusion",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "rollback_checkpoint",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_chamfer",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_add_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_add_two_sided",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_cut_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_fillet",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_counterbore",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_countersink",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_wizard",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_pattern_linear",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_pattern_mirror",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_reference_axis",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_reference_plane",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_revolve_add",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_revolve_cut",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_sphere_add",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      }
    ],
    "usage": {
      "prompt_tokens": 0,
      "completion_tokens": 0
    },
    "mode": "openai_compatible_tool_calling"
  },
  "reviewer_capability": {
    "schema_hash": "3935c6f40d3b8b9bb7c5672cdea37049a3cd9eb57c9f722b738397a45d77490c",
    "tool_count": 28,
    "supported": false,
    "failures": [
      {
        "tool": "submit_requirements_draft_batch",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "patch_requirements_draft",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "finalize_requirements_draft",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_requirements",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "propose_next_action",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "inspect_topology",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "get_cdsl_operation_contract",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_candidate",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "complete_task",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "review_final",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "record_geometry_conclusion",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "rollback_checkpoint",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_chamfer",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_add_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_add_two_sided",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_extrude_cut_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_fillet",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_blind",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_counterbore",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_countersink",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_hole_wizard",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_pattern_linear",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_pattern_mirror",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_reference_axis",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_reference_plane",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_revolve_add",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_revolve_cut",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      },
      {
        "tool": "conformance_sphere_add",
        "message": "Provider transport unavailable after 3 attempts: All connection attempts failed"
      }
    ],
    "usage": {
      "prompt_tokens": 0,
      "completion_tokens": 0
    },
    "mode": "openai_compatible_tool_calling"
  },
  "suite": "comprehensive",
  "require_live": true,
  "report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T174752Z"
}