386 lines
13 KiB
JSON
386 lines
13 KiB
JSON
{
|
|
"status": "failed",
|
|
"author": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.4-mini"
|
|
},
|
|
"reviewer": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.5"
|
|
},
|
|
"author_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 1247
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"reviewer_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 538
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"results": [
|
|
{
|
|
"scenario": "rectangular_plate",
|
|
"repetition": 1,
|
|
"success": false,
|
|
"started_at": "2026-08-31T09:00:56.612675+00:00",
|
|
"finished_at": "2026-08-31T09:02:11.406524+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_ccc4ec062035",
|
|
"lifecycle": "failed",
|
|
"code": "FAILED_AUTHOR_FORMAT",
|
|
"message": "Author repeatedly failed the same canonical schema."
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_ccc4ec062035",
|
|
"phase": "FAILED",
|
|
"lifecycle": "failed",
|
|
"state_version": 4,
|
|
"active_revision": "",
|
|
"current_revision": "",
|
|
"published_revision": "",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "FAILED_AUTHOR_FORMAT",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 1,
|
|
"at": "2026-08-31 09:01:04",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 2,
|
|
"at": "2026-08-31 09:01:30",
|
|
"draft_revision": 2,
|
|
"event": "requirements_draft_patched",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 3,
|
|
"at": "2026-08-31 09:01:53",
|
|
"draft_revision": 3,
|
|
"event": "requirements_draft_patched",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 4,
|
|
"at": "2026-08-31 09:02:11",
|
|
"event": "failed_author_format",
|
|
"field_errors": [
|
|
{
|
|
"message": "Field required",
|
|
"path": "/patches/0/target_draft_id"
|
|
},
|
|
{
|
|
"message": "Extra inputs are not permitted",
|
|
"path": "/target_draft_id"
|
|
}
|
|
],
|
|
"tool": "patch_requirements_draft"
|
|
}
|
|
],
|
|
"revisions": []
|
|
},
|
|
"usage": {
|
|
"calls": 9,
|
|
"prompt_tokens": 55142,
|
|
"completion_tokens": 2943,
|
|
"context_chars": 15133,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 385,
|
|
"context_chars": 836,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5910,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "c901d4c9931742387dfdd444508e5988a292648298da2b185857baea5349f213",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6295
|
|
},
|
|
{
|
|
"completion_tokens": 278,
|
|
"context_chars": 1518,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6098,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "54f21fc272547493351d211b86a1e4aeb420946bf7b65a56433f9096b9cf9f5c",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6376
|
|
},
|
|
{
|
|
"completion_tokens": 303,
|
|
"context_chars": 1919,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6178,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "23f239dab50941a8e43c1d717b7791115146edab9b9718d1478c9ae012fb1782",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6481
|
|
},
|
|
{
|
|
"completion_tokens": 252,
|
|
"context_chars": 1993,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6202,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "33da3f40fbd2ce0cb9dad052f8d4f140dea658fbb0fdb475d3625c347621f4db",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6454
|
|
},
|
|
{
|
|
"completion_tokens": 291,
|
|
"context_chars": 1518,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6098,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "6642d29cd838aaa3f6709e8f2a5edc1920e8c3caf9239e433ed66c321bb3af2b",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6389
|
|
},
|
|
{
|
|
"completion_tokens": 303,
|
|
"context_chars": 1919,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6178,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "232052bc3c6da6582f57a81f4061433188d4838c3f938ad1c7fff7c66049c653",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6481
|
|
},
|
|
{
|
|
"completion_tokens": 334,
|
|
"context_chars": 1993,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6202,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "05d75ea8566bc896be811b0b2801ec3665ded0db0271a053e0ac6bb4be6cfd59",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6536
|
|
},
|
|
{
|
|
"completion_tokens": 398,
|
|
"context_chars": 1518,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6098,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "7d6b3e0925d86ca6fbec794861924a8df0f4b51f56d074d558b864a43a2bc67e",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6496
|
|
},
|
|
{
|
|
"completion_tokens": 399,
|
|
"context_chars": 1919,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6178,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "1d13a8e2e056ed9b1b49885a9ab05599f58f51e4cecbd22a0640f3dc974b4872",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6577
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": false,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": false,
|
|
"required_operations": false,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": false,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": false
|
|
},
|
|
"rejection_codes": [
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID"
|
|
],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z/artifacts/cad_ccc4ec062035"
|
|
},
|
|
{
|
|
"scenario": "simple_flange",
|
|
"repetition": 1,
|
|
"success": false,
|
|
"started_at": "2026-08-31T09:02:11.409883+00:00",
|
|
"finished_at": "2026-08-31T09:03:23.606881+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_1a1c78be107f",
|
|
"lifecycle": "failed",
|
|
"code": "FAILED_AUTHOR_FORMAT",
|
|
"message": "Author repeatedly failed the same canonical schema."
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_1a1c78be107f",
|
|
"phase": "FAILED",
|
|
"lifecycle": "failed",
|
|
"state_version": 3,
|
|
"active_revision": "",
|
|
"current_revision": "",
|
|
"published_revision": "",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "FAILED_AUTHOR_FORMAT",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 5,
|
|
"at": "2026-08-31 09:02:22",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 6,
|
|
"at": "2026-08-31 09:03:07",
|
|
"draft_revision": 2,
|
|
"event": "requirements_draft_patched",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 7,
|
|
"at": "2026-08-31 09:03:23",
|
|
"event": "failed_author_format",
|
|
"field_errors": [
|
|
{
|
|
"message": "Field required",
|
|
"path": "/patches/0/target_draft_id"
|
|
},
|
|
{
|
|
"message": "Extra inputs are not permitted",
|
|
"path": "/target_draft_id"
|
|
}
|
|
],
|
|
"tool": "patch_requirements_draft"
|
|
}
|
|
],
|
|
"revisions": []
|
|
},
|
|
"usage": {
|
|
"calls": 7,
|
|
"prompt_tokens": 43222,
|
|
"completion_tokens": 3167,
|
|
"context_chars": 13018,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 412,
|
|
"context_chars": 892,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5918,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "5030e50bc35c2569181a2866da0ab2f10a7b23a88d6ef6e134c92f9976b6d949",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6330
|
|
},
|
|
{
|
|
"completion_tokens": 366,
|
|
"context_chars": 1729,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6156,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "29b6b77e6f42a319a373f7050e291647a60f4fd20940775f09411ac63140170d",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6522
|
|
},
|
|
{
|
|
"completion_tokens": 329,
|
|
"context_chars": 2130,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6236,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "be0d9112723c27fbae5139e29d93a5a88b0dd6b7a666e041816db9e515e40a86",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6565
|
|
},
|
|
{
|
|
"completion_tokens": 751,
|
|
"context_chars": 2204,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6260,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "83f8db722fbbfebf45ad2a39588e9ea3c5d9281803287cf4eef4c80c1070c52c",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 7011
|
|
},
|
|
{
|
|
"completion_tokens": 638,
|
|
"context_chars": 2204,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6260,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "7dffc30397b3bb660a206bc55945db9a643e2221c0c83e1fb06de2fe880163fa",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6898
|
|
},
|
|
{
|
|
"completion_tokens": 340,
|
|
"context_chars": 1729,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6156,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "29b6b77e6f42a319a373f7050e291647a60f4fd20940775f09411ac63140170d",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 6496
|
|
},
|
|
{
|
|
"completion_tokens": 331,
|
|
"context_chars": 2130,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6236,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "6660f5b34c43201605c0b9b2d948bd928770f3cc185991687703193c61566ec9",
|
|
"tool": "patch_requirements_draft",
|
|
"total_tokens": 6567
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": false,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": false,
|
|
"required_operations": false,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": false,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": false
|
|
},
|
|
"rejection_codes": [
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID",
|
|
"AUTHOR_FORMAT_INVALID"
|
|
],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z/artifacts/cad_1a1c78be107f"
|
|
}
|
|
],
|
|
"suite": "smoke",
|
|
"require_live": true,
|
|
"report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z"
|
|
} |