重构 改为标准化输出
This commit is contained in:
@@ -0,0 +1,386 @@
|
||||
{
|
||||
"status": "failed",
|
||||
"author": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.4-mini"
|
||||
},
|
||||
"reviewer": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.5"
|
||||
},
|
||||
"author_capability": {
|
||||
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
||||
"tool_count": 9,
|
||||
"supported": true,
|
||||
"failures": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 41789,
|
||||
"completion_tokens": 1247
|
||||
},
|
||||
"mode": "openai_compatible_tool_calling"
|
||||
},
|
||||
"reviewer_capability": {
|
||||
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
||||
"tool_count": 9,
|
||||
"supported": true,
|
||||
"failures": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 41789,
|
||||
"completion_tokens": 538
|
||||
},
|
||||
"mode": "openai_compatible_tool_calling"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"scenario": "rectangular_plate",
|
||||
"repetition": 1,
|
||||
"success": false,
|
||||
"started_at": "2026-08-31T09:00:56.612675+00:00",
|
||||
"finished_at": "2026-08-31T09:02:11.406524+00:00",
|
||||
"terminal": {
|
||||
"taskId": "cad_ccc4ec062035",
|
||||
"lifecycle": "failed",
|
||||
"code": "FAILED_AUTHOR_FORMAT",
|
||||
"message": "Author repeatedly failed the same canonical schema."
|
||||
},
|
||||
"projection": {
|
||||
"schema_version": "3.0",
|
||||
"task_id": "cad_ccc4ec062035",
|
||||
"phase": "FAILED",
|
||||
"lifecycle": "failed",
|
||||
"state_version": 4,
|
||||
"active_revision": "",
|
||||
"current_revision": "",
|
||||
"published_revision": "",
|
||||
"pending_action": null,
|
||||
"active_candidate_id": "",
|
||||
"repair_required": false,
|
||||
"last_error": "FAILED_AUTHOR_FORMAT",
|
||||
"action_ledger_summary": [
|
||||
{
|
||||
"sequence": 1,
|
||||
"at": "2026-08-31 09:01:04",
|
||||
"draft_revision": 1,
|
||||
"event": "requirements_draft_updated",
|
||||
"item_count": 1
|
||||
},
|
||||
{
|
||||
"sequence": 2,
|
||||
"at": "2026-08-31 09:01:30",
|
||||
"draft_revision": 2,
|
||||
"event": "requirements_draft_patched",
|
||||
"item_count": 1
|
||||
},
|
||||
{
|
||||
"sequence": 3,
|
||||
"at": "2026-08-31 09:01:53",
|
||||
"draft_revision": 3,
|
||||
"event": "requirements_draft_patched",
|
||||
"item_count": 1
|
||||
},
|
||||
{
|
||||
"sequence": 4,
|
||||
"at": "2026-08-31 09:02:11",
|
||||
"event": "failed_author_format",
|
||||
"field_errors": [
|
||||
{
|
||||
"message": "Field required",
|
||||
"path": "/patches/0/target_draft_id"
|
||||
},
|
||||
{
|
||||
"message": "Extra inputs are not permitted",
|
||||
"path": "/target_draft_id"
|
||||
}
|
||||
],
|
||||
"tool": "patch_requirements_draft"
|
||||
}
|
||||
],
|
||||
"revisions": []
|
||||
},
|
||||
"usage": {
|
||||
"calls": 9,
|
||||
"prompt_tokens": 55142,
|
||||
"completion_tokens": 2943,
|
||||
"context_chars": 15133,
|
||||
"records": [
|
||||
{
|
||||
"completion_tokens": 385,
|
||||
"context_chars": 836,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 5910,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "c901d4c9931742387dfdd444508e5988a292648298da2b185857baea5349f213",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6295
|
||||
},
|
||||
{
|
||||
"completion_tokens": 278,
|
||||
"context_chars": 1518,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6098,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "54f21fc272547493351d211b86a1e4aeb420946bf7b65a56433f9096b9cf9f5c",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6376
|
||||
},
|
||||
{
|
||||
"completion_tokens": 303,
|
||||
"context_chars": 1919,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6178,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "23f239dab50941a8e43c1d717b7791115146edab9b9718d1478c9ae012fb1782",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6481
|
||||
},
|
||||
{
|
||||
"completion_tokens": 252,
|
||||
"context_chars": 1993,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6202,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "33da3f40fbd2ce0cb9dad052f8d4f140dea658fbb0fdb475d3625c347621f4db",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6454
|
||||
},
|
||||
{
|
||||
"completion_tokens": 291,
|
||||
"context_chars": 1518,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6098,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "6642d29cd838aaa3f6709e8f2a5edc1920e8c3caf9239e433ed66c321bb3af2b",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6389
|
||||
},
|
||||
{
|
||||
"completion_tokens": 303,
|
||||
"context_chars": 1919,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6178,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "232052bc3c6da6582f57a81f4061433188d4838c3f938ad1c7fff7c66049c653",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6481
|
||||
},
|
||||
{
|
||||
"completion_tokens": 334,
|
||||
"context_chars": 1993,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6202,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "05d75ea8566bc896be811b0b2801ec3665ded0db0271a053e0ac6bb4be6cfd59",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6536
|
||||
},
|
||||
{
|
||||
"completion_tokens": 398,
|
||||
"context_chars": 1518,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6098,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "7d6b3e0925d86ca6fbec794861924a8df0f4b51f56d074d558b864a43a2bc67e",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6496
|
||||
},
|
||||
{
|
||||
"completion_tokens": 399,
|
||||
"context_chars": 1919,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6178,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "1d13a8e2e056ed9b1b49885a9ab05599f58f51e4cecbd22a0640f3dc974b4872",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6577
|
||||
}
|
||||
]
|
||||
},
|
||||
"checks": {
|
||||
"expected_terminal_phase": false,
|
||||
"token_budget": true,
|
||||
"call_budget": true,
|
||||
"author_turn_budget": true,
|
||||
"reviewer_turn_budget": true,
|
||||
"required_claims": false,
|
||||
"required_operations": false,
|
||||
"required_operation_alternative": true,
|
||||
"immutable_artifacts": false,
|
||||
"unique_revisions": true,
|
||||
"raw_argument_audit": true,
|
||||
"no_schema_or_decision_rejections": false
|
||||
},
|
||||
"rejection_codes": [
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID"
|
||||
],
|
||||
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z/artifacts/cad_ccc4ec062035"
|
||||
},
|
||||
{
|
||||
"scenario": "simple_flange",
|
||||
"repetition": 1,
|
||||
"success": false,
|
||||
"started_at": "2026-08-31T09:02:11.409883+00:00",
|
||||
"finished_at": "2026-08-31T09:03:23.606881+00:00",
|
||||
"terminal": {
|
||||
"taskId": "cad_1a1c78be107f",
|
||||
"lifecycle": "failed",
|
||||
"code": "FAILED_AUTHOR_FORMAT",
|
||||
"message": "Author repeatedly failed the same canonical schema."
|
||||
},
|
||||
"projection": {
|
||||
"schema_version": "3.0",
|
||||
"task_id": "cad_1a1c78be107f",
|
||||
"phase": "FAILED",
|
||||
"lifecycle": "failed",
|
||||
"state_version": 3,
|
||||
"active_revision": "",
|
||||
"current_revision": "",
|
||||
"published_revision": "",
|
||||
"pending_action": null,
|
||||
"active_candidate_id": "",
|
||||
"repair_required": false,
|
||||
"last_error": "FAILED_AUTHOR_FORMAT",
|
||||
"action_ledger_summary": [
|
||||
{
|
||||
"sequence": 5,
|
||||
"at": "2026-08-31 09:02:22",
|
||||
"draft_revision": 1,
|
||||
"event": "requirements_draft_updated",
|
||||
"item_count": 1
|
||||
},
|
||||
{
|
||||
"sequence": 6,
|
||||
"at": "2026-08-31 09:03:07",
|
||||
"draft_revision": 2,
|
||||
"event": "requirements_draft_patched",
|
||||
"item_count": 1
|
||||
},
|
||||
{
|
||||
"sequence": 7,
|
||||
"at": "2026-08-31 09:03:23",
|
||||
"event": "failed_author_format",
|
||||
"field_errors": [
|
||||
{
|
||||
"message": "Field required",
|
||||
"path": "/patches/0/target_draft_id"
|
||||
},
|
||||
{
|
||||
"message": "Extra inputs are not permitted",
|
||||
"path": "/target_draft_id"
|
||||
}
|
||||
],
|
||||
"tool": "patch_requirements_draft"
|
||||
}
|
||||
],
|
||||
"revisions": []
|
||||
},
|
||||
"usage": {
|
||||
"calls": 7,
|
||||
"prompt_tokens": 43222,
|
||||
"completion_tokens": 3167,
|
||||
"context_chars": 13018,
|
||||
"records": [
|
||||
{
|
||||
"completion_tokens": 412,
|
||||
"context_chars": 892,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 5918,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "5030e50bc35c2569181a2866da0ab2f10a7b23a88d6ef6e134c92f9976b6d949",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6330
|
||||
},
|
||||
{
|
||||
"completion_tokens": 366,
|
||||
"context_chars": 1729,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6156,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "29b6b77e6f42a319a373f7050e291647a60f4fd20940775f09411ac63140170d",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6522
|
||||
},
|
||||
{
|
||||
"completion_tokens": 329,
|
||||
"context_chars": 2130,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6236,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "be0d9112723c27fbae5139e29d93a5a88b0dd6b7a666e041816db9e515e40a86",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6565
|
||||
},
|
||||
{
|
||||
"completion_tokens": 751,
|
||||
"context_chars": 2204,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6260,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "83f8db722fbbfebf45ad2a39588e9ea3c5d9281803287cf4eef4c80c1070c52c",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 7011
|
||||
},
|
||||
{
|
||||
"completion_tokens": 638,
|
||||
"context_chars": 2204,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6260,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "7dffc30397b3bb660a206bc55945db9a643e2221c0c83e1fb06de2fe880163fa",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6898
|
||||
},
|
||||
{
|
||||
"completion_tokens": 340,
|
||||
"context_chars": 1729,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6156,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "29b6b77e6f42a319a373f7050e291647a60f4fd20940775f09411ac63140170d",
|
||||
"tool": "submit_requirements_draft_batch",
|
||||
"total_tokens": 6496
|
||||
},
|
||||
{
|
||||
"completion_tokens": 331,
|
||||
"context_chars": 2130,
|
||||
"model_id": "gpt-5.4-mini",
|
||||
"prompt_tokens": 6236,
|
||||
"provider_id": "openai",
|
||||
"raw_arguments_hash": "6660f5b34c43201605c0b9b2d948bd928770f3cc185991687703193c61566ec9",
|
||||
"tool": "patch_requirements_draft",
|
||||
"total_tokens": 6567
|
||||
}
|
||||
]
|
||||
},
|
||||
"checks": {
|
||||
"expected_terminal_phase": false,
|
||||
"token_budget": true,
|
||||
"call_budget": true,
|
||||
"author_turn_budget": true,
|
||||
"reviewer_turn_budget": true,
|
||||
"required_claims": false,
|
||||
"required_operations": false,
|
||||
"required_operation_alternative": true,
|
||||
"immutable_artifacts": false,
|
||||
"unique_revisions": true,
|
||||
"raw_argument_audit": true,
|
||||
"no_schema_or_decision_rejections": false
|
||||
},
|
||||
"rejection_codes": [
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID",
|
||||
"AUTHOR_FORMAT_INVALID"
|
||||
],
|
||||
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z/artifacts/cad_1a1c78be107f"
|
||||
}
|
||||
],
|
||||
"suite": "smoke",
|
||||
"require_live": true,
|
||||
"report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T085924Z"
|
||||
}
|
||||
Reference in New Issue
Block a user