795 lines
27 KiB
JSON
795 lines
27 KiB
JSON
{
|
|
"status": "failed",
|
|
"author": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.4-mini"
|
|
},
|
|
"reviewer": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.5"
|
|
},
|
|
"author_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 1156
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"reviewer_capability": {
|
|
"schema_hash": "fb899a5de2f869b648df88a32d04a5eee92ae3d193ab23ee641892d6b59cd8e7",
|
|
"tool_count": 9,
|
|
"supported": true,
|
|
"failures": [],
|
|
"usage": {
|
|
"prompt_tokens": 41789,
|
|
"completion_tokens": 541
|
|
},
|
|
"mode": "openai_compatible_tool_calling"
|
|
},
|
|
"results": [
|
|
{
|
|
"scenario": "rectangular_plate",
|
|
"repetition": 1,
|
|
"success": true,
|
|
"started_at": "2026-08-31T09:49:06.244495+00:00",
|
|
"finished_at": "2026-08-31T09:50:11.574252+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_52c2f993c7e5",
|
|
"lifecycle": "completed",
|
|
"revisionId": "rev_001"
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_52c2f993c7e5",
|
|
"phase": "COMPLETED",
|
|
"lifecycle": "completed",
|
|
"state_version": 9,
|
|
"active_revision": "rev_001",
|
|
"current_revision": "rev_001",
|
|
"published_revision": "rev_001",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": false,
|
|
"last_error": "",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 1,
|
|
"at": "2026-08-31 09:49:13",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 2,
|
|
"at": "2026-08-31 09:49:18",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 3,
|
|
"at": "2026-08-31 09:49:22",
|
|
"contract_hash": "42badf61b3707fe34319e2f2c145d0e64a781a656c8f37ad9552d96733095503",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 4,
|
|
"at": "2026-08-31 09:49:31",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create a single solid rectangular plate with footprint 80 mm by 50 mm and thickness 8 mm.",
|
|
"intent": "Build the required base solid for the plate in one verifiable step.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_52c2f993c7e5:root:v3"
|
|
},
|
|
{
|
|
"sequence": 5,
|
|
"at": "2026-08-31 09:49:39",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_4642dbd9452f51e3",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb"
|
|
},
|
|
{
|
|
"sequence": 6,
|
|
"at": "2026-08-31 09:49:40",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_4642dbd9452f51e3",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 7,
|
|
"at": "2026-08-31 09:49:49",
|
|
"action_id": "act_004",
|
|
"actual_atomic_id": "extrude_add_blind",
|
|
"candidate_id": "candidate_4642dbd9452f51e3",
|
|
"coverage": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "accepted",
|
|
"fragment_hash": "34b4567ab12fdc5a54b67ab39a791a6a0550daeb157fe6e40abd7f67726046bb",
|
|
"review_path": "revisions/rev_001/candidate-review.json",
|
|
"revision_id": "rev_001",
|
|
"selector_snapshot_id": "",
|
|
"working_head_before": "cad_52c2f993c7e5:root:v3"
|
|
},
|
|
{
|
|
"sequence": 8,
|
|
"at": "2026-08-31 09:50:00",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "final_validation_requested",
|
|
"revision_id": "rev_001"
|
|
},
|
|
{
|
|
"sequence": 9,
|
|
"at": "2026-08-31 09:50:11",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "solid_count_equals",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 80.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 50.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 8.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.0
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_006",
|
|
"claim_kind": "visual",
|
|
"deterministic": false,
|
|
"evidence": {
|
|
"reason": "requires independent visual review"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "completed",
|
|
"final_review_path": "revisions/rev_001/final-review.json",
|
|
"revision_id": "rev_001"
|
|
}
|
|
],
|
|
"revisions": [
|
|
{
|
|
"revision_id": "rev_001",
|
|
"status": "success",
|
|
"visibility": "final",
|
|
"cdsl_path": "revisions/rev_001/model.cdsl.json",
|
|
"step_path": "revisions/rev_001/model.step",
|
|
"glb_path": "revisions/rev_001/model.glb",
|
|
"report_path": "revisions/rev_001/rebuild-report.json",
|
|
"candidate_review_path": "revisions/rev_001/candidate-review.json"
|
|
}
|
|
]
|
|
},
|
|
"usage": {
|
|
"calls": 8,
|
|
"prompt_tokens": 67140,
|
|
"completion_tokens": 2343,
|
|
"context_chars": 44387,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 256,
|
|
"context_chars": 878,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5234,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "2e18aabb4b24b0d6a4d9f1e99aac7c625e266fb9de1b6cecc9ae0ff110b32f3a",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5490
|
|
},
|
|
{
|
|
"completion_tokens": 84,
|
|
"context_chars": 1715,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5450,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5534
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1195,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5013,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5084
|
|
},
|
|
{
|
|
"completion_tokens": 412,
|
|
"context_chars": 1301,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4972,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "14880afd2112d4805a5e96df077239ec12139c82dbfce41f563fce53471cf537",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5384
|
|
},
|
|
{
|
|
"completion_tokens": 366,
|
|
"context_chars": 2253,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5651,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "d52011a37183174b1d6e1937fdbede565bda754adb6894e35e85e2fe0883774a",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 6017
|
|
},
|
|
{
|
|
"completion_tokens": 258,
|
|
"context_chars": 21618,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19414,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "077328095f2b06d7dc28fa29ad1fcf5b876c88bc7f32b7298db29d24f048e722",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19672
|
|
},
|
|
{
|
|
"completion_tokens": 612,
|
|
"context_chars": 4419,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6172,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "complete_task",
|
|
"total_tokens": 6784
|
|
},
|
|
{
|
|
"completion_tokens": 284,
|
|
"context_chars": 11008,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 15234,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "f67fcf857c12765312b6a70ca5ccac33ddd5a9c84d1075142a92f3c0eb572870",
|
|
"role": "reviewer",
|
|
"tool": "review_final",
|
|
"total_tokens": 15518
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": true,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": true,
|
|
"required_operations": true,
|
|
"required_operation_alternative": true,
|
|
"immutable_artifacts": true,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": true
|
|
},
|
|
"rejection_codes": [],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z/artifacts/cad_52c2f993c7e5"
|
|
},
|
|
{
|
|
"scenario": "simple_flange",
|
|
"repetition": 1,
|
|
"success": false,
|
|
"started_at": "2026-08-31T09:50:11.576785+00:00",
|
|
"finished_at": "2026-08-31T09:52:12.813990+00:00",
|
|
"terminal": {
|
|
"taskId": "cad_e7a33b780edd",
|
|
"lifecycle": "failed",
|
|
"code": "FAILED_AUTHOR_FORMAT",
|
|
"message": "Author repeatedly failed the same canonical schema or state contract."
|
|
},
|
|
"projection": {
|
|
"schema_version": "3.0",
|
|
"task_id": "cad_e7a33b780edd",
|
|
"phase": "FAILED",
|
|
"lifecycle": "failed",
|
|
"state_version": 9,
|
|
"active_revision": "",
|
|
"current_revision": "",
|
|
"published_revision": "",
|
|
"pending_action": null,
|
|
"active_candidate_id": "",
|
|
"repair_required": true,
|
|
"last_error": "FAILED_AUTHOR_FORMAT",
|
|
"action_ledger_summary": [
|
|
{
|
|
"sequence": 10,
|
|
"at": "2026-08-31 09:50:25",
|
|
"draft_revision": 1,
|
|
"event": "requirements_draft_updated",
|
|
"item_count": 1
|
|
},
|
|
{
|
|
"sequence": 11,
|
|
"at": "2026-08-31 09:50:28",
|
|
"draft_revision": 1,
|
|
"event": "requirements_review_requested"
|
|
},
|
|
{
|
|
"sequence": 12,
|
|
"at": "2026-08-31 09:50:34",
|
|
"contract_hash": "5e2b5bbbc2e75d41288e49ae3eb844a50e370d781cb9ce70cac8122ce7576ccb",
|
|
"event": "requirements_contract_frozen",
|
|
"requirement_count": 1
|
|
},
|
|
{
|
|
"sequence": 13,
|
|
"at": "2026-08-31 09:50:45",
|
|
"action_id": "act_004",
|
|
"atomic_id": "extrude_add_blind",
|
|
"contract_hash": "fa9ffc9fd2141453fa18a389813dde2e2ef3fb4c7ef5d05e61d51d614309e95f",
|
|
"event": "proposed",
|
|
"expected_change": "Create the base flange solid as a centered 120 mm diameter, 12 mm thick round blank in millimetres.",
|
|
"intent": "Establish the primary connected body that will later receive the centered 40 mm through bore.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_e7a33b780edd:root:v3"
|
|
},
|
|
{
|
|
"sequence": 14,
|
|
"at": "2026-08-31 09:50:53",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_b92d6c008fb7c644",
|
|
"event": "candidate_building",
|
|
"fragment_hash": "95bd1e739aed6d4a791a162b2ce05d84d0a52f4d18dc87c11dc7d6e8d7761078"
|
|
},
|
|
{
|
|
"sequence": 15,
|
|
"at": "2026-08-31 09:50:55",
|
|
"action_id": "act_004",
|
|
"candidate_id": "candidate_b92d6c008fb7c644",
|
|
"claim_results": [
|
|
{
|
|
"claim_id": "claim_001",
|
|
"claim_kind": "single_connected_body",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"solid_count": 1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_002",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "x",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_003",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 120.0,
|
|
"axis": "y",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_004",
|
|
"claim_kind": "bbox_dimension_mm",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"actual_mm": 12.0,
|
|
"axis": "z",
|
|
"tolerance_mm": 0.1
|
|
},
|
|
"status": "pass"
|
|
},
|
|
{
|
|
"claim_id": "claim_005",
|
|
"claim_kind": "through_cylindrical_bore",
|
|
"deterministic": true,
|
|
"evidence": {
|
|
"reason": "the target bore has not been introduced at this checkpoint"
|
|
},
|
|
"status": "pending"
|
|
}
|
|
],
|
|
"event": "candidate_built"
|
|
},
|
|
{
|
|
"sequence": 16,
|
|
"at": "2026-08-31 09:51:16",
|
|
"candidate_id": "candidate_b92d6c008fb7c644",
|
|
"deterministic_fail": [],
|
|
"event": "candidate_rejected",
|
|
"review_path": "candidate-review.json"
|
|
},
|
|
{
|
|
"sequence": 17,
|
|
"at": "2026-08-31 09:51:22",
|
|
"action_id": "act_008",
|
|
"atomic_id": "extrude_cut_blind",
|
|
"contract_hash": "fe00ac225362469afe00cd4aee9b48259e8ceb74aa11d9a54ff038c6d12f0d50",
|
|
"event": "proposed",
|
|
"expected_change": "Cut a centered 40 mm diameter cylindrical void through the full 12 mm thickness of the flange, leaving the outer 120 mm round body intact.",
|
|
"intent": "Introduce the centered through bore required by the specification while preserving the single connected flange body.",
|
|
"requirement_ids": [
|
|
"req_001"
|
|
],
|
|
"working_head": "cad_e7a33b780edd:root:v7"
|
|
},
|
|
{
|
|
"sequence": 18,
|
|
"at": "2026-08-31 09:52:12",
|
|
"event": "failed_author_format",
|
|
"field_errors": [],
|
|
"tool": "submit_cdsl_fragment"
|
|
}
|
|
],
|
|
"revisions": []
|
|
},
|
|
"usage": {
|
|
"calls": 10,
|
|
"prompt_tokens": 70265,
|
|
"completion_tokens": 5557,
|
|
"context_chars": 44063,
|
|
"records": [
|
|
{
|
|
"completion_tokens": 745,
|
|
"context_chars": 934,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5242,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "bea9adcee320f20c42e965058dc3d445a4772c88cf97d8881b39c0703c1f640d",
|
|
"tool": "submit_requirements_draft_batch",
|
|
"total_tokens": 5987
|
|
},
|
|
{
|
|
"completion_tokens": 73,
|
|
"context_chars": 1877,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5499,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
"tool": "finalize_requirements_draft",
|
|
"total_tokens": 5572
|
|
},
|
|
{
|
|
"completion_tokens": 71,
|
|
"context_chars": 1331,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 5058,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00996cae282886750e3169d668db3145d5aad3fa73c825b1b9d5d423613bdb52",
|
|
"role": "reviewer",
|
|
"tool": "review_requirements",
|
|
"total_tokens": 5129
|
|
},
|
|
{
|
|
"completion_tokens": 544,
|
|
"context_chars": 1329,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 4970,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "0c61ad302145316bcf37a93f2320442bf7f98bcfaf97f0007d02aa46f2bd4b38",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5514
|
|
},
|
|
{
|
|
"completion_tokens": 393,
|
|
"context_chars": 2354,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5660,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "502aa2808018698f3ef18837ee58198e069831fe477e7ce8e02bb9b579b30696",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 6053
|
|
},
|
|
{
|
|
"completion_tokens": 396,
|
|
"context_chars": 21651,
|
|
"model_id": "gpt-5.5",
|
|
"prompt_tokens": 19233,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "cb2915023d05ca8ee84de886fecf34aab773d93ec642f57e4c5ab0579411a283",
|
|
"role": "reviewer",
|
|
"tool": "review_candidate",
|
|
"total_tokens": 19629
|
|
},
|
|
{
|
|
"completion_tokens": 265,
|
|
"context_chars": 3019,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 5719,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "00b71ef952389c68fcfbbd7871941cfcb755e180486f7420f01272158acd42d0",
|
|
"tool": "propose_next_action",
|
|
"total_tokens": 5984
|
|
},
|
|
{
|
|
"completion_tokens": 1167,
|
|
"context_chars": 3614,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6244,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 7411
|
|
},
|
|
{
|
|
"completion_tokens": 1271,
|
|
"context_chars": 3977,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6320,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 7591
|
|
},
|
|
{
|
|
"completion_tokens": 632,
|
|
"context_chars": 3977,
|
|
"model_id": "gpt-5.4-mini",
|
|
"prompt_tokens": 6320,
|
|
"provider_id": "openai",
|
|
"raw_arguments_hash": "a61be86bfd477b481a340d77c8cca329acedd0ca32b69cf7df218e44435af80a",
|
|
"tool": "submit_cdsl_fragment",
|
|
"total_tokens": 6952
|
|
}
|
|
]
|
|
},
|
|
"checks": {
|
|
"expected_terminal_phase": false,
|
|
"token_budget": true,
|
|
"call_budget": true,
|
|
"author_turn_budget": true,
|
|
"reviewer_turn_budget": true,
|
|
"required_claims": false,
|
|
"required_operations": false,
|
|
"required_operation_alternative": false,
|
|
"immutable_artifacts": false,
|
|
"unique_revisions": true,
|
|
"raw_argument_audit": true,
|
|
"no_schema_or_decision_rejections": true
|
|
},
|
|
"rejection_codes": [],
|
|
"artifact_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z/artifacts/cad_e7a33b780edd"
|
|
}
|
|
],
|
|
"suite": "smoke",
|
|
"require_live": true,
|
|
"report_root": "/Users/lk/Projects/cdsl-cad/backend/live-evals/20260831T094745Z"
|
|
} |