"""Run the real provider, engine, reviewer and v3 state machine end to end. ``--require-live`` deliberately fails closed: missing credentials, blocked network, a skipped model capability, a timeout, or any scenario failure emits ``LIVE_EVAL_BLOCKED``/failure details and exits non-zero. """ from __future__ import annotations import argparse import asyncio from dataclasses import replace from datetime import datetime, timezone from hashlib import sha256 import json from pathlib import Path import secrets import subprocess import sys from typing import Any from app.cad_agent.application.capabilities import verify_model_capability from app.cad_agent.application.workflow import ModelIdentity from app.cad_agent.composition import compose_v3 from app.cad_agent.domain.claim_matching import contains_expected from app.cad_agent.domain.operation_contract import canonical_hash from app.cad_agent.domain.verifier_registry import default_registry from app.cad_agent.evals.token_baseline import ( TokenBaselineError, author_request_identity, compare_token_baseline, load_token_baseline, profile_sha256, validate_token_baseline_provenance, ) from app.settings import BACKEND_ROOT, get_settings _CANDIDATE_EVIDENCE_FILES = frozenset({ "candidate.json", "candidate-review.json", "model.cdsl.json", "model.step", "model.glb", "model.topology.json", "rebuild-report.json", "renders/render-manifest.json", "renders/contact-sheet.jpg", }) _FAILURE_LAYERS = frozenset({ "model_format_or_decision", "v3_contract_or_verifier", "cdsl_expression", "engine_execution", "independent_visual_review", "configuration_or_network", }) _ERROR_FAILURE_LAYERS = { "AUTHOR_FORMAT_INVALID": "model_format_or_decision", "AUTHOR_DECISION_REJECTED": "model_format_or_decision", "STALE_WORKING_HEAD": "model_format_or_decision", "FAILED_AUTHOR_FORMAT": "model_format_or_decision", "MODEL_STRUCTURED_OUTPUT_UNSUPPORTED": "model_format_or_decision", "RUNTIME_PRECONDITION_FAILED": "v3_contract_or_verifier", "RUNTIME_CONTRACT_INVALID": "v3_contract_or_verifier", "VERIFIER_UNAVAILABLE": "v3_contract_or_verifier", "CLAIM_VERIFICATION_FAILED": "cdsl_expression", "CANDIDATE_BUILD_FAILED": "engine_execution", "CANDIDATE_REVIEW_REJECTED": "independent_visual_review", "AUTHOR_TRANSPORT_UNAVAILABLE": "configuration_or_network", "REVIEW_SERVICE_UNAVAILABLE": "configuration_or_network", "RENDER_SERVICE_UNAVAILABLE": "configuration_or_network", "STORAGE_FAILURE": "configuration_or_network", "LIVE_EVAL_TIMEOUT": "configuration_or_network", "FAILED_INTERNAL": "engine_execution", } def _arguments() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run live protocol-v3 CAD evaluations.") parser.add_argument("--suite", choices=("smoke", "release", "comprehensive"), default="smoke") parser.add_argument("--require-live", action="store_true") parser.add_argument("--allow-skip", action="store_true") parser.add_argument("--author-provider") parser.add_argument("--author-model") parser.add_argument("--review-provider") parser.add_argument("--review-model") parser.add_argument("--scenario", help="Run one fixture scenario by its stable ID for targeted regression validation.") parser.add_argument("--baseline-report", type=Path, help="Measured pre-v3 token baseline JSON for a release run.") return parser.parse_args() def _fixture(suite: str, scenario_id: str | None = None) -> list[dict[str, Any]]: fixture_name = "comprehensive.json" if suite == "comprehensive" else "release.json" value = json.loads((Path(__file__).parent / "fixtures" / fixture_name).read_text(encoding="utf-8")) if fixture_name == "comprehensive.json": source_document = str(value.get("source_document") or "") expected_digest = str(value.get("source_document_sha256") or "") source_path = (BACKEND_ROOT.parent / source_document).resolve() workspace_root = BACKEND_ROOT.parent.resolve() if ( not source_document or workspace_root not in source_path.parents or not source_path.is_file() or sha256(source_path.read_bytes()).hexdigest() != expected_digest ): raise ValueError("Comprehensive fixture is not synchronized with its source document.") values = [item for item in value.get("scenarios") or () if isinstance(item, dict)] values = values[:2] if suite == "smoke" else values if scenario_id is None: return values selected = [item for item in values if str(item.get("id") or "") == scenario_id] if not selected: raise ValueError(f"Unknown scenario {scenario_id!r} for suite {suite!r}") return selected def _rejection_codes(events: list[dict[str, Any]]) -> list[str]: codes: list[str] = [] for event in events: payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} result = payload.get("result") if isinstance(payload.get("result"), dict) else payload code = result.get("code") if isinstance(result, dict) else None if code in {"AUTHOR_FORMAT_INVALID", "AUTHOR_DECISION_REJECTED", "STALE_WORKING_HEAD"}: codes.append(str(code)) return codes def _acceptance_coverage( scenario: dict[str, Any], requirements_contract: dict[str, Any] | None, ) -> dict[str, Any]: """Assess whether a fixture's acceptance contract is representable today. Comprehensive prompts deliberately include manufacturing relationships that may not yet have a deterministic verifier. A healthy-looking solid is not evidence for those relationships, so they remain explicit capability gaps instead of silently passing a scenario. """ actual = [ { "claim_kind": str(claim.get("claim_kind") or ""), "expected": claim.get("expected") if isinstance(claim.get("expected"), dict) else {}, } for requirement in (requirements_contract or {}).get("requirements") or () if isinstance(requirement, dict) for claim in requirement.get("acceptance_claims") or () if isinstance(claim, dict) ] required = {str(value) for value in scenario.get("required_claim_kinds") or ()} expected_claims = [ { "claim_kind": str(item.get("claim_kind") or ""), "expected": item.get("expected") if isinstance(item.get("expected"), dict) else {}, } for item in scenario.get("required_claims") or () if isinstance(item, dict) and isinstance(item.get("claim_kind"), str) ] expected_kinds = {claim["claim_kind"] for claim in expected_claims} expected_claims.extend( {"claim_kind": claim_kind, "expected": {}} for claim_kind in sorted(required - expected_kinds) ) gaps = [ {"id": str(item.get("id") or ""), "description": str(item.get("description") or "")} for item in scenario.get("validation_capability_gaps") or () if isinstance(item, dict) ] missing = [ claim for claim in expected_claims if not any( actual_claim["claim_kind"] == claim["claim_kind"] and contains_expected(actual_claim["expected"], claim["expected"]) for actual_claim in actual ) ] return { "required_claim_kinds": sorted(required), "covered_claim_kinds": sorted(required.intersection({claim["claim_kind"] for claim in actual})), "required_claims": expected_claims, "missing_claims": missing, "validation_capability_gaps": gaps, "complete": not missing and not gaps, } def _contains_expected(actual: Any, required: Any) -> bool: """Backward-compatible evaluation helper for canonical claim matching.""" return contains_expected(actual, required) def _capability_block_reason(*reports: dict[str, Any]) -> tuple[str, str]: """Classify a failed conformance probe without confusing outages for gaps. A provider transport failure means the probe did not establish either support or non-support. Only a complete response that violates one of the exposed tool contracts is evidence of a model structured-output limit. """ messages = [ str(failure.get("message") or "").casefold() for report in reports for failure in report.get("failures") or () if isinstance(failure, dict) ] unavailable_markers = ( "transport unavailable", "connection", "network", "timeout", "timed out", "temporarily unavailable", ) if any(any(marker in message for marker in unavailable_markers) for message in messages): return "MODEL_CAPABILITY_PROBE_UNAVAILABLE", "configuration_or_network" return "MODEL_STRUCTURED_OUTPUT_UNSUPPORTED", "model_format_or_decision" def _artifact_evidence_complete( artifact_root: Path, revisions: list[str], active_revision: str, artifact_manifest: dict[str, Any] | None, ) -> bool: """Require every published CAD decision to retain its reviewable evidence. A non-empty report manifest is not enough: a task could otherwise report only a rendered contract view while silently losing the STEP, topology, or candidate review used to accept a revision. Candidate manifests protect the immutable build inputs/outputs; the report manifest additionally protects the frozen contract and final independent review written later. """ if not revisions or not active_revision: return False report_files = { str(item.get("path") or "") for item in (artifact_manifest or {}).get("files") or () if isinstance(item, dict) } required_report_files = { "requirements-contract.json", f"reviews/final/{active_revision}/final-review.json", } if artifact_manifest is not None and not required_report_files.issubset(report_files): return False if not all((artifact_root / path).is_file() for path in required_report_files): return False for revision_id in sorted(set(revisions)): revision_root = artifact_root / "revisions" / revision_id required_paths = {revision_root / relative for relative in _CANDIDATE_EVIDENCE_FILES} manifest_path = revision_root / "manifest.json" if not manifest_path.is_file() or not all(path.is_file() for path in required_paths): return False try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return False declared = manifest.get("files") if isinstance(manifest, dict) else None if not isinstance(declared, dict) or not _CANDIDATE_EVIDENCE_FILES.issubset(declared): return False for relative, digest in declared.items(): path = revision_root / str(relative) if not isinstance(digest, str) or len(digest) != 64 or not path.is_file(): return False if sha256(path.read_bytes()).hexdigest() != digest: return False if artifact_manifest is not None: expected_report_paths = {f"revisions/{revision_id}/{relative}" for relative in _CANDIDATE_EVIDENCE_FILES} if not expected_report_paths.issubset(report_files): return False return True def _failure_attribution( *, checks: dict[str, bool], outcome: str, events: list[dict[str, Any]], projection: dict[str, Any], ) -> dict[str, Any] | None: """Return one evidence-backed failure layer required by the test target.""" if outcome == "passed": return None codes: list[str] = [] for event in events: payload = event.get("payload") if isinstance(event, dict) and isinstance(event.get("payload"), dict) else {} result = payload.get("result") if isinstance(payload.get("result"), dict) else payload code = result.get("code") if isinstance(result, dict) else "" if isinstance(code, str) and code: codes.append(code) last_error = projection.get("last_error") if isinstance(projection, dict) else "" if isinstance(last_error, str) and last_error: codes.append(last_error) for code in reversed(codes): layer = _ERROR_FAILURE_LAYERS.get(code) if layer: return { "layer": layer, "reason_code": code, "evidence_codes": list(dict.fromkeys(codes)), "failed_checks": [name for name, passed in checks.items() if not passed], } failed_checks = [name for name, passed in checks.items() if not passed] if outcome == "validation_capability_gap" or any(name in {"required_claims", "acceptance_contract_coverage"} for name in failed_checks): layer, reason = "v3_contract_or_verifier", "VALIDATION_CAPABILITY_GAP" elif any(name in {"deterministic_claims_pass", "required_operations", "required_operation_alternative"} for name in failed_checks): layer, reason = "cdsl_expression", "CAD_ACCEPTANCE_GATE_FAILED" elif any(name in {"token_budget", "call_budget", "author_turn_budget", "reviewer_turn_budget"} for name in failed_checks): layer, reason = "configuration_or_network", "EVALUATION_BUDGET_EXCEEDED" elif "raw_argument_audit" in failed_checks: layer, reason = "v3_contract_or_verifier", "RAW_ARGUMENT_AUDIT_FAILED" elif any(name in {"immutable_artifacts", "artifact_manifest", "required_artifact_evidence", "action_ledger"} for name in failed_checks): layer, reason = "configuration_or_network", "ARTIFACT_EVIDENCE_INCOMPLETE" else: layer, reason = "model_format_or_decision", "WORKFLOW_TERMINAL_STATE_MISMATCH" return { "layer": layer, "reason_code": reason, "evidence_codes": list(dict.fromkeys(codes)), "failed_checks": failed_checks, } def _run_checks( scenario: dict[str, Any], projection: dict[str, Any], usage: dict[str, Any], requirements_contract: dict[str, Any] | None, events: list[dict[str, Any]], artifact_root: Path, ledger: list[dict[str, Any]] | None = None, tool_audits: list[dict[str, Any]] | None = None, artifact_manifest: dict[str, Any] | None = None, ) -> dict[str, bool]: records = usage.get("records") if isinstance(usage.get("records"), list) else [] author_calls = [item for item in records if isinstance(item, dict) and item.get("role") != "reviewer"] reviewer_calls = [item for item in records if isinstance(item, dict) and item.get("role") == "reviewer"] total_tokens = int(usage.get("prompt_tokens") or 0) + int(usage.get("completion_tokens") or 0) audit_ledger = ledger if ledger is not None else [item for item in projection.get("action_ledger_summary") or () if isinstance(item, dict)] accepted = [item for item in audit_ledger if isinstance(item, dict) and item.get("event") == "accepted"] revisions = [str(item.get("revision_id") or "") for item in accepted] operation_ids = {str(item.get("actual_atomic_id") or "") for item in accepted} required_operation_groups = [ {str(atomic_id) for atomic_id in group if isinstance(atomic_id, str) and atomic_id} for group in scenario.get("required_any_atomic_id_groups") or () if isinstance(group, list) ] active_revision = str(projection.get("active_revision") or "") manifest = artifact_root / "revisions" / active_revision / "manifest.json" jsonl = artifact_root / "actions" / "action-ledger.jsonl" final_claims = next((item.get("claim_results") for item in reversed(audit_ledger) if isinstance(item, dict) and item.get("event") == "completed" and isinstance(item.get("claim_results"), list)), []) audit_records = tool_audits if isinstance(tool_audits, list) else [] audit_valid = bool(audit_records) and all( isinstance(item, dict) and isinstance(item.get("raw_arguments_hash"), str) and len(item["raw_arguments_hash"]) == 64 and item.get("canonical_schema_valid") is True and isinstance(item.get("state_binding"), dict) and bool(item["state_binding"].get("phase")) and item["state_binding"].get("binding_valid") is True and item.get("single_allowed_call") is True for item in audit_records ) acceptance = _acceptance_coverage(scenario, requirements_contract) artifact_evidence = _artifact_evidence_complete( artifact_root, revisions, active_revision, artifact_manifest, ) return { "expected_terminal_phase": projection.get("phase") == scenario.get("expected_phase", "COMPLETED"), "token_budget": total_tokens <= int(scenario["max_total_tokens"]), "call_budget": len(records) <= int(scenario["max_total_calls"]), "author_turn_budget": len(author_calls) <= int(scenario["max_author_turns"]), "reviewer_turn_budget": len(reviewer_calls) <= int(scenario["max_reviewer_turns"]), "required_claims": not acceptance["missing_claims"], "acceptance_contract_coverage": acceptance["complete"], "required_operations": {str(value) for value in scenario.get("required_atomic_ids") or ()}.issubset(operation_ids), "required_operation_alternative": ( (not scenario.get("required_any_atomic_ids") or bool({str(value) for value in scenario["required_any_atomic_ids"]}.intersection(operation_ids))) and all(group.intersection(operation_ids) for group in required_operation_groups) ), "immutable_artifacts": bool(active_revision) and manifest.is_file(), "required_artifact_evidence": artifact_evidence, "artifact_manifest": ( artifact_manifest is None or ( isinstance(artifact_manifest.get("files"), list) and bool(artifact_manifest["files"]) and all( isinstance(item, dict) and isinstance(item.get("path"), str) and isinstance(item.get("sha256"), str) and len(item["sha256"]) == 64 for item in artifact_manifest["files"] ) ) ), "action_ledger": jsonl.is_file(), "unique_revisions": len(revisions) == len(set(revisions)), "deterministic_claims_pass": bool(final_claims) and all( not isinstance(item, dict) or not item.get("deterministic") or item.get("status") == "pass" for item in final_claims ), "raw_argument_audit": audit_valid if tool_audits is not None else bool(records) and all(isinstance(item, dict) and isinstance(item.get("raw_arguments_hash"), str) and len(item["raw_arguments_hash"]) == 64 for item in records), "no_schema_or_decision_rejections": not _rejection_codes(events), } def _git_revision() -> str: try: completed = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=BACKEND_ROOT.parent, capture_output=True, text=True, check=True, timeout=5, ) return completed.stdout.strip() except (OSError, subprocess.SubprocessError): return "unknown" def _event_audit(events: list[dict[str, Any]]) -> list[dict[str, Any]]: """Persist event metadata without prompts, tool arguments, or tool facts.""" audit: list[dict[str, Any]] = [] for item in events: payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} result = payload.get("result") if isinstance(payload.get("result"), dict) else {} usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {} audit.append({ "name": str(item.get("name") or ""), "tool": str(payload.get("tool") or ""), "status": str(payload.get("status") or ""), "code": str(result.get("code") or payload.get("code") or ""), "raw_arguments_hash": str(usage.get("raw_arguments_hash") or ""), "field_errors": result.get("field_errors") if isinstance(result.get("field_errors"), list) else [], }) return audit def _report_tool_audits(audits: list[dict[str, Any]]) -> list[dict[str, Any]]: """Return report-safe audit metadata without argument values or prompts.""" return [ { "audit_id": item.get("audit_id"), "at": item.get("at"), "actor": item.get("actor"), "tool": item.get("tool"), "raw_arguments_hash": item.get("raw_arguments_hash"), "canonical_schema_valid": item.get("canonical_schema_valid"), "field_errors": item.get("field_errors") if isinstance(item.get("field_errors"), list) else [], "state_binding": item.get("state_binding") if isinstance(item.get("state_binding"), dict) else {}, "returned_tool": item.get("returned_tool"), "single_allowed_call": item.get("single_allowed_call"), } for item in audits if isinstance(item, dict) ] def _ledger_identifiers(ledger: list[dict[str, Any]]) -> dict[str, list[str]]: """Expose audit identifiers explicitly without copying provider payloads.""" candidate_ids = sorted({ str(item.get("candidate_id")) for item in ledger if isinstance(item.get("candidate_id"), str) and item.get("candidate_id") }) revision_ids = sorted({ str(item.get("revision_id")) for item in ledger if isinstance(item.get("revision_id"), str) and item.get("revision_id") }) return {"candidate_ids": candidate_ids, "revision_ids": revision_ids} def _final_claim_results(ledger: list[dict[str, Any]]) -> list[dict[str, Any]]: for item in reversed(ledger): results = item.get("claim_results") if item.get("event") == "completed" and isinstance(results, list): return [value for value in results if isinstance(value, dict)] return [] def _redacted_correlation_ids(task_id: str, invocations: list[dict[str, Any]]) -> list[str]: """Keep run linkage without publishing task or provider correlation values.""" return [ canonical_hash({"task_id": task_id, "invocation_id": str(item.get("invocation_id") or "")})[:20] for item in invocations if isinstance(item, dict) and item.get("invocation_id") ] def _safe_artifact_manifest(artifact_root: Path) -> dict[str, Any]: """Hash reviewable CAD evidence without copying source or prompt content.""" allowed_exact = {"requirements-contract.json", "requirements-index.json", "requirements.md", "completion.md"} allowed_prefixes = ("actions/", "revisions/", "reviews/", "documents/requirements-") files: list[dict[str, str]] = [] if artifact_root.is_dir(): for path in sorted(artifact_root.rglob("*")): if not path.is_file(): continue relative = path.relative_to(artifact_root).as_posix() if relative not in allowed_exact and not relative.startswith(allowed_prefixes): continue files.append({"path": relative, "sha256": sha256(path.read_bytes()).hexdigest()}) return {"schema_version": "cad.live-eval-artifact-manifest.v1", "artifact_root": str(artifact_root), "files": files} async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, Any]: try: scenarios = _fixture(arguments.suite, arguments.scenario) except ValueError as error: return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)} settings = get_settings() try: author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model) if arguments.review_provider or arguments.review_model: review_provider, review_model = settings.resolve_model(arguments.review_provider, arguments.review_model) if review_provider.id == author_provider.id and review_model.id == author_model.id: raise ValueError("Reviewer must differ from author") else: review_provider, review_model = settings.resolve_independent_review_model(author_provider, author_model) except ValueError as error: return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)} baseline_path = arguments.baseline_report.resolve() if arguments.baseline_report else None if arguments.suite == "release" and baseline_path is None: return {"status": "LIVE_EVAL_BLOCKED", "error": "TOKEN_BASELINE_REQUIRED: release requires a provenance-checked pre-v3 token baseline report."} try: baseline = load_token_baseline(baseline_path) if baseline_path else None current_author_identity = author_request_identity(author_provider, author_model) runtime_profile_hash = profile_sha256(settings.engine_root / "profile_schema.json") except TokenBaselineError as error: return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {error}"} if baseline is not None: baseline_errors = validate_token_baseline_provenance( baseline, scenarios=scenarios, author_identity=current_author_identity, runtime_profile_hash=runtime_profile_hash, ) if baseline_errors: return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {baseline_errors[0]}", "baseline_errors": baseline_errors} isolated = replace(settings, task_root=report_root / "artifacts", conversation_root=report_root / "conversations") services = compose_v3(isolated) contracts = [ { "atomic_id": atomic_id, "contract_hash": services.workflow.runtime.operation_contract(atomic_id)["contract_hash"], "contract_version": services.workflow.runtime.operation_contract(atomic_id)["contract_version"], "registry_revision": services.workflow.runtime.operation_contract(atomic_id)["registry_revision"], } for atomic_id in services.workflow.runtime.supported_atomic_ids() ] verifier_schema_hash = canonical_hash(default_registry().expected_one_of_schema()) try: author_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=author_provider.id, model_id=author_model.id, force=True) reviewer_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=review_provider.id, model_id=review_model.id, force=True) except Exception as error: return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)[:1000]} if not author_capability.get("supported") or not reviewer_capability.get("supported"): error, failure_layer = _capability_block_reason(author_capability, reviewer_capability) return { "status": "LIVE_EVAL_BLOCKED", "error": error, "failure_layer": failure_layer, "author_capability": author_capability, "reviewer_capability": reviewer_capability, } results: list[dict[str, Any]] = [] for scenario in scenarios: for repetition in range(1, 4 if arguments.suite == "release" else 2): services.workflow.config = replace( services.workflow.config, # State transitions include local candidate recovery, so the # loop guard is deliberately independent from the externally # measured model-call budgets below. max_turns=max(8, int(scenario["max_total_calls"]) * 3), max_author_turns=int(scenario["max_author_turns"]), max_reviewer_turns=int(scenario["max_reviewer_turns"]), max_model_calls=int(scenario["max_total_calls"]), ) task_id = f"cad_{secrets.token_hex(6)}" oracle_claims = [item for item in scenario.get("required_claims") or () if isinstance(item, dict)] if oracle_claims: services.workflow.requirements.register_evaluation_contract_oracle( task_id, oracle_claims, validation_capability_gaps=[item for item in scenario.get("validation_capability_gaps") or () if isinstance(item, dict)], ) services.workflow.create_task(task_id, str(scenario["request"])) events: list[dict[str, Any]] = [] started = datetime.now(timezone.utc) try: async with asyncio.timeout(int(scenario["max_wall_seconds"])): async for name, payload in services.workflow.run( task_id=task_id, author=ModelIdentity(author_provider.id, author_model.id), reviewer=ModelIdentity(review_provider.id, review_model.id), ): events.append({"name": name, "payload": payload}) except TimeoutError: events.append({"name": "timeout", "payload": {"code": "LIVE_EVAL_TIMEOUT"}}) projection = services.repository.get_task_projection(task_id) or {} usage = services.repository.usage_summary(task_id) ledger = services.repository.ledger_events(task_id) invocations = services.repository.invocation_records(task_id) tool_audits = services.repository.tool_audits(task_id) terminal = next((item["payload"] for item in reversed(events) if item["name"] == "task_terminal"), {}) artifact_root = (isolated.task_root / task_id).resolve() artifact_manifest = _safe_artifact_manifest(artifact_root) finished = datetime.now(timezone.utc) checks = _run_checks( scenario, projection, usage, services.artifacts.read_requirements_contract( task_id, services.repository.get_state(task_id).requirements_contract_path if services.repository.get_state(task_id) is not None else "", ), events, artifact_root, ledger, tool_audits, artifact_manifest, ) success = all(checks.values()) acceptance = _acceptance_coverage( scenario, services.artifacts.read_requirements_contract( task_id, services.repository.get_state(task_id).requirements_contract_path if services.repository.get_state(task_id) is not None else "", ), ) outcome = ( "passed" if success else "validation_capability_gap" if not acceptance["complete"] and all(value for key, value in checks.items() if key != "acceptance_contract_coverage") else "failed" ) failure_attribution = _failure_attribution( checks=checks, outcome=outcome, events=events, projection=projection, ) results.append({ "scenario": scenario["id"], "repetition": repetition, "success": success, "outcome": outcome, "started_at": started.isoformat(), "finished_at": finished.isoformat(), "duration_ms": round((finished - started).total_seconds() * 1000), "terminal": terminal, "projection": projection, "usage": usage, "checks": checks, "acceptance_coverage": acceptance, "failure_attribution": failure_attribution, "rejection_codes": _rejection_codes(events), "schema_rejection_count": len(_rejection_codes(events)), "retry_count": sum(1 for event in events if (event.get("payload") or {}).get("status") == "error"), "rollback_count": sum(1 for item in ledger if item.get("event") == "rollback"), **_ledger_identifiers(ledger), "final_claim_results": _final_claim_results(ledger), "redacted_correlation_ids": _redacted_correlation_ids(task_id, invocations), "tool_audits": _report_tool_audits(tool_audits), "artifact_root": str(artifact_root), "action_ledger_path": str(artifact_root / "actions" / "action-ledger.jsonl"), "artifact_manifest": artifact_manifest, "ledger": ledger, "invocations": invocations, "event_audit": _event_audit(events), }) token_comparison = ( compare_token_baseline( baseline, scenarios=scenarios, v3_results=results, author_identity=current_author_identity, runtime_profile_hash=runtime_profile_hash, ) if baseline is not None else {"schema_version": "cad.token-comparison.v1", "status": "not_required"} ) token_gate = all(token_comparison.get("checks", {}).values()) if baseline is not None else True return { "status": "passed" if results and all(item["success"] for item in results) and token_gate else "failed", "author": {"provider": author_provider.id, "model": author_model.id}, "author_request_identity": current_author_identity, "reviewer": {"provider": review_provider.id, "model": review_model.id}, "author_capability": author_capability, "reviewer_capability": reviewer_capability, "structured_output_mode": { "author": str(author_capability.get("mode") or ""), "reviewer": str(reviewer_capability.get("mode") or ""), }, "git_revision": _git_revision(), "protocol_version": "3.0", "runtime_profile_sha256": runtime_profile_hash, "operation_contracts": contracts, "verifier_registry_version": "cad.verifier-registry.v1", "verifier_schema_hash": verifier_schema_hash, "token_comparison": token_comparison, "baseline_report": str(baseline_path) if baseline_path else "", "results": results, } def main() -> int: arguments = _arguments() report_root = BACKEND_ROOT / "live-evals" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") report_root.mkdir(parents=True, exist_ok=True) try: result = asyncio.run(_run(arguments, report_root)) except KeyboardInterrupt: # Let an explicit operator interruption retain its normal CLI # semantics. An external kill cannot be reported reliably either. raise except BaseException as error: # A live evaluation may fail before it creates a task (for example # during conformance). Its report is still the release gate's audit # artifact, so an unexpected evaluator failure must not disappear. result = { "status": "LIVE_EVAL_BLOCKED", "error": f"UNEXPECTED_LIVE_EVAL_ERROR: {type(error).__name__}: {str(error)[:900]}", "failure_layer": "configuration_or_network", } if result["status"] == "LIVE_EVAL_BLOCKED" and arguments.allow_skip and not arguments.require_live: result["skip_reason"] = result.get("error", "live provider access is unavailable") result["status"] = "skipped" result.update({"suite": arguments.suite, "require_live": arguments.require_live, "report_root": str(report_root.resolve())}) (report_root / "report.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps({"status": result["status"], "report": str((report_root / "report.json").resolve())}, ensure_ascii=False)) if result["status"] == "passed": return 0 if result["status"] == "skipped": return 0 return 2 if __name__ == "__main__": sys.exit(main())