991 lines
48 KiB
Python
991 lines
48 KiB
Python
"""Run the real provider, engine, reviewer and v3 state machine end to end.
|
|
|
|
``--require-live`` deliberately fails closed: missing credentials, blocked
|
|
network, a skipped model capability, a timeout, or any scenario failure emits
|
|
``LIVE_EVAL_BLOCKED``/failure details and exits non-zero.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
from dataclasses import replace
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import secrets
|
|
from statistics import median
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
from app.cad_agent.application.capabilities import verify_model_capability
|
|
from app.cad_agent.application.workflow import ModelIdentity
|
|
from app.cad_agent.composition import compose_v3
|
|
from app.cad_agent.domain.claim_matching import contains_expected
|
|
from app.cad_agent.domain.operation_contract import canonical_hash
|
|
from app.cad_agent.domain.verifier_registry import default_registry
|
|
from app.cad_agent.evals.token_baseline import (
|
|
TokenBaselineError,
|
|
author_request_identity,
|
|
compare_token_baseline,
|
|
load_token_baseline,
|
|
profile_sha256,
|
|
validate_token_baseline_provenance,
|
|
)
|
|
from app.settings import BACKEND_ROOT, get_settings
|
|
|
|
|
|
_LEGACY_CANDIDATE_EVIDENCE_FILES = frozenset({
|
|
"candidate.json",
|
|
"candidate-review.json",
|
|
"model.cdsl.json",
|
|
"model.step",
|
|
"model.glb",
|
|
"model.topology.json",
|
|
"rebuild-report.json",
|
|
"renders/render-manifest.json",
|
|
"renders/contact-sheet.jpg",
|
|
})
|
|
|
|
# A v3.2 Feature DAG publishes one atomic checkpoint after local verification.
|
|
# It deliberately has no candidate review or technical render bundle; the
|
|
# final review owns that later evidence. GLB is optional because a preview
|
|
# conversion outage must not invalidate an otherwise sound STEP checkpoint.
|
|
_FEATURE_NODE_EVIDENCE_FILES = frozenset({
|
|
"input.json",
|
|
"model.cdsl.json",
|
|
"model.step",
|
|
"model.topology.json",
|
|
"node-verification.json",
|
|
"rebuild-report.json",
|
|
})
|
|
|
|
_FAILURE_LAYERS = frozenset({
|
|
"model_format_or_decision",
|
|
"v3_contract_or_verifier",
|
|
"cdsl_expression",
|
|
"engine_execution",
|
|
"independent_visual_review",
|
|
"configuration_or_network",
|
|
})
|
|
|
|
_ERROR_FAILURE_LAYERS = {
|
|
"AUTHOR_FORMAT_INVALID": "model_format_or_decision",
|
|
"AUTHOR_DECISION_REJECTED": "model_format_or_decision",
|
|
"STALE_WORKING_HEAD": "model_format_or_decision",
|
|
"FAILED_AUTHOR_FORMAT": "model_format_or_decision",
|
|
"MODEL_STRUCTURED_OUTPUT_UNSUPPORTED": "model_format_or_decision",
|
|
"RUNTIME_PRECONDITION_FAILED": "v3_contract_or_verifier",
|
|
"RUNTIME_CONTRACT_INVALID": "v3_contract_or_verifier",
|
|
"VERIFIER_UNAVAILABLE": "v3_contract_or_verifier",
|
|
"CLAIM_VERIFICATION_FAILED": "cdsl_expression",
|
|
"CANDIDATE_BUILD_FAILED": "engine_execution",
|
|
"CANDIDATE_REVIEW_REJECTED": "independent_visual_review",
|
|
"AUTHOR_TRANSPORT_UNAVAILABLE": "configuration_or_network",
|
|
"REVIEW_SERVICE_UNAVAILABLE": "configuration_or_network",
|
|
"RENDER_SERVICE_UNAVAILABLE": "configuration_or_network",
|
|
"STORAGE_FAILURE": "configuration_or_network",
|
|
"LIVE_EVAL_TIMEOUT": "configuration_or_network",
|
|
"FAILED_INTERNAL": "engine_execution",
|
|
}
|
|
|
|
|
|
def _arguments() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run live protocol-v3 CAD evaluations.")
|
|
parser.add_argument("--suite", choices=("smoke", "release", "comprehensive"), default="smoke")
|
|
parser.add_argument("--require-live", action="store_true")
|
|
parser.add_argument("--allow-skip", action="store_true")
|
|
parser.add_argument("--author-provider")
|
|
parser.add_argument("--author-model")
|
|
parser.add_argument("--review-provider")
|
|
parser.add_argument("--review-model")
|
|
parser.add_argument("--scenario", action="append", dest="scenarios", help="Run a fixture scenario by stable ID. Repeat this option to select a comparison set.")
|
|
parser.add_argument("--repetitions", type=int, help="Run every selected scenario this many times.")
|
|
parser.add_argument("--author-guidance", choices=("on", "off"), help="Override CDSL author guidance for this run.")
|
|
parser.add_argument("--compare-guidance-reports", nargs=2, type=Path, metavar=("CONTROL", "TREATMENT"), help="Compare matched --author-guidance off/on report.json files without invoking providers.")
|
|
parser.add_argument("--baseline-report", type=Path, help="Measured pre-v3 token baseline JSON for a release run.")
|
|
return parser.parse_args()
|
|
|
|
|
|
def _fixture(suite: str, scenario_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
|
fixture_name = "comprehensive.json" if suite == "comprehensive" else "release.json"
|
|
value = json.loads((Path(__file__).parent / "fixtures" / fixture_name).read_text(encoding="utf-8"))
|
|
if fixture_name == "comprehensive.json":
|
|
source_document = str(value.get("source_document") or "")
|
|
expected_digest = str(value.get("source_document_sha256") or "")
|
|
source_path = (BACKEND_ROOT.parent / source_document).resolve()
|
|
workspace_root = BACKEND_ROOT.parent.resolve()
|
|
if (
|
|
not source_document
|
|
or workspace_root not in source_path.parents
|
|
or not source_path.is_file()
|
|
or sha256(source_path.read_bytes()).hexdigest() != expected_digest
|
|
):
|
|
raise ValueError("Comprehensive fixture is not synchronized with its source document.")
|
|
values = [item for item in value.get("scenarios") or () if isinstance(item, dict)]
|
|
values = values[:2] if suite == "smoke" else values
|
|
if not scenario_ids:
|
|
return values
|
|
requested = list(dict.fromkeys(scenario_ids))
|
|
available = {str(item.get("id") or "") for item in values}
|
|
unknown = [scenario_id for scenario_id in requested if scenario_id not in available]
|
|
if unknown:
|
|
raise ValueError(f"Unknown scenario {unknown[0]!r} for suite {suite!r}")
|
|
selected = [item for item in values if str(item.get("id") or "") in set(requested)]
|
|
return selected
|
|
|
|
|
|
def _rejection_codes(events: list[dict[str, Any]]) -> list[str]:
|
|
codes: list[str] = []
|
|
for event in events:
|
|
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else payload
|
|
code = result.get("code") if isinstance(result, dict) else None
|
|
if code in {"AUTHOR_FORMAT_INVALID", "AUTHOR_DECISION_REJECTED", "STALE_WORKING_HEAD"}:
|
|
codes.append(str(code))
|
|
return codes
|
|
|
|
|
|
def _acceptance_coverage(
|
|
scenario: dict[str, Any],
|
|
requirements_contract: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Assess whether a fixture's acceptance contract is representable today.
|
|
|
|
Comprehensive prompts deliberately include manufacturing relationships that
|
|
may not yet have a deterministic verifier. A healthy-looking solid is
|
|
not evidence for those relationships, so they remain explicit capability
|
|
gaps instead of silently passing a scenario.
|
|
"""
|
|
actual = [
|
|
{
|
|
"claim_kind": str(claim.get("claim_kind") or ""),
|
|
"expected": claim.get("expected") if isinstance(claim.get("expected"), dict) else {},
|
|
}
|
|
for requirement in (requirements_contract or {}).get("requirements") or ()
|
|
if isinstance(requirement, dict)
|
|
for claim in requirement.get("acceptance_claims") or ()
|
|
if isinstance(claim, dict)
|
|
]
|
|
required = {str(value) for value in scenario.get("required_claim_kinds") or ()}
|
|
expected_claims = [
|
|
{
|
|
"claim_kind": str(item.get("claim_kind") or ""),
|
|
"expected": item.get("expected") if isinstance(item.get("expected"), dict) else {},
|
|
}
|
|
for item in scenario.get("required_claims") or ()
|
|
if isinstance(item, dict) and isinstance(item.get("claim_kind"), str)
|
|
]
|
|
expected_kinds = {claim["claim_kind"] for claim in expected_claims}
|
|
expected_claims.extend(
|
|
{"claim_kind": claim_kind, "expected": {}}
|
|
for claim_kind in sorted(required - expected_kinds)
|
|
)
|
|
gaps = [
|
|
{"id": str(item.get("id") or ""), "description": str(item.get("description") or "")}
|
|
for item in scenario.get("validation_capability_gaps") or ()
|
|
if isinstance(item, dict)
|
|
]
|
|
missing = [
|
|
claim for claim in expected_claims
|
|
if not any(
|
|
actual_claim["claim_kind"] == claim["claim_kind"]
|
|
and _contains_business_expected(actual_claim["expected"], claim["expected"])
|
|
for actual_claim in actual
|
|
)
|
|
]
|
|
return {
|
|
"required_claim_kinds": sorted(required),
|
|
"covered_claim_kinds": sorted(required.intersection({claim["claim_kind"] for claim in actual})),
|
|
"required_claims": expected_claims,
|
|
"missing_claims": missing,
|
|
"validation_capability_gaps": gaps,
|
|
"complete": not missing and not gaps,
|
|
}
|
|
|
|
|
|
def _contains_business_expected(actual: Any, required: Any) -> bool:
|
|
"""Match fixture business values without coupling to verifier tolerances.
|
|
|
|
Tolerances are executable verifier parameters chosen within the schema's
|
|
safe range. They are not a separate user requirement and should not make
|
|
a valid generated contract fail release evaluation merely because the
|
|
author used the registry default instead of the fixture's tighter value.
|
|
"""
|
|
if isinstance(actual, dict) and isinstance(required, dict):
|
|
return all(
|
|
key in actual
|
|
and _contains_business_expected(actual[key], value)
|
|
for key, value in required.items()
|
|
if key not in {"tolerance_mm", "tolerance"}
|
|
)
|
|
return contains_expected(actual, required)
|
|
|
|
|
|
def _contains_expected(actual: Any, required: Any) -> bool:
|
|
"""Match canonical claim values in release evaluation."""
|
|
return contains_expected(actual, required)
|
|
|
|
|
|
def _capability_block_reason(*reports: dict[str, Any]) -> tuple[str, str]:
|
|
"""Classify a failed conformance probe without confusing outages for gaps.
|
|
|
|
A provider transport failure means the probe did not establish either
|
|
support or non-support. Only a complete response that violates one of the
|
|
exposed tool contracts is evidence of a model structured-output limit.
|
|
"""
|
|
messages = [
|
|
str(failure.get("message") or "").casefold()
|
|
for report in reports
|
|
for failure in report.get("failures") or ()
|
|
if isinstance(failure, dict)
|
|
]
|
|
unavailable_markers = (
|
|
"transport unavailable",
|
|
"connection",
|
|
"network",
|
|
"timeout",
|
|
"timed out",
|
|
"temporarily unavailable",
|
|
)
|
|
if any(any(marker in message for marker in unavailable_markers) for message in messages):
|
|
return "MODEL_CAPABILITY_PROBE_UNAVAILABLE", "configuration_or_network"
|
|
return "MODEL_STRUCTURED_OUTPUT_UNSUPPORTED", "model_format_or_decision"
|
|
|
|
|
|
def _artifact_evidence_complete(
|
|
artifact_root: Path,
|
|
revisions: list[str],
|
|
active_revision: str,
|
|
artifact_manifest: dict[str, Any] | None,
|
|
ledger: list[dict[str, Any]],
|
|
) -> bool:
|
|
"""Require every published CAD decision to retain its reviewable evidence.
|
|
|
|
A non-empty report manifest is not enough: a task could otherwise report
|
|
only a rendered contract view while silently losing the STEP, topology, or
|
|
node verification used to accept a revision. Checkpoint manifests protect
|
|
immutable build inputs and outputs; the report manifest additionally
|
|
protects the frozen contract and final independent review written later.
|
|
"""
|
|
if not revisions or not active_revision:
|
|
return False
|
|
report_files = {
|
|
str(item.get("path") or "")
|
|
for item in (artifact_manifest or {}).get("files") or ()
|
|
if isinstance(item, dict)
|
|
}
|
|
required_report_files = {
|
|
"requirements-contract.json",
|
|
f"reviews/final/{active_revision}/final-review.json",
|
|
}
|
|
if artifact_manifest is not None and not required_report_files.issubset(report_files):
|
|
return False
|
|
if not all((artifact_root / path).is_file() for path in required_report_files):
|
|
return False
|
|
feature_revisions = {
|
|
str(item.get("revision_id") or "")
|
|
for item in ledger
|
|
if isinstance(item, dict) and item.get("event") == "feature_node_verified"
|
|
}
|
|
for revision_id in sorted(set(revisions)):
|
|
revision_root = artifact_root / "revisions" / revision_id
|
|
evidence_files = (
|
|
_FEATURE_NODE_EVIDENCE_FILES
|
|
if revision_id in feature_revisions
|
|
else _LEGACY_CANDIDATE_EVIDENCE_FILES
|
|
)
|
|
required_paths = {revision_root / relative for relative in evidence_files}
|
|
manifest_path = revision_root / "manifest.json"
|
|
if not manifest_path.is_file() or not all(path.is_file() for path in required_paths):
|
|
return False
|
|
try:
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return False
|
|
declared = manifest.get("files") if isinstance(manifest, dict) else None
|
|
if not isinstance(declared, dict) or not evidence_files.issubset(declared):
|
|
return False
|
|
for relative, digest in declared.items():
|
|
path = revision_root / str(relative)
|
|
if not isinstance(digest, str) or len(digest) != 64 or not path.is_file():
|
|
return False
|
|
if sha256(path.read_bytes()).hexdigest() != digest:
|
|
return False
|
|
if artifact_manifest is not None:
|
|
expected_report_paths = {f"revisions/{revision_id}/{relative}" for relative in evidence_files}
|
|
if not expected_report_paths.issubset(report_files):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _failure_attribution(
|
|
*,
|
|
checks: dict[str, bool],
|
|
outcome: str,
|
|
events: list[dict[str, Any]],
|
|
projection: dict[str, Any],
|
|
) -> dict[str, Any] | None:
|
|
"""Return one evidence-backed failure layer required by the test target."""
|
|
if outcome == "passed":
|
|
return None
|
|
codes: list[str] = []
|
|
for event in events:
|
|
payload = event.get("payload") if isinstance(event, dict) and isinstance(event.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else payload
|
|
code = result.get("code") if isinstance(result, dict) else ""
|
|
if isinstance(code, str) and code:
|
|
codes.append(code)
|
|
last_error = projection.get("last_error") if isinstance(projection, dict) else ""
|
|
if isinstance(last_error, str) and last_error:
|
|
codes.append(last_error)
|
|
for code in reversed(codes):
|
|
layer = _ERROR_FAILURE_LAYERS.get(code)
|
|
if layer:
|
|
return {
|
|
"layer": layer,
|
|
"reason_code": code,
|
|
"evidence_codes": list(dict.fromkeys(codes)),
|
|
"failed_checks": [name for name, passed in checks.items() if not passed],
|
|
}
|
|
failed_checks = [name for name, passed in checks.items() if not passed]
|
|
if outcome == "validation_capability_gap" or any(name in {"required_claims", "acceptance_contract_coverage"} for name in failed_checks):
|
|
layer, reason = "v3_contract_or_verifier", "VALIDATION_CAPABILITY_GAP"
|
|
elif any(name in {"deterministic_claims_pass", "required_operations", "required_operation_alternative"} for name in failed_checks):
|
|
layer, reason = "cdsl_expression", "CAD_ACCEPTANCE_GATE_FAILED"
|
|
elif any(name in {"token_budget", "call_budget", "author_turn_budget", "reviewer_turn_budget"} for name in failed_checks):
|
|
layer, reason = "configuration_or_network", "EVALUATION_BUDGET_EXCEEDED"
|
|
elif "raw_argument_audit" in failed_checks:
|
|
layer, reason = "v3_contract_or_verifier", "RAW_ARGUMENT_AUDIT_FAILED"
|
|
elif any(name in {"immutable_artifacts", "artifact_manifest", "required_artifact_evidence", "action_ledger"} for name in failed_checks):
|
|
layer, reason = "configuration_or_network", "ARTIFACT_EVIDENCE_INCOMPLETE"
|
|
else:
|
|
layer, reason = "model_format_or_decision", "WORKFLOW_TERMINAL_STATE_MISMATCH"
|
|
return {
|
|
"layer": layer,
|
|
"reason_code": reason,
|
|
"evidence_codes": list(dict.fromkeys(codes)),
|
|
"failed_checks": failed_checks,
|
|
}
|
|
|
|
|
|
def _run_checks(
|
|
scenario: dict[str, Any],
|
|
projection: dict[str, Any],
|
|
usage: dict[str, Any],
|
|
requirements_contract: dict[str, Any] | None,
|
|
events: list[dict[str, Any]],
|
|
artifact_root: Path,
|
|
ledger: list[dict[str, Any]] | None = None,
|
|
tool_audits: list[dict[str, Any]] | None = None,
|
|
artifact_manifest: dict[str, Any] | None = None,
|
|
) -> dict[str, bool]:
|
|
records = usage.get("records") if isinstance(usage.get("records"), list) else []
|
|
author_calls = [item for item in records if isinstance(item, dict) and item.get("role") != "reviewer"]
|
|
reviewer_calls = [item for item in records if isinstance(item, dict) and item.get("role") == "reviewer"]
|
|
total_tokens = int(usage.get("prompt_tokens") or 0) + int(usage.get("completion_tokens") or 0)
|
|
audit_ledger = ledger if ledger is not None else [item for item in projection.get("action_ledger_summary") or () if isinstance(item, dict)]
|
|
published = [
|
|
item for item in audit_ledger
|
|
if isinstance(item, dict) and item.get("event") in {"accepted", "feature_node_verified"}
|
|
]
|
|
revisions = [str(item.get("revision_id") or "") for item in published]
|
|
scheduled_atomic_ids = {
|
|
str(item.get("node_id") or ""): str(item.get("atomic_id") or "")
|
|
for item in audit_ledger
|
|
if isinstance(item, dict) and item.get("event") == "feature_node_scheduled"
|
|
}
|
|
operation_ids = {
|
|
str(
|
|
item.get("actual_atomic_id")
|
|
or item.get("atomic_id")
|
|
or scheduled_atomic_ids.get(str(item.get("node_id") or ""), "")
|
|
)
|
|
for item in published
|
|
}
|
|
required_operation_groups = [
|
|
{str(atomic_id) for atomic_id in group if isinstance(atomic_id, str) and atomic_id}
|
|
for group in scenario.get("required_any_atomic_id_groups") or ()
|
|
if isinstance(group, list)
|
|
]
|
|
active_revision = str(projection.get("active_revision") or "")
|
|
manifest = artifact_root / "revisions" / active_revision / "manifest.json"
|
|
jsonl = artifact_root / "actions" / "action-ledger.jsonl"
|
|
final_claims = next((item.get("claim_results") for item in reversed(audit_ledger) if isinstance(item, dict) and item.get("event") == "completed" and isinstance(item.get("claim_results"), list)), [])
|
|
audit_records = tool_audits if isinstance(tool_audits, list) else []
|
|
audit_valid = bool(audit_records) and all(
|
|
isinstance(item, dict)
|
|
and isinstance(item.get("raw_arguments_hash"), str)
|
|
and len(item["raw_arguments_hash"]) == 64
|
|
and item.get("canonical_schema_valid") is True
|
|
and isinstance(item.get("state_binding"), dict)
|
|
and bool(item["state_binding"].get("phase"))
|
|
and item["state_binding"].get("binding_valid") is True
|
|
and item.get("single_allowed_call") is True
|
|
for item in audit_records
|
|
)
|
|
acceptance = _acceptance_coverage(scenario, requirements_contract)
|
|
artifact_evidence = _artifact_evidence_complete(
|
|
artifact_root,
|
|
revisions,
|
|
active_revision,
|
|
artifact_manifest,
|
|
audit_ledger,
|
|
)
|
|
return {
|
|
"expected_terminal_phase": projection.get("phase") == scenario.get("expected_phase", "COMPLETED"),
|
|
"token_budget": total_tokens <= int(scenario["max_total_tokens"]),
|
|
"call_budget": len(records) <= int(scenario["max_total_calls"]),
|
|
"author_turn_budget": len(author_calls) <= int(scenario["max_author_turns"]),
|
|
"reviewer_turn_budget": len(reviewer_calls) <= int(scenario["max_reviewer_turns"]),
|
|
"required_claims": not acceptance["missing_claims"],
|
|
"acceptance_contract_coverage": acceptance["complete"],
|
|
"required_operations": {str(value) for value in scenario.get("required_atomic_ids") or ()}.issubset(operation_ids),
|
|
"required_operation_alternative": (
|
|
(not scenario.get("required_any_atomic_ids") or bool({str(value) for value in scenario["required_any_atomic_ids"]}.intersection(operation_ids)))
|
|
and all(group.intersection(operation_ids) for group in required_operation_groups)
|
|
),
|
|
"immutable_artifacts": bool(active_revision) and manifest.is_file(),
|
|
"required_artifact_evidence": artifact_evidence,
|
|
"artifact_manifest": (
|
|
artifact_manifest is None
|
|
or (
|
|
isinstance(artifact_manifest.get("files"), list)
|
|
and bool(artifact_manifest["files"])
|
|
and all(
|
|
isinstance(item, dict)
|
|
and isinstance(item.get("path"), str)
|
|
and isinstance(item.get("sha256"), str)
|
|
and len(item["sha256"]) == 64
|
|
for item in artifact_manifest["files"]
|
|
)
|
|
)
|
|
),
|
|
"action_ledger": jsonl.is_file(),
|
|
"unique_revisions": len(revisions) == len(set(revisions)),
|
|
"deterministic_claims_pass": bool(final_claims) and all(
|
|
not isinstance(item, dict) or not item.get("deterministic") or item.get("status") == "pass"
|
|
for item in final_claims
|
|
),
|
|
"raw_argument_audit": audit_valid if tool_audits is not None else bool(records) and all(isinstance(item, dict) and isinstance(item.get("raw_arguments_hash"), str) and len(item["raw_arguments_hash"]) == 64 for item in records),
|
|
"no_schema_or_decision_rejections": not _rejection_codes(events),
|
|
}
|
|
|
|
|
|
def _git_revision() -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], cwd=BACKEND_ROOT.parent, capture_output=True,
|
|
text=True, check=True, timeout=5,
|
|
)
|
|
return completed.stdout.strip()
|
|
except (OSError, subprocess.SubprocessError):
|
|
return "unknown"
|
|
|
|
|
|
def _event_audit(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Persist event metadata without prompts, tool arguments, or tool facts."""
|
|
audit: list[dict[str, Any]] = []
|
|
for item in events:
|
|
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else {}
|
|
usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {}
|
|
audit.append({
|
|
"name": str(item.get("name") or ""), "tool": str(payload.get("tool") or ""),
|
|
"status": str(payload.get("status") or ""), "code": str(result.get("code") or payload.get("code") or ""),
|
|
"raw_arguments_hash": str(usage.get("raw_arguments_hash") or ""),
|
|
"field_errors": result.get("field_errors") if isinstance(result.get("field_errors"), list) else [],
|
|
})
|
|
return audit
|
|
|
|
|
|
def _report_tool_audits(audits: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Return report-safe audit metadata without argument values or prompts."""
|
|
return [
|
|
{
|
|
"audit_id": item.get("audit_id"),
|
|
"at": item.get("at"),
|
|
"actor": item.get("actor"),
|
|
"tool": item.get("tool"),
|
|
"raw_arguments_hash": item.get("raw_arguments_hash"),
|
|
"canonical_schema_valid": item.get("canonical_schema_valid"),
|
|
"field_errors": item.get("field_errors") if isinstance(item.get("field_errors"), list) else [],
|
|
"state_binding": item.get("state_binding") if isinstance(item.get("state_binding"), dict) else {},
|
|
"returned_tool": item.get("returned_tool"),
|
|
"single_allowed_call": item.get("single_allowed_call"),
|
|
}
|
|
for item in audits
|
|
if isinstance(item, dict)
|
|
]
|
|
|
|
|
|
def _ledger_identifiers(ledger: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
"""Expose audit identifiers explicitly without copying provider payloads."""
|
|
candidate_ids = sorted({
|
|
str(item.get("candidate_id"))
|
|
for item in ledger
|
|
if isinstance(item.get("candidate_id"), str) and item.get("candidate_id")
|
|
})
|
|
revision_ids = sorted({
|
|
str(item.get("revision_id"))
|
|
for item in ledger
|
|
if isinstance(item.get("revision_id"), str) and item.get("revision_id")
|
|
})
|
|
return {"candidate_ids": candidate_ids, "revision_ids": revision_ids}
|
|
|
|
|
|
def _final_claim_results(ledger: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
for item in reversed(ledger):
|
|
results = item.get("claim_results")
|
|
if item.get("event") == "completed" and isinstance(results, list):
|
|
return [value for value in results if isinstance(value, dict)]
|
|
return []
|
|
|
|
|
|
def _redacted_correlation_ids(task_id: str, invocations: list[dict[str, Any]]) -> list[str]:
|
|
"""Keep run linkage without publishing task or provider correlation values."""
|
|
return [
|
|
canonical_hash({"task_id": task_id, "invocation_id": str(item.get("invocation_id") or "")})[:20]
|
|
for item in invocations
|
|
if isinstance(item, dict) and item.get("invocation_id")
|
|
]
|
|
|
|
|
|
def _safe_artifact_manifest(artifact_root: Path) -> dict[str, Any]:
|
|
"""Hash reviewable CAD evidence without copying source or prompt content."""
|
|
allowed_exact = {
|
|
"requirements-contract.json",
|
|
"requirements.md",
|
|
"completion-target.md",
|
|
"completion-result.md",
|
|
}
|
|
allowed_prefixes = ("actions/", "revisions/", "reviews/", "documents/requirements-")
|
|
files: list[dict[str, str]] = []
|
|
if artifact_root.is_dir():
|
|
for path in sorted(artifact_root.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
relative = path.relative_to(artifact_root).as_posix()
|
|
if relative not in allowed_exact and not relative.startswith(allowed_prefixes):
|
|
continue
|
|
files.append({"path": relative, "sha256": sha256(path.read_bytes()).hexdigest()})
|
|
return {"schema_version": "cad.live-eval-artifact-manifest.v1", "artifact_root": str(artifact_root), "files": files}
|
|
|
|
|
|
def _guidance_metadata(usage: dict[str, Any]) -> dict[str, Any]:
|
|
"""Summarize author-only guidance audit metadata without retaining prompts."""
|
|
records = [
|
|
item for item in usage.get("records") or ()
|
|
if isinstance(item, dict) and item.get("role") != "reviewer"
|
|
]
|
|
sections = sorted({
|
|
section_id
|
|
for item in records
|
|
for section_id in item.get("guidance_section_ids") or ()
|
|
if isinstance(section_id, str)
|
|
})
|
|
versions = sorted({
|
|
str(item.get("guidance_version") or "")
|
|
for item in records
|
|
if str(item.get("guidance_version") or "")
|
|
})
|
|
return {
|
|
"enabled": bool(records) and any(item.get("guidance_enabled") is True for item in records),
|
|
"versions": versions,
|
|
"section_ids": sections,
|
|
"chars_total": sum(int(item.get("guidance_chars") or 0) for item in records),
|
|
"fallback_reasons": sorted({
|
|
str(item.get("guidance_fallback_reason") or "")
|
|
for item in records
|
|
if str(item.get("guidance_fallback_reason") or "")
|
|
}),
|
|
}
|
|
|
|
|
|
def _has_unsupported_capability(row: dict[str, Any]) -> bool:
|
|
"""Recognize engine-declared unsupported capability without hiding model errors."""
|
|
for event in row.get("ledger") or ():
|
|
if not isinstance(event, dict):
|
|
continue
|
|
for failure in event.get("operation_failures") or ():
|
|
if isinstance(failure, dict) and "unsupported_" in str(failure.get("message") or ""):
|
|
return True
|
|
if "unsupported_" in str(event.get("message") or ""):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _guidance_metric_rows(report: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
results = [item for item in report.get("results") or () if isinstance(item, dict)]
|
|
capability_gaps = [
|
|
item for item in results
|
|
if item.get("outcome") == "validation_capability_gap" or _has_unsupported_capability(item)
|
|
]
|
|
return [item for item in results if item not in capability_gaps], capability_gaps
|
|
|
|
|
|
def _guidance_metrics(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
if not rows:
|
|
return {"eligible_runs": 0}
|
|
author_calls = [
|
|
sum(1 for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
|
|
for row in rows
|
|
]
|
|
context_chars = [
|
|
sum(int(item.get("context_chars") or 0) for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
|
|
for row in rows
|
|
]
|
|
prompt_tokens = [
|
|
sum(int(item.get("prompt_tokens") or 0) for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
|
|
for row in rows
|
|
]
|
|
failure_layers: dict[str, int] = {}
|
|
for row in rows:
|
|
layer = str((row.get("failure_attribution") or {}).get("layer") or "passed")
|
|
failure_layers[layer] = failure_layers.get(layer, 0) + 1
|
|
def rate(predicate: Any) -> float:
|
|
return sum(1 for row in rows if predicate(row)) / len(rows)
|
|
return {
|
|
"eligible_runs": len(rows),
|
|
"executable_checkpoint_rate": rate(lambda row: bool(row.get("revision_ids"))),
|
|
"completion_rate": rate(lambda row: str((row.get("projection") or {}).get("phase") or "") == "COMPLETED"),
|
|
"deterministic_claim_success_rate": rate(lambda row: bool((row.get("checks") or {}).get("deterministic_claims_pass"))),
|
|
"schema_or_decision_rejections": sum(int(row.get("schema_rejection_count") or 0) for row in rows),
|
|
"cdsl_expression_failures": sum(1 for row in rows if str((row.get("failure_attribution") or {}).get("layer") or "") == "cdsl_expression"),
|
|
"median_author_calls": median(author_calls),
|
|
"median_author_context_chars": median(context_chars),
|
|
"total_author_prompt_tokens": sum(prompt_tokens),
|
|
"median_author_prompt_tokens": median(prompt_tokens),
|
|
"failure_layers": dict(sorted(failure_layers.items())),
|
|
}
|
|
|
|
|
|
def compare_guidance_reports(control: dict[str, Any], treatment: dict[str, Any]) -> dict[str, Any]:
|
|
"""Compare paired guidance-off/on runs without treating engine gaps as prompt results."""
|
|
control_rows, control_gaps = _guidance_metric_rows(control)
|
|
treatment_rows, treatment_gaps = _guidance_metric_rows(treatment)
|
|
control_by_key = {(str(row.get("scenario") or ""), int(row.get("repetition") or 0)): row for row in control_rows}
|
|
treatment_by_key = {(str(row.get("scenario") or ""), int(row.get("repetition") or 0)): row for row in treatment_rows}
|
|
paired = sorted(set(control_by_key).intersection(treatment_by_key))
|
|
control_only = sorted(set(control_by_key).difference(treatment_by_key))
|
|
treatment_only = sorted(set(treatment_by_key).difference(control_by_key))
|
|
control_pairs = [control_by_key[key] for key in paired]
|
|
treatment_pairs = [treatment_by_key[key] for key in paired]
|
|
control_metrics = _guidance_metrics(control_pairs)
|
|
treatment_metrics = _guidance_metrics(treatment_pairs)
|
|
same_runtime = (
|
|
control.get("author") == treatment.get("author")
|
|
and control.get("reviewer") == treatment.get("reviewer")
|
|
and control.get("runtime_profile_sha256") == treatment.get("runtime_profile_sha256")
|
|
and control.get("operation_contracts") == treatment.get("operation_contracts")
|
|
)
|
|
control_guidance = bool((control.get("author_guidance") or {}).get("enabled"))
|
|
treatment_guidance = bool((treatment.get("author_guidance") or {}).get("enabled"))
|
|
same_budgets = all(
|
|
control_by_key[key].get("scenario_budget") == treatment_by_key[key].get("scenario_budget")
|
|
for key in paired
|
|
)
|
|
calls_control = control_metrics.get("median_author_calls")
|
|
calls_treatment = treatment_metrics.get("median_author_calls")
|
|
calls_within_limit = (
|
|
isinstance(calls_control, (int, float))
|
|
and isinstance(calls_treatment, (int, float))
|
|
and calls_treatment <= calls_control * 1.10
|
|
)
|
|
improved = (
|
|
treatment_metrics.get("schema_or_decision_rejections", 0) < control_metrics.get("schema_or_decision_rejections", 0)
|
|
or treatment_metrics.get("cdsl_expression_failures", 0) < control_metrics.get("cdsl_expression_failures", 0)
|
|
)
|
|
gates = {
|
|
"complete_pairing": bool(paired) and not control_only and not treatment_only,
|
|
"control_off_treatment_on": not control_guidance and treatment_guidance,
|
|
"same_author_reviewer_runtime_and_contracts": same_runtime,
|
|
"same_per_scenario_budgets": same_budgets,
|
|
"checkpoint_rate_not_lower": treatment_metrics.get("executable_checkpoint_rate", -1) >= control_metrics.get("executable_checkpoint_rate", 0),
|
|
"completion_rate_not_lower": treatment_metrics.get("completion_rate", -1) >= control_metrics.get("completion_rate", 0),
|
|
"median_author_calls_within_ten_percent": calls_within_limit,
|
|
"model_or_cdsl_failure_improved": improved,
|
|
}
|
|
return {
|
|
"schema_version": "cad.author-guidance-comparison.v1",
|
|
"status": "passed" if all(gates.values()) else "failed",
|
|
"gates": gates,
|
|
"paired_runs": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in paired],
|
|
"unpaired_runs": {
|
|
"control_only": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in control_only],
|
|
"treatment_only": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in treatment_only],
|
|
},
|
|
"control": control_metrics,
|
|
"treatment": treatment_metrics,
|
|
"excluded_capability_gaps": {
|
|
"control": [{"scenario": item.get("scenario"), "repetition": item.get("repetition")} for item in control_gaps],
|
|
"treatment": [{"scenario": item.get("scenario"), "repetition": item.get("repetition")} for item in treatment_gaps],
|
|
},
|
|
}
|
|
|
|
|
|
def compare_guidance_report_paths(control_path: Path, treatment_path: Path) -> dict[str, Any]:
|
|
try:
|
|
control = json.loads(control_path.read_text(encoding="utf-8"))
|
|
treatment = json.loads(treatment_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
return {"status": "failed", "error": f"GUIDANCE_COMPARISON_INPUT_INVALID: {type(error).__name__}"}
|
|
if not isinstance(control, dict) or not isinstance(treatment, dict):
|
|
return {"status": "failed", "error": "GUIDANCE_COMPARISON_INPUT_INVALID: report must be an object"}
|
|
return compare_guidance_reports(control, treatment)
|
|
|
|
|
|
async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, Any]:
|
|
try:
|
|
scenarios = _fixture(arguments.suite, arguments.scenarios)
|
|
except ValueError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)}
|
|
repetitions = arguments.repetitions if arguments.repetitions is not None else 3 if arguments.suite == "release" else 1
|
|
if repetitions < 1:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": "--repetitions must be at least 1"}
|
|
settings = get_settings()
|
|
if arguments.author_guidance is not None:
|
|
settings = replace(settings, agent_author_guidance_enabled=arguments.author_guidance == "on")
|
|
try:
|
|
author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model)
|
|
if arguments.review_provider or arguments.review_model:
|
|
review_provider, review_model = settings.resolve_model(arguments.review_provider, arguments.review_model)
|
|
if review_provider.id == author_provider.id and review_model.id == author_model.id:
|
|
raise ValueError("Reviewer must differ from author")
|
|
else:
|
|
review_provider, review_model = settings.resolve_independent_review_model(author_provider, author_model)
|
|
except ValueError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)}
|
|
baseline_path = arguments.baseline_report.resolve() if arguments.baseline_report else None
|
|
if arguments.suite == "release" and baseline_path is None:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": "TOKEN_BASELINE_REQUIRED: release requires a provenance-checked pre-v3 token baseline report."}
|
|
try:
|
|
baseline = load_token_baseline(baseline_path) if baseline_path else None
|
|
current_author_identity = author_request_identity(author_provider, author_model)
|
|
runtime_profile_hash = profile_sha256(settings.engine_root / "profile_schema.json")
|
|
except TokenBaselineError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {error}"}
|
|
if baseline is not None:
|
|
baseline_errors = validate_token_baseline_provenance(
|
|
baseline,
|
|
scenarios=scenarios,
|
|
author_identity=current_author_identity,
|
|
runtime_profile_hash=runtime_profile_hash,
|
|
)
|
|
if baseline_errors:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {baseline_errors[0]}", "baseline_errors": baseline_errors}
|
|
isolated = replace(settings, task_root=report_root / "artifacts", conversation_root=report_root / "conversations")
|
|
services = compose_v3(isolated)
|
|
contracts = [
|
|
{
|
|
"atomic_id": atomic_id, "contract_hash": services.workflow.runtime.operation_contract(atomic_id)["contract_hash"],
|
|
"contract_version": services.workflow.runtime.operation_contract(atomic_id)["contract_version"],
|
|
"registry_revision": services.workflow.runtime.operation_contract(atomic_id)["registry_revision"],
|
|
}
|
|
for atomic_id in services.workflow.runtime.supported_atomic_ids()
|
|
]
|
|
verifier_schema_hash = canonical_hash(default_registry().expected_one_of_schema())
|
|
try:
|
|
author_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=author_provider.id, model_id=author_model.id, role="author", force=True)
|
|
reviewer_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=review_provider.id, model_id=review_model.id, role="reviewer", force=True)
|
|
except Exception as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)[:1000]}
|
|
if not author_capability.get("supported") or not reviewer_capability.get("supported"):
|
|
error, failure_layer = _capability_block_reason(author_capability, reviewer_capability)
|
|
return {
|
|
"status": "LIVE_EVAL_BLOCKED",
|
|
"error": error,
|
|
"failure_layer": failure_layer,
|
|
"author_capability": author_capability,
|
|
"reviewer_capability": reviewer_capability,
|
|
}
|
|
results: list[dict[str, Any]] = []
|
|
for scenario in scenarios:
|
|
for repetition in range(1, repetitions + 1):
|
|
services.workflow.config = replace(
|
|
services.workflow.config,
|
|
# State transitions include local candidate recovery, so the
|
|
# loop guard is deliberately independent from the externally
|
|
# measured model-call budgets below.
|
|
max_turns=max(8, int(scenario["max_total_calls"]) * 3),
|
|
max_author_turns=int(scenario["max_author_turns"]),
|
|
max_reviewer_turns=int(scenario["max_reviewer_turns"]),
|
|
max_model_calls=int(scenario["max_total_calls"]),
|
|
)
|
|
task_id = f"cad_{secrets.token_hex(6)}"
|
|
oracle_claims = [item for item in scenario.get("required_claims") or () if isinstance(item, dict)]
|
|
if oracle_claims:
|
|
services.workflow.requirements.register_evaluation_contract_oracle(
|
|
task_id,
|
|
oracle_claims,
|
|
validation_capability_gaps=[item for item in scenario.get("validation_capability_gaps") or () if isinstance(item, dict)],
|
|
)
|
|
services.workflow.create_task(task_id, str(scenario["request"]))
|
|
events: list[dict[str, Any]] = []
|
|
started = datetime.now(timezone.utc)
|
|
try:
|
|
async with asyncio.timeout(int(scenario["max_wall_seconds"])):
|
|
async for name, payload in services.workflow.run(
|
|
task_id=task_id,
|
|
author=ModelIdentity(author_provider.id, author_model.id),
|
|
reviewer=ModelIdentity(review_provider.id, review_model.id),
|
|
):
|
|
events.append({"name": name, "payload": payload})
|
|
except TimeoutError:
|
|
events.append({"name": "timeout", "payload": {"code": "LIVE_EVAL_TIMEOUT"}})
|
|
projection = services.repository.get_task_projection(task_id) or {}
|
|
usage = services.repository.usage_summary(task_id)
|
|
ledger = services.repository.ledger_events(task_id)
|
|
invocations = services.repository.invocation_records(task_id)
|
|
tool_audits = services.repository.tool_audits(task_id)
|
|
terminal = next((item["payload"] for item in reversed(events) if item["name"] == "task_terminal"), {})
|
|
artifact_root = (isolated.task_root / task_id).resolve()
|
|
artifact_manifest = _safe_artifact_manifest(artifact_root)
|
|
finished = datetime.now(timezone.utc)
|
|
checks = _run_checks(
|
|
scenario,
|
|
projection,
|
|
usage,
|
|
services.artifacts.read_requirements_contract(
|
|
task_id,
|
|
services.repository.get_state(task_id).requirements_contract_path
|
|
if services.repository.get_state(task_id) is not None
|
|
else "",
|
|
),
|
|
events,
|
|
artifact_root,
|
|
ledger,
|
|
tool_audits,
|
|
artifact_manifest,
|
|
)
|
|
success = all(checks.values())
|
|
acceptance = _acceptance_coverage(
|
|
scenario,
|
|
services.artifacts.read_requirements_contract(
|
|
task_id,
|
|
services.repository.get_state(task_id).requirements_contract_path
|
|
if services.repository.get_state(task_id) is not None
|
|
else "",
|
|
),
|
|
)
|
|
outcome = (
|
|
"passed"
|
|
if success
|
|
else "validation_capability_gap"
|
|
if not acceptance["complete"]
|
|
and all(value for key, value in checks.items() if key != "acceptance_contract_coverage")
|
|
else "failed"
|
|
)
|
|
failure_attribution = _failure_attribution(
|
|
checks=checks,
|
|
outcome=outcome,
|
|
events=events,
|
|
projection=projection,
|
|
)
|
|
results.append({
|
|
"scenario": scenario["id"], "repetition": repetition, "success": success, "outcome": outcome,
|
|
"started_at": started.isoformat(), "finished_at": finished.isoformat(),
|
|
"duration_ms": round((finished - started).total_seconds() * 1000),
|
|
"terminal": terminal, "projection": projection, "usage": usage, "checks": checks,
|
|
"acceptance_coverage": acceptance,
|
|
"failure_attribution": failure_attribution,
|
|
"guidance": _guidance_metadata(usage),
|
|
"scenario_budget": {
|
|
"max_author_turns": int(scenario["max_author_turns"]),
|
|
"max_reviewer_turns": int(scenario["max_reviewer_turns"]),
|
|
"max_total_calls": int(scenario["max_total_calls"]),
|
|
"max_total_tokens": int(scenario["max_total_tokens"]),
|
|
},
|
|
"rejection_codes": _rejection_codes(events),
|
|
"schema_rejection_count": len(_rejection_codes(events)),
|
|
"retry_count": sum(1 for event in events if (event.get("payload") or {}).get("status") == "error"),
|
|
"rollback_count": sum(1 for item in ledger if item.get("event") == "rollback"),
|
|
**_ledger_identifiers(ledger),
|
|
"final_claim_results": _final_claim_results(ledger),
|
|
"redacted_correlation_ids": _redacted_correlation_ids(task_id, invocations),
|
|
"tool_audits": _report_tool_audits(tool_audits),
|
|
"artifact_root": str(artifact_root), "action_ledger_path": str(artifact_root / "actions" / "action-ledger.jsonl"),
|
|
"artifact_manifest": artifact_manifest,
|
|
"ledger": ledger, "invocations": invocations, "event_audit": _event_audit(events),
|
|
})
|
|
token_comparison = (
|
|
compare_token_baseline(
|
|
baseline,
|
|
scenarios=scenarios,
|
|
v3_results=results,
|
|
author_identity=current_author_identity,
|
|
runtime_profile_hash=runtime_profile_hash,
|
|
)
|
|
if baseline is not None
|
|
else {"schema_version": "cad.token-comparison.v1", "status": "not_required"}
|
|
)
|
|
token_gate = all(token_comparison.get("checks", {}).values()) if baseline is not None else True
|
|
return {
|
|
"status": "passed" if results and all(item["success"] for item in results) and token_gate else "failed",
|
|
"author": {"provider": author_provider.id, "model": author_model.id},
|
|
"author_request_identity": current_author_identity,
|
|
"reviewer": {"provider": review_provider.id, "model": review_model.id},
|
|
"author_guidance": {
|
|
"enabled": settings.agent_author_guidance_enabled,
|
|
"max_chars": settings.agent_author_guidance_max_chars,
|
|
},
|
|
"author_capability": author_capability,
|
|
"reviewer_capability": reviewer_capability,
|
|
"structured_output_mode": {
|
|
"author": str(author_capability.get("mode") or ""),
|
|
"reviewer": str(reviewer_capability.get("mode") or ""),
|
|
},
|
|
"git_revision": _git_revision(),
|
|
"protocol_version": "3.2",
|
|
"runtime_profile_sha256": runtime_profile_hash,
|
|
"operation_contracts": contracts,
|
|
"verifier_registry_version": "cad.verifier-registry.v1",
|
|
"verifier_schema_hash": verifier_schema_hash,
|
|
"token_comparison": token_comparison,
|
|
"baseline_report": str(baseline_path) if baseline_path else "",
|
|
"repetitions": repetitions,
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
arguments = _arguments()
|
|
report_root = BACKEND_ROOT / "live-evals" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
report_root.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
result = (
|
|
compare_guidance_report_paths(*arguments.compare_guidance_reports)
|
|
if arguments.compare_guidance_reports
|
|
else asyncio.run(_run(arguments, report_root))
|
|
)
|
|
except KeyboardInterrupt:
|
|
# Let an explicit operator interruption retain its normal CLI
|
|
# semantics. An external kill cannot be reported reliably either.
|
|
raise
|
|
except BaseException as error:
|
|
# A live evaluation may fail before it creates a task (for example
|
|
# during conformance). Its report is still the release gate's audit
|
|
# artifact, so an unexpected evaluator failure must not disappear.
|
|
result = {
|
|
"status": "LIVE_EVAL_BLOCKED",
|
|
"error": f"UNEXPECTED_LIVE_EVAL_ERROR: {type(error).__name__}: {str(error)[:900]}",
|
|
"failure_layer": "configuration_or_network",
|
|
}
|
|
if result["status"] == "LIVE_EVAL_BLOCKED" and arguments.allow_skip and not arguments.require_live:
|
|
result["skip_reason"] = result.get("error", "live provider access is unavailable")
|
|
result["status"] = "skipped"
|
|
result.update({"suite": arguments.suite, "require_live": arguments.require_live, "report_root": str(report_root.resolve())})
|
|
(report_root / "report.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps({"status": result["status"], "report": str((report_root / "report.json").resolve())}, ensure_ascii=False))
|
|
if result["status"] == "passed":
|
|
return 0
|
|
if result["status"] == "skipped":
|
|
return 0
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|