737 lines
35 KiB
Python
737 lines
35 KiB
Python
"""Run the real provider, engine, reviewer and v3 state machine end to end.
|
|
|
|
``--require-live`` deliberately fails closed: missing credentials, blocked
|
|
network, a skipped model capability, a timeout, or any scenario failure emits
|
|
``LIVE_EVAL_BLOCKED``/failure details and exits non-zero.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
from dataclasses import replace
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
from app.cad_agent.application.capabilities import verify_model_capability
|
|
from app.cad_agent.application.workflow import ModelIdentity
|
|
from app.cad_agent.composition import compose_v3
|
|
from app.cad_agent.domain.claim_matching import contains_expected
|
|
from app.cad_agent.domain.operation_contract import canonical_hash
|
|
from app.cad_agent.domain.verifier_registry import default_registry
|
|
from app.cad_agent.evals.token_baseline import (
|
|
TokenBaselineError,
|
|
author_request_identity,
|
|
compare_token_baseline,
|
|
load_token_baseline,
|
|
profile_sha256,
|
|
validate_token_baseline_provenance,
|
|
)
|
|
from app.settings import BACKEND_ROOT, get_settings
|
|
|
|
|
|
_CANDIDATE_EVIDENCE_FILES = frozenset({
|
|
"candidate.json",
|
|
"candidate-review.json",
|
|
"model.cdsl.json",
|
|
"model.step",
|
|
"model.glb",
|
|
"model.topology.json",
|
|
"rebuild-report.json",
|
|
"renders/render-manifest.json",
|
|
"renders/contact-sheet.jpg",
|
|
})
|
|
|
|
_FAILURE_LAYERS = frozenset({
|
|
"model_format_or_decision",
|
|
"v3_contract_or_verifier",
|
|
"cdsl_expression",
|
|
"engine_execution",
|
|
"independent_visual_review",
|
|
"configuration_or_network",
|
|
})
|
|
|
|
_ERROR_FAILURE_LAYERS = {
|
|
"AUTHOR_FORMAT_INVALID": "model_format_or_decision",
|
|
"AUTHOR_DECISION_REJECTED": "model_format_or_decision",
|
|
"STALE_WORKING_HEAD": "model_format_or_decision",
|
|
"FAILED_AUTHOR_FORMAT": "model_format_or_decision",
|
|
"MODEL_STRUCTURED_OUTPUT_UNSUPPORTED": "model_format_or_decision",
|
|
"RUNTIME_PRECONDITION_FAILED": "v3_contract_or_verifier",
|
|
"RUNTIME_CONTRACT_INVALID": "v3_contract_or_verifier",
|
|
"VERIFIER_UNAVAILABLE": "v3_contract_or_verifier",
|
|
"CLAIM_VERIFICATION_FAILED": "cdsl_expression",
|
|
"CANDIDATE_BUILD_FAILED": "engine_execution",
|
|
"CANDIDATE_REVIEW_REJECTED": "independent_visual_review",
|
|
"AUTHOR_TRANSPORT_UNAVAILABLE": "configuration_or_network",
|
|
"REVIEW_SERVICE_UNAVAILABLE": "configuration_or_network",
|
|
"RENDER_SERVICE_UNAVAILABLE": "configuration_or_network",
|
|
"STORAGE_FAILURE": "configuration_or_network",
|
|
"LIVE_EVAL_TIMEOUT": "configuration_or_network",
|
|
"FAILED_INTERNAL": "engine_execution",
|
|
}
|
|
|
|
|
|
def _arguments() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run live protocol-v3 CAD evaluations.")
|
|
parser.add_argument("--suite", choices=("smoke", "release", "comprehensive"), default="smoke")
|
|
parser.add_argument("--require-live", action="store_true")
|
|
parser.add_argument("--allow-skip", action="store_true")
|
|
parser.add_argument("--author-provider")
|
|
parser.add_argument("--author-model")
|
|
parser.add_argument("--review-provider")
|
|
parser.add_argument("--review-model")
|
|
parser.add_argument("--scenario", help="Run one fixture scenario by its stable ID for targeted regression validation.")
|
|
parser.add_argument("--baseline-report", type=Path, help="Measured pre-v3 token baseline JSON for a release run.")
|
|
return parser.parse_args()
|
|
|
|
|
|
def _fixture(suite: str, scenario_id: str | None = None) -> list[dict[str, Any]]:
|
|
fixture_name = "comprehensive.json" if suite == "comprehensive" else "release.json"
|
|
value = json.loads((Path(__file__).parent / "fixtures" / fixture_name).read_text(encoding="utf-8"))
|
|
if fixture_name == "comprehensive.json":
|
|
source_document = str(value.get("source_document") or "")
|
|
expected_digest = str(value.get("source_document_sha256") or "")
|
|
source_path = (BACKEND_ROOT.parent / source_document).resolve()
|
|
workspace_root = BACKEND_ROOT.parent.resolve()
|
|
if (
|
|
not source_document
|
|
or workspace_root not in source_path.parents
|
|
or not source_path.is_file()
|
|
or sha256(source_path.read_bytes()).hexdigest() != expected_digest
|
|
):
|
|
raise ValueError("Comprehensive fixture is not synchronized with its source document.")
|
|
values = [item for item in value.get("scenarios") or () if isinstance(item, dict)]
|
|
values = values[:2] if suite == "smoke" else values
|
|
if scenario_id is None:
|
|
return values
|
|
selected = [item for item in values if str(item.get("id") or "") == scenario_id]
|
|
if not selected:
|
|
raise ValueError(f"Unknown scenario {scenario_id!r} for suite {suite!r}")
|
|
return selected
|
|
|
|
|
|
def _rejection_codes(events: list[dict[str, Any]]) -> list[str]:
|
|
codes: list[str] = []
|
|
for event in events:
|
|
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else payload
|
|
code = result.get("code") if isinstance(result, dict) else None
|
|
if code in {"AUTHOR_FORMAT_INVALID", "AUTHOR_DECISION_REJECTED", "STALE_WORKING_HEAD"}:
|
|
codes.append(str(code))
|
|
return codes
|
|
|
|
|
|
def _acceptance_coverage(
|
|
scenario: dict[str, Any],
|
|
requirements_contract: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Assess whether a fixture's acceptance contract is representable today.
|
|
|
|
Comprehensive prompts deliberately include manufacturing relationships that
|
|
may not yet have a deterministic verifier. A healthy-looking solid is
|
|
not evidence for those relationships, so they remain explicit capability
|
|
gaps instead of silently passing a scenario.
|
|
"""
|
|
actual = [
|
|
{
|
|
"claim_kind": str(claim.get("claim_kind") or ""),
|
|
"expected": claim.get("expected") if isinstance(claim.get("expected"), dict) else {},
|
|
}
|
|
for requirement in (requirements_contract or {}).get("requirements") or ()
|
|
if isinstance(requirement, dict)
|
|
for claim in requirement.get("acceptance_claims") or ()
|
|
if isinstance(claim, dict)
|
|
]
|
|
required = {str(value) for value in scenario.get("required_claim_kinds") or ()}
|
|
expected_claims = [
|
|
{
|
|
"claim_kind": str(item.get("claim_kind") or ""),
|
|
"expected": item.get("expected") if isinstance(item.get("expected"), dict) else {},
|
|
}
|
|
for item in scenario.get("required_claims") or ()
|
|
if isinstance(item, dict) and isinstance(item.get("claim_kind"), str)
|
|
]
|
|
expected_kinds = {claim["claim_kind"] for claim in expected_claims}
|
|
expected_claims.extend(
|
|
{"claim_kind": claim_kind, "expected": {}}
|
|
for claim_kind in sorted(required - expected_kinds)
|
|
)
|
|
gaps = [
|
|
{"id": str(item.get("id") or ""), "description": str(item.get("description") or "")}
|
|
for item in scenario.get("validation_capability_gaps") or ()
|
|
if isinstance(item, dict)
|
|
]
|
|
missing = [
|
|
claim for claim in expected_claims
|
|
if not any(
|
|
actual_claim["claim_kind"] == claim["claim_kind"]
|
|
and contains_expected(actual_claim["expected"], claim["expected"])
|
|
for actual_claim in actual
|
|
)
|
|
]
|
|
return {
|
|
"required_claim_kinds": sorted(required),
|
|
"covered_claim_kinds": sorted(required.intersection({claim["claim_kind"] for claim in actual})),
|
|
"required_claims": expected_claims,
|
|
"missing_claims": missing,
|
|
"validation_capability_gaps": gaps,
|
|
"complete": not missing and not gaps,
|
|
}
|
|
|
|
|
|
def _contains_expected(actual: Any, required: Any) -> bool:
|
|
"""Backward-compatible evaluation helper for canonical claim matching."""
|
|
return contains_expected(actual, required)
|
|
|
|
|
|
def _capability_block_reason(*reports: dict[str, Any]) -> tuple[str, str]:
|
|
"""Classify a failed conformance probe without confusing outages for gaps.
|
|
|
|
A provider transport failure means the probe did not establish either
|
|
support or non-support. Only a complete response that violates one of the
|
|
exposed tool contracts is evidence of a model structured-output limit.
|
|
"""
|
|
messages = [
|
|
str(failure.get("message") or "").casefold()
|
|
for report in reports
|
|
for failure in report.get("failures") or ()
|
|
if isinstance(failure, dict)
|
|
]
|
|
unavailable_markers = (
|
|
"transport unavailable",
|
|
"connection",
|
|
"network",
|
|
"timeout",
|
|
"timed out",
|
|
"temporarily unavailable",
|
|
)
|
|
if any(any(marker in message for marker in unavailable_markers) for message in messages):
|
|
return "MODEL_CAPABILITY_PROBE_UNAVAILABLE", "configuration_or_network"
|
|
return "MODEL_STRUCTURED_OUTPUT_UNSUPPORTED", "model_format_or_decision"
|
|
|
|
|
|
def _artifact_evidence_complete(
|
|
artifact_root: Path,
|
|
revisions: list[str],
|
|
active_revision: str,
|
|
artifact_manifest: dict[str, Any] | None,
|
|
) -> bool:
|
|
"""Require every published CAD decision to retain its reviewable evidence.
|
|
|
|
A non-empty report manifest is not enough: a task could otherwise report
|
|
only a rendered contract view while silently losing the STEP, topology, or
|
|
candidate review used to accept a revision. Candidate manifests protect
|
|
the immutable build inputs/outputs; the report manifest additionally
|
|
protects the frozen contract and final independent review written later.
|
|
"""
|
|
if not revisions or not active_revision:
|
|
return False
|
|
report_files = {
|
|
str(item.get("path") or "")
|
|
for item in (artifact_manifest or {}).get("files") or ()
|
|
if isinstance(item, dict)
|
|
}
|
|
required_report_files = {
|
|
"requirements-contract.json",
|
|
f"reviews/final/{active_revision}/final-review.json",
|
|
}
|
|
if artifact_manifest is not None and not required_report_files.issubset(report_files):
|
|
return False
|
|
if not all((artifact_root / path).is_file() for path in required_report_files):
|
|
return False
|
|
for revision_id in sorted(set(revisions)):
|
|
revision_root = artifact_root / "revisions" / revision_id
|
|
required_paths = {revision_root / relative for relative in _CANDIDATE_EVIDENCE_FILES}
|
|
manifest_path = revision_root / "manifest.json"
|
|
if not manifest_path.is_file() or not all(path.is_file() for path in required_paths):
|
|
return False
|
|
try:
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return False
|
|
declared = manifest.get("files") if isinstance(manifest, dict) else None
|
|
if not isinstance(declared, dict) or not _CANDIDATE_EVIDENCE_FILES.issubset(declared):
|
|
return False
|
|
for relative, digest in declared.items():
|
|
path = revision_root / str(relative)
|
|
if not isinstance(digest, str) or len(digest) != 64 or not path.is_file():
|
|
return False
|
|
if sha256(path.read_bytes()).hexdigest() != digest:
|
|
return False
|
|
if artifact_manifest is not None:
|
|
expected_report_paths = {f"revisions/{revision_id}/{relative}" for relative in _CANDIDATE_EVIDENCE_FILES}
|
|
if not expected_report_paths.issubset(report_files):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _failure_attribution(
|
|
*,
|
|
checks: dict[str, bool],
|
|
outcome: str,
|
|
events: list[dict[str, Any]],
|
|
projection: dict[str, Any],
|
|
) -> dict[str, Any] | None:
|
|
"""Return one evidence-backed failure layer required by the test target."""
|
|
if outcome == "passed":
|
|
return None
|
|
codes: list[str] = []
|
|
for event in events:
|
|
payload = event.get("payload") if isinstance(event, dict) and isinstance(event.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else payload
|
|
code = result.get("code") if isinstance(result, dict) else ""
|
|
if isinstance(code, str) and code:
|
|
codes.append(code)
|
|
last_error = projection.get("last_error") if isinstance(projection, dict) else ""
|
|
if isinstance(last_error, str) and last_error:
|
|
codes.append(last_error)
|
|
for code in reversed(codes):
|
|
layer = _ERROR_FAILURE_LAYERS.get(code)
|
|
if layer:
|
|
return {
|
|
"layer": layer,
|
|
"reason_code": code,
|
|
"evidence_codes": list(dict.fromkeys(codes)),
|
|
"failed_checks": [name for name, passed in checks.items() if not passed],
|
|
}
|
|
failed_checks = [name for name, passed in checks.items() if not passed]
|
|
if outcome == "validation_capability_gap" or any(name in {"required_claims", "acceptance_contract_coverage"} for name in failed_checks):
|
|
layer, reason = "v3_contract_or_verifier", "VALIDATION_CAPABILITY_GAP"
|
|
elif any(name in {"deterministic_claims_pass", "required_operations", "required_operation_alternative"} for name in failed_checks):
|
|
layer, reason = "cdsl_expression", "CAD_ACCEPTANCE_GATE_FAILED"
|
|
elif any(name in {"token_budget", "call_budget", "author_turn_budget", "reviewer_turn_budget"} for name in failed_checks):
|
|
layer, reason = "configuration_or_network", "EVALUATION_BUDGET_EXCEEDED"
|
|
elif "raw_argument_audit" in failed_checks:
|
|
layer, reason = "v3_contract_or_verifier", "RAW_ARGUMENT_AUDIT_FAILED"
|
|
elif any(name in {"immutable_artifacts", "artifact_manifest", "required_artifact_evidence", "action_ledger"} for name in failed_checks):
|
|
layer, reason = "configuration_or_network", "ARTIFACT_EVIDENCE_INCOMPLETE"
|
|
else:
|
|
layer, reason = "model_format_or_decision", "WORKFLOW_TERMINAL_STATE_MISMATCH"
|
|
return {
|
|
"layer": layer,
|
|
"reason_code": reason,
|
|
"evidence_codes": list(dict.fromkeys(codes)),
|
|
"failed_checks": failed_checks,
|
|
}
|
|
|
|
|
|
def _run_checks(
|
|
scenario: dict[str, Any],
|
|
projection: dict[str, Any],
|
|
usage: dict[str, Any],
|
|
requirements_contract: dict[str, Any] | None,
|
|
events: list[dict[str, Any]],
|
|
artifact_root: Path,
|
|
ledger: list[dict[str, Any]] | None = None,
|
|
tool_audits: list[dict[str, Any]] | None = None,
|
|
artifact_manifest: dict[str, Any] | None = None,
|
|
) -> dict[str, bool]:
|
|
records = usage.get("records") if isinstance(usage.get("records"), list) else []
|
|
author_calls = [item for item in records if isinstance(item, dict) and item.get("role") != "reviewer"]
|
|
reviewer_calls = [item for item in records if isinstance(item, dict) and item.get("role") == "reviewer"]
|
|
total_tokens = int(usage.get("prompt_tokens") or 0) + int(usage.get("completion_tokens") or 0)
|
|
audit_ledger = ledger if ledger is not None else [item for item in projection.get("action_ledger_summary") or () if isinstance(item, dict)]
|
|
accepted = [item for item in audit_ledger if isinstance(item, dict) and item.get("event") == "accepted"]
|
|
revisions = [str(item.get("revision_id") or "") for item in accepted]
|
|
operation_ids = {str(item.get("actual_atomic_id") or "") for item in accepted}
|
|
required_operation_groups = [
|
|
{str(atomic_id) for atomic_id in group if isinstance(atomic_id, str) and atomic_id}
|
|
for group in scenario.get("required_any_atomic_id_groups") or ()
|
|
if isinstance(group, list)
|
|
]
|
|
active_revision = str(projection.get("active_revision") or "")
|
|
manifest = artifact_root / "revisions" / active_revision / "manifest.json"
|
|
jsonl = artifact_root / "actions" / "action-ledger.jsonl"
|
|
final_claims = next((item.get("claim_results") for item in reversed(audit_ledger) if isinstance(item, dict) and item.get("event") == "completed" and isinstance(item.get("claim_results"), list)), [])
|
|
audit_records = tool_audits if isinstance(tool_audits, list) else []
|
|
audit_valid = bool(audit_records) and all(
|
|
isinstance(item, dict)
|
|
and isinstance(item.get("raw_arguments_hash"), str)
|
|
and len(item["raw_arguments_hash"]) == 64
|
|
and item.get("canonical_schema_valid") is True
|
|
and isinstance(item.get("state_binding"), dict)
|
|
and bool(item["state_binding"].get("phase"))
|
|
and item["state_binding"].get("binding_valid") is True
|
|
and item.get("single_allowed_call") is True
|
|
for item in audit_records
|
|
)
|
|
acceptance = _acceptance_coverage(scenario, requirements_contract)
|
|
artifact_evidence = _artifact_evidence_complete(
|
|
artifact_root,
|
|
revisions,
|
|
active_revision,
|
|
artifact_manifest,
|
|
)
|
|
return {
|
|
"expected_terminal_phase": projection.get("phase") == scenario.get("expected_phase", "COMPLETED"),
|
|
"token_budget": total_tokens <= int(scenario["max_total_tokens"]),
|
|
"call_budget": len(records) <= int(scenario["max_total_calls"]),
|
|
"author_turn_budget": len(author_calls) <= int(scenario["max_author_turns"]),
|
|
"reviewer_turn_budget": len(reviewer_calls) <= int(scenario["max_reviewer_turns"]),
|
|
"required_claims": not acceptance["missing_claims"],
|
|
"acceptance_contract_coverage": acceptance["complete"],
|
|
"required_operations": {str(value) for value in scenario.get("required_atomic_ids") or ()}.issubset(operation_ids),
|
|
"required_operation_alternative": (
|
|
(not scenario.get("required_any_atomic_ids") or bool({str(value) for value in scenario["required_any_atomic_ids"]}.intersection(operation_ids)))
|
|
and all(group.intersection(operation_ids) for group in required_operation_groups)
|
|
),
|
|
"immutable_artifacts": bool(active_revision) and manifest.is_file(),
|
|
"required_artifact_evidence": artifact_evidence,
|
|
"artifact_manifest": (
|
|
artifact_manifest is None
|
|
or (
|
|
isinstance(artifact_manifest.get("files"), list)
|
|
and bool(artifact_manifest["files"])
|
|
and all(
|
|
isinstance(item, dict)
|
|
and isinstance(item.get("path"), str)
|
|
and isinstance(item.get("sha256"), str)
|
|
and len(item["sha256"]) == 64
|
|
for item in artifact_manifest["files"]
|
|
)
|
|
)
|
|
),
|
|
"action_ledger": jsonl.is_file(),
|
|
"unique_revisions": len(revisions) == len(set(revisions)),
|
|
"deterministic_claims_pass": bool(final_claims) and all(
|
|
not isinstance(item, dict) or not item.get("deterministic") or item.get("status") == "pass"
|
|
for item in final_claims
|
|
),
|
|
"raw_argument_audit": audit_valid if tool_audits is not None else bool(records) and all(isinstance(item, dict) and isinstance(item.get("raw_arguments_hash"), str) and len(item["raw_arguments_hash"]) == 64 for item in records),
|
|
"no_schema_or_decision_rejections": not _rejection_codes(events),
|
|
}
|
|
|
|
|
|
def _git_revision() -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], cwd=BACKEND_ROOT.parent, capture_output=True,
|
|
text=True, check=True, timeout=5,
|
|
)
|
|
return completed.stdout.strip()
|
|
except (OSError, subprocess.SubprocessError):
|
|
return "unknown"
|
|
|
|
|
|
def _event_audit(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Persist event metadata without prompts, tool arguments, or tool facts."""
|
|
audit: list[dict[str, Any]] = []
|
|
for item in events:
|
|
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
|
|
result = payload.get("result") if isinstance(payload.get("result"), dict) else {}
|
|
usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {}
|
|
audit.append({
|
|
"name": str(item.get("name") or ""), "tool": str(payload.get("tool") or ""),
|
|
"status": str(payload.get("status") or ""), "code": str(result.get("code") or payload.get("code") or ""),
|
|
"raw_arguments_hash": str(usage.get("raw_arguments_hash") or ""),
|
|
"field_errors": result.get("field_errors") if isinstance(result.get("field_errors"), list) else [],
|
|
})
|
|
return audit
|
|
|
|
|
|
def _report_tool_audits(audits: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Return report-safe audit metadata without argument values or prompts."""
|
|
return [
|
|
{
|
|
"audit_id": item.get("audit_id"),
|
|
"at": item.get("at"),
|
|
"actor": item.get("actor"),
|
|
"tool": item.get("tool"),
|
|
"raw_arguments_hash": item.get("raw_arguments_hash"),
|
|
"canonical_schema_valid": item.get("canonical_schema_valid"),
|
|
"field_errors": item.get("field_errors") if isinstance(item.get("field_errors"), list) else [],
|
|
"state_binding": item.get("state_binding") if isinstance(item.get("state_binding"), dict) else {},
|
|
"returned_tool": item.get("returned_tool"),
|
|
"single_allowed_call": item.get("single_allowed_call"),
|
|
}
|
|
for item in audits
|
|
if isinstance(item, dict)
|
|
]
|
|
|
|
|
|
def _ledger_identifiers(ledger: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
"""Expose audit identifiers explicitly without copying provider payloads."""
|
|
candidate_ids = sorted({
|
|
str(item.get("candidate_id"))
|
|
for item in ledger
|
|
if isinstance(item.get("candidate_id"), str) and item.get("candidate_id")
|
|
})
|
|
revision_ids = sorted({
|
|
str(item.get("revision_id"))
|
|
for item in ledger
|
|
if isinstance(item.get("revision_id"), str) and item.get("revision_id")
|
|
})
|
|
return {"candidate_ids": candidate_ids, "revision_ids": revision_ids}
|
|
|
|
|
|
def _final_claim_results(ledger: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
for item in reversed(ledger):
|
|
results = item.get("claim_results")
|
|
if item.get("event") == "completed" and isinstance(results, list):
|
|
return [value for value in results if isinstance(value, dict)]
|
|
return []
|
|
|
|
|
|
def _redacted_correlation_ids(task_id: str, invocations: list[dict[str, Any]]) -> list[str]:
|
|
"""Keep run linkage without publishing task or provider correlation values."""
|
|
return [
|
|
canonical_hash({"task_id": task_id, "invocation_id": str(item.get("invocation_id") or "")})[:20]
|
|
for item in invocations
|
|
if isinstance(item, dict) and item.get("invocation_id")
|
|
]
|
|
|
|
|
|
def _safe_artifact_manifest(artifact_root: Path) -> dict[str, Any]:
|
|
"""Hash reviewable CAD evidence without copying source or prompt content."""
|
|
allowed_exact = {"requirements-contract.json", "requirements-index.json", "requirements.md", "completion.md"}
|
|
allowed_prefixes = ("actions/", "revisions/", "reviews/", "documents/requirements-")
|
|
files: list[dict[str, str]] = []
|
|
if artifact_root.is_dir():
|
|
for path in sorted(artifact_root.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
relative = path.relative_to(artifact_root).as_posix()
|
|
if relative not in allowed_exact and not relative.startswith(allowed_prefixes):
|
|
continue
|
|
files.append({"path": relative, "sha256": sha256(path.read_bytes()).hexdigest()})
|
|
return {"schema_version": "cad.live-eval-artifact-manifest.v1", "artifact_root": str(artifact_root), "files": files}
|
|
|
|
|
|
async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, Any]:
|
|
try:
|
|
scenarios = _fixture(arguments.suite, arguments.scenario)
|
|
except ValueError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)}
|
|
settings = get_settings()
|
|
try:
|
|
author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model)
|
|
if arguments.review_provider or arguments.review_model:
|
|
review_provider, review_model = settings.resolve_model(arguments.review_provider, arguments.review_model)
|
|
if review_provider.id == author_provider.id and review_model.id == author_model.id:
|
|
raise ValueError("Reviewer must differ from author")
|
|
else:
|
|
review_provider, review_model = settings.resolve_independent_review_model(author_provider, author_model)
|
|
except ValueError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)}
|
|
baseline_path = arguments.baseline_report.resolve() if arguments.baseline_report else None
|
|
if arguments.suite == "release" and baseline_path is None:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": "TOKEN_BASELINE_REQUIRED: release requires a provenance-checked pre-v3 token baseline report."}
|
|
try:
|
|
baseline = load_token_baseline(baseline_path) if baseline_path else None
|
|
current_author_identity = author_request_identity(author_provider, author_model)
|
|
runtime_profile_hash = profile_sha256(settings.engine_root / "profile_schema.json")
|
|
except TokenBaselineError as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {error}"}
|
|
if baseline is not None:
|
|
baseline_errors = validate_token_baseline_provenance(
|
|
baseline,
|
|
scenarios=scenarios,
|
|
author_identity=current_author_identity,
|
|
runtime_profile_hash=runtime_profile_hash,
|
|
)
|
|
if baseline_errors:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": f"TOKEN_BASELINE_INVALID: {baseline_errors[0]}", "baseline_errors": baseline_errors}
|
|
isolated = replace(settings, task_root=report_root / "artifacts", conversation_root=report_root / "conversations")
|
|
services = compose_v3(isolated)
|
|
contracts = [
|
|
{
|
|
"atomic_id": atomic_id, "contract_hash": services.workflow.runtime.operation_contract(atomic_id)["contract_hash"],
|
|
"contract_version": services.workflow.runtime.operation_contract(atomic_id)["contract_version"],
|
|
"registry_revision": services.workflow.runtime.operation_contract(atomic_id)["registry_revision"],
|
|
}
|
|
for atomic_id in services.workflow.runtime.supported_atomic_ids()
|
|
]
|
|
verifier_schema_hash = canonical_hash(default_registry().expected_one_of_schema())
|
|
try:
|
|
author_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=author_provider.id, model_id=author_model.id, force=True)
|
|
reviewer_capability = await verify_model_capability(services.repository, services.workflow.runtime, services.models, provider_id=review_provider.id, model_id=review_model.id, force=True)
|
|
except Exception as error:
|
|
return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)[:1000]}
|
|
if not author_capability.get("supported") or not reviewer_capability.get("supported"):
|
|
error, failure_layer = _capability_block_reason(author_capability, reviewer_capability)
|
|
return {
|
|
"status": "LIVE_EVAL_BLOCKED",
|
|
"error": error,
|
|
"failure_layer": failure_layer,
|
|
"author_capability": author_capability,
|
|
"reviewer_capability": reviewer_capability,
|
|
}
|
|
results: list[dict[str, Any]] = []
|
|
for scenario in scenarios:
|
|
for repetition in range(1, 4 if arguments.suite == "release" else 2):
|
|
services.workflow.config = replace(
|
|
services.workflow.config,
|
|
# State transitions include local candidate recovery, so the
|
|
# loop guard is deliberately independent from the externally
|
|
# measured model-call budgets below.
|
|
max_turns=max(8, int(scenario["max_total_calls"]) * 3),
|
|
max_author_turns=int(scenario["max_author_turns"]),
|
|
max_reviewer_turns=int(scenario["max_reviewer_turns"]),
|
|
max_model_calls=int(scenario["max_total_calls"]),
|
|
)
|
|
task_id = f"cad_{secrets.token_hex(6)}"
|
|
oracle_claims = [item for item in scenario.get("required_claims") or () if isinstance(item, dict)]
|
|
if oracle_claims:
|
|
services.workflow.requirements.register_evaluation_contract_oracle(
|
|
task_id,
|
|
oracle_claims,
|
|
validation_capability_gaps=[item for item in scenario.get("validation_capability_gaps") or () if isinstance(item, dict)],
|
|
)
|
|
services.workflow.create_task(task_id, str(scenario["request"]))
|
|
events: list[dict[str, Any]] = []
|
|
started = datetime.now(timezone.utc)
|
|
try:
|
|
async with asyncio.timeout(int(scenario["max_wall_seconds"])):
|
|
async for name, payload in services.workflow.run(
|
|
task_id=task_id,
|
|
author=ModelIdentity(author_provider.id, author_model.id),
|
|
reviewer=ModelIdentity(review_provider.id, review_model.id),
|
|
):
|
|
events.append({"name": name, "payload": payload})
|
|
except TimeoutError:
|
|
events.append({"name": "timeout", "payload": {"code": "LIVE_EVAL_TIMEOUT"}})
|
|
projection = services.repository.get_task_projection(task_id) or {}
|
|
usage = services.repository.usage_summary(task_id)
|
|
ledger = services.repository.ledger_events(task_id)
|
|
invocations = services.repository.invocation_records(task_id)
|
|
tool_audits = services.repository.tool_audits(task_id)
|
|
terminal = next((item["payload"] for item in reversed(events) if item["name"] == "task_terminal"), {})
|
|
artifact_root = (isolated.task_root / task_id).resolve()
|
|
artifact_manifest = _safe_artifact_manifest(artifact_root)
|
|
finished = datetime.now(timezone.utc)
|
|
checks = _run_checks(
|
|
scenario,
|
|
projection,
|
|
usage,
|
|
services.artifacts.read_requirements_contract(
|
|
task_id,
|
|
services.repository.get_state(task_id).requirements_contract_path
|
|
if services.repository.get_state(task_id) is not None
|
|
else "",
|
|
),
|
|
events,
|
|
artifact_root,
|
|
ledger,
|
|
tool_audits,
|
|
artifact_manifest,
|
|
)
|
|
success = all(checks.values())
|
|
acceptance = _acceptance_coverage(
|
|
scenario,
|
|
services.artifacts.read_requirements_contract(
|
|
task_id,
|
|
services.repository.get_state(task_id).requirements_contract_path
|
|
if services.repository.get_state(task_id) is not None
|
|
else "",
|
|
),
|
|
)
|
|
outcome = (
|
|
"passed"
|
|
if success
|
|
else "validation_capability_gap"
|
|
if not acceptance["complete"]
|
|
and all(value for key, value in checks.items() if key != "acceptance_contract_coverage")
|
|
else "failed"
|
|
)
|
|
failure_attribution = _failure_attribution(
|
|
checks=checks,
|
|
outcome=outcome,
|
|
events=events,
|
|
projection=projection,
|
|
)
|
|
results.append({
|
|
"scenario": scenario["id"], "repetition": repetition, "success": success, "outcome": outcome,
|
|
"started_at": started.isoformat(), "finished_at": finished.isoformat(),
|
|
"duration_ms": round((finished - started).total_seconds() * 1000),
|
|
"terminal": terminal, "projection": projection, "usage": usage, "checks": checks,
|
|
"acceptance_coverage": acceptance,
|
|
"failure_attribution": failure_attribution,
|
|
"rejection_codes": _rejection_codes(events),
|
|
"schema_rejection_count": len(_rejection_codes(events)),
|
|
"retry_count": sum(1 for event in events if (event.get("payload") or {}).get("status") == "error"),
|
|
"rollback_count": sum(1 for item in ledger if item.get("event") == "rollback"),
|
|
**_ledger_identifiers(ledger),
|
|
"final_claim_results": _final_claim_results(ledger),
|
|
"redacted_correlation_ids": _redacted_correlation_ids(task_id, invocations),
|
|
"tool_audits": _report_tool_audits(tool_audits),
|
|
"artifact_root": str(artifact_root), "action_ledger_path": str(artifact_root / "actions" / "action-ledger.jsonl"),
|
|
"artifact_manifest": artifact_manifest,
|
|
"ledger": ledger, "invocations": invocations, "event_audit": _event_audit(events),
|
|
})
|
|
token_comparison = (
|
|
compare_token_baseline(
|
|
baseline,
|
|
scenarios=scenarios,
|
|
v3_results=results,
|
|
author_identity=current_author_identity,
|
|
runtime_profile_hash=runtime_profile_hash,
|
|
)
|
|
if baseline is not None
|
|
else {"schema_version": "cad.token-comparison.v1", "status": "not_required"}
|
|
)
|
|
token_gate = all(token_comparison.get("checks", {}).values()) if baseline is not None else True
|
|
return {
|
|
"status": "passed" if results and all(item["success"] for item in results) and token_gate else "failed",
|
|
"author": {"provider": author_provider.id, "model": author_model.id},
|
|
"author_request_identity": current_author_identity,
|
|
"reviewer": {"provider": review_provider.id, "model": review_model.id},
|
|
"author_capability": author_capability,
|
|
"reviewer_capability": reviewer_capability,
|
|
"structured_output_mode": {
|
|
"author": str(author_capability.get("mode") or ""),
|
|
"reviewer": str(reviewer_capability.get("mode") or ""),
|
|
},
|
|
"git_revision": _git_revision(),
|
|
"protocol_version": "3.0",
|
|
"runtime_profile_sha256": runtime_profile_hash,
|
|
"operation_contracts": contracts,
|
|
"verifier_registry_version": "cad.verifier-registry.v1",
|
|
"verifier_schema_hash": verifier_schema_hash,
|
|
"token_comparison": token_comparison,
|
|
"baseline_report": str(baseline_path) if baseline_path else "",
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
arguments = _arguments()
|
|
report_root = BACKEND_ROOT / "live-evals" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
report_root.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
result = asyncio.run(_run(arguments, report_root))
|
|
except KeyboardInterrupt:
|
|
# Let an explicit operator interruption retain its normal CLI
|
|
# semantics. An external kill cannot be reported reliably either.
|
|
raise
|
|
except BaseException as error:
|
|
# A live evaluation may fail before it creates a task (for example
|
|
# during conformance). Its report is still the release gate's audit
|
|
# artifact, so an unexpected evaluator failure must not disappear.
|
|
result = {
|
|
"status": "LIVE_EVAL_BLOCKED",
|
|
"error": f"UNEXPECTED_LIVE_EVAL_ERROR: {type(error).__name__}: {str(error)[:900]}",
|
|
"failure_layer": "configuration_or_network",
|
|
}
|
|
if result["status"] == "LIVE_EVAL_BLOCKED" and arguments.allow_skip and not arguments.require_live:
|
|
result["skip_reason"] = result.get("error", "live provider access is unavailable")
|
|
result["status"] = "skipped"
|
|
result.update({"suite": arguments.suite, "require_live": arguments.require_live, "report_root": str(report_root.resolve())})
|
|
(report_root / "report.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(json.dumps({"status": result["status"], "report": str((report_root / "report.json").resolve())}, ensure_ascii=False))
|
|
if result["status"] == "passed":
|
|
return 0
|
|
if result["status"] == "skipped":
|
|
return 0
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|