change

backend/app/cad_agent/evals/live.py

@@ -15,6 +15,7 @@ from hashlib import sha256
 import json
 from pathlib import Path
 import secrets
+from statistics import median
 import subprocess
 import sys
 from typing import Any
@@ -100,12 +101,15 @@ def _arguments() -> argparse.Namespace:
     parser.add_argument("--author-model")
     parser.add_argument("--review-provider")
     parser.add_argument("--review-model")
-    parser.add_argument("--scenario", help="Run one fixture scenario by its stable ID for targeted regression validation.")
+    parser.add_argument("--scenario", action="append", dest="scenarios", help="Run a fixture scenario by stable ID. Repeat this option to select a comparison set.")
+    parser.add_argument("--repetitions", type=int, help="Run every selected scenario this many times.")
+    parser.add_argument("--author-guidance", choices=("on", "off"), help="Override CDSL author guidance for this run.")
+    parser.add_argument("--compare-guidance-reports", nargs=2, type=Path, metavar=("CONTROL", "TREATMENT"), help="Compare matched --author-guidance off/on report.json files without invoking providers.")
     parser.add_argument("--baseline-report", type=Path, help="Measured pre-v3 token baseline JSON for a release run.")
     return parser.parse_args()
 
 
-def _fixture(suite: str, scenario_id: str | None = None) -> list[dict[str, Any]]:
+def _fixture(suite: str, scenario_ids: list[str] | None = None) -> list[dict[str, Any]]:
     fixture_name = "comprehensive.json" if suite == "comprehensive" else "release.json"
     value = json.loads((Path(__file__).parent / "fixtures" / fixture_name).read_text(encoding="utf-8"))
     if fixture_name == "comprehensive.json":
@@ -122,11 +126,14 @@ def _fixture(suite: str, scenario_id: str | None = None) -> list[dict[str, Any]]
             raise ValueError("Comprehensive fixture is not synchronized with its source document.")
     values = [item for item in value.get("scenarios") or () if isinstance(item, dict)]
     values = values[:2] if suite == "smoke" else values
-    if scenario_id is None:
+    if not scenario_ids:
         return values
-    selected = [item for item in values if str(item.get("id") or "") == scenario_id]
-    if not selected:
-        raise ValueError(f"Unknown scenario {scenario_id!r} for suite {suite!r}")
+    requested = list(dict.fromkeys(scenario_ids))
+    available = {str(item.get("id") or "") for item in values}
+    unknown = [scenario_id for scenario_id in requested if scenario_id not in available]
+    if unknown:
+        raise ValueError(f"Unknown scenario {unknown[0]!r} for suite {suite!r}")
+    selected = [item for item in values if str(item.get("id") or "") in set(requested)]
     return selected
 
 
@@ -567,12 +574,180 @@ def _safe_artifact_manifest(artifact_root: Path) -> dict[str, Any]:
     return {"schema_version": "cad.live-eval-artifact-manifest.v1", "artifact_root": str(artifact_root), "files": files}
 
 
+def _guidance_metadata(usage: dict[str, Any]) -> dict[str, Any]:
+    """Summarize author-only guidance audit metadata without retaining prompts."""
+    records = [
+        item for item in usage.get("records") or ()
+        if isinstance(item, dict) and item.get("role") != "reviewer"
+    ]
+    sections = sorted({
+        section_id
+        for item in records
+        for section_id in item.get("guidance_section_ids") or ()
+        if isinstance(section_id, str)
+    })
+    versions = sorted({
+        str(item.get("guidance_version") or "")
+        for item in records
+        if str(item.get("guidance_version") or "")
+    })
+    return {
+        "enabled": bool(records) and any(item.get("guidance_enabled") is True for item in records),
+        "versions": versions,
+        "section_ids": sections,
+        "chars_total": sum(int(item.get("guidance_chars") or 0) for item in records),
+        "fallback_reasons": sorted({
+            str(item.get("guidance_fallback_reason") or "")
+            for item in records
+            if str(item.get("guidance_fallback_reason") or "")
+        }),
+    }
+
+
+def _has_unsupported_capability(row: dict[str, Any]) -> bool:
+    """Recognize engine-declared unsupported capability without hiding model errors."""
+    for event in row.get("ledger") or ():
+        if not isinstance(event, dict):
+            continue
+        for failure in event.get("operation_failures") or ():
+            if isinstance(failure, dict) and "unsupported_" in str(failure.get("message") or ""):
+                return True
+        if "unsupported_" in str(event.get("message") or ""):
+            return True
+    return False
+
+
+def _guidance_metric_rows(report: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    results = [item for item in report.get("results") or () if isinstance(item, dict)]
+    capability_gaps = [
+        item for item in results
+        if item.get("outcome") == "validation_capability_gap" or _has_unsupported_capability(item)
+    ]
+    return [item for item in results if item not in capability_gaps], capability_gaps
+
+
+def _guidance_metrics(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    if not rows:
+        return {"eligible_runs": 0}
+    author_calls = [
+        sum(1 for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
+        for row in rows
+    ]
+    context_chars = [
+        sum(int(item.get("context_chars") or 0) for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
+        for row in rows
+    ]
+    prompt_tokens = [
+        sum(int(item.get("prompt_tokens") or 0) for item in (row.get("usage") or {}).get("records") or () if isinstance(item, dict) and item.get("role") != "reviewer")
+        for row in rows
+    ]
+    failure_layers: dict[str, int] = {}
+    for row in rows:
+        layer = str((row.get("failure_attribution") or {}).get("layer") or "passed")
+        failure_layers[layer] = failure_layers.get(layer, 0) + 1
+    def rate(predicate: Any) -> float:
+        return sum(1 for row in rows if predicate(row)) / len(rows)
+    return {
+        "eligible_runs": len(rows),
+        "executable_checkpoint_rate": rate(lambda row: bool(row.get("revision_ids"))),
+        "completion_rate": rate(lambda row: str((row.get("projection") or {}).get("phase") or "") == "COMPLETED"),
+        "deterministic_claim_success_rate": rate(lambda row: bool((row.get("checks") or {}).get("deterministic_claims_pass"))),
+        "schema_or_decision_rejections": sum(int(row.get("schema_rejection_count") or 0) for row in rows),
+        "cdsl_expression_failures": sum(1 for row in rows if str((row.get("failure_attribution") or {}).get("layer") or "") == "cdsl_expression"),
+        "median_author_calls": median(author_calls),
+        "median_author_context_chars": median(context_chars),
+        "total_author_prompt_tokens": sum(prompt_tokens),
+        "median_author_prompt_tokens": median(prompt_tokens),
+        "failure_layers": dict(sorted(failure_layers.items())),
+    }
+
+
+def compare_guidance_reports(control: dict[str, Any], treatment: dict[str, Any]) -> dict[str, Any]:
+    """Compare paired guidance-off/on runs without treating engine gaps as prompt results."""
+    control_rows, control_gaps = _guidance_metric_rows(control)
+    treatment_rows, treatment_gaps = _guidance_metric_rows(treatment)
+    control_by_key = {(str(row.get("scenario") or ""), int(row.get("repetition") or 0)): row for row in control_rows}
+    treatment_by_key = {(str(row.get("scenario") or ""), int(row.get("repetition") or 0)): row for row in treatment_rows}
+    paired = sorted(set(control_by_key).intersection(treatment_by_key))
+    control_only = sorted(set(control_by_key).difference(treatment_by_key))
+    treatment_only = sorted(set(treatment_by_key).difference(control_by_key))
+    control_pairs = [control_by_key[key] for key in paired]
+    treatment_pairs = [treatment_by_key[key] for key in paired]
+    control_metrics = _guidance_metrics(control_pairs)
+    treatment_metrics = _guidance_metrics(treatment_pairs)
+    same_runtime = (
+        control.get("author") == treatment.get("author")
+        and control.get("reviewer") == treatment.get("reviewer")
+        and control.get("runtime_profile_sha256") == treatment.get("runtime_profile_sha256")
+        and control.get("operation_contracts") == treatment.get("operation_contracts")
+    )
+    control_guidance = bool((control.get("author_guidance") or {}).get("enabled"))
+    treatment_guidance = bool((treatment.get("author_guidance") or {}).get("enabled"))
+    same_budgets = all(
+        control_by_key[key].get("scenario_budget") == treatment_by_key[key].get("scenario_budget")
+        for key in paired
+    )
+    calls_control = control_metrics.get("median_author_calls")
+    calls_treatment = treatment_metrics.get("median_author_calls")
+    calls_within_limit = (
+        isinstance(calls_control, (int, float))
+        and isinstance(calls_treatment, (int, float))
+        and calls_treatment <= calls_control * 1.10
+    )
+    improved = (
+        treatment_metrics.get("schema_or_decision_rejections", 0) < control_metrics.get("schema_or_decision_rejections", 0)
+        or treatment_metrics.get("cdsl_expression_failures", 0) < control_metrics.get("cdsl_expression_failures", 0)
+    )
+    gates = {
+        "complete_pairing": bool(paired) and not control_only and not treatment_only,
+        "control_off_treatment_on": not control_guidance and treatment_guidance,
+        "same_author_reviewer_runtime_and_contracts": same_runtime,
+        "same_per_scenario_budgets": same_budgets,
+        "checkpoint_rate_not_lower": treatment_metrics.get("executable_checkpoint_rate", -1) >= control_metrics.get("executable_checkpoint_rate", 0),
+        "completion_rate_not_lower": treatment_metrics.get("completion_rate", -1) >= control_metrics.get("completion_rate", 0),
+        "median_author_calls_within_ten_percent": calls_within_limit,
+        "model_or_cdsl_failure_improved": improved,
+    }
+    return {
+        "schema_version": "cad.author-guidance-comparison.v1",
+        "status": "passed" if all(gates.values()) else "failed",
+        "gates": gates,
+        "paired_runs": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in paired],
+        "unpaired_runs": {
+            "control_only": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in control_only],
+            "treatment_only": [{"scenario": scenario, "repetition": repetition} for scenario, repetition in treatment_only],
+        },
+        "control": control_metrics,
+        "treatment": treatment_metrics,
+        "excluded_capability_gaps": {
+            "control": [{"scenario": item.get("scenario"), "repetition": item.get("repetition")} for item in control_gaps],
+            "treatment": [{"scenario": item.get("scenario"), "repetition": item.get("repetition")} for item in treatment_gaps],
+        },
+    }
+
+
+def compare_guidance_report_paths(control_path: Path, treatment_path: Path) -> dict[str, Any]:
+    try:
+        control = json.loads(control_path.read_text(encoding="utf-8"))
+        treatment = json.loads(treatment_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        return {"status": "failed", "error": f"GUIDANCE_COMPARISON_INPUT_INVALID: {type(error).__name__}"}
+    if not isinstance(control, dict) or not isinstance(treatment, dict):
+        return {"status": "failed", "error": "GUIDANCE_COMPARISON_INPUT_INVALID: report must be an object"}
+    return compare_guidance_reports(control, treatment)
+
+
 async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, Any]:
     try:
-        scenarios = _fixture(arguments.suite, arguments.scenario)
+        scenarios = _fixture(arguments.suite, arguments.scenarios)
     except ValueError as error:
         return {"status": "LIVE_EVAL_BLOCKED", "error": str(error)}
+    repetitions = arguments.repetitions if arguments.repetitions is not None else 3 if arguments.suite == "release" else 1
+    if repetitions < 1:
+        return {"status": "LIVE_EVAL_BLOCKED", "error": "--repetitions must be at least 1"}
     settings = get_settings()
+    if arguments.author_guidance is not None:
+        settings = replace(settings, agent_author_guidance_enabled=arguments.author_guidance == "on")
     try:
         author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model)
         if arguments.review_provider or arguments.review_model:
@@ -628,7 +803,7 @@ async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, An
         }
     results: list[dict[str, Any]] = []
     for scenario in scenarios:
-        for repetition in range(1, 4 if arguments.suite == "release" else 2):
+        for repetition in range(1, repetitions + 1):
             services.workflow.config = replace(
                 services.workflow.config,
                 # State transitions include local candidate recovery, so the
@@ -716,6 +891,13 @@ async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, An
                 "terminal": terminal, "projection": projection, "usage": usage, "checks": checks,
                 "acceptance_coverage": acceptance,
                 "failure_attribution": failure_attribution,
+                "guidance": _guidance_metadata(usage),
+                "scenario_budget": {
+                    "max_author_turns": int(scenario["max_author_turns"]),
+                    "max_reviewer_turns": int(scenario["max_reviewer_turns"]),
+                    "max_total_calls": int(scenario["max_total_calls"]),
+                    "max_total_tokens": int(scenario["max_total_tokens"]),
+                },
                 "rejection_codes": _rejection_codes(events),
                 "schema_rejection_count": len(_rejection_codes(events)),
                 "retry_count": sum(1 for event in events if (event.get("payload") or {}).get("status") == "error"),
@@ -745,6 +927,10 @@ async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, An
         "author": {"provider": author_provider.id, "model": author_model.id},
         "author_request_identity": current_author_identity,
         "reviewer": {"provider": review_provider.id, "model": review_model.id},
+        "author_guidance": {
+            "enabled": settings.agent_author_guidance_enabled,
+            "max_chars": settings.agent_author_guidance_max_chars,
+        },
         "author_capability": author_capability,
         "reviewer_capability": reviewer_capability,
         "structured_output_mode": {
@@ -759,6 +945,7 @@ async def _run(arguments: argparse.Namespace, report_root: Path) -> dict[str, An
         "verifier_schema_hash": verifier_schema_hash,
         "token_comparison": token_comparison,
         "baseline_report": str(baseline_path) if baseline_path else "",
+        "repetitions": repetitions,
         "results": results,
     }
 
@@ -768,7 +955,11 @@ def main() -> int:
     report_root = BACKEND_ROOT / "live-evals" / datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
     report_root.mkdir(parents=True, exist_ok=True)
     try:
-        result = asyncio.run(_run(arguments, report_root))
+        result = (
+            compare_guidance_report_paths(*arguments.compare_guidance_reports)
+            if arguments.compare_guidance_reports
+            else asyncio.run(_run(arguments, report_root))
+        )
     except KeyboardInterrupt:
         # Let an explicit operator interruption retain its normal CLI
         # semantics. An external kill cannot be reported reliably either.