from __future__ import annotations

from pathlib import Path
import sys
import unittest


ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "backend"))

from app.cad_agent.evals.live import _fixture, compare_guidance_reports  # noqa: E402


def _result(*, scenario: str, repetition: int, author_calls: int, context_chars: int, schema_rejections: int, failure_layer: str = "") -> dict:
    return {
        "scenario": scenario,
        "repetition": repetition,
        "outcome": "passed",
        "revision_ids": ["revision_001"],
        "projection": {"phase": "COMPLETED"},
        "checks": {"deterministic_claims_pass": True},
        "schema_rejection_count": schema_rejections,
        "failure_attribution": {"layer": failure_layer} if failure_layer else None,
        "scenario_budget": {"max_author_turns": 8, "max_reviewer_turns": 2, "max_total_calls": 10, "max_total_tokens": 1000},
        "usage": {"records": [{"context_chars": context_chars} for _ in range(author_calls)]},
    }


def _report(results: list[dict]) -> dict:
    return {
        "author": {"provider": "author", "model": "model"},
        "reviewer": {"provider": "reviewer", "model": "review"},
        "runtime_profile_sha256": "a" * 64,
        "operation_contracts": [{"atomic_id": "extrude_add_blind", "contract_hash": "b" * 64}],
        "author_guidance": {"enabled": False, "max_chars": 3600},
        "results": results,
    }


class LiveGuidanceComparisonTests(unittest.TestCase):
    def test_fixture_accepts_multiple_stable_scenarios_in_fixture_order(self) -> None:
        selected = _fixture("comprehensive", ["l_bracket", "circular_flange_pcd"])
        self.assertEqual([item["id"] for item in selected], ["circular_flange_pcd", "l_bracket"])

    def test_comparison_enforces_matched_budget_and_quality_gates(self) -> None:
        control = _report([
            _result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=2, failure_layer="cdsl_expression"),
            _result(scenario="part_a", repetition=2, author_calls=10, context_chars=1000, schema_rejections=1),
        ])
        treatment = _report([
            _result(scenario="part_a", repetition=1, author_calls=11, context_chars=1300, schema_rejections=0),
            _result(scenario="part_a", repetition=2, author_calls=11, context_chars=1300, schema_rejections=0),
        ])
        treatment["author_guidance"]["enabled"] = True
        comparison = compare_guidance_reports(control, treatment)
        self.assertEqual(comparison["status"], "passed")
        self.assertTrue(comparison["gates"]["median_author_calls_within_ten_percent"])
        self.assertTrue(comparison["gates"]["model_or_cdsl_failure_improved"])

    def test_comparison_excludes_explicit_unsupported_runtime_capability(self) -> None:
        control = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=1)])
        treatment = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1200, schema_rejections=0)])
        treatment["author_guidance"]["enabled"] = True
        treatment["results"][0]["ledger"] = [{"operation_failures": [{"message": "unsupported_draft"}]}]
        comparison = compare_guidance_reports(control, treatment)
        self.assertEqual(comparison["treatment"]["eligible_runs"], 0)
        self.assertEqual(comparison["excluded_capability_gaps"]["treatment"], [{"scenario": "part_a", "repetition": 1}])


if __name__ == "__main__":
    unittest.main()