72 lines
3.5 KiB
Python
72 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
import sys
|
|
import unittest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.cad_agent.evals.live import _fixture, compare_guidance_reports # noqa: E402
|
|
|
|
|
|
def _result(*, scenario: str, repetition: int, author_calls: int, context_chars: int, schema_rejections: int, failure_layer: str = "") -> dict:
|
|
return {
|
|
"scenario": scenario,
|
|
"repetition": repetition,
|
|
"outcome": "passed",
|
|
"revision_ids": ["revision_001"],
|
|
"projection": {"phase": "COMPLETED"},
|
|
"checks": {"deterministic_claims_pass": True},
|
|
"schema_rejection_count": schema_rejections,
|
|
"failure_attribution": {"layer": failure_layer} if failure_layer else None,
|
|
"scenario_budget": {"max_author_turns": 8, "max_reviewer_turns": 2, "max_total_calls": 10, "max_total_tokens": 1000},
|
|
"usage": {"records": [{"context_chars": context_chars} for _ in range(author_calls)]},
|
|
}
|
|
|
|
|
|
def _report(results: list[dict]) -> dict:
|
|
return {
|
|
"author": {"provider": "author", "model": "model"},
|
|
"reviewer": {"provider": "reviewer", "model": "review"},
|
|
"runtime_profile_sha256": "a" * 64,
|
|
"operation_contracts": [{"atomic_id": "extrude_add_blind", "contract_hash": "b" * 64}],
|
|
"author_guidance": {"enabled": False, "max_chars": 3600},
|
|
"results": results,
|
|
}
|
|
|
|
|
|
class LiveGuidanceComparisonTests(unittest.TestCase):
|
|
def test_fixture_accepts_multiple_stable_scenarios_in_fixture_order(self) -> None:
|
|
selected = _fixture("comprehensive", ["l_bracket", "circular_flange_pcd"])
|
|
self.assertEqual([item["id"] for item in selected], ["circular_flange_pcd", "l_bracket"])
|
|
|
|
def test_comparison_enforces_matched_budget_and_quality_gates(self) -> None:
|
|
control = _report([
|
|
_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=2, failure_layer="cdsl_expression"),
|
|
_result(scenario="part_a", repetition=2, author_calls=10, context_chars=1000, schema_rejections=1),
|
|
])
|
|
treatment = _report([
|
|
_result(scenario="part_a", repetition=1, author_calls=11, context_chars=1300, schema_rejections=0),
|
|
_result(scenario="part_a", repetition=2, author_calls=11, context_chars=1300, schema_rejections=0),
|
|
])
|
|
treatment["author_guidance"]["enabled"] = True
|
|
comparison = compare_guidance_reports(control, treatment)
|
|
self.assertEqual(comparison["status"], "passed")
|
|
self.assertTrue(comparison["gates"]["median_author_calls_within_ten_percent"])
|
|
self.assertTrue(comparison["gates"]["model_or_cdsl_failure_improved"])
|
|
|
|
def test_comparison_excludes_explicit_unsupported_runtime_capability(self) -> None:
|
|
control = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1000, schema_rejections=1)])
|
|
treatment = _report([_result(scenario="part_a", repetition=1, author_calls=10, context_chars=1200, schema_rejections=0)])
|
|
treatment["author_guidance"]["enabled"] = True
|
|
treatment["results"][0]["ledger"] = [{"operation_failures": [{"message": "unsupported_draft"}]}]
|
|
comparison = compare_guidance_reports(control, treatment)
|
|
self.assertEqual(comparison["treatment"]["eligible_runs"], 0)
|
|
self.assertEqual(comparison["excluded_capability_gaps"]["treatment"], [{"scenario": "part_a", "repetition": 1}])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|