"""Fixed, offline requirement scenarios with explicit expected outcomes.""" from __future__ import annotations import copy import json import math from collections import Counter from datetime import datetime, timezone def equivalent(actual, expected): if isinstance(expected, (int, float)) and not isinstance(expected, bool): return isinstance(actual, (int, float)) and not isinstance(actual, bool) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9) return actual == expected def evaluate(kb): cases = json.loads((kb.root / "evals/cases.json").read_text(encoding="utf-8")) outcomes = [] for case in cases: failures = [] try: expect = case["expect"] if case["action"] == "search": actual = kb.search(case["query"]) ids = {c["family_id"] for c in actual["candidates"]} if not set(expect.get("includes", [])).issubset(ids): failures.append("Missing expected family candidate") if expect.get("no_candidates") and ids: failures.append("Unexpected specialized family for uncovered/broad query") if not set(expect.get("flags", [])).issubset(actual["flags"]): failures.append("Missing semantic-review flag") if not set(expect.get("category_ids", [])).issubset({c["node_id"] for c in actual["category_hits"]}): failures.append("Missing expected catalog entry") if actual["selected_family_id"] is not None or not actual["needs_semantic_review"]: failures.append("Retrieval must not make final selection") elif case["action"] == "validate": fid = case["family_id"] example_id = kb.families[fid]["meta"]["example_id"] params = copy.deepcopy(kb.documents[example_id]["meta"]["params"]) for edit in case.get("edits", []): target = params for key in edit["path"][:-1]: target = target[key] if edit.get("remove"): del target[edit["path"][-1]] else: target[edit["path"][-1]] = edit["value"] actual = kb.validate_params(fid, params) if actual["valid"] != expect["valid"]: failures.append(f"Expected valid={expect['valid']}, got {actual['valid']}") if expect.get("group") and not actual[expect["group"]]: failures.append(f"Expected diagnostic group: {expect['group']}") for key, expected in expect.get("derived", {}).items(): if not equivalent(actual["derived"].get(key), expected): failures.append(f"Derived expectation failed: {key}") if actual["geometry_validated"]: failures.append("Offline validation cannot claim CAD evidence") else: raise ValueError(f"Unknown action: {case['action']}") outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"], "passed": not failures, "failures": failures, "actual": actual}) except (ValueError, KeyError, TypeError, IndexError) as exc: outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"], "passed": False, "failures": [str(exc)]}) unique_ids = len({case["id"] for case in cases}) == len(cases) counts = dict(Counter(case["group"] for case in cases)) structure_valid = len(cases) == 60 and unique_ids and counts == {"family": 36, "general": 12, "complex": 12} return {"version": "0.1.0", "evaluated_at": datetime.now(timezone.utc).isoformat(), "passed": structure_valid and all(o["passed"] for o in outcomes), "case_count": len(cases), "passed_count": sum(o["passed"] for o in outcomes), "group_counts": counts, "case_structure_valid": structure_valid, "geometry_validated": False, "outcomes": outcomes}