75 lines
4.1 KiB
Python
75 lines
4.1 KiB
Python
"""Fixed, offline requirement scenarios with explicit expected outcomes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import math
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
def equivalent(actual, expected):
|
|
if isinstance(expected, (int, float)) and not isinstance(expected, bool):
|
|
return isinstance(actual, (int, float)) and not isinstance(actual, bool) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9)
|
|
return actual == expected
|
|
|
|
|
|
def evaluate(kb):
|
|
cases = json.loads((kb.root / "evals/cases.json").read_text(encoding="utf-8"))
|
|
outcomes = []
|
|
for case in cases:
|
|
failures = []
|
|
try:
|
|
expect = case["expect"]
|
|
if case["action"] == "search":
|
|
actual = kb.search(case["query"])
|
|
ids = {c["family_id"] for c in actual["candidates"]}
|
|
if not set(expect.get("includes", [])).issubset(ids):
|
|
failures.append("Missing expected family candidate")
|
|
if expect.get("no_candidates") and ids:
|
|
failures.append("Unexpected specialized family for uncovered/broad query")
|
|
if not set(expect.get("flags", [])).issubset(actual["flags"]):
|
|
failures.append("Missing semantic-review flag")
|
|
if not set(expect.get("category_ids", [])).issubset({c["node_id"] for c in actual["category_hits"]}):
|
|
failures.append("Missing expected catalog entry")
|
|
if actual["selected_family_id"] is not None or not actual["needs_semantic_review"]:
|
|
failures.append("Retrieval must not make final selection")
|
|
elif case["action"] == "validate":
|
|
fid = case["family_id"]
|
|
example_id = kb.families[fid]["meta"]["example_id"]
|
|
params = copy.deepcopy(kb.documents[example_id]["meta"]["params"])
|
|
for edit in case.get("edits", []):
|
|
target = params
|
|
for key in edit["path"][:-1]:
|
|
target = target[key]
|
|
if edit.get("remove"):
|
|
del target[edit["path"][-1]]
|
|
else:
|
|
target[edit["path"][-1]] = edit["value"]
|
|
actual = kb.validate_params(fid, params)
|
|
if actual["valid"] != expect["valid"]:
|
|
failures.append(f"Expected valid={expect['valid']}, got {actual['valid']}")
|
|
if expect.get("group") and not actual[expect["group"]]:
|
|
failures.append(f"Expected diagnostic group: {expect['group']}")
|
|
for key, expected in expect.get("derived", {}).items():
|
|
if not equivalent(actual["derived"].get(key), expected):
|
|
failures.append(f"Derived expectation failed: {key}")
|
|
if actual["geometry_validated"]:
|
|
failures.append("Offline validation cannot claim CAD evidence")
|
|
else:
|
|
raise ValueError(f"Unknown action: {case['action']}")
|
|
outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"],
|
|
"passed": not failures, "failures": failures, "actual": actual})
|
|
except (ValueError, KeyError, TypeError, IndexError) as exc:
|
|
outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"],
|
|
"passed": False, "failures": [str(exc)]})
|
|
unique_ids = len({case["id"] for case in cases}) == len(cases)
|
|
counts = dict(Counter(case["group"] for case in cases))
|
|
structure_valid = len(cases) == 60 and unique_ids and counts == {"family": 36, "general": 12, "complex": 12}
|
|
return {"version": "0.1.0", "evaluated_at": datetime.now(timezone.utc).isoformat(),
|
|
"passed": structure_valid and all(o["passed"] for o in outcomes),
|
|
"case_count": len(cases), "passed_count": sum(o["passed"] for o in outcomes),
|
|
"group_counts": counts, "case_structure_valid": structure_valid,
|
|
"geometry_validated": False, "outcomes": outcomes}
|