Files
2026-11-12 20:40:21 +08:00

75 lines
4.1 KiB
Python

"""Fixed, offline requirement scenarios with explicit expected outcomes."""
from __future__ import annotations
import copy
import json
import math
from collections import Counter
from datetime import datetime, timezone
def equivalent(actual, expected):
if isinstance(expected, (int, float)) and not isinstance(expected, bool):
return isinstance(actual, (int, float)) and not isinstance(actual, bool) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9)
return actual == expected
def evaluate(kb):
cases = json.loads((kb.root / "evals/cases.json").read_text(encoding="utf-8"))
outcomes = []
for case in cases:
failures = []
try:
expect = case["expect"]
if case["action"] == "search":
actual = kb.search(case["query"])
ids = {c["family_id"] for c in actual["candidates"]}
if not set(expect.get("includes", [])).issubset(ids):
failures.append("Missing expected family candidate")
if expect.get("no_candidates") and ids:
failures.append("Unexpected specialized family for uncovered/broad query")
if not set(expect.get("flags", [])).issubset(actual["flags"]):
failures.append("Missing semantic-review flag")
if not set(expect.get("category_ids", [])).issubset({c["node_id"] for c in actual["category_hits"]}):
failures.append("Missing expected catalog entry")
if actual["selected_family_id"] is not None or not actual["needs_semantic_review"]:
failures.append("Retrieval must not make final selection")
elif case["action"] == "validate":
fid = case["family_id"]
example_id = kb.families[fid]["meta"]["example_id"]
params = copy.deepcopy(kb.documents[example_id]["meta"]["params"])
for edit in case.get("edits", []):
target = params
for key in edit["path"][:-1]:
target = target[key]
if edit.get("remove"):
del target[edit["path"][-1]]
else:
target[edit["path"][-1]] = edit["value"]
actual = kb.validate_params(fid, params)
if actual["valid"] != expect["valid"]:
failures.append(f"Expected valid={expect['valid']}, got {actual['valid']}")
if expect.get("group") and not actual[expect["group"]]:
failures.append(f"Expected diagnostic group: {expect['group']}")
for key, expected in expect.get("derived", {}).items():
if not equivalent(actual["derived"].get(key), expected):
failures.append(f"Derived expectation failed: {key}")
if actual["geometry_validated"]:
failures.append("Offline validation cannot claim CAD evidence")
else:
raise ValueError(f"Unknown action: {case['action']}")
outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"],
"passed": not failures, "failures": failures, "actual": actual})
except (ValueError, KeyError, TypeError, IndexError) as exc:
outcomes.append({"id": case["id"], "group": case["group"], "scenario": case["scenario"],
"passed": False, "failures": [str(exc)]})
unique_ids = len({case["id"] for case in cases}) == len(cases)
counts = dict(Counter(case["group"] for case in cases))
structure_valid = len(cases) == 60 and unique_ids and counts == {"family": 36, "general": 12, "complex": 12}
return {"version": "0.1.0", "evaluated_at": datetime.now(timezone.utc).isoformat(),
"passed": structure_valid and all(o["passed"] for o in outcomes),
"case_count": len(cases), "passed_count": sum(o["passed"] for o in outcomes),
"group_counts": counts, "case_structure_valid": structure_valid,
"geometry_validated": False, "outcomes": outcomes}