271 lines
13 KiB
Python
271 lines
13 KiB
Python
"""Registry, deterministic candidate retrieval and progressive context loading."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
from jsonschema import Draft202012Validator
|
||
|
||
from .validation import validate_parameters
|
||
from .generation import build_generation_context
|
||
|
||
|
||
class UniqueLoader(yaml.SafeLoader):
|
||
"""Reject duplicate keys instead of silently replacing engineering values."""
|
||
|
||
|
||
def _mapping(loader, node, deep=False):
|
||
result = {}
|
||
for key_node, value_node in node.value:
|
||
key = loader.construct_object(key_node, deep=deep)
|
||
if key in result:
|
||
raise ValueError(f"Duplicate YAML key: {key}")
|
||
result[key] = loader.construct_object(value_node, deep=deep)
|
||
return result
|
||
|
||
|
||
UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _mapping)
|
||
|
||
|
||
def normalized(value):
|
||
return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", value).casefold()).strip()
|
||
|
||
|
||
def matches(query, term):
|
||
term = normalized(term)
|
||
if not term:
|
||
return False
|
||
if re.search(r"[\u3400-\u9fff]", term):
|
||
return term in query
|
||
return re.search(r"(?<![a-z0-9])" + re.escape(term) + r"(?![a-z0-9])", query) is not None
|
||
|
||
|
||
def read_document(path):
|
||
text = path.read_text(encoding="utf-8")
|
||
if not text.startswith("---\n"):
|
||
return None
|
||
parts = text.split("\n---\n", 1)
|
||
if len(parts) != 2:
|
||
raise ValueError(f"Unterminated frontmatter: {path}")
|
||
meta = yaml.load(parts[0][4:], Loader=UniqueLoader)
|
||
if not isinstance(meta, dict):
|
||
raise ValueError(f"Frontmatter must be a mapping: {path}")
|
||
return meta, parts[1].strip()
|
||
|
||
|
||
class KnowledgeBase:
|
||
def __init__(self, root=None):
|
||
self.root = Path(root or Path(__file__).resolve().parents[1]).resolve()
|
||
self.documents = {}
|
||
schema = json.loads((self.root / "schemas/document.schema.json").read_text())
|
||
validator = Draft202012Validator(schema)
|
||
for path in sorted((self.root / "references").rglob("*.md")):
|
||
resolved = path.resolve()
|
||
if not resolved.is_relative_to(self.root / "references"):
|
||
raise ValueError(f"Document outside registry: {path}")
|
||
parsed = read_document(path)
|
||
if parsed is None:
|
||
continue
|
||
meta, body = parsed
|
||
validator.validate(meta)
|
||
doc_id = meta["doc_id"]
|
||
if doc_id in self.documents:
|
||
raise ValueError(f"Duplicate document ID: {doc_id}")
|
||
self.documents[doc_id] = {"meta": meta, "body": body,
|
||
"path": path.relative_to(self.root).as_posix()}
|
||
if meta["kind"] == "family":
|
||
Draft202012Validator.check_schema(meta["parameter_schema"])
|
||
self.families = {k: v for k, v in self.documents.items() if v["meta"]["kind"] == "family"}
|
||
self.catalog = json.loads((self.root / "references/classification/mcb-taxonomy.json").read_text())
|
||
self.nodes = {n["id"]: n for n in self.catalog["nodes"]}
|
||
self._check_references()
|
||
|
||
def _check_references(self):
|
||
def visit(doc_id, stack):
|
||
if doc_id not in self.documents:
|
||
raise ValueError(f"Unknown document reference: {doc_id}")
|
||
if doc_id in stack:
|
||
raise ValueError(f"Circular reference: {' -> '.join([*stack, doc_id])}")
|
||
meta = self.documents[doc_id]["meta"]
|
||
dependencies = meta.get("requires", [])
|
||
if dependencies and meta["kind"] != "feature":
|
||
raise ValueError(f"Only shared features may declare requires: {doc_id}")
|
||
for dep in dependencies:
|
||
if dep in self.documents and self.documents[dep]["meta"]["kind"] not in {"common", "feature"}:
|
||
raise ValueError(f"Shared dependency cannot load a family or example: {dep}")
|
||
visit(dep, [*stack, doc_id])
|
||
|
||
for doc_id, doc in self.documents.items():
|
||
visit(doc_id, [])
|
||
meta = doc["meta"]
|
||
if meta["kind"] == "family":
|
||
for fid in meta["feature_ids"]:
|
||
self._require(fid, "feature")
|
||
example = self._require(meta["example_id"], "example")
|
||
if example["meta"]["family_id"] != doc_id:
|
||
raise ValueError(f"Example belongs to another family: {doc_id}")
|
||
for nid in meta["classification"]["node_ids"]:
|
||
if nid not in self.nodes or nid == "mcb-root":
|
||
raise ValueError(f"Unknown classification node: {nid}")
|
||
if meta["kind"] == "example":
|
||
self._require(meta["family_id"], "family")
|
||
|
||
def _require(self, doc_id, kind):
|
||
if not isinstance(doc_id, str) or doc_id not in self.documents:
|
||
raise ValueError(f"Unknown registered ID: {doc_id}")
|
||
doc = self.documents[doc_id]
|
||
if doc["meta"]["kind"] != kind:
|
||
raise ValueError(f"Expected {kind}: {doc_id}")
|
||
return doc
|
||
|
||
def search(self, query):
|
||
if not isinstance(query, str) or not query.strip():
|
||
raise ValueError("Query must be non-empty text")
|
||
query = normalized(query)
|
||
candidates = []
|
||
for family_id, doc in self.families.items():
|
||
meta = doc["meta"]
|
||
terms = [meta["name_zh"], meta["name_en"], *meta["aliases"]]
|
||
hits = sorted({term for term in terms if matches(query, term)}, key=lambda s: (-len(s), s))
|
||
if hits:
|
||
candidates.append({"family_id": family_id, "name_zh": meta["name_zh"],
|
||
"matched_terms": hits, "knowledge_status": meta["status"]})
|
||
candidates.sort(key=lambda c: (-len(c["matched_terms"][0]), c["family_id"]))
|
||
category_hits = []
|
||
for node in self.catalog["nodes"]:
|
||
if node["id"] == "mcb-root":
|
||
continue
|
||
terms = [node["label_en"], re.sub(r"〔.*?〕", "", node["label_zh"]),
|
||
*node.get("aliases", [])]
|
||
hits = [term for term in terms if matches(query, term)]
|
||
if hits:
|
||
related = [{"family_id": fid, "relation": doc["meta"]["classification"]["relation"]}
|
||
for fid, doc in self.families.items()
|
||
if node["id"] in doc["meta"]["classification"]["node_ids"]]
|
||
category_hits.append({"node_id": node["id"], "name_zh": node["label_zh"],
|
||
"matched_terms": hits, "related_families": related,
|
||
"coverage": "related_family_available" if related else "catalog_only"})
|
||
flags = []
|
||
if re.search(r"不要|不需要|不是|排除|不含|\b(?:not|without|exclude)\b", query):
|
||
flags.append("negation")
|
||
if len(candidates) > 1:
|
||
flags.append("multi_family")
|
||
if re.search(r"或者|还是|或是|不确定|\bor\b", query):
|
||
flags.append("ambiguous")
|
||
if not candidates:
|
||
flags.append("catalog_only" if category_hits else "no_match")
|
||
return {"query": query, "candidates": candidates, "category_hits": category_hits,
|
||
"flags": flags, "needs_semantic_review": True, "selected_family_id": None}
|
||
|
||
def build_context(self, family_ids, feature_ids=None, include_examples=False):
|
||
if isinstance(family_ids, str) or isinstance(feature_ids, str):
|
||
raise ValueError("IDs must be lists, not strings")
|
||
family_ids = list(dict.fromkeys(family_ids))
|
||
feature_ids = list(dict.fromkeys(feature_ids or []))
|
||
for fid in family_ids:
|
||
self._require(fid, "family")
|
||
related_features = {dep for fid in family_ids for dep in self.families[fid]["meta"]["feature_ids"]}
|
||
for feature_id in feature_ids:
|
||
self._require(feature_id, "feature")
|
||
if family_ids and feature_id not in related_features:
|
||
raise ValueError(f"Feature unrelated to selected families: {feature_id}")
|
||
ordered = []
|
||
seen = set()
|
||
|
||
def add(doc_id):
|
||
if doc_id in seen:
|
||
return
|
||
for dep in self.documents[doc_id]["meta"].get("requires", []):
|
||
add(dep)
|
||
seen.add(doc_id)
|
||
ordered.append(doc_id)
|
||
|
||
add("common-conventions")
|
||
for fid in family_ids:
|
||
add(fid)
|
||
for feature_id in sorted(related_features | set(feature_ids)):
|
||
add(feature_id)
|
||
if include_examples:
|
||
for fid in family_ids:
|
||
add(self.families[fid]["meta"]["example_id"])
|
||
manifest, sections = [], []
|
||
for doc_id in ordered:
|
||
doc = self.documents[doc_id]
|
||
meta = doc["meta"]
|
||
manifest.append({"doc_id": doc_id, "version": meta["knowledge_version"],
|
||
"kind": meta["kind"], "path": doc["path"]})
|
||
# Parameter schemas and example inputs are part of usable knowledge.
|
||
# Stripping frontmatter would hide the very data the agent needs.
|
||
metadata = yaml.safe_dump(meta, allow_unicode=True, sort_keys=False).strip()
|
||
sections.append(f"<!-- {doc_id}@{meta['knowledge_version']} -->\n```yaml\n{metadata}\n```\n\n{doc['body']}")
|
||
return {"manifest": manifest, "text": "\n\n---\n\n".join(sections)}
|
||
|
||
def validate_params(self, family_id, params):
|
||
doc = self._require(family_id, "family")
|
||
return validate_parameters(family_id, doc["meta"]["parameter_schema"], params)
|
||
|
||
def build_generation_context(self, family_ids, engine_build=None, *, experimental=False):
|
||
"""No fallback to the archival family/feature context."""
|
||
return build_generation_context(self, family_ids, engine_build, experimental=experimental)
|
||
|
||
def check(self):
|
||
errors = []
|
||
nodes = self.catalog["nodes"]
|
||
if len(nodes) != 204 or len(self.nodes) != 204:
|
||
errors.append("Expected 203 unique categories plus one root")
|
||
counts = {d: sum(n["depth"] == d for n in nodes) for d in range(4)}
|
||
if counts != {0: 1, 1: 15, 2: 109, 3: 79}:
|
||
errors.append(f"Unexpected classification depths: {counts}")
|
||
for node in nodes:
|
||
visited = set()
|
||
current = node
|
||
while current["parent_id"] is not None:
|
||
if current["id"] in visited or current["parent_id"] not in self.nodes:
|
||
errors.append(f"Broken/circular classification: {node['id']}")
|
||
break
|
||
visited.add(current["id"])
|
||
parent = self.nodes[current["parent_id"]]
|
||
if current["depth"] != parent["depth"] + 1:
|
||
errors.append(f"Invalid depth: {current['id']}")
|
||
current = parent
|
||
if current["id"] != "mcb-root":
|
||
errors.append(f"Classification not rooted: {node['id']}")
|
||
leaves = sum(not any(child["parent_id"] == node["id"] for child in nodes) for node in nodes)
|
||
if leaves != 173:
|
||
errors.append(f"Expected 173 leaves, found {leaves}")
|
||
if len(self.families) != 6:
|
||
errors.append("Expected six pilot families")
|
||
examples = {}
|
||
for fid, doc in self.families.items():
|
||
def check_attributes(schema, path):
|
||
for key, attribute in schema.get("properties", {}).items():
|
||
required = {"title", "description", "x-unit", "x-datum", "x-constraints"}
|
||
if not required.issubset(attribute) or not any(k in attribute for k in ("type", "const")):
|
||
errors.append(f"Incomplete attribute dictionary: {fid}.{path}{key}")
|
||
check_attributes(attribute, f"{path}{key}.")
|
||
if "items" in attribute:
|
||
check_attributes(attribute["items"], f"{path}{key}[].")
|
||
check_attributes(doc["meta"]["parameter_schema"], "")
|
||
example = self.documents[doc["meta"]["example_id"]]["meta"]
|
||
result = self.validate_params(fid, example["params"])
|
||
examples[fid] = result
|
||
if not result["valid"]:
|
||
errors.append(f"Invalid example: {fid}")
|
||
for key, expected in example.get("expected_derived", {}).items():
|
||
actual = result["derived"].get(key)
|
||
if isinstance(expected, (int, float)):
|
||
import math
|
||
equal = isinstance(actual, (int, float)) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9)
|
||
else:
|
||
equal = actual == expected
|
||
if not equal:
|
||
errors.append(f"Example expectation mismatch: {fid}.{key}")
|
||
return {"passed": not errors, "errors": errors, "categories": len(nodes) - 1,
|
||
"depth_counts": counts, "leaves": leaves, "families": len(self.families),
|
||
"documents": len(self.documents), "examples": examples,
|
||
"geometry_validated": False}
|