feat(cadfs): expand selector provenance coverage

2026-09-14 11:00:14 +08:00
parent 994d06aaea
commit de21ee8a5b
45 changed files with 19837 additions and 842 deletions
@@ -1,6 +1,7 @@
 from __future__ import annotations

 import hashlib, multiprocessing, random
+from copy import deepcopy
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
@@ -8,6 +9,7 @@ from .compare import compare_steps
 from .dataset import Sample, scan_dataset
 from .featurescript_parser import parse_featurescript
 from .lowering import lower_model
+from .operation_registry import build_operation_registry
 from .rebuild import rebuild_candidate
 from .reports import generate_reports, read_json, write_json, write_manifest

@@ -22,9 +24,34 @@ def _fingerprint(sample: Sample) -> str:
 def _sample_dir(output: Path, sample_id: str) -> Path: return output / "samples" / sample_id


+def _semantic_valid_prefix(
+    cdsl: dict[str, Any],
+    validate_semantic_cdsl: Any,
+) -> tuple[dict[str, Any], dict[str, Any]] | None:
+    """Return the longest contiguous semantically valid feature prefix.
+
+    This is an artifact-recovery boundary, not permission to execute an
+    invalid candidate.  The selected prefix is independently validated and
+    saved under an explicit prefix filename; the complete rejected CDSL stays
+    diagnostic-only.
+    """
+    features = cdsl.get("features") or []
+    for count in range(len(features) - 1, 0, -1):
+        prefix = deepcopy(cdsl)
+        prefix["features"] = prefix["features"][:count]
+        try:
+            semantic = validate_semantic_cdsl(prefix)
+        except Exception:
+            continue
+        return prefix, semantic
+    return None
+
+
 def scan(input_root: Path, output: Path) -> list[dict[str, Any]]:
-    records = [sample.as_dict() for sample in scan_dataset(input_root)]
+    samples = scan_dataset(input_root)
+    records = [sample.as_dict() for sample in samples]
    write_json(output / "dataset_index.json", {"schema": "cadfs_to_cdsl.dataset_index.v1", "input": str(input_root), "sample_count": len(records), "records": records})
+    write_json(output / "operation_registry.json", build_operation_registry(samples))
    initial = [{"sample_id": item["sample_id"], "status": "scanned", "diagnostics": item["diagnostics"]} for item in records]
    write_manifest(output / "manifest.jsonl", initial)
    return records
@@ -56,26 +83,91 @@ def convert_one(sample: Sample, output: Path, *, force: bool = False) -> dict[st
        if cached.get("input_fingerprint") == fingerprint and cached.get("conversion_status"):
            return cached
    if force:
-        for name in ("candidate.cdsl.json", "bound.cdsl.json", "rebuild.step", "rebuild.json", "rebuild.worker.json", "comparison.json", "comparison.worker.json"):
+        for name in (
+            "candidate.cdsl.json", "candidate.invalid.cdsl.json", "candidate.prefix.cdsl.json",
+            "bound.cdsl.json", "prefix.bound.cdsl.json", "rebuild.step", "prefix.rebuild.step",
+            "rebuild.json", "prefix.rebuild.json", "rebuild.worker.json", "comparison.json",
+            "comparison.worker.json",
+        ):
            (directory / name).unlink(missing_ok=True)
    diagnostics = list(sample.diagnostics)
    try:
        feature_path = Path(sample.files["featurescript"])
        model = parse_featurescript(feature_path.read_text(encoding="utf-8"), sample.sample_id)
-        provenance = {"source_featurescript": str(feature_path), **{f"source_{key}_sha256": value for key, value in sample.hashes.items()}, "jsonl": sample.metadata}
-        result = lower_model(model, provenance); diagnostics.extend(result.diagnostics)
-        write_json(directory / "history.json", result.history); write_json(directory / "diagnostics.json", diagnostics)
-        if result.cdsl is not None:
-            from engine.cdsl_engine.semantic_validation import validate_semantic_cdsl
-            semantic = validate_semantic_cdsl(result.cdsl); write_json(directory / "candidate.cdsl.json", result.cdsl)
-        else: semantic = None
-        status = {"schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id, "status": result.status, "conversion_status": result.status, "input_fingerprint": fingerprint, "semantic_validation": semantic, "diagnostic_count": len(diagnostics)}
    except Exception as exc:
        missing = isinstance(exc, FileNotFoundError)
-        diagnostics.append({"code": "source_missing" if missing else "parse_or_lowering_failed", "message": str(exc), "type": type(exc).__name__})
+        diagnostics.append({"code": "source_missing" if missing else "parse_failed", "message": str(exc), "type": type(exc).__name__})
        write_json(directory / "diagnostics.json", diagnostics)
        final_status = "source_missing" if missing else "parse_failed"
        status = {"schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id, "status": final_status, "conversion_status": final_status, "input_fingerprint": fingerprint, "diagnostic_count": len(diagnostics)}
+        write_json(status_path, status); return status
+    provenance = {
+        "source_featurescript": str(feature_path),
+        **{f"source_{key}_sha256": value for key, value in sample.hashes.items()},
+        "jsonl": sample.metadata,
+    }
+    try:
+        result = lower_model(model, provenance)
+    except Exception as exc:
+        diagnostics.append({"code": "lowering_failed", "message": str(exc), "type": type(exc).__name__})
+        write_json(directory / "diagnostics.json", diagnostics)
+        status = {
+            "schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id,
+            "status": "lowering_failed", "conversion_status": "lowering_failed",
+            "input_fingerprint": fingerprint, "diagnostic_count": len(diagnostics),
+        }
+        write_json(status_path, status); return status
+    diagnostics.extend(result.diagnostics)
+    write_json(directory / "history.json", result.history)
+    if result.cdsl is None:
+        write_json(directory / "diagnostics.json", diagnostics)
+        status = {
+            "schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id,
+            "status": result.status, "conversion_status": result.status,
+            "input_fingerprint": fingerprint, "semantic_validation": None, "diagnostic_count": len(diagnostics),
+        }
+        write_json(status_path, status); return status
+    try:
+        from engine.cdsl_engine.semantic_validation import validate_semantic_cdsl
+        semantic = validate_semantic_cdsl(result.cdsl)
+    except Exception as exc:
+        diagnostics.append({"code": "semantic_validation_failed", "message": str(exc), "type": type(exc).__name__})
+        write_json(directory / "candidate.invalid.cdsl.json", result.cdsl)
+        prefix_result = _semantic_valid_prefix(result.cdsl, validate_semantic_cdsl)
+        prefix_status: dict[str, Any] | None = None
+        if prefix_result is not None:
+            prefix, prefix_semantic = prefix_result
+            prefix_features = prefix.get("features") or []
+            write_json(directory / "candidate.prefix.cdsl.json", prefix)
+            prefix_status = {
+                "feature_count": len(prefix_features),
+                "last_feature_id": prefix_features[-1].get("id"),
+                "semantic_validation": prefix_semantic,
+            }
+            diagnostics.append({
+                "code": "semantic_valid_prefix_preserved",
+                "feature_count": len(prefix_features),
+                "last_feature_id": prefix_features[-1].get("id"),
+                "message": "longest contiguous semantic-valid CDSL prefix was preserved separately",
+            })
+        write_json(directory / "diagnostics.json", diagnostics)
+        status = {
+            "schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id,
+            "status": "semantic_validation_failed", "conversion_status": "semantic_validation_failed",
+            "input_fingerprint": fingerprint,
+            "semantic_validation": {"valid": False, "error": {"type": type(exc).__name__, "message": str(exc)}},
+            "diagnostic_count": len(diagnostics),
+        }
+        if prefix_status is not None:
+            status["semantic_valid_prefix"] = prefix_status
+        write_json(status_path, status); return status
+    write_json(directory / "candidate.cdsl.json", result.cdsl)
+    write_json(directory / "diagnostics.json", diagnostics)
+    status = {
+        "schema": "cadfs_to_cdsl.status.v1", "sample_id": sample.sample_id,
+        "status": result.status, "conversion_status": result.status,
+        "input_fingerprint": fingerprint, "semantic_validation": semantic, "diagnostic_count": len(diagnostics),
+    }
    write_json(status_path, status); return status


@@ -108,12 +200,19 @@ def rebuild_one(sample: Sample, output: Path, *, force: bool = False, timeout_se
    # must be rebuilt instead of being hidden behind the conversion label.
    # ``rebuild_candidate`` keeps the two outcomes distinct by reporting a
    # runtime-ineligible candidate when no executable body can be produced.
-    if not (directory / "candidate.cdsl.json").exists(): return status
-    rebuild_path = directory / "rebuild.json"; step_path = directory / "rebuild.step"
-    if not force and rebuild_path.exists() and status.get("rebuild_status"):
-        if status.get("rebuild_status") != "rebuilt" or step_path.exists(): return status
+    candidate_path = directory / "candidate.cdsl.json"
+    is_prefix = False
+    if not candidate_path.exists():
+        candidate_path = directory / "candidate.prefix.cdsl.json"
+        is_prefix = candidate_path.exists()
+    if not candidate_path.exists(): return status
+    rebuild_path = directory / ("prefix.rebuild.json" if is_prefix else "rebuild.json")
+    step_path = directory / ("prefix.rebuild.step" if is_prefix else "rebuild.step")
+    status_key = "prefix_rebuild_status" if is_prefix else "rebuild_status"
+    if not force and rebuild_path.exists() and status.get(status_key):
+        if status.get(status_key) != "rebuilt" or step_path.exists(): return status
    worker_result = directory / "rebuild.worker.json"
-    outcome = _isolated(_rebuild_worker, (str(directory / "candidate.cdsl.json"), str(step_path), str(worker_result)), worker_result, timeout_seconds)
+    outcome = _isolated(_rebuild_worker, (str(candidate_path), str(step_path), str(worker_result)), worker_result, timeout_seconds)
    if outcome == "completed":
        result = read_json(worker_result); worker_result.unlink(missing_ok=True)
        bound_cdsl = result.pop("bound_cdsl", None)
@@ -123,14 +222,17 @@ def rebuild_one(sample: Sample, output: Path, *, force: bool = False, timeout_se
            if bound_cdsl is None and prefix_bound_cdsl is not None:
                bound_cdsl = prefix_bound_cdsl
                prefix["bound_feature_count"] = prefix.get("feature_count")
-        if bound_cdsl is not None: write_json(directory / "bound.cdsl.json", bound_cdsl)
+        if bound_cdsl is not None:
+            write_json(directory / ("prefix.bound.cdsl.json" if is_prefix else "bound.cdsl.json"), bound_cdsl)
    else:
        step_path.unlink(missing_ok=True)
        error_type = "TimeoutError" if outcome == "timeout" else "WorkerProcessError"
        message = f"rebuild exceeded {timeout_seconds:g} seconds" if outcome == "timeout" else "rebuild worker exited without a result"
        result = {"status": "rebuild_timeout" if outcome == "timeout" else "rebuild_failed", "error": {"type": error_type, "message": message}}
    write_json(rebuild_path, result)
-    status["rebuild_status"] = result["status"]; status["status"] = result["status"]
+    status[status_key] = result["status"]
+    if not is_prefix:
+        status["status"] = result["status"]
    write_json(status_path, status); return status