"""Immutable v3 file artifacts with staging manifests and atomic publication.""" from __future__ import annotations from hashlib import sha256 import json import os from pathlib import Path import re import secrets import shutil from typing import Any from app.cad_agent.ports import CandidateStage _SAFE_RELATIVE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,240}$") class FileArtifactStore: def __init__(self, root: Path) -> None: # Keep every externally reported artifact path in the same canonical # form as staging and publication paths (notably /var vs /private/var # on macOS). self.root = root.resolve() self.root.mkdir(parents=True, exist_ok=True) def task_dir(self, task_id: str) -> Path: if not re.fullmatch(r"cad_[a-z0-9]{12}", task_id): raise ValueError("Invalid v3 task id") return self.root / task_id def artifact_path(self, task_id: str, relative_path: str) -> Path: return self._path(task_id, relative_path) def initialize_task( self, task_id: str, request: str, *, source_blocks: list[dict[str, Any]] | None = None, image_inputs: list[dict[str, str]] | None = None, ) -> None: root = self.task_dir(task_id) (root / "documents").mkdir(parents=True, exist_ok=True) (root / "actions").mkdir(exist_ok=True) (root / "revisions").mkdir(exist_ok=True) (root / ".staging").mkdir(exist_ok=True) source = root / "source-requirements.md" blocks = self._source_blocks(request, source_blocks) if not source.exists(): self._write_once(source, "\n\n".join(block["text"] for block in blocks) + "\n") self.write_source_index(task_id, request, source_blocks=blocks) images: list[dict[str, str]] = [] for position, item in enumerate(image_inputs or (), 1): source_path = Path(str(item.get("path") or "")).resolve() digest = str(item.get("sha256") or "") if not source_path.is_file() or not re.fullmatch(r"[a-f0-9]{64}", digest): raise ValueError("Image input is unavailable or has no valid checksum") data = source_path.read_bytes() if sha256(data).hexdigest() != digest: raise ValueError("Image input checksum mismatch") suffix = source_path.suffix.lower() if source_path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"} else ".bin" relative = f"inputs/reference-{position:03d}{suffix}" self._write_bytes_once(self._path(task_id, relative), data) images.append({"path": relative, "mime": str(item.get("mime") or "image/*"), "sha256": digest}) if images: self.write_json_once(task_id, "documents/source-images.json", {"schema_version": "cad.source-images.v1", "images": images}) def sync_action_ledger(self, task_id: str, events: list[dict[str, Any]]) -> str: """Mirror committed SQLite events into an append-only JSONL audit log. SQLite remains authoritative. Replaying this method after an interruption appends only missing committed sequences and rejects a divergent line instead of rewriting audit history. """ path = self._path(task_id, "actions/action-ledger.jsonl") existing: dict[int, dict[str, Any]] = {} if path.is_file(): for raw in path.read_text(encoding="utf-8").splitlines(): if not raw.strip(): continue value = json.loads(raw) sequence = value.get("sequence") if isinstance(value, dict) else None if not isinstance(sequence, int) or sequence < 1: raise ValueError("Action ledger contains an invalid sequence") existing[sequence] = value missing: list[str] = [] for event in events: sequence = event.get("sequence") if not isinstance(sequence, int) or sequence < 1: raise ValueError("Action ledger event has an invalid sequence") entry = {"schema_version": "cad.action-ledger.v1", **event} previous = existing.get(sequence) if previous is not None: if previous != entry: raise ValueError("Action ledger diverges from committed SQLite event") continue missing.append(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n") if missing: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.writelines(missing) handle.flush() os.fsync(handle.fileno()) return "actions/action-ledger.jsonl" def write_source_index( self, task_id: str, request: str, *, source_blocks: list[dict[str, Any]] | None = None, ) -> dict[str, str]: root = self.task_dir(task_id) source = root / "source-requirements.md" if source_blocks is None: text = source.read_text(encoding="utf-8") if source.is_file() else request blocks = self._source_blocks(text, None) else: blocks = self._source_blocks(request, source_blocks) index = {f"src_{position:03d}": block["text"] for position, block in enumerate(blocks, 1)} attachment_blocks = { f"src_{position:03d}": block["attachment"] for position, block in enumerate(blocks, 1) if isinstance(block.get("attachment"), dict) } payload: dict[str, Any] = {"schema_version": "cad.source-index.v1", "sources": index} if attachment_blocks: payload["attachment_blocks"] = attachment_blocks self._write_json_once(root / "documents" / "source-index.json", payload) return index @staticmethod def _source_blocks(request: str, source_blocks: list[dict[str, Any]] | None) -> list[dict[str, Any]]: """Return immutable source paragraphs and attachment blocks. Source identifiers are assigned only after this method returns, so callers cannot choose IDs. Attachment metadata is deliberately a closed, server-provided projection: it lets a reviewer trace the source without treating upload metadata as arbitrary LLM input. """ supplied = source_blocks if supplied is None: supplied = [{"text": value.strip()} for value in re.split(r"\n\s*\n", request) if value.strip()] blocks: list[dict[str, Any]] = [] for item in supplied: if not isinstance(item, dict): raise ValueError("Source blocks must be objects") text = item.get("text") if not isinstance(text, str) or not text.strip(): raise ValueError("Every source block requires non-empty text") if len(text) > 30_000: raise ValueError("Source block exceeds the 30000 character limit") block: dict[str, Any] = {"text": text.strip()} attachment = item.get("attachment") if attachment is not None: if not isinstance(attachment, dict): raise ValueError("Source attachment metadata must be an object") allowed = {"attachment_id", "name", "kind", "mime", "sha256"} if set(attachment) - allowed: raise ValueError("Source attachment metadata contains unsupported fields") attachment_id = attachment.get("attachment_id") digest = attachment.get("sha256") if not isinstance(attachment_id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", attachment_id): raise ValueError("Source attachment id is invalid") if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): raise ValueError("Source attachment SHA-256 is invalid") block["attachment"] = { "attachment_id": attachment_id, "name": str(attachment.get("name") or "attachment")[:255], "kind": str(attachment.get("kind") or "document")[:32], "mime": str(attachment.get("mime") or "application/octet-stream")[:128], "sha256": digest, } blocks.append(block) if not blocks: raise ValueError("At least one source block is required") return blocks def read_source_index(self, task_id: str) -> dict[str, str]: payload = self.read_json(task_id, "documents/source-index.json") or {} sources = payload.get("sources") if isinstance(payload.get("sources"), dict) else {} return {str(key): str(value) for key, value in sources.items()} def read_source_requirements(self, task_id: str) -> str: path = self.task_dir(task_id) / "source-requirements.md" return path.read_text(encoding="utf-8") if path.is_file() else "" def source_image_paths(self, task_id: str) -> list[str]: manifest = self.read_json(task_id, "documents/source-images.json") or {} return [ str(self._path(task_id, str(item.get("path") or ""))) for item in manifest.get("images") or () if isinstance(item, dict) and item.get("path") and self._path(task_id, str(item["path"])).is_file() ] def read_requirements_spec(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None: return self.read_json(task_id, artifact_path or "documents/requirements-spec.json") def read_requirements_contract(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None: # State points to the immutable artifact used as program input. The # fixed name is a read-only convenience view for terminal tasks. return self.read_json(task_id, artifact_path or "requirements-contract.json") def write_requirements_contract(self, task_id: str, payload: dict[str, Any], *, invocation_id: str = "") -> str: if not invocation_id: return self.write_json_once(task_id, "requirements-contract.json", payload) return self._write_invocation_json(task_id, "requirements-contract", payload, invocation_id) def read_json(self, task_id: str, relative_path: str) -> dict[str, Any] | None: path = self._path(task_id, relative_path) if not path.is_file(): return None value = json.loads(path.read_text(encoding="utf-8")) return value if isinstance(value, dict) else None def write_json_once(self, task_id: str, relative_path: str, payload: dict[str, Any]) -> str: path = self._path(task_id, relative_path) self._write_json_once(path, payload) return relative_path def write_text_once(self, task_id: str, relative_path: str, text: str) -> str: path = self._path(task_id, relative_path) self._write_once(path, text) return relative_path def read_active_cdsl(self, task_id: str, revision_id: str) -> dict[str, Any] | None: return self.read_json(task_id, f"revisions/{revision_id}/model.cdsl.json") if revision_id else None def read_topology(self, task_id: str, revision_id: str) -> dict[str, Any] | None: return self.read_json(task_id, f"revisions/{revision_id}/model.topology.json") if revision_id else None def start_candidate_stage(self, task_id: str, idempotency_key: str, payload: dict[str, Any]) -> CandidateStage: root = self.task_dir(task_id) / ".staging" stable_id = "stage_" + sha256(idempotency_key.encode("utf-8")).hexdigest()[:20] directory = root / stable_id directory.mkdir(parents=True, exist_ok=True) self._write_json_once(directory / "input.json", payload) return CandidateStage(stable_id, str(directory.resolve())) def stage_output_dir(self, task_id: str, stage_id: str) -> str: return str(self._stage_path(task_id, stage_id, "")) def write_stage_json(self, task_id: str, stage_id: str, relative_path: str, payload: dict[str, Any]) -> str: path = self._stage_path(task_id, stage_id, relative_path) self._write_json_once(path, payload) return relative_path def read_stage_json(self, task_id: str, stage_id: str, relative_path: str) -> dict[str, Any] | None: path = self._stage_path(task_id, stage_id, relative_path) if not path.is_file(): return None value = json.loads(path.read_text(encoding="utf-8")) return value if isinstance(value, dict) else None def publish_candidate(self, task_id: str, stage_id: str, revision_id: str) -> dict[str, str]: source = self._stage_path(task_id, stage_id, "") target = self._path(task_id, f"revisions/{revision_id}") if target.exists(): manifest = self._manifest(target) if manifest is None: raise RuntimeError("Published revision has no valid manifest") return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]} # Render/rebuild reports contain paths for the reviewer. Rebase those # paths while artifacts are still mutable staging output, so they point # at the revision after the atomic directory rename. self._rebase_staged_paths(source, target) manifest = self._create_manifest(source) self._write_json_once(source / "manifest.json", manifest) target.parent.mkdir(parents=True, exist_ok=True) os.replace(source, target) return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]} def find_published_candidate(self, task_id: str, stage_id: str) -> tuple[str, dict[str, Any]] | None: """Find a manifest-verified candidate already renamed before its DB CAS. Publishing artifacts and advancing SQLite cannot share a transaction. The stage id persisted inside ``candidate.json`` makes a post-rename recovery deterministic and prevents another build or revision. """ revisions = self.task_dir(task_id) / "revisions" if not revisions.is_dir(): return None for revision in sorted(revisions.iterdir()): if not revision.is_dir() or self._manifest(revision) is None: continue candidate_path = revision / "candidate.json" if not candidate_path.is_file(): continue try: candidate = json.loads(candidate_path.read_text(encoding="utf-8")) except json.JSONDecodeError: continue if isinstance(candidate, dict) and candidate.get("stage_id") == stage_id: return revision.name, candidate return None def recover_staged_candidates(self, task_id: str, referenced_stage_ids: set[str]) -> None: root = self.task_dir(task_id) / ".staging" if not root.is_dir(): return for directory in root.iterdir(): if not directory.is_dir() or directory.name in referenced_stage_ids: continue shutil.rmtree(directory) def _path(self, task_id: str, relative_path: str) -> Path: if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts): raise ValueError("Invalid artifact relative path") root = self.task_dir(task_id).resolve() target = (root / relative_path).resolve() if target != root and root not in target.parents: raise ValueError("Artifact path escapes its task") return target def _stage_path(self, task_id: str, stage_id: str, relative_path: str) -> Path: if not re.fullmatch(r"stage_[a-f0-9]{20}", stage_id): raise ValueError("Invalid candidate stage id") root = self.task_dir(task_id).resolve() stage = (root / ".staging" / stage_id).resolve() if root not in stage.parents: raise ValueError("Stage path escapes task directory") if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts): raise ValueError("Invalid stage artifact path") path = (stage / relative_path).resolve() if path != stage and stage not in path.parents: raise ValueError("Stage artifact path escapes staging directory") return path @staticmethod def _write_once(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) encoded = text.encode("utf-8") if path.exists(): if path.read_bytes() != encoded: raise ValueError(f"Immutable artifact already exists: {path.name}") return temporary = path.with_name(path.name + ".tmp-" + secrets.token_hex(4)) temporary.write_bytes(encoded) os.replace(temporary, path) @staticmethod def _write_bytes_once(path: Path, data: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) if path.exists(): if path.read_bytes() != data: raise ValueError(f"Immutable artifact already exists: {path.name}") return temporary = path.with_name(path.name + ".tmp-" + secrets.token_hex(4)) temporary.write_bytes(data) os.replace(temporary, path) def _write_invocation_json(self, task_id: str, stem: str, payload: dict[str, Any], invocation_id: str) -> str: digest = sha256(json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()[:12] safe_invocation = re.sub(r"[^A-Za-z0-9_-]", "", invocation_id)[:32] if not safe_invocation: raise ValueError("Artifact invocation ID is invalid") return self.write_json_once(task_id, f"documents/{stem}-{digest}-{safe_invocation}.json", payload) @classmethod def _write_json_once(cls, path: Path, payload: dict[str, Any]) -> None: cls._write_once(path, json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n") @staticmethod def _create_manifest(directory: Path) -> dict[str, Any]: files: dict[str, str] = {} for path in sorted(directory.rglob("*")): if not path.is_file() or path.name == "manifest.json": continue relative = path.relative_to(directory).as_posix() files[relative] = sha256(path.read_bytes()).hexdigest() if not files: raise RuntimeError("Candidate staging directory has no artifacts") return {"schema_version": "cad.v3.artifact-manifest.v1", "files": files} @staticmethod def _manifest(directory: Path) -> dict[str, Any] | None: path = directory / "manifest.json" if not path.is_file(): return None value = json.loads(path.read_text(encoding="utf-8")) files = value.get("files") if isinstance(value, dict) else None if not isinstance(files, dict): return None for relative, digest in files.items(): candidate = directory / str(relative) if not candidate.is_file() or sha256(candidate.read_bytes()).hexdigest() != digest: return None return value @classmethod def _rebase_staged_paths(cls, source: Path, target: Path) -> None: source_text = str(source) target_text = str(target) def rebase(value: Any) -> Any: if isinstance(value, str): return target_text + value.removeprefix(source_text) if value.startswith(source_text) else value if isinstance(value, list): return [rebase(item) for item in value] if isinstance(value, dict): return {key: rebase(item) for key, item in value.items()} return value for path in source.rglob("*.json"): try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue rebased = rebase(value) if rebased != value: path.write_text(json.dumps(rebased, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")