414 lines
20 KiB
Python
414 lines
20 KiB
Python
"""Immutable v3 file artifacts with staging manifests and atomic publication."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from hashlib import sha256
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
from typing import Any
|
|
|
|
from app.cad_agent.ports import CandidateStage
|
|
|
|
|
|
_SAFE_RELATIVE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,240}$")
|
|
|
|
|
|
class FileArtifactStore:
|
|
def __init__(self, root: Path) -> None:
|
|
# Keep every externally reported artifact path in the same canonical
|
|
# form as staging and publication paths (notably /var vs /private/var
|
|
# on macOS).
|
|
self.root = root.resolve()
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
|
|
def task_dir(self, task_id: str) -> Path:
|
|
if not re.fullmatch(r"cad_[a-z0-9]{12}", task_id):
|
|
raise ValueError("Invalid v3 task id")
|
|
return self.root / task_id
|
|
|
|
def artifact_path(self, task_id: str, relative_path: str) -> Path:
|
|
return self._path(task_id, relative_path)
|
|
|
|
def initialize_task(
|
|
self,
|
|
task_id: str,
|
|
request: str,
|
|
*,
|
|
source_blocks: list[dict[str, Any]] | None = None,
|
|
image_inputs: list[dict[str, str]] | None = None,
|
|
) -> None:
|
|
root = self.task_dir(task_id)
|
|
(root / "documents").mkdir(parents=True, exist_ok=True)
|
|
(root / "actions").mkdir(exist_ok=True)
|
|
(root / "revisions").mkdir(exist_ok=True)
|
|
(root / ".staging").mkdir(exist_ok=True)
|
|
source = root / "source-requirements.md"
|
|
blocks = self._source_blocks(request, source_blocks)
|
|
if not source.exists():
|
|
self._write_once(source, "\n\n".join(block["text"] for block in blocks) + "\n")
|
|
self.write_source_index(task_id, request, source_blocks=blocks)
|
|
images: list[dict[str, str]] = []
|
|
for position, item in enumerate(image_inputs or (), 1):
|
|
source_path = Path(str(item.get("path") or "")).resolve()
|
|
digest = str(item.get("sha256") or "")
|
|
if not source_path.is_file() or not re.fullmatch(r"[a-f0-9]{64}", digest):
|
|
raise ValueError("Image input is unavailable or has no valid checksum")
|
|
data = source_path.read_bytes()
|
|
if sha256(data).hexdigest() != digest:
|
|
raise ValueError("Image input checksum mismatch")
|
|
suffix = source_path.suffix.lower() if source_path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"} else ".bin"
|
|
relative = f"inputs/reference-{position:03d}{suffix}"
|
|
self._write_bytes_once(self._path(task_id, relative), data)
|
|
images.append({"path": relative, "mime": str(item.get("mime") or "image/*"), "sha256": digest})
|
|
if images:
|
|
self.write_json_once(task_id, "documents/source-images.json", {"schema_version": "cad.source-images.v1", "images": images})
|
|
|
|
def sync_action_ledger(self, task_id: str, events: list[dict[str, Any]]) -> str:
|
|
"""Mirror committed SQLite events into an append-only JSONL audit log.
|
|
|
|
SQLite remains authoritative. Replaying this method after an
|
|
interruption appends only missing committed sequences and rejects a
|
|
divergent line instead of rewriting audit history.
|
|
"""
|
|
path = self._path(task_id, "actions/action-ledger.jsonl")
|
|
existing: dict[int, dict[str, Any]] = {}
|
|
if path.is_file():
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
if not raw.strip():
|
|
continue
|
|
value = json.loads(raw)
|
|
sequence = value.get("sequence") if isinstance(value, dict) else None
|
|
if not isinstance(sequence, int) or sequence < 1:
|
|
raise ValueError("Action ledger contains an invalid sequence")
|
|
existing[sequence] = value
|
|
missing: list[str] = []
|
|
for event in events:
|
|
sequence = event.get("sequence")
|
|
if not isinstance(sequence, int) or sequence < 1:
|
|
raise ValueError("Action ledger event has an invalid sequence")
|
|
entry = {"schema_version": "cad.action-ledger.v1", **event}
|
|
previous = existing.get(sequence)
|
|
if previous is not None:
|
|
if previous != entry:
|
|
raise ValueError("Action ledger diverges from committed SQLite event")
|
|
continue
|
|
missing.append(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")
|
|
if missing:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a", encoding="utf-8") as handle:
|
|
handle.writelines(missing)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
return "actions/action-ledger.jsonl"
|
|
|
|
def write_source_index(
|
|
self,
|
|
task_id: str,
|
|
request: str,
|
|
*,
|
|
source_blocks: list[dict[str, Any]] | None = None,
|
|
) -> dict[str, str]:
|
|
root = self.task_dir(task_id)
|
|
source = root / "source-requirements.md"
|
|
if source_blocks is None:
|
|
text = source.read_text(encoding="utf-8") if source.is_file() else request
|
|
blocks = self._source_blocks(text, None)
|
|
else:
|
|
blocks = self._source_blocks(request, source_blocks)
|
|
index = {f"src_{position:03d}": block["text"] for position, block in enumerate(blocks, 1)}
|
|
attachment_blocks = {
|
|
f"src_{position:03d}": block["attachment"]
|
|
for position, block in enumerate(blocks, 1)
|
|
if isinstance(block.get("attachment"), dict)
|
|
}
|
|
payload: dict[str, Any] = {"schema_version": "cad.source-index.v1", "sources": index}
|
|
if attachment_blocks:
|
|
payload["attachment_blocks"] = attachment_blocks
|
|
self._write_json_once(root / "documents" / "source-index.json", payload)
|
|
return index
|
|
|
|
@staticmethod
|
|
def _source_blocks(request: str, source_blocks: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
|
"""Return immutable source paragraphs and attachment blocks.
|
|
|
|
Source identifiers are assigned only after this method returns, so
|
|
callers cannot choose IDs. Attachment metadata is deliberately a
|
|
closed, server-provided projection: it lets a reviewer trace the
|
|
source without treating upload metadata as arbitrary LLM input.
|
|
"""
|
|
supplied = source_blocks
|
|
if supplied is None:
|
|
supplied = [{"text": value.strip()} for value in re.split(r"\n\s*\n", request) if value.strip()]
|
|
blocks: list[dict[str, Any]] = []
|
|
for item in supplied:
|
|
if not isinstance(item, dict):
|
|
raise ValueError("Source blocks must be objects")
|
|
text = item.get("text")
|
|
if not isinstance(text, str) or not text.strip():
|
|
raise ValueError("Every source block requires non-empty text")
|
|
if len(text) > 30_000:
|
|
raise ValueError("Source block exceeds the 30000 character limit")
|
|
block: dict[str, Any] = {"text": text.strip()}
|
|
attachment = item.get("attachment")
|
|
if attachment is not None:
|
|
if not isinstance(attachment, dict):
|
|
raise ValueError("Source attachment metadata must be an object")
|
|
allowed = {"attachment_id", "name", "kind", "mime", "sha256"}
|
|
if set(attachment) - allowed:
|
|
raise ValueError("Source attachment metadata contains unsupported fields")
|
|
attachment_id = attachment.get("attachment_id")
|
|
digest = attachment.get("sha256")
|
|
if not isinstance(attachment_id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", attachment_id):
|
|
raise ValueError("Source attachment id is invalid")
|
|
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
|
|
raise ValueError("Source attachment SHA-256 is invalid")
|
|
block["attachment"] = {
|
|
"attachment_id": attachment_id,
|
|
"name": str(attachment.get("name") or "attachment")[:255],
|
|
"kind": str(attachment.get("kind") or "document")[:32],
|
|
"mime": str(attachment.get("mime") or "application/octet-stream")[:128],
|
|
"sha256": digest,
|
|
}
|
|
blocks.append(block)
|
|
if not blocks:
|
|
raise ValueError("At least one source block is required")
|
|
return blocks
|
|
|
|
def read_source_index(self, task_id: str) -> dict[str, str]:
|
|
payload = self.read_json(task_id, "documents/source-index.json") or {}
|
|
sources = payload.get("sources") if isinstance(payload.get("sources"), dict) else {}
|
|
return {str(key): str(value) for key, value in sources.items()}
|
|
|
|
def read_source_requirements(self, task_id: str) -> str:
|
|
path = self.task_dir(task_id) / "source-requirements.md"
|
|
return path.read_text(encoding="utf-8") if path.is_file() else ""
|
|
|
|
def source_image_paths(self, task_id: str) -> list[str]:
|
|
manifest = self.read_json(task_id, "documents/source-images.json") or {}
|
|
return [
|
|
str(self._path(task_id, str(item.get("path") or "")))
|
|
for item in manifest.get("images") or ()
|
|
if isinstance(item, dict) and item.get("path") and self._path(task_id, str(item["path"])).is_file()
|
|
]
|
|
|
|
def read_requirements_spec(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None:
|
|
return self.read_json(task_id, artifact_path or "documents/requirements-spec.json")
|
|
|
|
def read_requirements_contract(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None:
|
|
# State points to the immutable artifact used as program input. The
|
|
# fixed name is a read-only convenience view for terminal tasks.
|
|
return self.read_json(task_id, artifact_path or "requirements-contract.json")
|
|
|
|
def write_requirements_contract(self, task_id: str, payload: dict[str, Any], *, invocation_id: str = "") -> str:
|
|
if not invocation_id:
|
|
return self.write_json_once(task_id, "requirements-contract.json", payload)
|
|
return self._write_invocation_json(task_id, "requirements-contract", payload, invocation_id)
|
|
|
|
def read_json(self, task_id: str, relative_path: str) -> dict[str, Any] | None:
|
|
path = self._path(task_id, relative_path)
|
|
if not path.is_file():
|
|
return None
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else None
|
|
|
|
def write_json_once(self, task_id: str, relative_path: str, payload: dict[str, Any]) -> str:
|
|
path = self._path(task_id, relative_path)
|
|
self._write_json_once(path, payload)
|
|
return relative_path
|
|
|
|
def write_text_once(self, task_id: str, relative_path: str, text: str) -> str:
|
|
path = self._path(task_id, relative_path)
|
|
self._write_once(path, text)
|
|
return relative_path
|
|
|
|
def read_active_cdsl(self, task_id: str, revision_id: str) -> dict[str, Any] | None:
|
|
return self.read_json(task_id, f"revisions/{revision_id}/model.cdsl.json") if revision_id else None
|
|
|
|
def read_topology(self, task_id: str, revision_id: str) -> dict[str, Any] | None:
|
|
return self.read_json(task_id, f"revisions/{revision_id}/model.topology.json") if revision_id else None
|
|
|
|
def start_candidate_stage(self, task_id: str, idempotency_key: str, payload: dict[str, Any]) -> CandidateStage:
|
|
root = self.task_dir(task_id) / ".staging"
|
|
stable_id = "stage_" + sha256(idempotency_key.encode("utf-8")).hexdigest()[:20]
|
|
directory = root / stable_id
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
self._write_json_once(directory / "input.json", payload)
|
|
return CandidateStage(stable_id, str(directory.resolve()))
|
|
|
|
def stage_output_dir(self, task_id: str, stage_id: str) -> str:
|
|
return str(self._stage_path(task_id, stage_id, ""))
|
|
|
|
def write_stage_json(self, task_id: str, stage_id: str, relative_path: str, payload: dict[str, Any]) -> str:
|
|
path = self._stage_path(task_id, stage_id, relative_path)
|
|
self._write_json_once(path, payload)
|
|
return relative_path
|
|
|
|
def read_stage_json(self, task_id: str, stage_id: str, relative_path: str) -> dict[str, Any] | None:
|
|
path = self._stage_path(task_id, stage_id, relative_path)
|
|
if not path.is_file():
|
|
return None
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
return value if isinstance(value, dict) else None
|
|
|
|
def publish_candidate(self, task_id: str, stage_id: str, revision_id: str) -> dict[str, str]:
|
|
source = self._stage_path(task_id, stage_id, "")
|
|
target = self._path(task_id, f"revisions/{revision_id}")
|
|
if target.exists():
|
|
manifest = self._manifest(target)
|
|
if manifest is None:
|
|
raise RuntimeError("Published revision has no valid manifest")
|
|
return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]}
|
|
# Render/rebuild reports contain paths for the reviewer. Rebase those
|
|
# paths while artifacts are still mutable staging output, so they point
|
|
# at the revision after the atomic directory rename.
|
|
self._rebase_staged_paths(source, target)
|
|
manifest = self._create_manifest(source)
|
|
self._write_json_once(source / "manifest.json", manifest)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
os.replace(source, target)
|
|
return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]}
|
|
|
|
def find_published_candidate(self, task_id: str, stage_id: str) -> tuple[str, dict[str, Any]] | None:
|
|
"""Find a manifest-verified candidate already renamed before its DB CAS.
|
|
|
|
Publishing artifacts and advancing SQLite cannot share a transaction.
|
|
The stage id persisted inside ``candidate.json`` makes a post-rename
|
|
recovery deterministic and prevents another build or revision.
|
|
"""
|
|
revisions = self.task_dir(task_id) / "revisions"
|
|
if not revisions.is_dir():
|
|
return None
|
|
for revision in sorted(revisions.iterdir()):
|
|
if not revision.is_dir() or self._manifest(revision) is None:
|
|
continue
|
|
candidate_path = revision / "candidate.json"
|
|
if not candidate_path.is_file():
|
|
continue
|
|
try:
|
|
candidate = json.loads(candidate_path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(candidate, dict) and candidate.get("stage_id") == stage_id:
|
|
return revision.name, candidate
|
|
return None
|
|
|
|
def recover_staged_candidates(self, task_id: str, referenced_stage_ids: set[str]) -> None:
|
|
root = self.task_dir(task_id) / ".staging"
|
|
if not root.is_dir():
|
|
return
|
|
for directory in root.iterdir():
|
|
if not directory.is_dir() or directory.name in referenced_stage_ids:
|
|
continue
|
|
shutil.rmtree(directory)
|
|
|
|
def _path(self, task_id: str, relative_path: str) -> Path:
|
|
if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts):
|
|
raise ValueError("Invalid artifact relative path")
|
|
root = self.task_dir(task_id).resolve()
|
|
target = (root / relative_path).resolve()
|
|
if target != root and root not in target.parents:
|
|
raise ValueError("Artifact path escapes its task")
|
|
return target
|
|
|
|
def _stage_path(self, task_id: str, stage_id: str, relative_path: str) -> Path:
|
|
if not re.fullmatch(r"stage_[a-f0-9]{20}", stage_id):
|
|
raise ValueError("Invalid candidate stage id")
|
|
root = self.task_dir(task_id).resolve()
|
|
stage = (root / ".staging" / stage_id).resolve()
|
|
if root not in stage.parents:
|
|
raise ValueError("Stage path escapes task directory")
|
|
if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts):
|
|
raise ValueError("Invalid stage artifact path")
|
|
path = (stage / relative_path).resolve()
|
|
if path != stage and stage not in path.parents:
|
|
raise ValueError("Stage artifact path escapes staging directory")
|
|
return path
|
|
|
|
@staticmethod
|
|
def _write_once(path: Path, text: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
encoded = text.encode("utf-8")
|
|
if path.exists():
|
|
if path.read_bytes() != encoded:
|
|
raise ValueError(f"Immutable artifact already exists: {path.name}")
|
|
return
|
|
temporary = path.with_name(path.name + ".tmp-" + secrets.token_hex(4))
|
|
temporary.write_bytes(encoded)
|
|
os.replace(temporary, path)
|
|
|
|
@staticmethod
|
|
def _write_bytes_once(path: Path, data: bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.exists():
|
|
if path.read_bytes() != data:
|
|
raise ValueError(f"Immutable artifact already exists: {path.name}")
|
|
return
|
|
temporary = path.with_name(path.name + ".tmp-" + secrets.token_hex(4))
|
|
temporary.write_bytes(data)
|
|
os.replace(temporary, path)
|
|
|
|
def _write_invocation_json(self, task_id: str, stem: str, payload: dict[str, Any], invocation_id: str) -> str:
|
|
digest = sha256(json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()[:12]
|
|
safe_invocation = re.sub(r"[^A-Za-z0-9_-]", "", invocation_id)[:32]
|
|
if not safe_invocation:
|
|
raise ValueError("Artifact invocation ID is invalid")
|
|
return self.write_json_once(task_id, f"documents/{stem}-{digest}-{safe_invocation}.json", payload)
|
|
|
|
@classmethod
|
|
def _write_json_once(cls, path: Path, payload: dict[str, Any]) -> None:
|
|
cls._write_once(path, json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n")
|
|
|
|
@staticmethod
|
|
def _create_manifest(directory: Path) -> dict[str, Any]:
|
|
files: dict[str, str] = {}
|
|
for path in sorted(directory.rglob("*")):
|
|
if not path.is_file() or path.name == "manifest.json":
|
|
continue
|
|
relative = path.relative_to(directory).as_posix()
|
|
files[relative] = sha256(path.read_bytes()).hexdigest()
|
|
if not files:
|
|
raise RuntimeError("Candidate staging directory has no artifacts")
|
|
return {"schema_version": "cad.v3.artifact-manifest.v1", "files": files}
|
|
|
|
@staticmethod
|
|
def _manifest(directory: Path) -> dict[str, Any] | None:
|
|
path = directory / "manifest.json"
|
|
if not path.is_file():
|
|
return None
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
files = value.get("files") if isinstance(value, dict) else None
|
|
if not isinstance(files, dict):
|
|
return None
|
|
for relative, digest in files.items():
|
|
candidate = directory / str(relative)
|
|
if not candidate.is_file() or sha256(candidate.read_bytes()).hexdigest() != digest:
|
|
return None
|
|
return value
|
|
|
|
@classmethod
|
|
def _rebase_staged_paths(cls, source: Path, target: Path) -> None:
|
|
source_text = str(source)
|
|
target_text = str(target)
|
|
|
|
def rebase(value: Any) -> Any:
|
|
if isinstance(value, str):
|
|
return target_text + value.removeprefix(source_text) if value.startswith(source_text) else value
|
|
if isinstance(value, list):
|
|
return [rebase(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return {key: rebase(item) for key, item in value.items()}
|
|
return value
|
|
|
|
for path in source.rglob("*.json"):
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
continue
|
|
rebased = rebase(value)
|
|
if rebased != value:
|
|
path.write_text(json.dumps(rebased, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|