重构 改为标准化输出
This commit is contained in:
@@ -0,0 +1,400 @@
|
||||
"""Immutable v3 file artifacts with staging manifests and atomic publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
from app.cad_agent.ports import CandidateStage
|
||||
|
||||
|
||||
_SAFE_RELATIVE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,240}$")
|
||||
|
||||
|
||||
class FileArtifactStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
# Keep every externally reported artifact path in the same canonical
|
||||
# form as staging and publication paths (notably /var vs /private/var
|
||||
# on macOS).
|
||||
self.root = root.resolve()
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def task_dir(self, task_id: str) -> Path:
|
||||
if not re.fullmatch(r"cad_[a-z0-9]{12}", task_id):
|
||||
raise ValueError("Invalid v3 task id")
|
||||
return self.root / task_id
|
||||
|
||||
def artifact_path(self, task_id: str, relative_path: str) -> Path:
|
||||
return self._path(task_id, relative_path)
|
||||
|
||||
def initialize_task(
|
||||
self,
|
||||
task_id: str,
|
||||
request: str,
|
||||
*,
|
||||
source_blocks: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
root = self.task_dir(task_id)
|
||||
(root / "documents").mkdir(parents=True, exist_ok=True)
|
||||
(root / "actions").mkdir(exist_ok=True)
|
||||
(root / "revisions").mkdir(exist_ok=True)
|
||||
(root / ".staging").mkdir(exist_ok=True)
|
||||
source = root / "source-requirements.md"
|
||||
blocks = self._source_blocks(request, source_blocks)
|
||||
if not source.exists():
|
||||
self._write_once(source, "\n\n".join(block["text"] for block in blocks) + "\n")
|
||||
self.write_source_index(task_id, request, source_blocks=blocks)
|
||||
|
||||
def sync_action_ledger(self, task_id: str, events: list[dict[str, Any]]) -> str:
|
||||
"""Mirror committed SQLite events into an append-only JSONL audit log.
|
||||
|
||||
SQLite remains authoritative. Replaying this method after an
|
||||
interruption appends only missing committed sequences and rejects a
|
||||
divergent line instead of rewriting audit history.
|
||||
"""
|
||||
path = self._path(task_id, "actions/action-ledger.jsonl")
|
||||
existing: dict[int, dict[str, Any]] = {}
|
||||
if path.is_file():
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
if not raw.strip():
|
||||
continue
|
||||
value = json.loads(raw)
|
||||
sequence = value.get("sequence") if isinstance(value, dict) else None
|
||||
if not isinstance(sequence, int) or sequence < 1:
|
||||
raise ValueError("Action ledger contains an invalid sequence")
|
||||
existing[sequence] = value
|
||||
missing: list[str] = []
|
||||
for event in events:
|
||||
sequence = event.get("sequence")
|
||||
if not isinstance(sequence, int) or sequence < 1:
|
||||
raise ValueError("Action ledger event has an invalid sequence")
|
||||
entry = {"schema_version": "cad.action-ledger.v1", **event}
|
||||
previous = existing.get(sequence)
|
||||
if previous is not None:
|
||||
if previous != entry:
|
||||
raise ValueError("Action ledger diverges from committed SQLite event")
|
||||
continue
|
||||
missing.append(json.dumps(entry, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
if missing:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.writelines(missing)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
return "actions/action-ledger.jsonl"
|
||||
|
||||
def write_source_index(
|
||||
self,
|
||||
task_id: str,
|
||||
request: str,
|
||||
*,
|
||||
source_blocks: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, str]:
|
||||
root = self.task_dir(task_id)
|
||||
source = root / "source-requirements.md"
|
||||
if source_blocks is None:
|
||||
text = source.read_text(encoding="utf-8") if source.is_file() else request
|
||||
blocks = self._source_blocks(text, None)
|
||||
else:
|
||||
blocks = self._source_blocks(request, source_blocks)
|
||||
index = {f"src_{position:03d}": block["text"] for position, block in enumerate(blocks, 1)}
|
||||
attachment_blocks = {
|
||||
f"src_{position:03d}": block["attachment"]
|
||||
for position, block in enumerate(blocks, 1)
|
||||
if isinstance(block.get("attachment"), dict)
|
||||
}
|
||||
payload: dict[str, Any] = {"schema_version": "cad.source-index.v1", "sources": index}
|
||||
if attachment_blocks:
|
||||
payload["attachment_blocks"] = attachment_blocks
|
||||
self._write_json_once(root / "documents" / "source-index.json", payload)
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _source_blocks(request: str, source_blocks: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
||||
"""Return immutable source paragraphs and attachment blocks.
|
||||
|
||||
Source identifiers are assigned only after this method returns, so
|
||||
callers cannot choose IDs. Attachment metadata is deliberately a
|
||||
closed, server-provided projection: it lets a reviewer trace the
|
||||
source without treating upload metadata as arbitrary LLM input.
|
||||
"""
|
||||
supplied = source_blocks
|
||||
if supplied is None:
|
||||
supplied = [{"text": value.strip()} for value in re.split(r"\n\s*\n", request) if value.strip()]
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for item in supplied:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Source blocks must be objects")
|
||||
text = item.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("Every source block requires non-empty text")
|
||||
if len(text) > 30_000:
|
||||
raise ValueError("Source block exceeds the 30000 character limit")
|
||||
block: dict[str, Any] = {"text": text.strip()}
|
||||
attachment = item.get("attachment")
|
||||
if attachment is not None:
|
||||
if not isinstance(attachment, dict):
|
||||
raise ValueError("Source attachment metadata must be an object")
|
||||
allowed = {"attachment_id", "name", "kind", "mime", "sha256"}
|
||||
if set(attachment) - allowed:
|
||||
raise ValueError("Source attachment metadata contains unsupported fields")
|
||||
attachment_id = attachment.get("attachment_id")
|
||||
digest = attachment.get("sha256")
|
||||
if not isinstance(attachment_id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", attachment_id):
|
||||
raise ValueError("Source attachment id is invalid")
|
||||
if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest):
|
||||
raise ValueError("Source attachment SHA-256 is invalid")
|
||||
block["attachment"] = {
|
||||
"attachment_id": attachment_id,
|
||||
"name": str(attachment.get("name") or "attachment")[:255],
|
||||
"kind": str(attachment.get("kind") or "document")[:32],
|
||||
"mime": str(attachment.get("mime") or "application/octet-stream")[:128],
|
||||
"sha256": digest,
|
||||
}
|
||||
blocks.append(block)
|
||||
if not blocks:
|
||||
raise ValueError("At least one source block is required")
|
||||
return blocks
|
||||
|
||||
def read_source_index(self, task_id: str) -> dict[str, str]:
|
||||
payload = self.read_json(task_id, "documents/source-index.json") or {}
|
||||
sources = payload.get("sources") if isinstance(payload.get("sources"), dict) else {}
|
||||
return {str(key): str(value) for key, value in sources.items()}
|
||||
|
||||
def read_source_requirements(self, task_id: str) -> str:
|
||||
path = self.task_dir(task_id) / "source-requirements.md"
|
||||
return path.read_text(encoding="utf-8") if path.is_file() else ""
|
||||
|
||||
def read_requirements_draft(self, task_id: str, artifact_path: str = "") -> dict[str, Any]:
|
||||
if artifact_path:
|
||||
value = self.read_json(task_id, artifact_path)
|
||||
if value is None:
|
||||
raise ValueError("Committed requirements draft artifact is unavailable")
|
||||
return value
|
||||
documents = self.task_dir(task_id) / "documents"
|
||||
candidates = sorted(documents.glob("requirements-draft-v*.json"))
|
||||
if not candidates:
|
||||
return {"schema_version": "cad.requirements-draft.v1", "revision": 0, "items": []}
|
||||
value = json.loads(candidates[-1].read_text(encoding="utf-8"))
|
||||
return value if isinstance(value, dict) else {"schema_version": "cad.requirements-draft.v1", "revision": 0, "items": []}
|
||||
|
||||
def write_requirements_draft(self, task_id: str, payload: dict[str, Any], *, invocation_id: str) -> str:
|
||||
revision = int(payload.get("revision") or 0)
|
||||
if revision < 1:
|
||||
raise ValueError("Requirements draft revision is invalid")
|
||||
return self._write_invocation_json(task_id, f"requirements-draft-v{revision}", payload, invocation_id)
|
||||
|
||||
def read_requirements_review(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None:
|
||||
return self.read_json(task_id, artifact_path) if artifact_path else None
|
||||
|
||||
def write_requirements_review(self, task_id: str, payload: dict[str, Any], *, invocation_id: str) -> str:
|
||||
return self._write_invocation_json(task_id, "requirements-review", payload, invocation_id)
|
||||
|
||||
def read_requirements_contract(self, task_id: str, artifact_path: str = "") -> dict[str, Any] | None:
|
||||
# State points to the immutable artifact used as program input. The
|
||||
# fixed name is a read-only convenience view for terminal tasks.
|
||||
return self.read_json(task_id, artifact_path or "requirements-contract.json")
|
||||
|
||||
def write_requirements_contract(self, task_id: str, payload: dict[str, Any], *, invocation_id: str = "") -> str:
|
||||
if not invocation_id:
|
||||
return self.write_json_once(task_id, "requirements-contract.json", payload)
|
||||
return self._write_invocation_json(task_id, "requirements-contract", payload, invocation_id)
|
||||
|
||||
def read_json(self, task_id: str, relative_path: str) -> dict[str, Any] | None:
|
||||
path = self._path(task_id, relative_path)
|
||||
if not path.is_file():
|
||||
return None
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
def write_json_once(self, task_id: str, relative_path: str, payload: dict[str, Any]) -> str:
|
||||
path = self._path(task_id, relative_path)
|
||||
self._write_json_once(path, payload)
|
||||
return relative_path
|
||||
|
||||
def write_text_once(self, task_id: str, relative_path: str, text: str) -> str:
|
||||
path = self._path(task_id, relative_path)
|
||||
self._write_once(path, text)
|
||||
return relative_path
|
||||
|
||||
def read_active_cdsl(self, task_id: str, revision_id: str) -> dict[str, Any] | None:
|
||||
return self.read_json(task_id, f"revisions/{revision_id}/model.cdsl.json") if revision_id else None
|
||||
|
||||
def read_topology(self, task_id: str, revision_id: str) -> dict[str, Any] | None:
|
||||
return self.read_json(task_id, f"revisions/{revision_id}/model.topology.json") if revision_id else None
|
||||
|
||||
def start_candidate_stage(self, task_id: str, idempotency_key: str, payload: dict[str, Any]) -> CandidateStage:
|
||||
root = self.task_dir(task_id) / ".staging"
|
||||
stable_id = "stage_" + sha256(idempotency_key.encode("utf-8")).hexdigest()[:20]
|
||||
directory = root / stable_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
self._write_json_once(directory / "input.json", payload)
|
||||
return CandidateStage(stable_id, str(directory.resolve()))
|
||||
|
||||
def stage_output_dir(self, task_id: str, stage_id: str) -> str:
|
||||
return str(self._stage_path(task_id, stage_id, ""))
|
||||
|
||||
def write_stage_json(self, task_id: str, stage_id: str, relative_path: str, payload: dict[str, Any]) -> str:
|
||||
path = self._stage_path(task_id, stage_id, relative_path)
|
||||
self._write_json_once(path, payload)
|
||||
return relative_path
|
||||
|
||||
def read_stage_json(self, task_id: str, stage_id: str, relative_path: str) -> dict[str, Any] | None:
|
||||
path = self._stage_path(task_id, stage_id, relative_path)
|
||||
if not path.is_file():
|
||||
return None
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
def publish_candidate(self, task_id: str, stage_id: str, revision_id: str) -> dict[str, str]:
|
||||
source = self._stage_path(task_id, stage_id, "")
|
||||
target = self._path(task_id, f"revisions/{revision_id}")
|
||||
if target.exists():
|
||||
manifest = self._manifest(target)
|
||||
if manifest is None:
|
||||
raise RuntimeError("Published revision has no valid manifest")
|
||||
return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]}
|
||||
# Render/rebuild reports contain paths for the reviewer. Rebase those
|
||||
# paths while artifacts are still mutable staging output, so they point
|
||||
# at the revision after the atomic directory rename.
|
||||
self._rebase_staged_paths(source, target)
|
||||
manifest = self._create_manifest(source)
|
||||
self._write_json_once(source / "manifest.json", manifest)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.replace(source, target)
|
||||
return {key: f"revisions/{revision_id}/{key}" for key in manifest["files"]}
|
||||
|
||||
def find_published_candidate(self, task_id: str, stage_id: str) -> tuple[str, dict[str, Any]] | None:
|
||||
"""Find a manifest-verified candidate already renamed before its DB CAS.
|
||||
|
||||
Publishing artifacts and advancing SQLite cannot share a transaction.
|
||||
The stage id persisted inside ``candidate.json`` makes a post-rename
|
||||
recovery deterministic and prevents another build or revision.
|
||||
"""
|
||||
revisions = self.task_dir(task_id) / "revisions"
|
||||
if not revisions.is_dir():
|
||||
return None
|
||||
for revision in sorted(revisions.iterdir()):
|
||||
if not revision.is_dir() or self._manifest(revision) is None:
|
||||
continue
|
||||
candidate_path = revision / "candidate.json"
|
||||
if not candidate_path.is_file():
|
||||
continue
|
||||
try:
|
||||
candidate = json.loads(candidate_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(candidate, dict) and candidate.get("stage_id") == stage_id:
|
||||
return revision.name, candidate
|
||||
return None
|
||||
|
||||
def recover_staged_candidates(self, task_id: str, referenced_stage_ids: set[str]) -> None:
|
||||
root = self.task_dir(task_id) / ".staging"
|
||||
if not root.is_dir():
|
||||
return
|
||||
for directory in root.iterdir():
|
||||
if not directory.is_dir() or directory.name in referenced_stage_ids:
|
||||
continue
|
||||
shutil.rmtree(directory)
|
||||
|
||||
def _path(self, task_id: str, relative_path: str) -> Path:
|
||||
if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts):
|
||||
raise ValueError("Invalid artifact relative path")
|
||||
root = self.task_dir(task_id).resolve()
|
||||
target = (root / relative_path).resolve()
|
||||
if target != root and root not in target.parents:
|
||||
raise ValueError("Artifact path escapes its task")
|
||||
return target
|
||||
|
||||
def _stage_path(self, task_id: str, stage_id: str, relative_path: str) -> Path:
|
||||
if not re.fullmatch(r"stage_[a-f0-9]{20}", stage_id):
|
||||
raise ValueError("Invalid candidate stage id")
|
||||
root = self.task_dir(task_id).resolve()
|
||||
stage = (root / ".staging" / stage_id).resolve()
|
||||
if root not in stage.parents:
|
||||
raise ValueError("Stage path escapes task directory")
|
||||
if relative_path and (not _SAFE_RELATIVE.fullmatch(relative_path) or ".." in Path(relative_path).parts):
|
||||
raise ValueError("Invalid stage artifact path")
|
||||
path = (stage / relative_path).resolve()
|
||||
if path != stage and stage not in path.parents:
|
||||
raise ValueError("Stage artifact path escapes staging directory")
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _write_once(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
encoded = text.encode("utf-8")
|
||||
if path.exists():
|
||||
if path.read_bytes() != encoded:
|
||||
raise ValueError(f"Immutable artifact already exists: {path.name}")
|
||||
return
|
||||
temporary = path.with_name(path.name + ".tmp-" + secrets.token_hex(4))
|
||||
temporary.write_bytes(encoded)
|
||||
os.replace(temporary, path)
|
||||
|
||||
def _write_invocation_json(self, task_id: str, stem: str, payload: dict[str, Any], invocation_id: str) -> str:
|
||||
digest = sha256(json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()[:12]
|
||||
safe_invocation = re.sub(r"[^A-Za-z0-9_-]", "", invocation_id)[:32]
|
||||
if not safe_invocation:
|
||||
raise ValueError("Artifact invocation ID is invalid")
|
||||
return self.write_json_once(task_id, f"documents/{stem}-{digest}-{safe_invocation}.json", payload)
|
||||
|
||||
@classmethod
|
||||
def _write_json_once(cls, path: Path, payload: dict[str, Any]) -> None:
|
||||
cls._write_once(path, json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n")
|
||||
|
||||
@staticmethod
|
||||
def _create_manifest(directory: Path) -> dict[str, Any]:
|
||||
files: dict[str, str] = {}
|
||||
for path in sorted(directory.rglob("*")):
|
||||
if not path.is_file() or path.name == "manifest.json":
|
||||
continue
|
||||
relative = path.relative_to(directory).as_posix()
|
||||
files[relative] = sha256(path.read_bytes()).hexdigest()
|
||||
if not files:
|
||||
raise RuntimeError("Candidate staging directory has no artifacts")
|
||||
return {"schema_version": "cad.v3.artifact-manifest.v1", "files": files}
|
||||
|
||||
@staticmethod
|
||||
def _manifest(directory: Path) -> dict[str, Any] | None:
|
||||
path = directory / "manifest.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
files = value.get("files") if isinstance(value, dict) else None
|
||||
if not isinstance(files, dict):
|
||||
return None
|
||||
for relative, digest in files.items():
|
||||
candidate = directory / str(relative)
|
||||
if not candidate.is_file() or sha256(candidate.read_bytes()).hexdigest() != digest:
|
||||
return None
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _rebase_staged_paths(cls, source: Path, target: Path) -> None:
|
||||
source_text = str(source)
|
||||
target_text = str(target)
|
||||
|
||||
def rebase(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return target_text + value.removeprefix(source_text) if value.startswith(source_text) else value
|
||||
if isinstance(value, list):
|
||||
return [rebase(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: rebase(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
for path in source.rglob("*.json"):
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
rebased = rebase(value)
|
||||
if rebased != value:
|
||||
path.write_text(json.dumps(rebased, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
Reference in New Issue
Block a user