253 lines
11 KiB
Python
253 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import shutil
|
|
import stat
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any
|
|
|
|
from .config import save_hardware_profile, validate_hardware_profile, validate_robot_manifest
|
|
from .controller import MicroduckController
|
|
from .policy_runner import PolicySet
|
|
|
|
|
|
MAX_ZIP_FILES = 512
|
|
MAX_ZIP_UNCOMPRESSED_BYTES = 128 * 1024 * 1024
|
|
|
|
|
|
def _safe_rel_path(member_name: str) -> PurePosixPath:
|
|
if not member_name or "\\" in member_name:
|
|
raise ValueError(f"Unsafe zip member path: {member_name!r}")
|
|
if member_name.startswith("/") or ":" in member_name:
|
|
raise ValueError(f"Absolute or drive path forbidden in zip: {member_name!r}")
|
|
parts = PurePosixPath(member_name).parts
|
|
if not parts or any(part in ("..", ".", "") for part in parts):
|
|
raise ValueError(f"Path traversal forbidden in zip: {member_name!r}")
|
|
return PurePosixPath(*parts)
|
|
|
|
|
|
class DeployService:
|
|
"""Atomic, validated hot-reload deployment of Microduck manifest + ONNX policies."""
|
|
|
|
def __init__(
|
|
self,
|
|
base_dir: str | Path,
|
|
controller: MicroduckController,
|
|
policy_backend: str = "auto",
|
|
) -> None:
|
|
self.base_dir = Path(base_dir)
|
|
self.controller = controller
|
|
self.policy_backend = policy_backend
|
|
self._lock = threading.Lock()
|
|
self.last_deploy: dict[str, Any] | None = self._read_last_deploy()
|
|
|
|
def _read_last_deploy(self) -> dict[str, Any] | None:
|
|
meta_path = self.base_dir / "config" / ".last_deploy.json"
|
|
if not meta_path.is_file():
|
|
return None
|
|
try:
|
|
with meta_path.open("r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
return None
|
|
|
|
def _extract_bundle_to_stage(self, raw_bytes: bytes, stage_dir: Path) -> None:
|
|
if not raw_bytes:
|
|
raise ValueError("Empty deployment bundle")
|
|
try:
|
|
zf = zipfile.ZipFile(io.BytesIO(raw_bytes), "r")
|
|
except zipfile.BadZipFile as exc:
|
|
raise ValueError("Deployment payload is not a valid ZIP archive") from exc
|
|
|
|
with zf:
|
|
infos = zf.infolist()
|
|
if len(infos) > MAX_ZIP_FILES:
|
|
raise ValueError(f"Too many files in deployment zip ({len(infos)} > {MAX_ZIP_FILES})")
|
|
|
|
total_size = 0
|
|
entries: list[tuple[zipfile.ZipInfo, PurePosixPath]] = []
|
|
for info in infos:
|
|
if info.is_dir():
|
|
if info.filename.rstrip("/"):
|
|
_safe_rel_path(info.filename.rstrip("/"))
|
|
continue
|
|
mode = (info.external_attr >> 16) & 0o170000
|
|
if mode == stat.S_IFLNK:
|
|
raise ValueError(f"Symlinks are forbidden in deployment zip: {info.filename!r}")
|
|
total_size += max(0, int(info.file_size))
|
|
if total_size > MAX_ZIP_UNCOMPRESSED_BYTES:
|
|
raise ValueError("Deployment zip exceeds maximum uncompressed size (128 MB)")
|
|
entries.append((info, _safe_rel_path(info.filename)))
|
|
|
|
if not entries:
|
|
raise ValueError("Deployment zip contains no files")
|
|
|
|
top_dirs = {p.parts[0] for _, p in entries if len(p.parts) > 1}
|
|
has_root_manifest = any(
|
|
str(p) in ("config/robot_manifest.json", "robot_manifest.json") for _, p in entries
|
|
)
|
|
strip_prefix: str | None = None
|
|
if not has_root_manifest and len(top_dirs) == 1:
|
|
only_top = next(iter(top_dirs))
|
|
if any(
|
|
str(p) in (f"{only_top}/config/robot_manifest.json", f"{only_top}/robot_manifest.json")
|
|
for _, p in entries
|
|
):
|
|
strip_prefix = only_top
|
|
|
|
for info, rel in entries:
|
|
if strip_prefix and rel.parts[0] == strip_prefix:
|
|
if len(rel.parts) == 1:
|
|
continue
|
|
rel = PurePosixPath(*rel.parts[1:])
|
|
|
|
dest = (stage_dir / Path(*rel.parts)).resolve()
|
|
if not dest.is_relative_to(stage_dir.resolve()):
|
|
raise ValueError(f"Escaping path rejected: {info.filename!r}")
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
with zf.open(info, "r") as src, dest.open("wb") as dst:
|
|
shutil.copyfileobj(src, dst)
|
|
|
|
def apply_zip_bytes(self, raw_bytes: bytes, overwrite_profile: bool = False) -> dict[str, Any]:
|
|
with self._lock, tempfile.TemporaryDirectory(prefix="microduck_deploy_") as tmp:
|
|
stage = Path(tmp) / "stage"
|
|
backup = Path(tmp) / "backup"
|
|
stage.mkdir(parents=True, exist_ok=True)
|
|
backup.mkdir(parents=True, exist_ok=True)
|
|
|
|
self._extract_bundle_to_stage(raw_bytes, stage)
|
|
|
|
manifest_src = stage / "config" / "robot_manifest.json"
|
|
if not manifest_src.is_file():
|
|
alt = stage / "robot_manifest.json"
|
|
if alt.is_file():
|
|
manifest_src = alt
|
|
else:
|
|
raise ValueError("Deployment zip missing config/robot_manifest.json")
|
|
|
|
try:
|
|
manifest = validate_robot_manifest(json.loads(manifest_src.read_text(encoding="utf-8")))
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"Invalid robot_manifest.json: {exc}") from exc
|
|
|
|
# Normalize into stage/config & verify every referenced ONNX exists & is non-empty
|
|
(stage / "config").mkdir(parents=True, exist_ok=True)
|
|
(stage / "config" / "robot_manifest.json").write_text(
|
|
json.dumps(manifest, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
stage_policies: dict[str, Path] = {}
|
|
for key, spec in manifest["skills"].items():
|
|
rel = spec.get("path")
|
|
if not rel:
|
|
continue
|
|
rel_posix = _safe_rel_path(str(rel))
|
|
pol_file = (stage / Path(*rel_posix.parts)).resolve()
|
|
if not pol_file.is_relative_to(stage.resolve()) or not pol_file.is_file():
|
|
raise ValueError(f"Deployment zip missing policy '{key}' at '{rel}'")
|
|
if pol_file.stat().st_size == 0:
|
|
raise ValueError(f"Policy '{key}' at '{rel}' is empty")
|
|
stage_policies[key] = pol_file
|
|
|
|
# Optional custom move script
|
|
custom_script: dict[str, Any] | None = None
|
|
script_src = stage / "config" / "custom_move_script.json"
|
|
if not script_src.is_file():
|
|
alt_script = stage / "custom_move_script.json"
|
|
if alt_script.is_file():
|
|
script_src = alt_script
|
|
if script_src.is_file():
|
|
try:
|
|
custom_script = json.loads(script_src.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"Invalid custom_move_script.json: {exc}") from exc
|
|
if not isinstance(custom_script, dict) or not isinstance(custom_script.get("keyframes"), list):
|
|
raise ValueError("custom_move_script.json must contain a 'keyframes' list")
|
|
elif isinstance(manifest.get("custom_move"), dict) and manifest["custom_move"].get("slot") == "script":
|
|
raise ValueError("Manifest references custom_move script, but config/custom_move_script.json is missing")
|
|
|
|
# Optional hardware profile
|
|
target_profile_path = self.base_dir / "config" / "hardware_profile.json"
|
|
incoming_profile: dict[str, Any] | None = None
|
|
profile_src = stage / "config" / "hardware_profile.json"
|
|
if not profile_src.is_file():
|
|
alt_prof = stage / "hardware_profile.json"
|
|
if alt_prof.is_file():
|
|
profile_src = alt_prof
|
|
if (overwrite_profile or not target_profile_path.is_file()) and profile_src.is_file():
|
|
try:
|
|
incoming_profile = validate_hardware_profile(
|
|
json.loads(profile_src.read_text(encoding="utf-8")),
|
|
joint_names=manifest["joint_names"],
|
|
)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"Invalid hardware_profile.json: {exc}") from exc
|
|
|
|
# Pre-validate ONNX graph & metadata inside stage before touching live files
|
|
PolicySet.load_from_manifest(stage, manifest, backend=self.policy_backend)
|
|
|
|
# Backup existing config + policies for atomic rollback
|
|
live_config = self.base_dir / "config"
|
|
live_policies = self.base_dir / "policies"
|
|
live_config.mkdir(parents=True, exist_ok=True)
|
|
live_policies.mkdir(parents=True, exist_ok=True)
|
|
for sub in ("config", "policies"):
|
|
src_dir = self.base_dir / sub
|
|
if src_dir.is_dir():
|
|
shutil.copytree(src_dir, backup / sub, dirs_exist_ok=True)
|
|
|
|
try:
|
|
shutil.copy2(stage / "config" / "robot_manifest.json", live_config / "robot_manifest.json")
|
|
|
|
target_script = live_config / "custom_move_script.json"
|
|
if custom_script is not None:
|
|
target_script.write_text(json.dumps(custom_script, indent=2) + "\n", encoding="utf-8")
|
|
elif target_script.is_file():
|
|
target_script.unlink()
|
|
|
|
for _, src_pol in stage_policies.items():
|
|
rel_to_stage = src_pol.relative_to(stage)
|
|
dst_pol = self.base_dir / rel_to_stage
|
|
dst_pol.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src_pol, dst_pol)
|
|
|
|
if incoming_profile is not None:
|
|
save_hardware_profile(target_profile_path, incoming_profile, joint_names=manifest["joint_names"])
|
|
|
|
live_policies_set = PolicySet.load_from_manifest(
|
|
self.base_dir, manifest, backend=self.policy_backend
|
|
)
|
|
self.controller.reload_runtime(
|
|
manifest=manifest,
|
|
policies=live_policies_set,
|
|
profile=incoming_profile,
|
|
custom_script=custom_script,
|
|
)
|
|
except Exception:
|
|
# Roll back live config + policies on reload failure
|
|
for sub in ("config", "policies"):
|
|
bak_dir = backup / sub
|
|
live_dir = self.base_dir / sub
|
|
if bak_dir.is_dir():
|
|
shutil.copytree(bak_dir, live_dir, dirs_exist_ok=True)
|
|
raise
|
|
|
|
deploy_meta = {
|
|
"ok": True,
|
|
"deployed_at_s": round(time.time(), 3),
|
|
"robot_id": manifest.get("robot_id", "microduck"),
|
|
"loaded_skills": sorted(live_policies_set.policies.keys()),
|
|
"profile_overwritten": incoming_profile is not None,
|
|
"custom_move": manifest.get("custom_move"),
|
|
}
|
|
(live_config / ".last_deploy.json").write_text(
|
|
json.dumps(deploy_meta, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
self.last_deploy = deploy_meta
|
|
return deploy_meta
|