Files
VibemakerSandbox/hardware/unoq/python/core/deploy_service.py
T

253 lines
11 KiB
Python

from __future__ import annotations
import io
import json
import shutil
import stat
import tempfile
import threading
import time
import zipfile
from pathlib import Path, PurePosixPath
from typing import Any
from .config import save_hardware_profile, validate_hardware_profile, validate_robot_manifest
from .controller import MicroduckController
from .policy_runner import PolicySet
MAX_ZIP_FILES = 512
MAX_ZIP_UNCOMPRESSED_BYTES = 128 * 1024 * 1024
def _safe_rel_path(member_name: str) -> PurePosixPath:
if not member_name or "\\" in member_name:
raise ValueError(f"Unsafe zip member path: {member_name!r}")
if member_name.startswith("/") or ":" in member_name:
raise ValueError(f"Absolute or drive path forbidden in zip: {member_name!r}")
parts = PurePosixPath(member_name).parts
if not parts or any(part in ("..", ".", "") for part in parts):
raise ValueError(f"Path traversal forbidden in zip: {member_name!r}")
return PurePosixPath(*parts)
class DeployService:
"""Atomic, validated hot-reload deployment of Microduck manifest + ONNX policies."""
def __init__(
self,
base_dir: str | Path,
controller: MicroduckController,
policy_backend: str = "auto",
) -> None:
self.base_dir = Path(base_dir)
self.controller = controller
self.policy_backend = policy_backend
self._lock = threading.Lock()
self.last_deploy: dict[str, Any] | None = self._read_last_deploy()
def _read_last_deploy(self) -> dict[str, Any] | None:
meta_path = self.base_dir / "config" / ".last_deploy.json"
if not meta_path.is_file():
return None
try:
with meta_path.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return None
def _extract_bundle_to_stage(self, raw_bytes: bytes, stage_dir: Path) -> None:
if not raw_bytes:
raise ValueError("Empty deployment bundle")
try:
zf = zipfile.ZipFile(io.BytesIO(raw_bytes), "r")
except zipfile.BadZipFile as exc:
raise ValueError("Deployment payload is not a valid ZIP archive") from exc
with zf:
infos = zf.infolist()
if len(infos) > MAX_ZIP_FILES:
raise ValueError(f"Too many files in deployment zip ({len(infos)} > {MAX_ZIP_FILES})")
total_size = 0
entries: list[tuple[zipfile.ZipInfo, PurePosixPath]] = []
for info in infos:
if info.is_dir():
if info.filename.rstrip("/"):
_safe_rel_path(info.filename.rstrip("/"))
continue
mode = (info.external_attr >> 16) & 0o170000
if mode == stat.S_IFLNK:
raise ValueError(f"Symlinks are forbidden in deployment zip: {info.filename!r}")
total_size += max(0, int(info.file_size))
if total_size > MAX_ZIP_UNCOMPRESSED_BYTES:
raise ValueError("Deployment zip exceeds maximum uncompressed size (128 MB)")
entries.append((info, _safe_rel_path(info.filename)))
if not entries:
raise ValueError("Deployment zip contains no files")
top_dirs = {p.parts[0] for _, p in entries if len(p.parts) > 1}
has_root_manifest = any(
str(p) in ("config/robot_manifest.json", "robot_manifest.json") for _, p in entries
)
strip_prefix: str | None = None
if not has_root_manifest and len(top_dirs) == 1:
only_top = next(iter(top_dirs))
if any(
str(p) in (f"{only_top}/config/robot_manifest.json", f"{only_top}/robot_manifest.json")
for _, p in entries
):
strip_prefix = only_top
for info, rel in entries:
if strip_prefix and rel.parts[0] == strip_prefix:
if len(rel.parts) == 1:
continue
rel = PurePosixPath(*rel.parts[1:])
dest = (stage_dir / Path(*rel.parts)).resolve()
if not dest.is_relative_to(stage_dir.resolve()):
raise ValueError(f"Escaping path rejected: {info.filename!r}")
dest.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info, "r") as src, dest.open("wb") as dst:
shutil.copyfileobj(src, dst)
def apply_zip_bytes(self, raw_bytes: bytes, overwrite_profile: bool = False) -> dict[str, Any]:
with self._lock, tempfile.TemporaryDirectory(prefix="microduck_deploy_") as tmp:
stage = Path(tmp) / "stage"
backup = Path(tmp) / "backup"
stage.mkdir(parents=True, exist_ok=True)
backup.mkdir(parents=True, exist_ok=True)
self._extract_bundle_to_stage(raw_bytes, stage)
manifest_src = stage / "config" / "robot_manifest.json"
if not manifest_src.is_file():
alt = stage / "robot_manifest.json"
if alt.is_file():
manifest_src = alt
else:
raise ValueError("Deployment zip missing config/robot_manifest.json")
try:
manifest = validate_robot_manifest(json.loads(manifest_src.read_text(encoding="utf-8")))
except json.JSONDecodeError as exc:
raise ValueError(f"Invalid robot_manifest.json: {exc}") from exc
# Normalize into stage/config & verify every referenced ONNX exists & is non-empty
(stage / "config").mkdir(parents=True, exist_ok=True)
(stage / "config" / "robot_manifest.json").write_text(
json.dumps(manifest, indent=2) + "\n", encoding="utf-8"
)
stage_policies: dict[str, Path] = {}
for key, spec in manifest["skills"].items():
rel = spec.get("path")
if not rel:
continue
rel_posix = _safe_rel_path(str(rel))
pol_file = (stage / Path(*rel_posix.parts)).resolve()
if not pol_file.is_relative_to(stage.resolve()) or not pol_file.is_file():
raise ValueError(f"Deployment zip missing policy '{key}' at '{rel}'")
if pol_file.stat().st_size == 0:
raise ValueError(f"Policy '{key}' at '{rel}' is empty")
stage_policies[key] = pol_file
# Optional custom move script
custom_script: dict[str, Any] | None = None
script_src = stage / "config" / "custom_move_script.json"
if not script_src.is_file():
alt_script = stage / "custom_move_script.json"
if alt_script.is_file():
script_src = alt_script
if script_src.is_file():
try:
custom_script = json.loads(script_src.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError(f"Invalid custom_move_script.json: {exc}") from exc
if not isinstance(custom_script, dict) or not isinstance(custom_script.get("keyframes"), list):
raise ValueError("custom_move_script.json must contain a 'keyframes' list")
elif isinstance(manifest.get("custom_move"), dict) and manifest["custom_move"].get("slot") == "script":
raise ValueError("Manifest references custom_move script, but config/custom_move_script.json is missing")
# Optional hardware profile
target_profile_path = self.base_dir / "config" / "hardware_profile.json"
incoming_profile: dict[str, Any] | None = None
profile_src = stage / "config" / "hardware_profile.json"
if not profile_src.is_file():
alt_prof = stage / "hardware_profile.json"
if alt_prof.is_file():
profile_src = alt_prof
if (overwrite_profile or not target_profile_path.is_file()) and profile_src.is_file():
try:
incoming_profile = validate_hardware_profile(
json.loads(profile_src.read_text(encoding="utf-8")),
joint_names=manifest["joint_names"],
)
except json.JSONDecodeError as exc:
raise ValueError(f"Invalid hardware_profile.json: {exc}") from exc
# Pre-validate ONNX graph & metadata inside stage before touching live files
PolicySet.load_from_manifest(stage, manifest, backend=self.policy_backend)
# Backup existing config + policies for atomic rollback
live_config = self.base_dir / "config"
live_policies = self.base_dir / "policies"
live_config.mkdir(parents=True, exist_ok=True)
live_policies.mkdir(parents=True, exist_ok=True)
for sub in ("config", "policies"):
src_dir = self.base_dir / sub
if src_dir.is_dir():
shutil.copytree(src_dir, backup / sub, dirs_exist_ok=True)
try:
shutil.copy2(stage / "config" / "robot_manifest.json", live_config / "robot_manifest.json")
target_script = live_config / "custom_move_script.json"
if custom_script is not None:
target_script.write_text(json.dumps(custom_script, indent=2) + "\n", encoding="utf-8")
elif target_script.is_file():
target_script.unlink()
for _, src_pol in stage_policies.items():
rel_to_stage = src_pol.relative_to(stage)
dst_pol = self.base_dir / rel_to_stage
dst_pol.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src_pol, dst_pol)
if incoming_profile is not None:
save_hardware_profile(target_profile_path, incoming_profile, joint_names=manifest["joint_names"])
live_policies_set = PolicySet.load_from_manifest(
self.base_dir, manifest, backend=self.policy_backend
)
self.controller.reload_runtime(
manifest=manifest,
policies=live_policies_set,
profile=incoming_profile,
custom_script=custom_script,
)
except Exception:
# Roll back live config + policies on reload failure
for sub in ("config", "policies"):
bak_dir = backup / sub
live_dir = self.base_dir / sub
if bak_dir.is_dir():
shutil.copytree(bak_dir, live_dir, dirs_exist_ok=True)
raise
deploy_meta = {
"ok": True,
"deployed_at_s": round(time.time(), 3),
"robot_id": manifest.get("robot_id", "microduck"),
"loaded_skills": sorted(live_policies_set.policies.keys()),
"profile_overwritten": incoming_profile is not None,
"custom_move": manifest.get("custom_move"),
}
(live_config / ".last_deploy.json").write_text(
json.dumps(deploy_meta, indent=2) + "\n", encoding="utf-8"
)
self.last_deploy = deploy_meta
return deploy_meta