Files
chenlin ceca51b0d8
Checks / Layout, translation and Colab checks (push) Has been cancelled
Checks / Ruff (push) Has been cancelled
Checks / Framework tests (native backend, CPU) (push) Has been cancelled
Checks / Portable Rust controller (push) Has been cancelled
Improve generic robot onboarding, walking controls and end-effector interaction
Default humanoid intake to command-following gait and support bounded Cartesian target dragging for arms. Carry model and actuator semantics through calibration, native verification and packaging. Document reusable intake decisions and cover them with regression tests.
2026-11-06 10:54:58 +08:00

510 lines
27 KiB
Python

"""Versioned, integrity-checked applications containing models and controllers."""
from __future__ import annotations
from contextlib import contextmanager
import hashlib
from importlib.resources import files
import json
from pathlib import Path, PurePosixPath
import re
import shutil
import stat
import tempfile
import zipfile
SCHEMA = "mjrl-robot-app/1"
MODES = {"rl_onnx", "ik_mocap", "wheeled_velocity", "cpg_keyframe", "model_viewer"}
KIND_MODE = {"ik": "ik_mocap", "joint_sliders": "ik_mocap", "trajectory": "cpg_keyframe",
"cpg": "cpg_keyframe", "differential": "wheeled_velocity", "omni": "wheeled_velocity",
"direct": "model_viewer", "rl_onnx": "rl_onnx"}
IK_MOUSE_HELP = (
"Double-click the end-effector target marker to select it. Hold Ctrl and drag "
"with the right mouse button to move it. Hold Shift as well to change the "
"translation plane. Release the mouse to hold the target."
)
def _trusted_schema_bytes() -> bytes:
# The application also ships a published schema for readers. Validation
# always uses the copy shipped with this installed or portable runtime.
return files(__package__).joinpath("app.schema").read_bytes()
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path: Path, obj) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(obj, indent=2, allow_nan=False, ensure_ascii=False) + "\n", "utf-8")
def safe_path(root: Path, name: str) -> Path:
part = PurePosixPath(name)
reserved = re.compile(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\.|$)", re.IGNORECASE)
if (not name or part.is_absolute() or ".." in part.parts or "\\" in name or ":" in name
or any(ord(char) < 32 or char in '<>"|?*' for char in name)
or any(piece.endswith((".", " ")) or reserved.match(piece) for piece in part.parts)):
raise ValueError(f"unsafe bundle path: {name!r}")
path = root.joinpath(*part.parts)
if not path.resolve().is_relative_to(root.resolve()):
raise ValueError(f"bundle dependency escapes its root: {name!r}")
return path
def _archive(root: Path, output: Path) -> None:
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for path in sorted(root.rglob("*")):
if path.is_file():
entry = zipfile.ZipInfo(path.relative_to(root).as_posix(), (1980, 1, 1, 0, 0, 0))
entry.compress_type = zipfile.ZIP_DEFLATED
entry.external_attr = 0o100644 << 16
archive.writestr(entry, path.read_bytes())
def _copy_package(source, target: Path) -> None:
target.mkdir()
for item in source.iterdir():
if item.name == "__pycache__" or item.name.endswith(".pyc"):
continue
destination = target / item.name
if item.is_dir():
_copy_package(item, destination)
elif item.is_file():
destination.write_bytes(item.read_bytes())
def _runtime(output: Path) -> None:
"""Ship the exact Python implementation, runnable without this checkout.
The host provides Python and declared third-party dependencies. No executable
code is imported from an app when opened by the workbench: the host uses its
installed implementation. Running the bundled pyz is an explicit operation.
"""
with tempfile.TemporaryDirectory(prefix="mjrl-runtime-") as temp:
root = Path(temp)
(root / "mjrl").mkdir()
(root / "mjrl/__init__.py").write_text('"""Robot application runtime."""\n')
for name in ("platform", "controllers"):
_copy_package(files(f"mjrl.{name}"), root / "mjrl" / name)
_copy_package(files("controller"), root / "controller")
(root / "mjrl/viewer").mkdir()
(root / "mjrl/viewer/__init__.py").write_text('"""Native viewer input."""\n')
viewer = files("mjrl.viewer")
for name in ("keys.py", "encoding.py"):
(root / "mjrl/viewer" / name).write_bytes(viewer.joinpath(name).read_bytes())
(root / "__main__.py").write_text("from mjrl.platform.play import main\nmain()\n")
_archive(root, output)
def _manual(root: Path, robot_id: str, modes: dict, default: str) -> dict:
from controller.controls import load_controls
def key_label(stroke):
return " + ".join(part.removeprefix("key_").replace("keypad_", "Num ").upper()
for part in stroke.split("+"))
entries = []
for name, mode in modes.items():
item = {"mode": name, "task": None, "default": name == default,
"summary": f"Native MuJoCo {mode['control_mode']} mode.",
"enter": f"Select --app-mode {name} before replay.",
"leave": "Close the viewer or stop replay.",
"keyboard": "This mode has no keyboard bindings.",
"controls": [], "keys": []}
contract = json.loads((root / mode["contract"]).read_text())
if contract.get("kind") == "ik":
item["summary"] = IK_MOUSE_HELP
if mode.get("controls"):
controls = load_controls(root / mode["controls"])
if mode["control_mode"] == "rl_onnx":
layout = json.loads((root / mode["layout"]).read_text())
ranges = layout.get("command_ranges", {}).get("twist", {})
else:
ranges = contract.get("command_ranges", {})
fixed = {axis: bounds[0] for axis, bounds in ranges.items()
if bounds[0] == bounds[1]}
item["keyboard"] = f"Hold a bound key to reach full command in {controls.full_after_s:g} seconds."
axes = {axis.name: axis for command in controls.commands for axis in command.axes}
for axis, bindings in controls.bindings.items():
for binding in bindings:
item["controls"].append({"control": binding.source, "pad": binding.source,
"does": (f"No effect: {axis} is fixed at {fixed[axis]:g}." if axis in fixed
else f"Drive {axis}; input sign {binding.sign:+g}; positive means {axes[axis].positive}.")})
for axis, binding in controls.key_axes.items():
for sign, strokes in (("Increase", binding.plus), ("Decrease", binding.minus)):
for stroke in strokes:
item["keys"].append({"key": key_label(stroke),
"does": (f"No effect: {axis} is fixed at {fixed[axis]:g}." if axis in fixed
else f"{sign} {axis} ({axes[axis].unit}).")})
item["controls"].append({"control": controls.release_button, "pad": controls.release_button,
"does": "Release operator commands."})
item["keys"].extend({"key": key_label(stroke), "does": "Release operator commands."}
for stroke in controls.key_release)
if controls.shift:
item["controls"].append({"control": controls.shift, "pad": controls.shift,
"does": f"{controls.shift_gesture} the alternate control layer."})
if controls.reset:
item["controls"].append({"control": controls.reset, "pad": controls.reset,
"does": "Tap to reset integrated command axes."})
entries.append(item)
return {"schema": "kk-bundle-manual/1", "language": "en", "bundle": robot_id,
"source": "Generated from the application mode manifest.", "modes": entries,
"switches": []}
def _copy_mode(root: Path, name: str, entry: dict) -> dict:
if not re.fullmatch(r"[a-z][a-z0-9_]*", name):
raise ValueError(f"invalid mode name: {name}")
config_path = Path(entry["controller"]).resolve() if entry.get("controller") else None
config = json.loads(config_path.read_text()) if config_path else {"kind": "direct", "actuators": []}
kind = entry.get("control_mode", KIND_MODE.get(config.get("kind")))
if kind not in MODES:
raise ValueError(f"unsupported control mode: {kind}")
contract = f"{name}.json"
if config.get("trajectory") and isinstance(config["trajectory"], str):
source = config_path.parent / config["trajectory"]
target = f"{name}.trajectory.json"
shutil.copyfile(source, root / target)
config["trajectory"] = target
result = {"control_mode": kind, "contract": contract}
if entry.get("controls"):
from controller.controls import load_controls
controls = Path(entry["controls"])
load_controls(controls)
target = f"controls/{name}.yaml"
(root / "controls").mkdir(exist_ok=True)
shutil.copyfile(controls, root / target)
result["controls"] = target
if kind == "rl_onnx":
if not entry.get("policy") or not entry.get("layout"):
raise ValueError("rl_onnx requires policy and layout")
(root / "models").mkdir(exist_ok=True)
shutil.copyfile(entry["policy"], root / f"models/{name}.onnx")
shutil.copyfile(entry["layout"], root / f"{name}.layout.json")
config.update(kind="rl_onnx", policy=f"models/{name}.onnx", layout=f"{name}.layout.json")
result["policy"] = config["policy"]
result["layout"] = config["layout"]
write_json(root / contract, config)
return result
def _check_evidence(report: dict, model: Path, entry: dict) -> None:
import mujoco
from mjrl.onboarding._physics import root_body
from mjrl.onboarding.verify import (
compiled_model_sha256, controller_dependencies, controller_sha256,
)
from .endpoint import ENDPOINT_TOLERANCES
from .mobility import missing_mobility_axes, mobility_error
from .velocity import VELOCITY_METRIC_NAMES
if report.get("schema") != "mjrl.onboarding.verify/1":
raise ValueError("controlled app requires a native onboarding verification report")
if report.get("model_sha256") != digest(model):
raise ValueError("verification does not match this exact model")
compiled = mujoco.MjModel.from_xml_path(str(model))
if report.get("compiled_model_sha256") != compiled_model_sha256(compiled):
raise ValueError("verification model dependencies have changed")
if report.get("passed") is not True or not all(report.get("stages", {}).get(stage, {}).get("passed") is True
for stage in ("physics", "control")):
raise ValueError("both physics and control verification must pass")
stages = report["stages"]
for stage in stages.values():
if not stage.get("checks") or any(item.get("passed") is not True for item in stage["checks"]):
raise ValueError("verification contains missing or failed checks")
initial = {item["name"]: item for item in stages["physics"]["checks"] if "probe" not in item}
required = {"native_compilation", "contact_dynamics_enabled", "dynamic_robot",
"finite_initial_state", "initial_joint_ranges", "initial_self_penetration"}
if not required <= initial.keys() or "minimum_margin_fraction" not in initial["initial_joint_ranges"]:
raise ValueError("verification is missing current initial-physics gates; rerun verify")
if not report.get("probes"):
raise ValueError("verification contains no measured control probes")
root_name = report["frames"]["base"]["name"]
root_id = root_body(compiled, root_name or None)
for probe in report["probes"]:
for stage, names in (("physics", {"finite_state", "native_steps", "joint_ranges", "root_motion"}),
("control", {"finite_control", "effort_saturation", "control_range",
"nonzero_generalized_actuation", "response_differs_with_actuators_disabled"})):
present = {item["name"] for item in stages[stage]["checks"] if item.get("probe") == probe["index"]}
if not names <= present:
raise ValueError("verification is missing current per-probe gates; rerun verify")
response = [
check for check in stages["control"]["checks"]
if check.get("probe") == probe["index"]
and check["name"] in ENDPOINT_TOLERANCES.keys() | VELOCITY_METRIC_NAMES
]
metrics = {check["name"]: check for check in response}
if not metrics or len(metrics) != len(response) or any(
not {"expected", "observed", "tolerance"} <= check.keys() for check in response
):
raise ValueError("verification requires explicit physical response expectations per probe")
for field, allowed in (
("endpoint_response", ENDPOINT_TOLERANCES),
("velocity_evaluation", VELOCITY_METRIC_NAMES),
):
evidence = probe.get(field, {})
recorded = {
name: {"name": name, "probe": probe["index"], **check}
for name, check in evidence.get("metrics", {}).items()
}
scoped = {name: check for name, check in metrics.items() if name in allowed}
if scoped != recorded or (
evidence
and (evidence.get("root_body"), evidence.get("root_body_id")) != (root_name, root_id)
):
raise ValueError(f"verification {field} evidence differs between probe and stage; rerun verify")
if not entry.get("controller"):
raise ValueError("controlled app requires the verified controller JSON")
path = Path(entry["controller"]).resolve()
config = json.loads(path.read_text())
if report.get("controller_sha256") != controller_sha256(config):
raise ValueError("verification controller configuration has changed")
if report.get("controller_dependencies_sha256") != controller_dependencies(config, path.parent):
raise ValueError("verification controller dependencies have changed")
expected_controls = digest(Path(entry["controls"])) if entry.get("controls") else None
if report.get("controls_sha256") != expected_controls:
raise ValueError("verification operator controls have changed")
layout = None
if config.get("kind") == "rl_onnx":
for key, field in (("policy", "policy"), ("layout", "layout")):
if digest(Path(entry[field])) != digest(path.parent / config[key]):
raise ValueError(f"packaged {field} differs from the verified controller")
layout = json.loads(Path(entry["layout"]).read_text("utf-8"))
if missing := missing_mobility_axes(compiled, config, report, layout):
raise ValueError(mobility_error(config["kind"], missing))
def build_bundle(model: Path, output: Path, *, robot_id: str, modes: dict[str, dict],
default_mode: str | None = None, verification: Path | None = None,
licenses: tuple[Path, ...] = ()) -> Path:
"""Serialize publication to one destination; never overwrite another build."""
destination = Path(output).resolve()
if destination.suffix == ".app":
destination = destination.with_suffix("")
destination.parent.mkdir(parents=True, exist_ok=True)
lock = destination.parent / f".{destination.name}.build-lock"
try:
lock.mkdir()
except FileExistsError as error:
raise FileExistsError(f"application build already in progress: {destination}") from error
try:
return _build_bundle(model, destination, robot_id=robot_id, modes=modes,
default_mode=default_mode, verification=verification, licenses=licenses)
finally:
lock.rmdir()
def _build_bundle(model: Path, output: Path, *, robot_id: str, modes: dict[str, dict],
default_mode: str | None = None, verification: Path | None = None,
licenses: tuple[Path, ...] = ()) -> Path:
"""Build atomically from explicit model, modes and matching verification.
Existing output is refused. Physical/control acceptance must come from the
exact source model, not a similarly named asset. Viewer packages claim only
successful model compilation. The generated runtime is tested by the caller.
"""
from .model import copy_model
if not re.fullmatch(r"[a-z][a-z0-9_]*", robot_id):
raise ValueError("robot_id must be a lowercase identifier")
if not modes:
raise ValueError("at least one mode is required")
default_mode = default_mode or next(iter(modes))
if default_mode not in modes:
raise ValueError("default_mode does not name a mode")
model, output = Path(model).resolve(), Path(output).resolve()
if output.suffix == ".app":
output = output.with_suffix("")
if output.exists() or output.with_suffix(".app").exists():
raise FileExistsError(f"refusing to replace application: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".mjrl-build-", dir=output.parent) as temp:
root = Path(temp) / robot_id
root.mkdir()
copy_model(model, root / f"model/{robot_id}.xml")
mode_records = {name: _copy_mode(root, name, entry) for name, entry in modes.items()}
reports = {}
for name, record in mode_records.items():
if record["control_mode"] == "model_viewer":
continue
report_path = modes[name].get("verification") or verification
if not report_path:
raise ValueError("controlled app requires verification for each mode")
report = json.loads(Path(report_path).read_text())
_check_evidence(report, model, modes[name])
reports[name] = report
record["verification"] = f"verification/{name}.json"
write_json(root / record["verification"], report)
if reports:
write_json(root / "verification.json", {"schema": "mjrl-app-verification/1", "passed": True,
"modes": {name: mode_records[name]["verification"] for name in reports}})
frames = {name: report.get("frames", {}) for name, report in reports.items()}
write_json(root / "reference.json", {"schema": "mjrl-app-reference/1",
"source_model_sha256": digest(model), "frames": frames,
"modes": {name: report["probes"] for name, report in reports.items()}})
(root / "controller.toml").write_text(f'schema = "mjrl-app-controller/1"\ndefault_mode = "{default_mode}"\n')
write_json(root / "manual.en.json", _manual(root, robot_id, mode_records, default_mode))
(root / "runtime").mkdir()
_runtime(root / "runtime/controller.pyz")
requirements = "mujoco~=3.11.0\nnumpy\nPyYAML\njsonschema\nonnx\nonnxruntime>=1.17\nimageio-ffmpeg\n"
python_requirement = ">=3.10,<3.14"
for record in mode_records.values():
if record.get("layout"):
layout = json.loads((root / record["layout"]).read_text())
if "motor_model" in layout:
from .motors import BAM_PYTHON_REQUIREMENT, BAM_REQUIREMENT, validate_motor_model
validate_motor_model(layout["motor_model"])
requirements += BAM_REQUIREMENT + "\n"
python_requirement = BAM_PYTHON_REQUIREMENT
break
(root / "runtime/requirements.txt").write_text(requirements)
(root / "app.schema").write_bytes(_trusted_schema_bytes())
(root / "LICENSE").write_bytes(files(__package__).joinpath("LICENSE").read_bytes())
license_files = ["LICENSE"]
for index, source in enumerate(licenses):
source = Path(source)
relative = f"licenses/{index:02d}-{source.name}"
target = safe_path(root, relative)
target.parent.mkdir(exist_ok=True)
shutil.copyfile(source, target)
license_files.append(relative)
(root / "README.md").write_text(
f"# {robot_id}\n\nInstall runtime/requirements.txt in a separate Python environment.\n"
"Run `python runtime/controller.pyz --app . --headless --steps 100`.\n"
"The model, meshes, controllers and policies are contained in this directory.\n"
"Python and third-party runtime dependencies are provided by the host.\n"
"Only the declared MuJoCo runtime is supported; this app is not a board binary.\n")
manifest = {"schema": SCHEMA, "robot_id": robot_id, "model": f"model/{robot_id}.xml",
"default_mode": default_mode, "modes": mode_records,
"controller": "controller.toml", "manuals": {"en": "manual.en.json"},
"reference": "reference.json", "verification": "verification.json" if reports else None,
"runtimes": {"native": {"file": "runtime/controller.pyz", "language": "python",
"python": python_requirement, "dependencies": "runtime/requirements.txt",
"platforms": ["linux", "windows", "macos"]}},
"capabilities": sorted({x["control_mode"] for x in mode_records.values()}),
"source": {"model_sha256": digest(model)}, "files": {}}
manifest["licenses"] = license_files
manifest["files"] = {p.relative_to(root).as_posix(): {"sha256": digest(p), "bytes": p.stat().st_size}
for p in sorted(root.rglob("*")) if p.is_file()}
write_json(root / "bundle.json", manifest)
validate_bundle(root, compile_model=True)
from .play import RobotApplication
for name in mode_records:
RobotApplication(root, manifest, name)
if reports:
from .reference import check_reference
measured = check_reference(root)
if measured["passed"] is not True:
raise ValueError(f"packaged controller did not reproduce verification: {measured}")
archive = Path(temp) / f"{robot_id}.app"
_archive(root, archive)
shutil.move(str(root), output)
shutil.move(str(archive), output.with_suffix(".app"))
return output.with_suffix(".app")
def validate_bundle(root: Path, *, compile_model: bool = False) -> dict:
"""Validate schema, every dependency and byte hash before constructing a runtime."""
import jsonschema
root = Path(root)
manifest = json.loads((root / "bundle.json").read_text())
schema = json.loads(_trusted_schema_bytes())
jsonschema.Draft202012Validator({"$schema": schema["$schema"], "$defs": schema["$defs"],
"$ref": "#/$defs/robotApp"}).validate(manifest)
files = manifest["files"]
if len({name.casefold() for name in files}) != len(files):
raise ValueError("application paths collide on case-insensitive hosts")
actual = {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()}
if any(p.is_symlink() for p in root.rglob("*")):
raise ValueError("application directories may not contain symbolic links")
if actual != set(files) | {"bundle.json"}:
raise ValueError("bundle inventory does not match files on disk")
for name, record in files.items():
path = safe_path(root, name)
if path.is_symlink() or path.stat().st_size != record["bytes"] or digest(path) != record["sha256"]:
raise ValueError(f"bundle integrity mismatch: {name}")
references = [manifest[k] for k in ("model", "controller", "reference", "verification") if manifest[k]]
references.extend(manifest["manuals"].values())
references.extend(manifest.get("licenses", []))
references += [manifest["runtimes"]["native"][k] for k in ("file", "dependencies")]
for entry in manifest["modes"].values():
references.extend(entry[k] for k in ("contract", "controls", "policy", "layout", "verification") if k in entry)
for name in references:
if name not in files or not safe_path(root, name).is_file():
raise ValueError(f"unlisted bundle dependency: {name}")
manual_validator = jsonschema.Draft202012Validator({"$schema": schema["$schema"],
"$defs": schema["$defs"], "$ref": "#/$defs/manual"})
for language, name in manifest["manuals"].items():
manual = json.loads((root / name).read_text())
manual_validator.validate(manual)
if manual["language"] != language or manual["bundle"] != manifest["robot_id"]:
raise ValueError("manual identity differs from the application")
if {mode["mode"] for mode in manual["modes"]} != set(manifest["modes"]):
raise ValueError("manual does not cover the application modes")
if manifest["default_mode"] not in manifest["modes"]:
raise ValueError("default_mode does not exist")
if sorted(set(x["control_mode"] for x in manifest["modes"].values())) != manifest["capabilities"]:
raise ValueError("capabilities disagree with modes")
from .model import validate_contained_model
validate_contained_model(root, root / manifest["model"])
for entry in manifest["modes"].values():
config = json.loads((root / entry["contract"]).read_text())
if KIND_MODE.get(config.get("kind")) != entry["control_mode"]:
raise ValueError("declared mode differs from its controller kind")
if entry["control_mode"] == "model_viewer" and config.get("actuators") != []:
raise ValueError("model_viewer cannot drive actuators; use a verified control mode")
for field in ("policy", "layout", "trajectory"):
if isinstance(config.get(field), str):
name = config[field]
safe_path(root, name)
if name not in files:
raise ValueError(f"unlisted controller dependency: {name}")
if compile_model:
import mujoco
mujoco.MjModel.from_xml_path(str(root / manifest["model"]))
return manifest
@contextmanager
def open_bundle(path: Path):
"""Read an app with traversal, symlink, duplicate and expansion limits."""
path = Path(path)
if path.is_dir():
yield path, validate_bundle(path)
return
with tempfile.TemporaryDirectory(prefix="mjrl-app-") as temp:
root = Path(temp)
with zipfile.ZipFile(path) as archive:
entries = archive.infolist()
names = [entry.filename for entry in entries]
canonical = [PurePosixPath(name).as_posix().casefold() for name in names]
if len(names) != len(set(canonical)) or len(names) > 20000:
raise ValueError("duplicate or excessive archive entries")
if sum(entry.file_size for entry in entries) > 4 * 1024**3:
raise ValueError("application expands beyond 4 GiB")
for entry in entries:
target = safe_path(root, entry.filename)
if stat.S_ISLNK(entry.external_attr >> 16):
raise ValueError("application archives may not contain symbolic links")
if entry.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
with archive.open(entry) as source, target.open("wb") as dest:
shutil.copyfileobj(source, dest)
yield root, validate_bundle(root)
def is_platform_app(path: Path) -> bool:
"""Read only the version tag; opening the app performs full validation."""
path = Path(path)
if path.is_dir():
source = (path / "bundle.json").read_text()
else:
with zipfile.ZipFile(path) as archive:
source = archive.read("bundle.json")
return json.loads(source).get("schema") == SCHEMA