Files
UniversalTrain-ai/scripts/export.py
T
chenlin c32df70326
Checks / Layout, translation and Colab checks (push) Has been cancelled
Checks / Ruff (push) Has been cancelled
Checks / Framework tests (native backend, CPU) (push) Has been cancelled
Checks / Portable Rust controller (push) Has been cancelled
release: v0.1 generic robot training and application platform
2026-11-05 11:05:01 +08:00

1254 lines
57 KiB
Python
Executable File

#!/usr/bin/env python3
"""Export a trained policy and its contract from a live mjlab environment.
The export contains the ONNX actor, the observation and action layout, a
checkpoint copy, and a README. The native layout also carries a replay model
and controller configuration. The exporter reads joint selections, target
types, gains, timing, and command ranges from the environment and checkpoint.
python scripts/export.py --task my_robot --checkpoint model_1999.pt \
--native-layout --out out/native_policy
"""
from __future__ import annotations
import argparse
import json
import shutil
from dataclasses import asdict
from datetime import datetime
from pathlib import Path
import numpy as np
import torch
from mjrl.platform.layout import COMMAND_ORDER, JOINT_TERMS, NATIVE_LAYOUT_SCHEMA, STANDARD_TERMS
from _cli import (
build_parser,
parse_with_task_args,
print_task_table,
resolve_all,
)
# ──────────────────────────────────────────────────────────────────────
# The deployment vocabulary
# ──────────────────────────────────────────────────────────────────────
#
# The contract declares `_schema: isaac_layout v1`, and what defines that schema
# is the consumer: the on-robot controller builds each observation term **by
# name**, and a name it does not know is a hard failure at load. mjlab's names
# are not that vocabulary -- it calls the velocity command `command` and the
# actuator torque `actuator_force` -- so the export translates, and refuses to
# write a contract whose terms the deployment could not build.
#
# The table lives here rather than in the task configs because the names come
# from vendored mjlab (`rl/mjlab/tasks/velocity/velocity_env_cfg.py`); a task
# that renamed them would drift from upstream at every rebase, and the rename
# belongs to the export, not to training.
#: mjlab's term name -> what the deployment calls the same quantity.
_TERM_RENAME = {
"command": "velocity_commands",
"actuator_force": "joint_torque",
}
#: Measurements unavailable through the legacy controller's sensor interface.
_UNMEASURABLE_TERMS = {
"base_lin_vel": "the legacy controller has no base linear velocity estimate",
}
#: Observation names supported by the Rust builder, plus its layout aliases.
#: tests/test_layout.py checks this against the Rust source.
_DEPLOY_TERMS = frozenset({
# obs.rs SUPPORTED
"base_lin_vel", "base_ang_vel", "projected_gravity", "base_pose",
"commands", "commands_diff",
"joint_pos", "joint_vel", "joint_torque",
"actions", "gait_phase",
# Buildable, but only from a recording: see `_REFERENCE_TERMS`.
"jump_phase", "ref_future",
"clip_phase", "ref_joint_pos", "ref_joint_vel", "ref_tilt_error",
# layout.rs rewrites these three onto `commands` before the builder sees them
"velocity_commands", "jump_command", "jump_go",
"posture_command",
})
#: Observation terms that need a compatible recorded trajectory contract.
_REFERENCE_TERMS = frozenset({
"jump_phase", "ref_future",
"clip_phase", "ref_joint_pos", "ref_joint_vel", "ref_tilt_error",
})
# ──────────────────────────────────────────────────────────────────────
# Building the contract
# ──────────────────────────────────────────────────────────────────────
def _uniform_or_map(values, names):
"""Return a scalar when every value is equal, otherwise a {name: value} map.
The map form means the deployment side never has to trust array order.
"""
vals = [float(v) for v in values]
if len(set(round(v, 12) for v in vals)) == 1:
return vals[0]
return {n: v for n, v in zip(names, vals)}
def _actuator_scalars(env, action_joint_order):
"""Read gains and effort limits from the first environment's live actuators.
Software PD may compile to a simple motor. The compiled motor's scalar
gain then describes its torque input, while the actuator object carries
the actual position and velocity gains.
"""
robot = env.scene["robot"]
found: dict[str, tuple[float, float, float]] = {}
for act in robot.actuators:
relevant = set(act.target_names) & set(action_joint_order)
if not relevant:
continue
if not {"stiffness", "damping", "force_limit"} <= set(act.param_names):
raise SystemExit(f"[export] {sorted(relevant)} lack a PD actuator contract")
for i, joint in enumerate(act.target_names):
if joint not in relevant:
continue
if joint in found:
raise SystemExit(f"[export] multiple PD actuators target {joint!r}")
values = (
float(act.stiffness[0, i]),
float(act.damping[0, i]),
float(act.force_limit[0, i]),
)
if not np.isfinite(values).all() or values[0] < 0 or values[1] < 0 or values[2] <= 0:
raise SystemExit(f"[export] invalid PD gains or effort limit for {joint!r}")
found[joint] = values
missing = [j for j in action_joint_order if j not in found]
if missing:
raise SystemExit(f"[export] no actuator found for these controlled joints: {missing}")
kp, kd, eff = (
np.array([found[j][k] for j in action_joint_order]) for k in range(3)
)
return (
_uniform_or_map(kp, action_joint_order),
_uniform_or_map(kd, action_joint_order),
_uniform_or_map(eff, action_joint_order),
)
#: The trajectory file a reference-guided policy ships beside its ONNX.
#:
#: `trajectory`, not `reference`: a bundle already has a `reference.json` and it
#: is a different thing -- the recorded frames three hosts replay to check they
#: agree. Two meanings for one word is what `bundle` was doing a week ago.
TRAJECTORY_SUFFIX = ".trajectory.json"
def _reference(env_cfg, out: Path, wire_joint_order: list[str]) -> dict:
"""Write a legacy reference-guided policy's companion trajectory in wire order.
The recorded state table `q` is used for observations. A separate
`q_cmd` table, if present, supplies the residual-action baseline and may
have a different rate and length.
"""
cmd = getattr(env_cfg, "reference_contract", None)
if callable(cmd):
cmd = cmd()
if not isinstance(cmd, dict):
return {}
ref = cmd["reference"]
src = list(cmd["joint_order"])
missing = [j for j in wire_joint_order if j not in src]
if missing:
raise SystemExit(
f"[export] the reference trajectory is missing joints {missing}.\n"
f" Its own order is {src}\n"
f" A permuted trajectory is a robot tracking the wrong joints, "
f"and nothing downstream can see it."
)
perm = [src.index(j) for j in wire_joint_order]
# Round the sampled table to five decimal places in its authored units.
places = 5
def rows(table, columns=None) -> list[list[float]]:
out = []
for row in table:
picked = row if columns is None else [row[i] for i in columns]
out.append([round(float(v), places) for v in picked])
return out
tables = {"joint_order": wire_joint_order, "q": rows(ref["q"], perm)}
for name in ("q_cmd", "qd"):
if name in ref:
tables[name] = rows(ref[name], perm)
if "root_quat" in ref:
# Not per-joint: one orientation per frame, `(w, x, y, z)`.
tables["root_quat"] = rows(ref["root_quat"])
path = out / (cmd["name"] + TRAJECTORY_SUFFIX)
path.write_text(
json.dumps({
"_source": f"scripts/export.py, from {cmd['npz']}",
"_note": "Every per-joint table is in **wire order**, already permuted. "
"`q` is the state reached; `q_cmd`, where present, is what the "
"recorded controller commanded and is the residual baseline -- "
"a different rate and a different length. `qd` and `root_quat` "
"are per-frame companions of `q`.",
**tables,
}, separators=(",", ":")) + "\n",
"utf-8",
)
carried = ["residual_action", "rec_hz", "duration", "lookahead_s"]
optional = ["starts_on_entry", "control_hz", "t_go", "go_frame", "span",
"baseline", "baseline_lead_s", "go_event", "n_cmd"]
block = {k: cmd[k] for k in carried}
block.update({k: cmd[k] for k in optional if cmd.get(k) is not None})
block["file"] = path.name
block["n_frames"] = len(tables["q"])
if "q_cmd" in tables:
block["n_cmd"] = len(tables["q_cmd"])
shapes = " ".join(f"{k} {len(v)}x{len(v[0])}"
for k, v in tables.items() if k != "joint_order")
kind = "residual on " + str(block.get("baseline")) if block["residual_action"] else "played"
print(f"[export] reference {cmd['name']}: {shapes} ({kind}, "
f"{block['duration']:.2f} s at {block['rec_hz']:.0f} Hz, "
f"{path.stat().st_size / 1e6:.2f} MB)")
return {"reference": block}
def _controller(env_cfg):
"""How an operator's input becomes the command, when the task describes it.
`command_ranges` says how far an axis may be pushed; it does not say which
way is positive, nor what a stick at full deflection means. Both were being
decided a second time by every consumer, from the comments in the task's
teleop modules -- and a consumer that reads a sign wrong drives the robot
sideways while everything else looks correct.
Read off the env config rather than assembled here: this file is generic and
the bindings are the robot's. A task that does not describe its controller
simply exports no `controller` block, which is honest -- better than a
default that a reader would take for a measurement.
"""
contract = getattr(env_cfg, "controller_contract", None)
if callable(contract):
contract = contract()
return {"controller": contract} if isinstance(contract, dict) else {}
def _command_ranges(env, env_cfg):
"""Command ranges are part of the contract too.
A deployment whose joystick spans more than the training range is pushing the
policy out of distribution; one that spans less quietly hides capability.
Both have happened, so this reads from the live cfg.
"""
out = {}
for name in env.command_manager.active_terms:
cfg = env_cfg.commands.get(name) if hasattr(env_cfg.commands, "get") else None
if cfg is None:
cfg = getattr(env_cfg.commands, name, None)
rng = getattr(cfg, "ranges", None)
if rng is None:
continue
fields = {}
for f, v in vars(rng).items():
if f.startswith("_"):
continue
if isinstance(v, (tuple, list)) and len(v) == 2:
fields[f] = [float(v[0]), float(v[1])]
# Some command configs express a symmetric range as a half-width.
elif isinstance(v, (int, float)) and not isinstance(v, bool) and v > 0:
fields[f] = [-float(v), float(v)]
if fields:
out[name] = fields
return out
def _json_params(params) -> dict:
"""A term's params, restricted to what JSON can carry.
Params hold live objects -- a resolved `SceneEntityCfg`, a sensor handle --
beside the scalars that describe the term's behaviour. Only the scalars mean
anything to a reader on the other side, and the rest are not serialisable.
"""
return {
k: v
for k, v in (params or {}).items()
if isinstance(v, (bool, int, float, str))
}
def _obs_terms(env):
om = env.observation_manager
names = list(om.active_terms["actor"])
dims = [int(d[0]) for d in om.group_obs_term_dim["actor"]]
terms, off = [], 0
for name, dim in zip(names, dims):
cfg = om.get_term_cfg("actor", name)
deploy_name = _TERM_RENAME.get(name, name)
entry = {"name": deploy_name, "dim": dim, "offset": off}
if deploy_name != name:
# Kept so the contract can still be diffed against the env it came
# from; the deployment ignores it.
entry["source_name"] = name
params = _json_params(getattr(cfg, "params", None))
# The legacy gait clock reads a period in seconds.
if deploy_name == "gait_phase" and params.get("freq_hz"):
params["period"] = 1.0 / float(params["freq_hz"])
if params:
entry["params"] = params
scale = getattr(cfg, "scale", None)
if scale is not None:
entry["scale"] = (
[float(s) for s in scale]
if isinstance(scale, (list, tuple))
else float(scale.cpu().tolist())
if isinstance(scale, torch.Tensor)
else float(scale)
)
clip = getattr(cfg, "clip", None)
if clip is not None:
entry["clip"] = [float(c) for c in clip]
# History may be stacked by the manager or by the observation callable.
# Record both the frame count and stride before checking the wire width.
hist = int(getattr(cfg, "history_length", 0) or 0)
stride = 1
own = int(getattr(getattr(cfg, "func", None), "history_frames", 0) or 0)
if own:
if hist:
raise SystemExit(
f"[export] term {name} stacks {own} frames itself and the "
f"manager stacks {hist} more; the contract cannot describe "
f"the result"
)
hist = own
stride = int(getattr(cfg.func, "history_stride", 1) or 1)
if hist > 0:
entry["history_length"] = hist
entry["flatten_history_dim"] = bool(getattr(cfg, "flatten_history_dim", True))
entry["base_dim"] = dim // hist
# **Present only when it is not 1**, so a builder that has never seen
# it cannot quietly default: an unstrided contract looks exactly as it
# always did, and a strided one carries a field an old reader will
# fail on rather than ignore.
if stride != 1:
entry["history_stride"] = stride
terms.append(entry)
off += dim
return terms, off
def _observed_joint_orders(env, robot):
"""Read each per-joint term's resolved selection in the term's own order."""
om = env.observation_manager
orders = {}
for name in om.active_terms["actor"]:
deploy_name = _TERM_RENAME.get(name, name)
if deploy_name not in JOINT_TERMS:
continue
asset_cfg = (getattr(om.get_term_cfg("actor", name), "params", None) or {}).get(
"asset_cfg"
)
if asset_cfg is not None and asset_cfg.name != "robot":
raise SystemExit(f"[export] {name} observes an entity other than the robot")
ids = getattr(asset_cfg, "joint_ids", None)
if ids is None:
ids = slice(None)
order = robot.joint_names[ids] if isinstance(ids, slice) else [
robot.joint_names[int(i)] for i in ids
]
orders[deploy_name] = list(order)
return orders
def _action_parameters(env):
"""Read the action manager's real concatenation, transforms and target types."""
from mjlab.envs.mdp.actions.actions import JointPositionAction, JointVelocityAction
targets, scales = {}, {}
for name in env.action_manager.active_terms:
term = env.action_manager.get_term(name)
if type(term) is JointPositionAction:
mode = "position"
elif type(term) is JointVelocityAction:
mode = "velocity"
else:
raise SystemExit(f"[export] unsupported joint action term {name!r} ({type(term).__name__})")
if term.cfg.entity_name != "robot":
raise SystemExit(f"[export] action term {name!r} does not drive the robot")
joints = term.target_names
if len(joints) != term.action_dim:
raise SystemExit(f"[export] action term {name!r} has an inconsistent target count")
scale = term.scale
scale = scale[0].cpu().tolist() if isinstance(scale, torch.Tensor) else [scale] * len(joints)
offset = term.offset
offset = offset[0].cpu().tolist() if isinstance(offset, torch.Tensor) else [offset] * len(joints)
clip = getattr(term, "_clip", None)
clips = clip[0].cpu().tolist() if clip is not None else [(None, None)] * len(joints)
for joint, gain, rest, (lo, hi) in zip(joints, scale, offset, clips, strict=True):
if joint in targets:
raise SystemExit(f"[export] multiple action channels drive joint {joint!r}")
if not np.isfinite(gain) or not np.isfinite(rest):
raise SystemExit(f"[export] {joint!r} has a non-finite action transform")
targets[joint] = {"kind": mode, "offset": float(rest)}
scales[joint] = float(gain)
if lo is not None and (lo, hi) != (-np.inf, np.inf):
if not np.isfinite([lo, hi]).all() or lo >= hi:
raise SystemExit(f"[export] {joint!r} needs finite, ordered action clip limits")
targets[joint]["clip"] = [float(lo), float(hi)]
if not targets:
raise SystemExit("[export] policy has no joint action")
return targets, scales
def _checkpoint_ref(ckpt: Path) -> dict:
"""Where a checkpoint is, said in a way another machine can use.
Repository-relative when it is inside one, because an absolute path is a
fact about the machine that exported rather than about the policy. `run` is
the training directory -- `logs/<model>/<task>/<date-time>` -- which is the
part a person recognises; `iteration` is the file.
"""
from tasks.paths import REPO_ROOT
ckpt = ckpt.resolve()
where = ckpt.relative_to(REPO_ROOT) if ckpt.is_relative_to(REPO_ROOT) else ckpt
return {"path": str(where), "run": str(where.parent), "iteration": ckpt.stem}
def build_contract(
env, env_cfg, agent_cfg, checkpoint: Path | None = None, *, native: bool = False,
) -> dict:
import mujoco
robot = env.scene["robot"]
targets, scales = _action_parameters(env)
action_joint_order = list(targets)
wire_joint_order = list(robot.joint_names)
orders = _observed_joint_orders(env, robot)
distinct_orders = {tuple(order) for order in orders.values()}
if len(distinct_orders) <= 1:
obs_joint_order = list(next(iter(distinct_orders), wire_joint_order))
else:
observed = set().union(*distinct_orders)
obs_joint_order = [name for name in wire_joint_order if name in observed]
default_jp = robot.data.default_joint_pos[0].detach().cpu().tolist()
default_joint_pos = dict(zip(wire_joint_order, default_jp, strict=True))
default_joint_vel = dict(zip(
wire_joint_order, robot.data.default_joint_vel[0].detach().cpu().tolist(), strict=True,
))
mj = env.sim.mj_model
ids = [int(i) for i in robot.indexing.joint_ids.cpu().tolist()]
joint_limits = {
name: mj.jnt_range[jid].tolist()
for name, jid in zip(wire_joint_order, ids, strict=True)
if mj.jnt_limited[jid]
}
joint_transmission = mj.actuator_trntype == mujoco.mjtTrn.mjTRN_JOINT
driven_ids = set(mj.actuator_trnid[joint_transmission, 0])
driven = [name for name, jid in zip(wire_joint_order, ids, strict=True) if jid in driven_ids]
passive = [name for name in wire_joint_order if name not in driven]
if set(targets) - set(driven):
raise SystemExit("[export] an action target has no joint actuator")
unactuated = {n: default_joint_pos[n] for n in driven if n not in targets}
kp, kd, eff = _actuator_scalars(env, action_joint_order)
obs_terms, obs_dim = _obs_terms(env)
action_scale = _uniform_or_map(scales.values(), action_joint_order)
wrapper_clip = getattr(agent_cfg, "clip_actions", None)
native = (
native or len(distinct_orders) > 1 or any(default_joint_vel.values()) or bool(passive)
or any(isinstance(x, dict) for x in (kp, kd, eff, action_scale))
or wrapper_clip is not None
or any(
info["kind"] != "position" or info["offset"] != default_joint_pos[name]
or "clip" in info for name, info in targets.items()
)
or any("scale" in term or "clip" in term for term in obs_terms)
)
action = {"dim": int(env.action_manager.total_action_dim)}
native_fields = {}
if native:
if len(driven) != mj.nu:
raise SystemExit("[export] native replay requires one actuator per driven joint")
kp, kd, eff = _actuator_scalars(env, driven)
action["targets"] = targets
if wrapper_clip is not None:
if not np.isfinite(wrapper_clip) or wrapper_clip <= 0:
raise SystemExit("[export] agent action clip must be finite and positive")
action["input_clip"] = float(wrapper_clip)
# Older layouts take only a scalar. The required map makes a reader
# that ignores _schema still reject this native action contract.
action_scale = scales
for entry in obs_terms:
name = entry["name"]
if name in orders:
entry["joint_order"] = orders[name]
if name == "velocity_commands":
entry["name"] = "commands"
native_fields = {
"default_joint_vel": default_joint_vel,
"passive_joints": passive,
"model_joint_names": {
name: mj.joint(jid).name
for name, jid in zip(wire_joint_order, ids, strict=True)
},
}
init_pos = env_cfg.scene.entities["robot"].init_state.pos
actor_cfg = agent_cfg.actor
return {
"_source": "scripts/export.py -- re-export whenever the env / robot / agent config changes",
# Which checkpoint this came from, so the directory's name does not have
# to carry it -- a name cannot hold the run as well as the iteration, and
# nothing can check a name. `deploy/manifests.json` finds an export by
# matching this rather than by rebuilding a path.
**({"_checkpoint": _checkpoint_ref(checkpoint)} if checkpoint else {}),
"_schema": NATIVE_LAYOUT_SCHEMA if native else "isaac_layout v1",
# The joints the observation carries, in order. Every per-joint term
# (joint_pos / joint_vel / joint_torque) is this wide and in this order.
"obs_joint_order": obs_joint_order,
# Every joint the robot has. Not read by the deployment -- it is here so
# that `default_joint_pos` and `unactuated_joints` can be checked against
# the robot rather than against the observation.
"wire_joint_order": wire_joint_order,
"action_joint_order": action_joint_order,
# The policy does not control these joints; lock them at the given positions
"unactuated_joints": unactuated,
"action_scale": action_scale,
**(native_fields if native else {"use_default_offset": True}),
"default_joint_pos": default_joint_pos,
# [lo, hi] per joint, keyed like default_joint_pos and covering every
# joint on the wire. A consumer that clamps targets should clamp to this
# rather than to a number in its own config -- see the note above.
"joint_limits": joint_limits,
"base_height": float(init_pos[2]),
"base_init_pos": [float(v) for v in init_pos],
"control": {
"kp": kp,
"kd": kd,
"effort_limit": eff,
"decimation": int(env_cfg.decimation),
"sim_dt": float(env.physics_dt),
"control_dt": float(env.step_dt),
"control_hz": round(1.0 / float(env.step_dt), 6),
},
"observation": {
"dim": obs_dim,
"history_length": 1,
"history_order": "oldest_first",
**({"command_order": list(COMMAND_ORDER)} if native else {}),
# **Spelled out here as well as in the README**, because a contract
# that carries a number a reader does not know to look for is how a
# robot ends up with a correct-shaped observation built from the
# wrong instants. Present only when some term is strided, so an
# unstrided bundle is byte-identical to what it always was.
**(
{
"history_stride_note": (
"One or more terms carry `history_stride` > 1. Those "
"terms' frames are NOT consecutive control steps: frame "
"i counts back i*stride steps from now, oldest first. "
"Build them by keeping a ring of stride*(length-1)+1 "
"frames and emitting every stride-th, newest included. "
"A builder that shifts one frame per inference -- which "
"is what rl-wbc-fsm's ObservationBuilder does today -- "
"produces the right SHAPE from a window stride times too "
"short, and nothing will report it."
)
}
if any(t.get("history_stride", 1) != 1 for t in obs_terms)
else {}
),
# The actor carries an EmpiricalNormalization and it is **baked into
# the ONNX graph** (see rsl_rl `_OnnxMLPModel.forward`: obs_normalizer
# first, then the mlp). Feed raw observations on the deployment side
# and **do not normalise again**.
"normalization": "baked_into_onnx",
"terms": obs_terms,
},
"action": action,
"command_ranges": _command_ranges(env, env_cfg),
**_controller(env_cfg),
"network": {
"actor_hidden_dims": [int(h) for h in actor_cfg.hidden_dims],
"activation": str(actor_cfg.activation),
"obs_normalization": bool(actor_cfg.obs_normalization),
},
"deploy_notes": _deploy_notes(obs_terms, native=native),
}
def _deploy_notes(obs_terms, *, native: bool = False) -> list[str]:
"""Known sim2real gaps, shipped with the contract so they are not discovered
during deployment."""
target_note = (
"Native-only targets use action.targets by joint, with a declared position or velocity "
"quantity, offset, and optional target clamp."
if native else
"Action transform: target = action * action_scale + default_joint_pos[joint], "
"paired joint by joint in action_joint_order."
)
notes = [
target_note,
"In simulation, apply_actions also subtracts encoder_bias (a domain "
"randomisation term modelling encoder offset). Do not reproduce it on hardware.",
"Joints in unactuated_joints are not policy-controlled and must be locked at "
"the given positions: they take part in collision, and freeing them changes "
"the foot contact geometry.",
"The ONNX already contains observation normalisation; feed raw observations.",
]
# What `posture_command` is, for a consumer that is not this repository's
# controller. `deploy/fsm` builds it from the term's own params and needs
# none of this; the note is for whoever builds it a second time.
if any(t["name"] == "posture_command" for t in obs_terms):
notes += [
"posture_command is 4 floats ordered [twist, pitch, roll, height - "
"params.neutral_height]: the command term's own order, the height "
"centred on standing. Not the builder's `base_pose`, which is three of "
"the same quantities ordered [pitch, roll, twist].",
]
# There used to be a `base_lin_vel` warning here. It is gone because a note is
# the wrong instrument: the bundle exported anyway, and the note was read by
# whoever already knew. `_validate_measurable` now refuses the export instead.
return notes
# ──────────────────────────────────────────────────────────────────────
# Validation: better to fail the export than to ship a wrong one
# ──────────────────────────────────────────────────────────────────────
def _validate_shapes(contract):
n_act = contract["action"]["dim"]
n_joint = len(contract["action_joint_order"])
if n_act != n_joint:
raise SystemExit(f"[export] action dim {n_act} != controlled joint count {n_joint}")
terms_sum = sum(t["dim"] for t in contract["observation"]["terms"])
if terms_sum != contract["observation"]["dim"]:
raise SystemExit(
f"[export] observation terms sum to {terms_sum} != observation dim "
f"{contract['observation']['dim']}"
)
n_wire = len(contract["wire_joint_order"])
n_un = len(contract["unactuated_joints"])
passive = set(contract.get("passive_joints", []))
if n_wire != n_joint + n_un + len(passive):
raise SystemExit(
f"[export] joint counts disagree: robot has {n_wire} != controlled "
f"{n_joint} + held {n_un} + passive {len(passive)}"
)
# The deployment refuses to start unless the home pose covers every joint on
# the wire, including the ones the policy never sees -- they still have to be
# held somewhere.
missing = [j for j in contract["wire_joint_order"] if j not in contract["default_joint_pos"]]
if missing:
raise SystemExit(f"[export] default_joint_pos does not cover these joints: {missing}")
native = contract["_schema"] == NATIVE_LAYOUT_SCHEMA
targets = contract["action"].get("targets", {})
position_joints = [j for j in contract["wire_joint_order"] if j not in passive
and targets.get(j, {}).get("kind", "position") == "position"]
missing = [j for j in position_joints if j not in contract["joint_limits"]]
if missing:
raise SystemExit(f"[export] joint_limits does not cover these joints: {missing}")
inverted = {j: v for j, v in contract["joint_limits"].items()
if not np.isfinite(v).all() or not v[0] < v[1]}
if inverted:
raise SystemExit(f"[export] joint_limits are not [lo, hi]: {inverted}")
outside = {
j: (contract["default_joint_pos"][j], contract["joint_limits"][j])
for j in contract["joint_limits"]
if not (contract["joint_limits"][j][0] - 1e-6
<= contract["default_joint_pos"][j]
<= contract["joint_limits"][j][1] + 1e-6)
}
if outside:
# The home pose is where every mode switch ramps to. Outside the clamp it
# is a pose the robot is asked to reach and then prevented from holding.
raise SystemExit(f"[export] the home pose is outside the joint limits: {outside}")
stray = [j for j in contract["obs_joint_order"] if j not in contract["default_joint_pos"]]
if stray:
raise SystemExit(f"[export] observed joints absent from the robot: {stray}")
n_obs_j = len(contract["obs_joint_order"])
for term in contract["observation"]["terms"]:
if term["name"] not in JOINT_TERMS:
continue
order = term["joint_order"] if native else contract["obs_joint_order"]
want = len(order) * term.get("history_length", 1)
if term["dim"] != want or len(order) != len(set(order)):
raise SystemExit(
f"[export] {term['name']} width {term['dim']} disagrees with its joint order "
f"and history ({want})"
)
print(f"[export] dimensions consistent obs={contract['observation']['dim']} act={n_act} "
f"(robot {n_wire} joints = controlled {n_joint} + held {n_un} "
f"+ passive {len(passive)}; "
f"observation carries {n_obs_j})")
def _validate_native(contract, env):
if contract.get("reference"):
raise SystemExit("[export] native policy layout does not support motion references")
terms = contract["observation"]["terms"]
unsupported = [t["name"] for t in terms if t["name"] not in STANDARD_TERMS]
if unsupported:
raise SystemExit(f"[export] unsupported native observation terms: {unsupported}")
for name in env.observation_manager.active_terms["actor"]:
cfg = env.observation_manager.get_term_cfg("actor", name)
if getattr(cfg, "delay_max_lag", 0) or not getattr(cfg, "flatten_history_dim", True):
raise SystemExit(f"[export] native replay cannot reproduce {name}'s delay/history")
gains = contract["control"]
for name, target in contract["action"]["targets"].items():
if target["kind"] == "velocity":
kp = gains["kp"][name] if isinstance(gains["kp"], dict) else gains["kp"]
kd = gains["kd"][name] if isinstance(gains["kd"], dict) else gains["kd"]
if kp != 0 or kd <= 0:
raise SystemExit(f"[export] velocity joint {name!r} needs kp=0 and kd>0")
def _validate_deployable(contract):
"""Every observation term must be one the deployment can build.
The on-robot observation builder constructs terms **by name** and throws on a
name it does not know, so an unknown term is a robot that will not start --
discovered on the bench, after the export looked fine. A new observation term
in a task is the way this happens: it costs nothing in training and nothing in
simulation, and only the deployment cares.
Failing here is the point. Either the term has a deployment name and belongs
in `_TERM_RENAME`, or the policy genuinely cannot be deployed until the
controller learns to build it.
"""
allowed = set(_DEPLOY_TERMS)
spec = contract.get("reference")
if spec:
# `ref_future` is one joint vector per declared offset. A width that
# disagrees means the term and the block describe different previews,
# and the controller would build the block's -- silently, at the right
# total width only by coincidence.
n_obs = len(contract["observation"].get("joint_order", contract["obs_joint_order"]))
want = len(spec["lookahead_s"]) * n_obs
for t in contract["observation"]["terms"]:
if t["name"] == "ref_future" and t["dim"] != want:
raise SystemExit(
f"[export] ref_future is {t['dim']} wide but the reference block "
f"declares {len(spec['lookahead_s'])} offsets over {n_obs} observed "
f"joints, which is {want}.\n"
f" The deployment builds the block's version, so these have to "
f"agree or the robot previews something the policy never saw."
)
else:
allowed -= _REFERENCE_TERMS
# The legacy consumer needs the source cadence and height parameters.
for t in contract["observation"]["terms"]:
params = t.get("params") or {}
if t["name"] == "gait_phase":
law = {"stride", "freq_min", "freq_max", "turn_radius"}
if "period" not in params and not law <= set(params):
raise SystemExit(
f"[export] gait_phase carries neither `period` nor the cadence law "
f"({', '.join(sorted(law))}); its params are {sorted(params)}.\n"
f" The controller would run it at a default tempo."
)
if t["name"] == "posture_command" and "neutral_height" not in params:
raise SystemExit(
"[export] posture_command carries no `neutral_height`, so the controller "
"cannot centre its height channel the way training did."
)
unknown = [t["name"] for t in contract["observation"]["terms"] if t["name"] not in allowed]
if unknown:
raise SystemExit(
f"[export] these observation terms have no deployment equivalent: {unknown}\n"
f" The on-robot observation builder constructs terms by name and "
f"throws on an unknown one, so this contract would not load.\n"
f" Add a mapping to _TERM_RENAME in this file if the deployment "
f"already builds the same quantity under another name; otherwise the "
f"controller has to gain the term first."
)
named = [f"{t['name']}({t['dim']})" for t in contract["observation"]["terms"]]
print(f"[export] deployable terms {' '.join(named)}")
def _validate_measurable(contract):
"""Reject actor terms missing from the legacy sensor interface."""
bad = [t["name"] for t in contract["observation"]["terms"] if t["name"] in _UNMEASURABLE_TERMS]
if bad:
why = "\n".join(f" {n}: {_UNMEASURABLE_TERMS[n]}" for n in bad)
raise SystemExit(
f"[export] the actor observes signals unavailable to the legacy controller: {bad}\n"
f"{why}\n"
f" These belong in the critic, which is discarded at export. Move "
f"them there and retrain for this controller."
)
print(f"[export] the legacy controller exposes every observed signal ({len(contract['observation']['terms'])} terms)")
def _validate_home_pose(contract, env_cfg, tol=1e-4):
"""The exported home pose must match `init_state.joint_pos` in the cfg.
They should share a source, but default_joint_pos can be altered by
post-processing such as soft_joint_pos_limit. A mismatch means something
changed the pose along the way and has to be understood first.
"""
authored = env_cfg.scene.entities["robot"].init_state.joint_pos or {}
bad, unspec = {}, []
for j, dumped in contract["default_joint_pos"].items():
if j in authored:
a = float(authored[j])
if abs(dumped - a) > tol:
bad[j] = (dumped, a)
else:
unspec.append(j)
if bad:
lines = "\n ".join(
f"{j}: env={d:.5f} cfg={a:.5f} (delta {d - a:+.2e})"
for j, (d, a) in sorted(bad.items())
)
raise SystemExit(f"[export] home pose disagrees with init_state ({len(bad)} joints, "
f"tolerance {tol}):\n {lines}")
note = f"; {len(unspec)} not specified in the cfg" if unspec else ""
print(f"[export] home pose {len(contract['default_joint_pos']) - len(unspec)} "
f"joints match init_state (tolerance {tol}){note}")
def _validate_checkpoint(contract, ckpt_path, state=None):
"""The checkpoint's dimensions must match the current task config.
Pairing weights trained on a different task with this contract is the easiest
mistake to make and the hardest to find.
"""
sd = state if state is not None else torch.load(ckpt_path, map_location="cpu", weights_only=False)
model = sd.get("actor_state_dict")
if model is None: # other rsl_rl versions name it differently
model = sd.get("model_state_dict") or sd.get("model") or sd
if isinstance(model, dict) and isinstance(model.get("actor"), dict):
model = model["actor"]
# mlp.<i>.weight, sorted by layer number -- lexicographic order would put
# mlp.10 before mlp.2
ws = sorted(
((k, tuple(v.shape)) for k, v in model.items()
if k.startswith("mlp.") and k.endswith(".weight")
and hasattr(v, "shape") and v.ndim == 2),
key=lambda kv: int(kv[0].split(".")[1]),
)
if not ws:
raise SystemExit(
f"[export] cannot infer the network shape from the checkpoint "
f"(top-level keys: {list(sd)[:6]}). This validation exists to stop "
f"weights from another task being paired with this contract and must "
f"not be skipped silently."
)
obs_dim = ws[0][1][1]
act_dim = ws[-1][1][0]
hidden = [shape[0] for _, shape in ws[:-1]]
mism = []
if obs_dim != contract["observation"]["dim"]:
mism.append(f"obs_dim: checkpoint={obs_dim} contract={contract['observation']['dim']}")
if act_dim != contract["action"]["dim"]:
mism.append(f"act_dim: checkpoint={act_dim} contract={contract['action']['dim']}")
if hidden != contract["network"]["actor_hidden_dims"]:
mism.append(f"hidden: checkpoint={hidden} contract={contract['network']['actor_hidden_dims']}")
if mism:
raise SystemExit(
"[export] checkpoint does not match the current task config:\n "
+ "\n ".join(mism)
+ "\nExport with the task this checkpoint was trained on."
)
print(f"[export] checkpoint dims obs={obs_dim} act={act_dim} hidden={hidden}")
def _export_onnx(runner, out: Path) -> None:
"""Write an actor graph with one observation input and one action output.
A tracking runner may export additional trajectory inputs and outputs.
Fall back to the base runner when the exported graph adds an input.
"""
runner.export_policy_to_onnx(str(out), "actor.onnx")
path = out / "actor.onnx"
import onnx
inputs = [i.name for i in onnx.load(str(path)).graph.input]
if len(inputs) == 1:
return
base = next(
(c.export_policy_to_onnx for c in type(runner).__mro__[1:]
if "export_policy_to_onnx" in c.__dict__),
None,
)
if base is None:
raise SystemExit(
f"[export] {type(runner).__name__} exported a graph taking {inputs}, and "
f"the deployment feeds exactly one tensor.\n"
f" No base class defines another export to fall back to."
)
print(f"[export] {type(runner).__name__} exported a graph taking {inputs}; the "
f"deployment feeds one tensor, so re-exporting with "
f"{base.__qualname__.split('.')[0]}'s. The reference travels as a "
f"trajectory file instead of inside the graph.")
base(runner, str(out), "actor.onnx")
def _validate_onnx(onnx_path, contract, policy, device):
"""The ONNX must match the torch policy in both shape **and** value.
Matching shapes do not imply matching values (whether the normaliser was baked
in, whether the mean rather than a sample was taken), so the same observations
are fed to both and compared element-wise. Without onnxruntime only shapes can
be checked, and that is stated explicitly.
"""
import onnx
import onnxruntime as ort
from mjrl.platform.graph import read_embedded_onnx
data, m = read_embedded_onnx(onnx_path)
if len(m.graph.input) != 1 or len(m.graph.output) != 1:
raise SystemExit("[export] ONNX needs exactly one observation input and action output")
shapes = []
for kind, node, width in (
("input", m.graph.input[0], contract["observation"]["dim"]),
("output", m.graph.output[0], contract["action"]["dim"]),
):
tensor = node.type.tensor_type
shape = [d.dim_value if d.HasField("dim_value") else d.dim_param or None
for d in tensor.shape.dim]
if (tensor.elem_type != onnx.TensorProto.FLOAT or len(shape) != 2
or isinstance(shape[0], int) and shape[0] != 1 or shape[-1] != width):
raise SystemExit(f"[export] ONNX {kind} must be float32 [1,{width}], got {shape}")
shapes.append(shape)
ishape, oshape = shapes
print(f"[export] ONNX shape {ishape} -> {oshape}")
options = ort.SessionOptions()
options.intra_op_num_threads = 1
options.inter_op_num_threads = 1
sess = ort.InferenceSession(data, sess_options=options, providers=["CPUExecutionProvider"])
iname = sess.get_inputs()[0].name
rng = np.random.default_rng(0)
worst = 0.0
for _ in range(8):
x = rng.normal(0, 1, size=(1, contract["observation"]["dim"])).astype(np.float32)
y_onnx = sess.run(None, {iname: x})[0]
with torch.inference_mode():
td = _as_policy_input(policy, torch.from_numpy(x).to(device))
y_torch = policy(td).cpu().numpy()
if y_onnx.shape != y_torch.shape or not np.isfinite(y_onnx).all() or not np.isfinite(y_torch).all():
raise SystemExit("[export] ONNX or torch produced invalid action values")
worst = max(worst, float(np.abs(y_onnx - y_torch).max()))
if worst > 1e-4:
raise SystemExit(f"[export] ONNX and torch policy outputs disagree, "
f"max difference {worst:.3e}")
print(f"[export] ONNX values max difference from the torch policy {worst:.2e} "
f"over 8 random observations")
def _as_policy_input(policy, x):
"""Wrap a flat observation tensor into the form the policy expects."""
from tensordict import TensorDict
groups = getattr(policy, "obs_groups", None) or ["actor"]
return TensorDict({g: x for g in groups}, batch_size=[x.shape[0]], device=x.device)
# ──────────────────────────────────────────────────────────────────────
# README
# ──────────────────────────────────────────────────────────────────────
def write_readme(path: Path, contract: dict, task: str, ckpt: str) -> None:
o = contract["observation"]
lines = [
f"# Deployment bundle: {task}",
"",
f"Source checkpoint: `{ckpt}`",
"",
"Generated from a live environment by `scripts/export.py`.",
"**Do not edit by hand** -- re-export whenever the config changes; values",
"edited by hand always drift.",
"",
"## Files",
"",
"| File | Contents |",
"|---|---|",
f"| `actor.onnx` | `[1, {o['dim']}] -> [1, {contract['action']['dim']}]`; "
f"takes raw observations, returns the mean action |",
"| `layout.json` | The deployment contract. Read every number below from "
"it rather than copying them into code |",
"",
"## Assembling the observation",
"",
f"Length {o['dim']}, concatenated in the order below. Joint observations use "
"the term's joint order when present, or `obs_joint_order`.",
"",
"| Range | Term | Dim |",
"|---|---|---|",
]
for t in o["terms"]:
lines.append(f"| `[{t['offset']}:{t['offset'] + t['dim']}]` | {t['name']} | {t['dim']} |")
strided = [t for t in o["terms"] if t.get("history_stride", 1) != 1]
if strided:
lines += [
"",
"### The history is strided -- read this before building it",
"",
"These terms stack several frames, and **the frames are not "
"consecutive control steps**:",
"",
"| Term | Frames | Stride | Spans |",
"|---|---|---|---|",
]
hz = contract["control"]["control_hz"]
for t in strided:
n, k = t["history_length"], t["history_stride"]
lines.append(
f"| {t['name']} | {n} | {k} | {(n - 1) * k / hz * 1000:.0f} ms "
f"({(n - 1) * k} control steps) |"
)
lines += [
"",
"Emit frames from oldest to newest, ending at the current observation. "
"Keep `stride * (frames - 1) + 1` frames and select every `stride`-th "
"frame. Using consecutive frames would preserve the input width "
"while changing its time span.",
]
lines += [
"",
f"Observation normalisation is already inside the ONNX "
f"(`normalization = {o['normalization']}`), so **feed raw values and do not "
f"normalise again**.",
"",
"## Applying the action",
"",
]
if contract["_schema"] == NATIVE_LAYOUT_SCHEMA:
lines += [
"In `action_joint_order`, multiply each output by its named `action_scale` "
"and add the named target's `offset`. Apply the target's optional `clip`.",
"Use the resulting position or velocity target according to `action.targets`.",
"Replay uses `robot.xml` and `controller.json` from this export.",
"",
]
else:
lines += [
"```",
"for i, joint in enumerate(contract['action_joint_order']):",
" target[joint] = action[i] * action_scale + default_joint_pos[joint]",
"```",
"",
]
lines += [
f"Then track with PD: kp = {contract['control']['kp']}, "
f"kd = {contract['control']['kd']}, "
f"effort limit {contract['control']['effort_limit']} in joint units, "
f"control rate {contract['control']['control_hz']} Hz.",
"",
"Joints in `unactuated_joints` are not policy-controlled and **must be locked "
"at the given positions**: they take part in collision, and freeing them "
"changes the foot contact geometry.",
"",
"## Read before deploying",
"",
]
for n in contract["deploy_notes"]:
lines.append(f"- {n}")
lines.append("")
path.write_text("\n".join(lines), encoding="utf-8")
# ──────────────────────────────────────────────────────────────────────
def main() -> None:
parser = build_parser("export.py", __doc__ or "")
g = parser.add_argument_group("export")
g.add_argument("--checkpoint", help="model_*.pt to export")
g.add_argument("--native-layout", action="store_true",
help="export a native replay layout and the live scene's MuJoCo model")
g.add_argument(
"--out", default=None,
help="output directory; defaults to tasks/<task path>/out/<date-time>/",
)
# Custom task hooks may produce optional media after the core export.
g.add_argument(
"--video", action=argparse.BooleanOptionalAction, default=True,
help="write the task's extra artifacts (rendered video, audio) when it has "
"any; --no-video writes only the policy and its contract",
)
# The selected task may take arguments of its own; see
# `tasks.load_cli_args`. Nothing here knows which, or how many.
args = parse_with_task_args(parser)
if args.list:
print_task_table()
return
spec, res, asset = resolve_all(args)
if args.dry_run:
print(f"[mjrl] resolved: task={spec.id} -- --dry-run, stopping here")
return
if not args.checkpoint:
raise SystemExit("[export] --checkpoint is required")
ckpt = Path(args.checkpoint).resolve()
if not ckpt.is_file():
raise SystemExit(f"[export] checkpoint does not exist: {ckpt}")
from mjrl.backend.select import use_backend
from mjrl.backend.warp_allocation import FILENAME as WARP_ALLOCATION_FILENAME
from mjrl.backend.warp_allocation import apply_warp_allocation
import tasks
# The backend has to be registered before the environment is built; see the
# note in scripts/train.py.
use_backend(res)
from mjlab.envs import ManagerBasedRlEnv
from mjlab.rl import RslRlVecEnvWrapper
from mjlab.rl.runner import MjlabOnPolicyRunner
from tasks.paths import out_dir_for
# `tasks/<task>/out/<date-time>/` -- beside the task it came from, and
# tracked, unlike `logs/`. See `tasks.paths.out_dir_for`.
#
# Timestamped rather than named after the checkpoint. Two exports of one
# checkpoint are two different things whenever anything between the config
# and this script has changed, and the old name meant the second silently
# replaced the first. The same `%Y-%m-%d_%H-%M-%S` as `logs/` and
# `out/bundle_*`, so there is one way to read a directory name here.
#
# Which checkpoint it came from is recorded *inside*, in `layout.json`,
# because a name is not a place to keep provenance: it cannot hold the run
# as well as the iteration, and nothing can check it.
out = Path(args.out) if args.out else (
out_dir_for(spec.id) / f"{datetime.now():%Y-%m-%d_%H-%M-%S}"
)
out.mkdir(parents=True, exist_ok=True)
# One environment is enough: the contract describes the robot, not a batch.
env_cfg = tasks.load_env_cfg(
spec.id, asset=asset, play=True, task_args=args.task_args
)
apply_warp_allocation(env_cfg.sim, args, res.backend, ckpt)
env_cfg.scene.num_envs = 1
checkpoint_state = None
command_evidence = None
if spec.robot is not None:
from mjrl.replay import configure_velocity_replay
checkpoint_state = torch.load(ckpt, map_location="cpu", weights_only=False)
training_cfg = tasks.load_env_cfg(spec.id, asset=asset, task_args=args.task_args)
command_evidence = configure_velocity_replay(env_cfg, training_cfg, checkpoint_state)
agent_cfg = tasks.load_agent_cfg(spec.id)
env = ManagerBasedRlEnv(cfg=env_cfg, device=res.device)
try:
wrapped = RslRlVecEnvWrapper(env, clip_actions=agent_cfg.clip_actions)
runner_cls = tasks.load_runner_cls(spec.id) or MjlabOnPolicyRunner
runner = runner_cls(wrapped, asdict(agent_cfg), device=res.device)
runner.load(
str(ckpt), load_cfg={"actor": True}, strict=True, map_location=res.device
)
policy = runner.get_inference_policy(device=res.device)
print()
contract = build_contract(env, env_cfg, agent_cfg, checkpoint=ckpt,
native=args.native_layout)
if command_evidence is not None:
contract["_command_evidence"] = command_evidence
# After the contract, because it needs the wire order the contract
# resolved; before validation, because the block is part of what is
# validated.
contract.update(_reference(env_cfg, out, contract["wire_joint_order"]))
_validate_shapes(contract)
native = contract["_schema"] == NATIVE_LAYOUT_SCHEMA
if native:
_validate_native(contract, env)
else:
_validate_deployable(contract)
_validate_measurable(contract)
_validate_home_pose(contract, env_cfg)
_validate_checkpoint(contract, ckpt, checkpoint_state)
_export_onnx(runner, out)
_validate_onnx(out / "actor.onnx", contract, policy, res.device)
if native:
from mjrl.native_export import write_native_model
write_native_model(env, out, contract)
finally:
env.close()
# The layout records which contract the export uses.
(out / "layout.json").write_text(
json.dumps(contract, indent=2, ensure_ascii=False), encoding="utf-8"
)
write_readme(out / "README.md", contract, spec.id, str(ckpt))
# **The checkpoint travels with the bundle.**
#
# A bundle is otherwise a one-way artifact: `layout.json` records which file
# it came from, and a month later that path is a run somebody deleted or a
# config that has moved underneath it. Both of the things anyone wants to do
# with a bundle afterwards need the weights -- comparing it against the policy
# it claims to be (`play.py --checkpoint <this directory>` replays the copy), and exporting
# it again after the contract gains a field, which is not hypothetical: this
# week a term was missing `neutral_height` and every bundle written before
# then had to be redone.
#
# Re-exporting from the *repository* is the trap that makes this worth the
# 10 MB. The observation width does not change when a gait constant does, so
# an old checkpoint re-exported against today's config produces a contract
# that loads, runs, and describes a cadence the policy never trained under.
# Keeping the weights beside the contract at least makes "which policy is
# this" answerable without git archaeology.
#
# Copied rather than linked: a bundle is a thing to send somewhere. Kept
# under its own name rather than a fixed one, so `CHECKPOINT_GLOB` finds
# it -- which makes the bundle directory something `--checkpoint` and
# `train --resume` can be pointed at directly.
shutil.copy2(ckpt, out / ckpt.name)
allocation_record = ckpt.parent / WARP_ALLOCATION_FILENAME
if allocation_record.is_file():
shutil.copy2(allocation_record, out / allocation_record.name)
print(f"\n[export] done -> {out}")
for f in sorted(out.iterdir()):
print(f" {f.name:<16}{f.stat().st_size / 1024:8.1f} KB")
# Extra artifacts, if this task has any. Deliberately after the bundle is
# complete and outside the env's lifetime: this hook builds its own
# environment (it needs a full-length episode, which the contract env is not),
# and a failure here must not leave a half-written actor.onnx behind.
#
# This file knows only that the hook may exist -- never what it writes. The
# rule that `scripts/` does not change when a task is added still holds; what
# changed once, here, is that there is now a place for a task to put artifacts
# that are not part of the deployment contract. See tasks.load_export_media.
if args.video:
extra = tasks.load_export_media(spec.id)
if extra is not None:
extra(out, task_id=spec.id, checkpoint=ckpt, asset=asset, res=res)
if __name__ == "__main__":
main()