1254 lines
57 KiB
Python
Executable File
1254 lines
57 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Export a trained policy and its contract from a live mjlab environment.
|
|
|
|
The export contains the ONNX actor, the observation and action layout, a
|
|
checkpoint copy, and a README. The native layout also carries a replay model
|
|
and controller configuration. The exporter reads joint selections, target
|
|
types, gains, timing, and command ranges from the environment and checkpoint.
|
|
|
|
python scripts/export.py --task my_robot --checkpoint model_1999.pt \
|
|
--native-layout --out out/native_policy
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import shutil
|
|
from dataclasses import asdict
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from mjrl.platform.layout import COMMAND_ORDER, JOINT_TERMS, NATIVE_LAYOUT_SCHEMA, STANDARD_TERMS
|
|
|
|
from _cli import (
|
|
build_parser,
|
|
parse_with_task_args,
|
|
print_task_table,
|
|
resolve_all,
|
|
)
|
|
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# The deployment vocabulary
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
#
|
|
# The contract declares `_schema: isaac_layout v1`, and what defines that schema
|
|
# is the consumer: the on-robot controller builds each observation term **by
|
|
# name**, and a name it does not know is a hard failure at load. mjlab's names
|
|
# are not that vocabulary -- it calls the velocity command `command` and the
|
|
# actuator torque `actuator_force` -- so the export translates, and refuses to
|
|
# write a contract whose terms the deployment could not build.
|
|
#
|
|
# The table lives here rather than in the task configs because the names come
|
|
# from vendored mjlab (`rl/mjlab/tasks/velocity/velocity_env_cfg.py`); a task
|
|
# that renamed them would drift from upstream at every rebase, and the rename
|
|
# belongs to the export, not to training.
|
|
|
|
#: mjlab's term name -> what the deployment calls the same quantity.
|
|
_TERM_RENAME = {
|
|
"command": "velocity_commands",
|
|
"actuator_force": "joint_torque",
|
|
}
|
|
|
|
#: Measurements unavailable through the legacy controller's sensor interface.
|
|
_UNMEASURABLE_TERMS = {
|
|
"base_lin_vel": "the legacy controller has no base linear velocity estimate",
|
|
}
|
|
|
|
#: Observation names supported by the Rust builder, plus its layout aliases.
|
|
#: tests/test_layout.py checks this against the Rust source.
|
|
_DEPLOY_TERMS = frozenset({
|
|
# obs.rs SUPPORTED
|
|
"base_lin_vel", "base_ang_vel", "projected_gravity", "base_pose",
|
|
"commands", "commands_diff",
|
|
"joint_pos", "joint_vel", "joint_torque",
|
|
"actions", "gait_phase",
|
|
# Buildable, but only from a recording: see `_REFERENCE_TERMS`.
|
|
"jump_phase", "ref_future",
|
|
"clip_phase", "ref_joint_pos", "ref_joint_vel", "ref_tilt_error",
|
|
# layout.rs rewrites these three onto `commands` before the builder sees them
|
|
"velocity_commands", "jump_command", "jump_go",
|
|
"posture_command",
|
|
})
|
|
|
|
#: Observation terms that need a compatible recorded trajectory contract.
|
|
_REFERENCE_TERMS = frozenset({
|
|
"jump_phase", "ref_future",
|
|
"clip_phase", "ref_joint_pos", "ref_joint_vel", "ref_tilt_error",
|
|
})
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# Building the contract
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _uniform_or_map(values, names):
|
|
"""Return a scalar when every value is equal, otherwise a {name: value} map.
|
|
|
|
The map form means the deployment side never has to trust array order.
|
|
"""
|
|
vals = [float(v) for v in values]
|
|
if len(set(round(v, 12) for v in vals)) == 1:
|
|
return vals[0]
|
|
return {n: v for n, v in zip(names, vals)}
|
|
|
|
|
|
def _actuator_scalars(env, action_joint_order):
|
|
"""Read gains and effort limits from the first environment's live actuators.
|
|
|
|
Software PD may compile to a simple motor. The compiled motor's scalar
|
|
gain then describes its torque input, while the actuator object carries
|
|
the actual position and velocity gains.
|
|
"""
|
|
robot = env.scene["robot"]
|
|
found: dict[str, tuple[float, float, float]] = {}
|
|
for act in robot.actuators:
|
|
relevant = set(act.target_names) & set(action_joint_order)
|
|
if not relevant:
|
|
continue
|
|
if not {"stiffness", "damping", "force_limit"} <= set(act.param_names):
|
|
raise SystemExit(f"[export] {sorted(relevant)} lack a PD actuator contract")
|
|
for i, joint in enumerate(act.target_names):
|
|
if joint not in relevant:
|
|
continue
|
|
if joint in found:
|
|
raise SystemExit(f"[export] multiple PD actuators target {joint!r}")
|
|
values = (
|
|
float(act.stiffness[0, i]),
|
|
float(act.damping[0, i]),
|
|
float(act.force_limit[0, i]),
|
|
)
|
|
if not np.isfinite(values).all() or values[0] < 0 or values[1] < 0 or values[2] <= 0:
|
|
raise SystemExit(f"[export] invalid PD gains or effort limit for {joint!r}")
|
|
found[joint] = values
|
|
|
|
missing = [j for j in action_joint_order if j not in found]
|
|
if missing:
|
|
raise SystemExit(f"[export] no actuator found for these controlled joints: {missing}")
|
|
|
|
kp, kd, eff = (
|
|
np.array([found[j][k] for j in action_joint_order]) for k in range(3)
|
|
)
|
|
return (
|
|
_uniform_or_map(kp, action_joint_order),
|
|
_uniform_or_map(kd, action_joint_order),
|
|
_uniform_or_map(eff, action_joint_order),
|
|
)
|
|
|
|
|
|
#: The trajectory file a reference-guided policy ships beside its ONNX.
|
|
#:
|
|
#: `trajectory`, not `reference`: a bundle already has a `reference.json` and it
|
|
#: is a different thing -- the recorded frames three hosts replay to check they
|
|
#: agree. Two meanings for one word is what `bundle` was doing a week ago.
|
|
TRAJECTORY_SUFFIX = ".trajectory.json"
|
|
|
|
|
|
def _reference(env_cfg, out: Path, wire_joint_order: list[str]) -> dict:
|
|
"""Write a legacy reference-guided policy's companion trajectory in wire order.
|
|
|
|
The recorded state table `q` is used for observations. A separate
|
|
`q_cmd` table, if present, supplies the residual-action baseline and may
|
|
have a different rate and length.
|
|
"""
|
|
cmd = getattr(env_cfg, "reference_contract", None)
|
|
if callable(cmd):
|
|
cmd = cmd()
|
|
if not isinstance(cmd, dict):
|
|
return {}
|
|
|
|
ref = cmd["reference"]
|
|
src = list(cmd["joint_order"])
|
|
missing = [j for j in wire_joint_order if j not in src]
|
|
if missing:
|
|
raise SystemExit(
|
|
f"[export] the reference trajectory is missing joints {missing}.\n"
|
|
f" Its own order is {src}\n"
|
|
f" A permuted trajectory is a robot tracking the wrong joints, "
|
|
f"and nothing downstream can see it."
|
|
)
|
|
perm = [src.index(j) for j in wire_joint_order]
|
|
|
|
# Round the sampled table to five decimal places in its authored units.
|
|
places = 5
|
|
|
|
def rows(table, columns=None) -> list[list[float]]:
|
|
out = []
|
|
for row in table:
|
|
picked = row if columns is None else [row[i] for i in columns]
|
|
out.append([round(float(v), places) for v in picked])
|
|
return out
|
|
|
|
tables = {"joint_order": wire_joint_order, "q": rows(ref["q"], perm)}
|
|
for name in ("q_cmd", "qd"):
|
|
if name in ref:
|
|
tables[name] = rows(ref[name], perm)
|
|
if "root_quat" in ref:
|
|
# Not per-joint: one orientation per frame, `(w, x, y, z)`.
|
|
tables["root_quat"] = rows(ref["root_quat"])
|
|
|
|
path = out / (cmd["name"] + TRAJECTORY_SUFFIX)
|
|
path.write_text(
|
|
json.dumps({
|
|
"_source": f"scripts/export.py, from {cmd['npz']}",
|
|
"_note": "Every per-joint table is in **wire order**, already permuted. "
|
|
"`q` is the state reached; `q_cmd`, where present, is what the "
|
|
"recorded controller commanded and is the residual baseline -- "
|
|
"a different rate and a different length. `qd` and `root_quat` "
|
|
"are per-frame companions of `q`.",
|
|
**tables,
|
|
}, separators=(",", ":")) + "\n",
|
|
"utf-8",
|
|
)
|
|
|
|
carried = ["residual_action", "rec_hz", "duration", "lookahead_s"]
|
|
optional = ["starts_on_entry", "control_hz", "t_go", "go_frame", "span",
|
|
"baseline", "baseline_lead_s", "go_event", "n_cmd"]
|
|
block = {k: cmd[k] for k in carried}
|
|
block.update({k: cmd[k] for k in optional if cmd.get(k) is not None})
|
|
block["file"] = path.name
|
|
block["n_frames"] = len(tables["q"])
|
|
if "q_cmd" in tables:
|
|
block["n_cmd"] = len(tables["q_cmd"])
|
|
|
|
shapes = " ".join(f"{k} {len(v)}x{len(v[0])}"
|
|
for k, v in tables.items() if k != "joint_order")
|
|
kind = "residual on " + str(block.get("baseline")) if block["residual_action"] else "played"
|
|
print(f"[export] reference {cmd['name']}: {shapes} ({kind}, "
|
|
f"{block['duration']:.2f} s at {block['rec_hz']:.0f} Hz, "
|
|
f"{path.stat().st_size / 1e6:.2f} MB)")
|
|
return {"reference": block}
|
|
|
|
|
|
def _controller(env_cfg):
|
|
"""How an operator's input becomes the command, when the task describes it.
|
|
|
|
`command_ranges` says how far an axis may be pushed; it does not say which
|
|
way is positive, nor what a stick at full deflection means. Both were being
|
|
decided a second time by every consumer, from the comments in the task's
|
|
teleop modules -- and a consumer that reads a sign wrong drives the robot
|
|
sideways while everything else looks correct.
|
|
|
|
Read off the env config rather than assembled here: this file is generic and
|
|
the bindings are the robot's. A task that does not describe its controller
|
|
simply exports no `controller` block, which is honest -- better than a
|
|
default that a reader would take for a measurement.
|
|
"""
|
|
contract = getattr(env_cfg, "controller_contract", None)
|
|
if callable(contract):
|
|
contract = contract()
|
|
return {"controller": contract} if isinstance(contract, dict) else {}
|
|
|
|
|
|
def _command_ranges(env, env_cfg):
|
|
"""Command ranges are part of the contract too.
|
|
|
|
A deployment whose joystick spans more than the training range is pushing the
|
|
policy out of distribution; one that spans less quietly hides capability.
|
|
Both have happened, so this reads from the live cfg.
|
|
"""
|
|
out = {}
|
|
for name in env.command_manager.active_terms:
|
|
cfg = env_cfg.commands.get(name) if hasattr(env_cfg.commands, "get") else None
|
|
if cfg is None:
|
|
cfg = getattr(env_cfg.commands, name, None)
|
|
rng = getattr(cfg, "ranges", None)
|
|
if rng is None:
|
|
continue
|
|
fields = {}
|
|
for f, v in vars(rng).items():
|
|
if f.startswith("_"):
|
|
continue
|
|
if isinstance(v, (tuple, list)) and len(v) == 2:
|
|
fields[f] = [float(v[0]), float(v[1])]
|
|
# Some command configs express a symmetric range as a half-width.
|
|
elif isinstance(v, (int, float)) and not isinstance(v, bool) and v > 0:
|
|
fields[f] = [-float(v), float(v)]
|
|
if fields:
|
|
out[name] = fields
|
|
return out
|
|
|
|
|
|
def _json_params(params) -> dict:
|
|
"""A term's params, restricted to what JSON can carry.
|
|
|
|
Params hold live objects -- a resolved `SceneEntityCfg`, a sensor handle --
|
|
beside the scalars that describe the term's behaviour. Only the scalars mean
|
|
anything to a reader on the other side, and the rest are not serialisable.
|
|
"""
|
|
return {
|
|
k: v
|
|
for k, v in (params or {}).items()
|
|
if isinstance(v, (bool, int, float, str))
|
|
}
|
|
|
|
|
|
def _obs_terms(env):
|
|
om = env.observation_manager
|
|
names = list(om.active_terms["actor"])
|
|
dims = [int(d[0]) for d in om.group_obs_term_dim["actor"]]
|
|
terms, off = [], 0
|
|
for name, dim in zip(names, dims):
|
|
cfg = om.get_term_cfg("actor", name)
|
|
deploy_name = _TERM_RENAME.get(name, name)
|
|
entry = {"name": deploy_name, "dim": dim, "offset": off}
|
|
if deploy_name != name:
|
|
# Kept so the contract can still be diffed against the env it came
|
|
# from; the deployment ignores it.
|
|
entry["source_name"] = name
|
|
params = _json_params(getattr(cfg, "params", None))
|
|
# The legacy gait clock reads a period in seconds.
|
|
if deploy_name == "gait_phase" and params.get("freq_hz"):
|
|
params["period"] = 1.0 / float(params["freq_hz"])
|
|
if params:
|
|
entry["params"] = params
|
|
scale = getattr(cfg, "scale", None)
|
|
if scale is not None:
|
|
entry["scale"] = (
|
|
[float(s) for s in scale]
|
|
if isinstance(scale, (list, tuple))
|
|
else float(scale.cpu().tolist())
|
|
if isinstance(scale, torch.Tensor)
|
|
else float(scale)
|
|
)
|
|
clip = getattr(cfg, "clip", None)
|
|
if clip is not None:
|
|
entry["clip"] = [float(c) for c in clip]
|
|
# History may be stacked by the manager or by the observation callable.
|
|
# Record both the frame count and stride before checking the wire width.
|
|
hist = int(getattr(cfg, "history_length", 0) or 0)
|
|
stride = 1
|
|
own = int(getattr(getattr(cfg, "func", None), "history_frames", 0) or 0)
|
|
if own:
|
|
if hist:
|
|
raise SystemExit(
|
|
f"[export] term {name} stacks {own} frames itself and the "
|
|
f"manager stacks {hist} more; the contract cannot describe "
|
|
f"the result"
|
|
)
|
|
hist = own
|
|
stride = int(getattr(cfg.func, "history_stride", 1) or 1)
|
|
if hist > 0:
|
|
entry["history_length"] = hist
|
|
entry["flatten_history_dim"] = bool(getattr(cfg, "flatten_history_dim", True))
|
|
entry["base_dim"] = dim // hist
|
|
# **Present only when it is not 1**, so a builder that has never seen
|
|
# it cannot quietly default: an unstrided contract looks exactly as it
|
|
# always did, and a strided one carries a field an old reader will
|
|
# fail on rather than ignore.
|
|
if stride != 1:
|
|
entry["history_stride"] = stride
|
|
terms.append(entry)
|
|
off += dim
|
|
return terms, off
|
|
|
|
|
|
def _observed_joint_orders(env, robot):
|
|
"""Read each per-joint term's resolved selection in the term's own order."""
|
|
om = env.observation_manager
|
|
orders = {}
|
|
for name in om.active_terms["actor"]:
|
|
deploy_name = _TERM_RENAME.get(name, name)
|
|
if deploy_name not in JOINT_TERMS:
|
|
continue
|
|
asset_cfg = (getattr(om.get_term_cfg("actor", name), "params", None) or {}).get(
|
|
"asset_cfg"
|
|
)
|
|
if asset_cfg is not None and asset_cfg.name != "robot":
|
|
raise SystemExit(f"[export] {name} observes an entity other than the robot")
|
|
ids = getattr(asset_cfg, "joint_ids", None)
|
|
if ids is None:
|
|
ids = slice(None)
|
|
order = robot.joint_names[ids] if isinstance(ids, slice) else [
|
|
robot.joint_names[int(i)] for i in ids
|
|
]
|
|
orders[deploy_name] = list(order)
|
|
return orders
|
|
|
|
|
|
def _action_parameters(env):
|
|
"""Read the action manager's real concatenation, transforms and target types."""
|
|
from mjlab.envs.mdp.actions.actions import JointPositionAction, JointVelocityAction
|
|
|
|
targets, scales = {}, {}
|
|
for name in env.action_manager.active_terms:
|
|
term = env.action_manager.get_term(name)
|
|
if type(term) is JointPositionAction:
|
|
mode = "position"
|
|
elif type(term) is JointVelocityAction:
|
|
mode = "velocity"
|
|
else:
|
|
raise SystemExit(f"[export] unsupported joint action term {name!r} ({type(term).__name__})")
|
|
if term.cfg.entity_name != "robot":
|
|
raise SystemExit(f"[export] action term {name!r} does not drive the robot")
|
|
joints = term.target_names
|
|
if len(joints) != term.action_dim:
|
|
raise SystemExit(f"[export] action term {name!r} has an inconsistent target count")
|
|
scale = term.scale
|
|
scale = scale[0].cpu().tolist() if isinstance(scale, torch.Tensor) else [scale] * len(joints)
|
|
offset = term.offset
|
|
offset = offset[0].cpu().tolist() if isinstance(offset, torch.Tensor) else [offset] * len(joints)
|
|
clip = getattr(term, "_clip", None)
|
|
clips = clip[0].cpu().tolist() if clip is not None else [(None, None)] * len(joints)
|
|
for joint, gain, rest, (lo, hi) in zip(joints, scale, offset, clips, strict=True):
|
|
if joint in targets:
|
|
raise SystemExit(f"[export] multiple action channels drive joint {joint!r}")
|
|
if not np.isfinite(gain) or not np.isfinite(rest):
|
|
raise SystemExit(f"[export] {joint!r} has a non-finite action transform")
|
|
targets[joint] = {"kind": mode, "offset": float(rest)}
|
|
scales[joint] = float(gain)
|
|
if lo is not None and (lo, hi) != (-np.inf, np.inf):
|
|
if not np.isfinite([lo, hi]).all() or lo >= hi:
|
|
raise SystemExit(f"[export] {joint!r} needs finite, ordered action clip limits")
|
|
targets[joint]["clip"] = [float(lo), float(hi)]
|
|
if not targets:
|
|
raise SystemExit("[export] policy has no joint action")
|
|
return targets, scales
|
|
|
|
|
|
def _checkpoint_ref(ckpt: Path) -> dict:
|
|
"""Where a checkpoint is, said in a way another machine can use.
|
|
|
|
Repository-relative when it is inside one, because an absolute path is a
|
|
fact about the machine that exported rather than about the policy. `run` is
|
|
the training directory -- `logs/<model>/<task>/<date-time>` -- which is the
|
|
part a person recognises; `iteration` is the file.
|
|
"""
|
|
from tasks.paths import REPO_ROOT
|
|
|
|
ckpt = ckpt.resolve()
|
|
where = ckpt.relative_to(REPO_ROOT) if ckpt.is_relative_to(REPO_ROOT) else ckpt
|
|
return {"path": str(where), "run": str(where.parent), "iteration": ckpt.stem}
|
|
|
|
|
|
def build_contract(
|
|
env, env_cfg, agent_cfg, checkpoint: Path | None = None, *, native: bool = False,
|
|
) -> dict:
|
|
import mujoco
|
|
|
|
robot = env.scene["robot"]
|
|
targets, scales = _action_parameters(env)
|
|
action_joint_order = list(targets)
|
|
wire_joint_order = list(robot.joint_names)
|
|
orders = _observed_joint_orders(env, robot)
|
|
distinct_orders = {tuple(order) for order in orders.values()}
|
|
if len(distinct_orders) <= 1:
|
|
obs_joint_order = list(next(iter(distinct_orders), wire_joint_order))
|
|
else:
|
|
observed = set().union(*distinct_orders)
|
|
obs_joint_order = [name for name in wire_joint_order if name in observed]
|
|
|
|
default_jp = robot.data.default_joint_pos[0].detach().cpu().tolist()
|
|
default_joint_pos = dict(zip(wire_joint_order, default_jp, strict=True))
|
|
default_joint_vel = dict(zip(
|
|
wire_joint_order, robot.data.default_joint_vel[0].detach().cpu().tolist(), strict=True,
|
|
))
|
|
mj = env.sim.mj_model
|
|
ids = [int(i) for i in robot.indexing.joint_ids.cpu().tolist()]
|
|
joint_limits = {
|
|
name: mj.jnt_range[jid].tolist()
|
|
for name, jid in zip(wire_joint_order, ids, strict=True)
|
|
if mj.jnt_limited[jid]
|
|
}
|
|
joint_transmission = mj.actuator_trntype == mujoco.mjtTrn.mjTRN_JOINT
|
|
driven_ids = set(mj.actuator_trnid[joint_transmission, 0])
|
|
driven = [name for name, jid in zip(wire_joint_order, ids, strict=True) if jid in driven_ids]
|
|
passive = [name for name in wire_joint_order if name not in driven]
|
|
if set(targets) - set(driven):
|
|
raise SystemExit("[export] an action target has no joint actuator")
|
|
unactuated = {n: default_joint_pos[n] for n in driven if n not in targets}
|
|
|
|
kp, kd, eff = _actuator_scalars(env, action_joint_order)
|
|
obs_terms, obs_dim = _obs_terms(env)
|
|
action_scale = _uniform_or_map(scales.values(), action_joint_order)
|
|
wrapper_clip = getattr(agent_cfg, "clip_actions", None)
|
|
native = (
|
|
native or len(distinct_orders) > 1 or any(default_joint_vel.values()) or bool(passive)
|
|
or any(isinstance(x, dict) for x in (kp, kd, eff, action_scale))
|
|
or wrapper_clip is not None
|
|
or any(
|
|
info["kind"] != "position" or info["offset"] != default_joint_pos[name]
|
|
or "clip" in info for name, info in targets.items()
|
|
)
|
|
or any("scale" in term or "clip" in term for term in obs_terms)
|
|
)
|
|
action = {"dim": int(env.action_manager.total_action_dim)}
|
|
native_fields = {}
|
|
if native:
|
|
if len(driven) != mj.nu:
|
|
raise SystemExit("[export] native replay requires one actuator per driven joint")
|
|
kp, kd, eff = _actuator_scalars(env, driven)
|
|
action["targets"] = targets
|
|
if wrapper_clip is not None:
|
|
if not np.isfinite(wrapper_clip) or wrapper_clip <= 0:
|
|
raise SystemExit("[export] agent action clip must be finite and positive")
|
|
action["input_clip"] = float(wrapper_clip)
|
|
# Older layouts take only a scalar. The required map makes a reader
|
|
# that ignores _schema still reject this native action contract.
|
|
action_scale = scales
|
|
for entry in obs_terms:
|
|
name = entry["name"]
|
|
if name in orders:
|
|
entry["joint_order"] = orders[name]
|
|
if name == "velocity_commands":
|
|
entry["name"] = "commands"
|
|
native_fields = {
|
|
"default_joint_vel": default_joint_vel,
|
|
"passive_joints": passive,
|
|
"model_joint_names": {
|
|
name: mj.joint(jid).name
|
|
for name, jid in zip(wire_joint_order, ids, strict=True)
|
|
},
|
|
}
|
|
|
|
init_pos = env_cfg.scene.entities["robot"].init_state.pos
|
|
actor_cfg = agent_cfg.actor
|
|
|
|
return {
|
|
"_source": "scripts/export.py -- re-export whenever the env / robot / agent config changes",
|
|
# Which checkpoint this came from, so the directory's name does not have
|
|
# to carry it -- a name cannot hold the run as well as the iteration, and
|
|
# nothing can check a name. `deploy/manifests.json` finds an export by
|
|
# matching this rather than by rebuilding a path.
|
|
**({"_checkpoint": _checkpoint_ref(checkpoint)} if checkpoint else {}),
|
|
"_schema": NATIVE_LAYOUT_SCHEMA if native else "isaac_layout v1",
|
|
# The joints the observation carries, in order. Every per-joint term
|
|
# (joint_pos / joint_vel / joint_torque) is this wide and in this order.
|
|
"obs_joint_order": obs_joint_order,
|
|
# Every joint the robot has. Not read by the deployment -- it is here so
|
|
# that `default_joint_pos` and `unactuated_joints` can be checked against
|
|
# the robot rather than against the observation.
|
|
"wire_joint_order": wire_joint_order,
|
|
"action_joint_order": action_joint_order,
|
|
# The policy does not control these joints; lock them at the given positions
|
|
"unactuated_joints": unactuated,
|
|
"action_scale": action_scale,
|
|
**(native_fields if native else {"use_default_offset": True}),
|
|
"default_joint_pos": default_joint_pos,
|
|
# [lo, hi] per joint, keyed like default_joint_pos and covering every
|
|
# joint on the wire. A consumer that clamps targets should clamp to this
|
|
# rather than to a number in its own config -- see the note above.
|
|
"joint_limits": joint_limits,
|
|
"base_height": float(init_pos[2]),
|
|
"base_init_pos": [float(v) for v in init_pos],
|
|
"control": {
|
|
"kp": kp,
|
|
"kd": kd,
|
|
"effort_limit": eff,
|
|
"decimation": int(env_cfg.decimation),
|
|
"sim_dt": float(env.physics_dt),
|
|
"control_dt": float(env.step_dt),
|
|
"control_hz": round(1.0 / float(env.step_dt), 6),
|
|
},
|
|
"observation": {
|
|
"dim": obs_dim,
|
|
"history_length": 1,
|
|
"history_order": "oldest_first",
|
|
**({"command_order": list(COMMAND_ORDER)} if native else {}),
|
|
# **Spelled out here as well as in the README**, because a contract
|
|
# that carries a number a reader does not know to look for is how a
|
|
# robot ends up with a correct-shaped observation built from the
|
|
# wrong instants. Present only when some term is strided, so an
|
|
# unstrided bundle is byte-identical to what it always was.
|
|
**(
|
|
{
|
|
"history_stride_note": (
|
|
"One or more terms carry `history_stride` > 1. Those "
|
|
"terms' frames are NOT consecutive control steps: frame "
|
|
"i counts back i*stride steps from now, oldest first. "
|
|
"Build them by keeping a ring of stride*(length-1)+1 "
|
|
"frames and emitting every stride-th, newest included. "
|
|
"A builder that shifts one frame per inference -- which "
|
|
"is what rl-wbc-fsm's ObservationBuilder does today -- "
|
|
"produces the right SHAPE from a window stride times too "
|
|
"short, and nothing will report it."
|
|
)
|
|
}
|
|
if any(t.get("history_stride", 1) != 1 for t in obs_terms)
|
|
else {}
|
|
),
|
|
# The actor carries an EmpiricalNormalization and it is **baked into
|
|
# the ONNX graph** (see rsl_rl `_OnnxMLPModel.forward`: obs_normalizer
|
|
# first, then the mlp). Feed raw observations on the deployment side
|
|
# and **do not normalise again**.
|
|
"normalization": "baked_into_onnx",
|
|
"terms": obs_terms,
|
|
},
|
|
"action": action,
|
|
"command_ranges": _command_ranges(env, env_cfg),
|
|
**_controller(env_cfg),
|
|
"network": {
|
|
"actor_hidden_dims": [int(h) for h in actor_cfg.hidden_dims],
|
|
"activation": str(actor_cfg.activation),
|
|
"obs_normalization": bool(actor_cfg.obs_normalization),
|
|
},
|
|
"deploy_notes": _deploy_notes(obs_terms, native=native),
|
|
}
|
|
|
|
|
|
def _deploy_notes(obs_terms, *, native: bool = False) -> list[str]:
|
|
"""Known sim2real gaps, shipped with the contract so they are not discovered
|
|
during deployment."""
|
|
target_note = (
|
|
"Native-only targets use action.targets by joint, with a declared position or velocity "
|
|
"quantity, offset, and optional target clamp."
|
|
if native else
|
|
"Action transform: target = action * action_scale + default_joint_pos[joint], "
|
|
"paired joint by joint in action_joint_order."
|
|
)
|
|
notes = [
|
|
target_note,
|
|
"In simulation, apply_actions also subtracts encoder_bias (a domain "
|
|
"randomisation term modelling encoder offset). Do not reproduce it on hardware.",
|
|
"Joints in unactuated_joints are not policy-controlled and must be locked at "
|
|
"the given positions: they take part in collision, and freeing them changes "
|
|
"the foot contact geometry.",
|
|
"The ONNX already contains observation normalisation; feed raw observations.",
|
|
]
|
|
# What `posture_command` is, for a consumer that is not this repository's
|
|
# controller. `deploy/fsm` builds it from the term's own params and needs
|
|
# none of this; the note is for whoever builds it a second time.
|
|
if any(t["name"] == "posture_command" for t in obs_terms):
|
|
notes += [
|
|
"posture_command is 4 floats ordered [twist, pitch, roll, height - "
|
|
"params.neutral_height]: the command term's own order, the height "
|
|
"centred on standing. Not the builder's `base_pose`, which is three of "
|
|
"the same quantities ordered [pitch, roll, twist].",
|
|
]
|
|
# There used to be a `base_lin_vel` warning here. It is gone because a note is
|
|
# the wrong instrument: the bundle exported anyway, and the note was read by
|
|
# whoever already knew. `_validate_measurable` now refuses the export instead.
|
|
return notes
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# Validation: better to fail the export than to ship a wrong one
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _validate_shapes(contract):
|
|
n_act = contract["action"]["dim"]
|
|
n_joint = len(contract["action_joint_order"])
|
|
if n_act != n_joint:
|
|
raise SystemExit(f"[export] action dim {n_act} != controlled joint count {n_joint}")
|
|
terms_sum = sum(t["dim"] for t in contract["observation"]["terms"])
|
|
if terms_sum != contract["observation"]["dim"]:
|
|
raise SystemExit(
|
|
f"[export] observation terms sum to {terms_sum} != observation dim "
|
|
f"{contract['observation']['dim']}"
|
|
)
|
|
n_wire = len(contract["wire_joint_order"])
|
|
n_un = len(contract["unactuated_joints"])
|
|
passive = set(contract.get("passive_joints", []))
|
|
if n_wire != n_joint + n_un + len(passive):
|
|
raise SystemExit(
|
|
f"[export] joint counts disagree: robot has {n_wire} != controlled "
|
|
f"{n_joint} + held {n_un} + passive {len(passive)}"
|
|
)
|
|
# The deployment refuses to start unless the home pose covers every joint on
|
|
# the wire, including the ones the policy never sees -- they still have to be
|
|
# held somewhere.
|
|
missing = [j for j in contract["wire_joint_order"] if j not in contract["default_joint_pos"]]
|
|
if missing:
|
|
raise SystemExit(f"[export] default_joint_pos does not cover these joints: {missing}")
|
|
native = contract["_schema"] == NATIVE_LAYOUT_SCHEMA
|
|
targets = contract["action"].get("targets", {})
|
|
position_joints = [j for j in contract["wire_joint_order"] if j not in passive
|
|
and targets.get(j, {}).get("kind", "position") == "position"]
|
|
missing = [j for j in position_joints if j not in contract["joint_limits"]]
|
|
if missing:
|
|
raise SystemExit(f"[export] joint_limits does not cover these joints: {missing}")
|
|
inverted = {j: v for j, v in contract["joint_limits"].items()
|
|
if not np.isfinite(v).all() or not v[0] < v[1]}
|
|
if inverted:
|
|
raise SystemExit(f"[export] joint_limits are not [lo, hi]: {inverted}")
|
|
outside = {
|
|
j: (contract["default_joint_pos"][j], contract["joint_limits"][j])
|
|
for j in contract["joint_limits"]
|
|
if not (contract["joint_limits"][j][0] - 1e-6
|
|
<= contract["default_joint_pos"][j]
|
|
<= contract["joint_limits"][j][1] + 1e-6)
|
|
}
|
|
if outside:
|
|
# The home pose is where every mode switch ramps to. Outside the clamp it
|
|
# is a pose the robot is asked to reach and then prevented from holding.
|
|
raise SystemExit(f"[export] the home pose is outside the joint limits: {outside}")
|
|
stray = [j for j in contract["obs_joint_order"] if j not in contract["default_joint_pos"]]
|
|
if stray:
|
|
raise SystemExit(f"[export] observed joints absent from the robot: {stray}")
|
|
|
|
n_obs_j = len(contract["obs_joint_order"])
|
|
for term in contract["observation"]["terms"]:
|
|
if term["name"] not in JOINT_TERMS:
|
|
continue
|
|
order = term["joint_order"] if native else contract["obs_joint_order"]
|
|
want = len(order) * term.get("history_length", 1)
|
|
if term["dim"] != want or len(order) != len(set(order)):
|
|
raise SystemExit(
|
|
f"[export] {term['name']} width {term['dim']} disagrees with its joint order "
|
|
f"and history ({want})"
|
|
)
|
|
print(f"[export] dimensions consistent obs={contract['observation']['dim']} act={n_act} "
|
|
f"(robot {n_wire} joints = controlled {n_joint} + held {n_un} "
|
|
f"+ passive {len(passive)}; "
|
|
f"observation carries {n_obs_j})")
|
|
|
|
|
|
def _validate_native(contract, env):
|
|
if contract.get("reference"):
|
|
raise SystemExit("[export] native policy layout does not support motion references")
|
|
terms = contract["observation"]["terms"]
|
|
unsupported = [t["name"] for t in terms if t["name"] not in STANDARD_TERMS]
|
|
if unsupported:
|
|
raise SystemExit(f"[export] unsupported native observation terms: {unsupported}")
|
|
for name in env.observation_manager.active_terms["actor"]:
|
|
cfg = env.observation_manager.get_term_cfg("actor", name)
|
|
if getattr(cfg, "delay_max_lag", 0) or not getattr(cfg, "flatten_history_dim", True):
|
|
raise SystemExit(f"[export] native replay cannot reproduce {name}'s delay/history")
|
|
gains = contract["control"]
|
|
for name, target in contract["action"]["targets"].items():
|
|
if target["kind"] == "velocity":
|
|
kp = gains["kp"][name] if isinstance(gains["kp"], dict) else gains["kp"]
|
|
kd = gains["kd"][name] if isinstance(gains["kd"], dict) else gains["kd"]
|
|
if kp != 0 or kd <= 0:
|
|
raise SystemExit(f"[export] velocity joint {name!r} needs kp=0 and kd>0")
|
|
|
|
|
|
def _validate_deployable(contract):
|
|
"""Every observation term must be one the deployment can build.
|
|
|
|
The on-robot observation builder constructs terms **by name** and throws on a
|
|
name it does not know, so an unknown term is a robot that will not start --
|
|
discovered on the bench, after the export looked fine. A new observation term
|
|
in a task is the way this happens: it costs nothing in training and nothing in
|
|
simulation, and only the deployment cares.
|
|
|
|
Failing here is the point. Either the term has a deployment name and belongs
|
|
in `_TERM_RENAME`, or the policy genuinely cannot be deployed until the
|
|
controller learns to build it.
|
|
"""
|
|
allowed = set(_DEPLOY_TERMS)
|
|
spec = contract.get("reference")
|
|
if spec:
|
|
# `ref_future` is one joint vector per declared offset. A width that
|
|
# disagrees means the term and the block describe different previews,
|
|
# and the controller would build the block's -- silently, at the right
|
|
# total width only by coincidence.
|
|
n_obs = len(contract["observation"].get("joint_order", contract["obs_joint_order"]))
|
|
want = len(spec["lookahead_s"]) * n_obs
|
|
for t in contract["observation"]["terms"]:
|
|
if t["name"] == "ref_future" and t["dim"] != want:
|
|
raise SystemExit(
|
|
f"[export] ref_future is {t['dim']} wide but the reference block "
|
|
f"declares {len(spec['lookahead_s'])} offsets over {n_obs} observed "
|
|
f"joints, which is {want}.\n"
|
|
f" The deployment builds the block's version, so these have to "
|
|
f"agree or the robot previews something the policy never saw."
|
|
)
|
|
else:
|
|
allowed -= _REFERENCE_TERMS
|
|
|
|
# The legacy consumer needs the source cadence and height parameters.
|
|
for t in contract["observation"]["terms"]:
|
|
params = t.get("params") or {}
|
|
if t["name"] == "gait_phase":
|
|
law = {"stride", "freq_min", "freq_max", "turn_radius"}
|
|
if "period" not in params and not law <= set(params):
|
|
raise SystemExit(
|
|
f"[export] gait_phase carries neither `period` nor the cadence law "
|
|
f"({', '.join(sorted(law))}); its params are {sorted(params)}.\n"
|
|
f" The controller would run it at a default tempo."
|
|
)
|
|
if t["name"] == "posture_command" and "neutral_height" not in params:
|
|
raise SystemExit(
|
|
"[export] posture_command carries no `neutral_height`, so the controller "
|
|
"cannot centre its height channel the way training did."
|
|
)
|
|
|
|
unknown = [t["name"] for t in contract["observation"]["terms"] if t["name"] not in allowed]
|
|
if unknown:
|
|
raise SystemExit(
|
|
f"[export] these observation terms have no deployment equivalent: {unknown}\n"
|
|
f" The on-robot observation builder constructs terms by name and "
|
|
f"throws on an unknown one, so this contract would not load.\n"
|
|
f" Add a mapping to _TERM_RENAME in this file if the deployment "
|
|
f"already builds the same quantity under another name; otherwise the "
|
|
f"controller has to gain the term first."
|
|
)
|
|
named = [f"{t['name']}({t['dim']})" for t in contract["observation"]["terms"]]
|
|
print(f"[export] deployable terms {' '.join(named)}")
|
|
|
|
|
|
def _validate_measurable(contract):
|
|
"""Reject actor terms missing from the legacy sensor interface."""
|
|
bad = [t["name"] for t in contract["observation"]["terms"] if t["name"] in _UNMEASURABLE_TERMS]
|
|
if bad:
|
|
why = "\n".join(f" {n}: {_UNMEASURABLE_TERMS[n]}" for n in bad)
|
|
raise SystemExit(
|
|
f"[export] the actor observes signals unavailable to the legacy controller: {bad}\n"
|
|
f"{why}\n"
|
|
f" These belong in the critic, which is discarded at export. Move "
|
|
f"them there and retrain for this controller."
|
|
)
|
|
print(f"[export] the legacy controller exposes every observed signal ({len(contract['observation']['terms'])} terms)")
|
|
|
|
|
|
def _validate_home_pose(contract, env_cfg, tol=1e-4):
|
|
"""The exported home pose must match `init_state.joint_pos` in the cfg.
|
|
|
|
They should share a source, but default_joint_pos can be altered by
|
|
post-processing such as soft_joint_pos_limit. A mismatch means something
|
|
changed the pose along the way and has to be understood first.
|
|
"""
|
|
authored = env_cfg.scene.entities["robot"].init_state.joint_pos or {}
|
|
bad, unspec = {}, []
|
|
for j, dumped in contract["default_joint_pos"].items():
|
|
if j in authored:
|
|
a = float(authored[j])
|
|
if abs(dumped - a) > tol:
|
|
bad[j] = (dumped, a)
|
|
else:
|
|
unspec.append(j)
|
|
if bad:
|
|
lines = "\n ".join(
|
|
f"{j}: env={d:.5f} cfg={a:.5f} (delta {d - a:+.2e})"
|
|
for j, (d, a) in sorted(bad.items())
|
|
)
|
|
raise SystemExit(f"[export] home pose disagrees with init_state ({len(bad)} joints, "
|
|
f"tolerance {tol}):\n {lines}")
|
|
note = f"; {len(unspec)} not specified in the cfg" if unspec else ""
|
|
print(f"[export] home pose {len(contract['default_joint_pos']) - len(unspec)} "
|
|
f"joints match init_state (tolerance {tol}){note}")
|
|
|
|
|
|
def _validate_checkpoint(contract, ckpt_path, state=None):
|
|
"""The checkpoint's dimensions must match the current task config.
|
|
|
|
Pairing weights trained on a different task with this contract is the easiest
|
|
mistake to make and the hardest to find.
|
|
"""
|
|
sd = state if state is not None else torch.load(ckpt_path, map_location="cpu", weights_only=False)
|
|
model = sd.get("actor_state_dict")
|
|
if model is None: # other rsl_rl versions name it differently
|
|
model = sd.get("model_state_dict") or sd.get("model") or sd
|
|
if isinstance(model, dict) and isinstance(model.get("actor"), dict):
|
|
model = model["actor"]
|
|
# mlp.<i>.weight, sorted by layer number -- lexicographic order would put
|
|
# mlp.10 before mlp.2
|
|
ws = sorted(
|
|
((k, tuple(v.shape)) for k, v in model.items()
|
|
if k.startswith("mlp.") and k.endswith(".weight")
|
|
and hasattr(v, "shape") and v.ndim == 2),
|
|
key=lambda kv: int(kv[0].split(".")[1]),
|
|
)
|
|
if not ws:
|
|
raise SystemExit(
|
|
f"[export] cannot infer the network shape from the checkpoint "
|
|
f"(top-level keys: {list(sd)[:6]}). This validation exists to stop "
|
|
f"weights from another task being paired with this contract and must "
|
|
f"not be skipped silently."
|
|
)
|
|
obs_dim = ws[0][1][1]
|
|
act_dim = ws[-1][1][0]
|
|
hidden = [shape[0] for _, shape in ws[:-1]]
|
|
mism = []
|
|
if obs_dim != contract["observation"]["dim"]:
|
|
mism.append(f"obs_dim: checkpoint={obs_dim} contract={contract['observation']['dim']}")
|
|
if act_dim != contract["action"]["dim"]:
|
|
mism.append(f"act_dim: checkpoint={act_dim} contract={contract['action']['dim']}")
|
|
if hidden != contract["network"]["actor_hidden_dims"]:
|
|
mism.append(f"hidden: checkpoint={hidden} contract={contract['network']['actor_hidden_dims']}")
|
|
if mism:
|
|
raise SystemExit(
|
|
"[export] checkpoint does not match the current task config:\n "
|
|
+ "\n ".join(mism)
|
|
+ "\nExport with the task this checkpoint was trained on."
|
|
)
|
|
print(f"[export] checkpoint dims obs={obs_dim} act={act_dim} hidden={hidden}")
|
|
|
|
|
|
def _export_onnx(runner, out: Path) -> None:
|
|
"""Write an actor graph with one observation input and one action output.
|
|
|
|
A tracking runner may export additional trajectory inputs and outputs.
|
|
Fall back to the base runner when the exported graph adds an input.
|
|
"""
|
|
runner.export_policy_to_onnx(str(out), "actor.onnx")
|
|
path = out / "actor.onnx"
|
|
|
|
import onnx
|
|
|
|
inputs = [i.name for i in onnx.load(str(path)).graph.input]
|
|
if len(inputs) == 1:
|
|
return
|
|
|
|
base = next(
|
|
(c.export_policy_to_onnx for c in type(runner).__mro__[1:]
|
|
if "export_policy_to_onnx" in c.__dict__),
|
|
None,
|
|
)
|
|
if base is None:
|
|
raise SystemExit(
|
|
f"[export] {type(runner).__name__} exported a graph taking {inputs}, and "
|
|
f"the deployment feeds exactly one tensor.\n"
|
|
f" No base class defines another export to fall back to."
|
|
)
|
|
print(f"[export] {type(runner).__name__} exported a graph taking {inputs}; the "
|
|
f"deployment feeds one tensor, so re-exporting with "
|
|
f"{base.__qualname__.split('.')[0]}'s. The reference travels as a "
|
|
f"trajectory file instead of inside the graph.")
|
|
base(runner, str(out), "actor.onnx")
|
|
|
|
|
|
def _validate_onnx(onnx_path, contract, policy, device):
|
|
"""The ONNX must match the torch policy in both shape **and** value.
|
|
|
|
Matching shapes do not imply matching values (whether the normaliser was baked
|
|
in, whether the mean rather than a sample was taken), so the same observations
|
|
are fed to both and compared element-wise. Without onnxruntime only shapes can
|
|
be checked, and that is stated explicitly.
|
|
"""
|
|
import onnx
|
|
import onnxruntime as ort
|
|
|
|
from mjrl.platform.graph import read_embedded_onnx
|
|
|
|
data, m = read_embedded_onnx(onnx_path)
|
|
if len(m.graph.input) != 1 or len(m.graph.output) != 1:
|
|
raise SystemExit("[export] ONNX needs exactly one observation input and action output")
|
|
shapes = []
|
|
for kind, node, width in (
|
|
("input", m.graph.input[0], contract["observation"]["dim"]),
|
|
("output", m.graph.output[0], contract["action"]["dim"]),
|
|
):
|
|
tensor = node.type.tensor_type
|
|
shape = [d.dim_value if d.HasField("dim_value") else d.dim_param or None
|
|
for d in tensor.shape.dim]
|
|
if (tensor.elem_type != onnx.TensorProto.FLOAT or len(shape) != 2
|
|
or isinstance(shape[0], int) and shape[0] != 1 or shape[-1] != width):
|
|
raise SystemExit(f"[export] ONNX {kind} must be float32 [1,{width}], got {shape}")
|
|
shapes.append(shape)
|
|
ishape, oshape = shapes
|
|
print(f"[export] ONNX shape {ishape} -> {oshape}")
|
|
options = ort.SessionOptions()
|
|
options.intra_op_num_threads = 1
|
|
options.inter_op_num_threads = 1
|
|
sess = ort.InferenceSession(data, sess_options=options, providers=["CPUExecutionProvider"])
|
|
iname = sess.get_inputs()[0].name
|
|
rng = np.random.default_rng(0)
|
|
worst = 0.0
|
|
for _ in range(8):
|
|
x = rng.normal(0, 1, size=(1, contract["observation"]["dim"])).astype(np.float32)
|
|
y_onnx = sess.run(None, {iname: x})[0]
|
|
with torch.inference_mode():
|
|
td = _as_policy_input(policy, torch.from_numpy(x).to(device))
|
|
y_torch = policy(td).cpu().numpy()
|
|
if y_onnx.shape != y_torch.shape or not np.isfinite(y_onnx).all() or not np.isfinite(y_torch).all():
|
|
raise SystemExit("[export] ONNX or torch produced invalid action values")
|
|
worst = max(worst, float(np.abs(y_onnx - y_torch).max()))
|
|
if worst > 1e-4:
|
|
raise SystemExit(f"[export] ONNX and torch policy outputs disagree, "
|
|
f"max difference {worst:.3e}")
|
|
print(f"[export] ONNX values max difference from the torch policy {worst:.2e} "
|
|
f"over 8 random observations")
|
|
|
|
|
|
def _as_policy_input(policy, x):
|
|
"""Wrap a flat observation tensor into the form the policy expects."""
|
|
from tensordict import TensorDict
|
|
|
|
groups = getattr(policy, "obs_groups", None) or ["actor"]
|
|
return TensorDict({g: x for g in groups}, batch_size=[x.shape[0]], device=x.device)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# README
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def write_readme(path: Path, contract: dict, task: str, ckpt: str) -> None:
|
|
o = contract["observation"]
|
|
lines = [
|
|
f"# Deployment bundle: {task}",
|
|
"",
|
|
f"Source checkpoint: `{ckpt}`",
|
|
"",
|
|
"Generated from a live environment by `scripts/export.py`.",
|
|
"**Do not edit by hand** -- re-export whenever the config changes; values",
|
|
"edited by hand always drift.",
|
|
"",
|
|
"## Files",
|
|
"",
|
|
"| File | Contents |",
|
|
"|---|---|",
|
|
f"| `actor.onnx` | `[1, {o['dim']}] -> [1, {contract['action']['dim']}]`; "
|
|
f"takes raw observations, returns the mean action |",
|
|
"| `layout.json` | The deployment contract. Read every number below from "
|
|
"it rather than copying them into code |",
|
|
"",
|
|
"## Assembling the observation",
|
|
"",
|
|
f"Length {o['dim']}, concatenated in the order below. Joint observations use "
|
|
"the term's joint order when present, or `obs_joint_order`.",
|
|
"",
|
|
"| Range | Term | Dim |",
|
|
"|---|---|---|",
|
|
]
|
|
for t in o["terms"]:
|
|
lines.append(f"| `[{t['offset']}:{t['offset'] + t['dim']}]` | {t['name']} | {t['dim']} |")
|
|
strided = [t for t in o["terms"] if t.get("history_stride", 1) != 1]
|
|
if strided:
|
|
lines += [
|
|
"",
|
|
"### The history is strided -- read this before building it",
|
|
"",
|
|
"These terms stack several frames, and **the frames are not "
|
|
"consecutive control steps**:",
|
|
"",
|
|
"| Term | Frames | Stride | Spans |",
|
|
"|---|---|---|---|",
|
|
]
|
|
hz = contract["control"]["control_hz"]
|
|
for t in strided:
|
|
n, k = t["history_length"], t["history_stride"]
|
|
lines.append(
|
|
f"| {t['name']} | {n} | {k} | {(n - 1) * k / hz * 1000:.0f} ms "
|
|
f"({(n - 1) * k} control steps) |"
|
|
)
|
|
lines += [
|
|
"",
|
|
"Emit frames from oldest to newest, ending at the current observation. "
|
|
"Keep `stride * (frames - 1) + 1` frames and select every `stride`-th "
|
|
"frame. Using consecutive frames would preserve the input width "
|
|
"while changing its time span.",
|
|
]
|
|
lines += [
|
|
"",
|
|
f"Observation normalisation is already inside the ONNX "
|
|
f"(`normalization = {o['normalization']}`), so **feed raw values and do not "
|
|
f"normalise again**.",
|
|
"",
|
|
"## Applying the action",
|
|
"",
|
|
]
|
|
if contract["_schema"] == NATIVE_LAYOUT_SCHEMA:
|
|
lines += [
|
|
"In `action_joint_order`, multiply each output by its named `action_scale` "
|
|
"and add the named target's `offset`. Apply the target's optional `clip`.",
|
|
"Use the resulting position or velocity target according to `action.targets`.",
|
|
"Replay uses `robot.xml` and `controller.json` from this export.",
|
|
"",
|
|
]
|
|
else:
|
|
lines += [
|
|
"```",
|
|
"for i, joint in enumerate(contract['action_joint_order']):",
|
|
" target[joint] = action[i] * action_scale + default_joint_pos[joint]",
|
|
"```",
|
|
"",
|
|
]
|
|
lines += [
|
|
f"Then track with PD: kp = {contract['control']['kp']}, "
|
|
f"kd = {contract['control']['kd']}, "
|
|
f"effort limit {contract['control']['effort_limit']} in joint units, "
|
|
f"control rate {contract['control']['control_hz']} Hz.",
|
|
"",
|
|
"Joints in `unactuated_joints` are not policy-controlled and **must be locked "
|
|
"at the given positions**: they take part in collision, and freeing them "
|
|
"changes the foot contact geometry.",
|
|
"",
|
|
"## Read before deploying",
|
|
"",
|
|
]
|
|
for n in contract["deploy_notes"]:
|
|
lines.append(f"- {n}")
|
|
lines.append("")
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def main() -> None:
|
|
parser = build_parser("export.py", __doc__ or "")
|
|
g = parser.add_argument_group("export")
|
|
g.add_argument("--checkpoint", help="model_*.pt to export")
|
|
g.add_argument("--native-layout", action="store_true",
|
|
help="export a native replay layout and the live scene's MuJoCo model")
|
|
g.add_argument(
|
|
"--out", default=None,
|
|
help="output directory; defaults to tasks/<task path>/out/<date-time>/",
|
|
)
|
|
# Custom task hooks may produce optional media after the core export.
|
|
g.add_argument(
|
|
"--video", action=argparse.BooleanOptionalAction, default=True,
|
|
help="write the task's extra artifacts (rendered video, audio) when it has "
|
|
"any; --no-video writes only the policy and its contract",
|
|
)
|
|
# The selected task may take arguments of its own; see
|
|
# `tasks.load_cli_args`. Nothing here knows which, or how many.
|
|
args = parse_with_task_args(parser)
|
|
|
|
if args.list:
|
|
print_task_table()
|
|
return
|
|
|
|
spec, res, asset = resolve_all(args)
|
|
if args.dry_run:
|
|
print(f"[mjrl] resolved: task={spec.id} -- --dry-run, stopping here")
|
|
return
|
|
if not args.checkpoint:
|
|
raise SystemExit("[export] --checkpoint is required")
|
|
|
|
ckpt = Path(args.checkpoint).resolve()
|
|
if not ckpt.is_file():
|
|
raise SystemExit(f"[export] checkpoint does not exist: {ckpt}")
|
|
|
|
from mjrl.backend.select import use_backend
|
|
from mjrl.backend.warp_allocation import FILENAME as WARP_ALLOCATION_FILENAME
|
|
from mjrl.backend.warp_allocation import apply_warp_allocation
|
|
|
|
import tasks
|
|
|
|
# The backend has to be registered before the environment is built; see the
|
|
# note in scripts/train.py.
|
|
use_backend(res)
|
|
|
|
from mjlab.envs import ManagerBasedRlEnv
|
|
from mjlab.rl import RslRlVecEnvWrapper
|
|
from mjlab.rl.runner import MjlabOnPolicyRunner
|
|
|
|
from tasks.paths import out_dir_for
|
|
|
|
# `tasks/<task>/out/<date-time>/` -- beside the task it came from, and
|
|
# tracked, unlike `logs/`. See `tasks.paths.out_dir_for`.
|
|
#
|
|
# Timestamped rather than named after the checkpoint. Two exports of one
|
|
# checkpoint are two different things whenever anything between the config
|
|
# and this script has changed, and the old name meant the second silently
|
|
# replaced the first. The same `%Y-%m-%d_%H-%M-%S` as `logs/` and
|
|
# `out/bundle_*`, so there is one way to read a directory name here.
|
|
#
|
|
# Which checkpoint it came from is recorded *inside*, in `layout.json`,
|
|
# because a name is not a place to keep provenance: it cannot hold the run
|
|
# as well as the iteration, and nothing can check it.
|
|
out = Path(args.out) if args.out else (
|
|
out_dir_for(spec.id) / f"{datetime.now():%Y-%m-%d_%H-%M-%S}"
|
|
)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
|
|
# One environment is enough: the contract describes the robot, not a batch.
|
|
env_cfg = tasks.load_env_cfg(
|
|
spec.id, asset=asset, play=True, task_args=args.task_args
|
|
)
|
|
apply_warp_allocation(env_cfg.sim, args, res.backend, ckpt)
|
|
env_cfg.scene.num_envs = 1
|
|
checkpoint_state = None
|
|
command_evidence = None
|
|
if spec.robot is not None:
|
|
from mjrl.replay import configure_velocity_replay
|
|
|
|
checkpoint_state = torch.load(ckpt, map_location="cpu", weights_only=False)
|
|
training_cfg = tasks.load_env_cfg(spec.id, asset=asset, task_args=args.task_args)
|
|
command_evidence = configure_velocity_replay(env_cfg, training_cfg, checkpoint_state)
|
|
agent_cfg = tasks.load_agent_cfg(spec.id)
|
|
env = ManagerBasedRlEnv(cfg=env_cfg, device=res.device)
|
|
try:
|
|
wrapped = RslRlVecEnvWrapper(env, clip_actions=agent_cfg.clip_actions)
|
|
runner_cls = tasks.load_runner_cls(spec.id) or MjlabOnPolicyRunner
|
|
runner = runner_cls(wrapped, asdict(agent_cfg), device=res.device)
|
|
runner.load(
|
|
str(ckpt), load_cfg={"actor": True}, strict=True, map_location=res.device
|
|
)
|
|
policy = runner.get_inference_policy(device=res.device)
|
|
|
|
print()
|
|
contract = build_contract(env, env_cfg, agent_cfg, checkpoint=ckpt,
|
|
native=args.native_layout)
|
|
if command_evidence is not None:
|
|
contract["_command_evidence"] = command_evidence
|
|
# After the contract, because it needs the wire order the contract
|
|
# resolved; before validation, because the block is part of what is
|
|
# validated.
|
|
contract.update(_reference(env_cfg, out, contract["wire_joint_order"]))
|
|
|
|
_validate_shapes(contract)
|
|
native = contract["_schema"] == NATIVE_LAYOUT_SCHEMA
|
|
if native:
|
|
_validate_native(contract, env)
|
|
else:
|
|
_validate_deployable(contract)
|
|
_validate_measurable(contract)
|
|
_validate_home_pose(contract, env_cfg)
|
|
_validate_checkpoint(contract, ckpt, checkpoint_state)
|
|
|
|
_export_onnx(runner, out)
|
|
_validate_onnx(out / "actor.onnx", contract, policy, res.device)
|
|
if native:
|
|
from mjrl.native_export import write_native_model
|
|
|
|
write_native_model(env, out, contract)
|
|
finally:
|
|
env.close()
|
|
|
|
# The layout records which contract the export uses.
|
|
(out / "layout.json").write_text(
|
|
json.dumps(contract, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
write_readme(out / "README.md", contract, spec.id, str(ckpt))
|
|
|
|
# **The checkpoint travels with the bundle.**
|
|
#
|
|
# A bundle is otherwise a one-way artifact: `layout.json` records which file
|
|
# it came from, and a month later that path is a run somebody deleted or a
|
|
# config that has moved underneath it. Both of the things anyone wants to do
|
|
# with a bundle afterwards need the weights -- comparing it against the policy
|
|
# it claims to be (`play.py --checkpoint <this directory>` replays the copy), and exporting
|
|
# it again after the contract gains a field, which is not hypothetical: this
|
|
# week a term was missing `neutral_height` and every bundle written before
|
|
# then had to be redone.
|
|
#
|
|
# Re-exporting from the *repository* is the trap that makes this worth the
|
|
# 10 MB. The observation width does not change when a gait constant does, so
|
|
# an old checkpoint re-exported against today's config produces a contract
|
|
# that loads, runs, and describes a cadence the policy never trained under.
|
|
# Keeping the weights beside the contract at least makes "which policy is
|
|
# this" answerable without git archaeology.
|
|
#
|
|
# Copied rather than linked: a bundle is a thing to send somewhere. Kept
|
|
# under its own name rather than a fixed one, so `CHECKPOINT_GLOB` finds
|
|
# it -- which makes the bundle directory something `--checkpoint` and
|
|
# `train --resume` can be pointed at directly.
|
|
shutil.copy2(ckpt, out / ckpt.name)
|
|
allocation_record = ckpt.parent / WARP_ALLOCATION_FILENAME
|
|
if allocation_record.is_file():
|
|
shutil.copy2(allocation_record, out / allocation_record.name)
|
|
|
|
print(f"\n[export] done -> {out}")
|
|
for f in sorted(out.iterdir()):
|
|
print(f" {f.name:<16}{f.stat().st_size / 1024:8.1f} KB")
|
|
|
|
# Extra artifacts, if this task has any. Deliberately after the bundle is
|
|
# complete and outside the env's lifetime: this hook builds its own
|
|
# environment (it needs a full-length episode, which the contract env is not),
|
|
# and a failure here must not leave a half-written actor.onnx behind.
|
|
#
|
|
# This file knows only that the hook may exist -- never what it writes. The
|
|
# rule that `scripts/` does not change when a task is added still holds; what
|
|
# changed once, here, is that there is now a place for a task to put artifacts
|
|
# that are not part of the deployment contract. See tasks.load_export_media.
|
|
if args.video:
|
|
extra = tasks.load_export_media(spec.id)
|
|
if extra is not None:
|
|
extra(out, task_id=spec.id, checkpoint=ckpt, asset=asset, res=res)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|