commit 7e1ff58b4665cbde07cae528ce7009cee3be40bb Author: cen617-code <1057290604@qq.com> Date: Tue Nov 3 19:46:08 2026 +0800 Initialize portable humanoid RL framework with licenses, locks and CI diff --git a/.agents/skills/controls/SKILL.md b/.agents/skills/controls/SKILL.md new file mode 100644 index 0000000..812976f --- /dev/null +++ b/.agents/skills/controls/SKILL.md @@ -0,0 +1,32 @@ +--- +name: controls +description: Inspect simulation observation/action contracts, joint ordering, PD/effort settings and measured joint replay without enabling physical robot control. +--- + +# Inspect controls and measured replay + +This manual concerns **simulation/inference controls**, not a ROS/DDS or motor-bus implementation. +Read [Robot checks](../../../docs/NEW_ROBOT.md), [Training operations](../../../docs/TRAINING_OPERATIONS.md) +and [Deployment contract](../../../docs/DEPLOYMENT.md). + +## Procedure + +1. Re-run `./scripts/uh check-robot --config FILE --output NEW_REPORT` after changes to joints, + gains, torque limits, sensors, reset pose or model assets. Inspect resolved order, not regex order. +2. Distinguish observation joint order from action order; check frames, radians/rad/s, action + scaling/offset/clipping, physics dt, policy dt and normalizer ownership. Never normalize twice. +3. Record `./scripts/uh play --checkpoint CHECKPOINT --measure FILE.csv --steps 1000 --profile stress`. + Use `--video FILE.mp4` for visual context. Repeat randomized clean-observation evaluation to + separate observation-noise effects. Interactive `play --viewer native|viser` is a different mode. +4. Inspect joint positions, velocities, projected actuator torque, raw/applied actions, targets and + terminal flags. Measurements occur after each control step BEFORE reset; torque is an endpoint + actuator contribution, not a desired torque, substep peak or physical sensor measurement. +5. Look for saturation, large action oscillations, limit proximity, sliding and collapsed posture. + State the evidence; do not silently change gain/clip settings to make the replay appear better. + +## Boundaries and handoff + +A sensor named IMU does not validate a real estimator. Simulated PD gains are not a hardware tuning +recipe. There is no joystick-to-motor path, bus mapping, E-stop or autonomous robot operation in this +repository. Provide contract/check/report/CSV/video paths and delegate actual hardware integration +only after explicit operator authorization and a separately designed safety system. diff --git a/.agents/skills/deploy/SKILL.md b/.agents/skills/deploy/SKILL.md new file mode 100644 index 0000000..732bacf --- /dev/null +++ b/.agents/skills/deploy/SKILL.md @@ -0,0 +1,32 @@ +--- +name: deploy +description: Export and validate research-only ONNX inference bundles with matching evaluation provenance, numerical parity and explicit hardware exclusions. +--- + +# Package a policy for research inference + +Read [Deployment contract](../../../docs/DEPLOYMENT.md). Complete train/controls evidence first. +This is not a hardware deployment skill: no motor enable, robot transport or firmware operation. + +## Procedure + +1. Select a committed checkpoint from a completed run. Inspect its training curves, conformance + report, fixed-protocol evaluation, censoring, CSV and video. Export can package a poor policy; + it does not certify quality. +2. Finish baseline evaluation/export before editing source. Restore historical sources/dependencies + in a separate trusted environment when needed; do not bypass provenance checks. +3. Run `./scripts/uh export --checkpoint CHECKPOINT --evaluation REPORT --output NEW_BUNDLE`. + The evaluator must match checkpoint, experiment and source identity. ONNX parity uses actual + rollout observations, not just random input vectors. +4. Run `./scripts/uh verify-bundle BUNDLE --inference`. Inspect observation fields, normalizer, + joint order, action scale/offset/clip, PD metadata, units and control timing. +5. Review asset/code licenses and manifests for private paths before publishing artifacts. + Source archives are trusted restoration material, not automatically executable downloads. + +## Failure handling and handoff + +Hash mismatch, source drift, unsupported action schema or failed parity is a blocker. Do not +edit hashes or tolerance to turn an unknown failure into success. Hand over bundle/report paths, +measured parity, training/evaluation limitations and the fact that `hardware_approved` remains +false. Physical use additionally needs validated estimation, transport, joint mapping, watchdog, +limits, emergency stop and operator-approved testing; those are absent here. diff --git a/.agents/skills/new-task/SKILL.md b/.agents/skills/new-task/SKILL.md new file mode 100644 index 0000000..b39b5ec --- /dev/null +++ b/.agents/skills/new-task/SKILL.md @@ -0,0 +1,39 @@ +--- +name: new-task +description: Add a robot or task with explicit mechanics, independent conformance declarations, negative tests and bounded simulation acceptance. +--- + +# Add a robot or task + +Read [Robot checks](../../../docs/NEW_ROBOT.md) and [Architecture](../../../docs/ARCHITECTURE.md). +Use setup-env first if runtime origins or CUDA are unverified. + +## Procedure + +1. Identify the exact model variant and asset license. Run `./scripts/uh inspect-asset PATH`. + Compilation does not validate sensor frames, contact semantics or control assumptions. +2. Implement independent robot/environment/PPO factories. Establish foot sites, IMU placement, + collision filters, gains/effort limits, joint order and nominal posture from reviewed sources. + Do not derive motor ratings from names or copy another robot's gains without justification. +3. Register the task in `src/universal_humanoid/registry.py`; create a bounded experiment JSON. + Keep the CLI robot-independent and do not change an existing task ID's mechanics silently. +4. Add `src/universal_humanoid/robot_specs/.json` with independently reviewed expected values. + It is a contract, not an auto-blessed dump of the current model. +5. Run `./scripts/uh check-robot --config FILE --output NEW_REPORT` and read every failed check ID. + Add negative tests for a missing site, wrong IMU frame, bad PD gain, swapped order and lost contact. +6. Run `./scripts/check`, zero-action smoke, short PPO, randomized/stress evaluation, measured + replay and export parity. Record all output paths and failures. + +## Boundaries + +Current conformance/export assumes flat-ground velocity tracking and one position actuator per +controlled hinge. Extend and version the contract for coupled drives or new task semantics; +do not disable checks to fit a different robot. A passing check means adapter conformance, not +walking ability, mechanical accuracy or hardware approval. + +## Handoff + +Supply factories, registry/spec/experiment paths, asset attribution, conformance report, negative +tests and measured pipeline evidence. State which properties remain unvalidated. Changes to +vendored code additionally require the vendor-review procedure; routine new adapters should live +in the first-party package. diff --git a/.agents/skills/rl-engineer/SKILL.md b/.agents/skills/rl-engineer/SKILL.md new file mode 100644 index 0000000..e0b55c9 --- /dev/null +++ b/.agents/skills/rl-engineer/SKILL.md @@ -0,0 +1,105 @@ +--- +name: rl-engineer +description: Engineer humanoid reward revisions, bounded mjlab training, measured evaluation, replay and research-only ONNX packaging in Universal Humanoid. +--- + +# RL engineer + +## 1. Establish a contract + +Use this as the cross-stage coordinator; consult setup-env, new-task, train, controls and deploy +through the manual index in `AGENTS.md` for stage-specific work. + +Read `AGENTS.md`, `README.md`, and the target experiment. Confirm the robot/task, desired +behavior, resource budget and acceptance metrics. A request to build architecture authorizes +short smoke validation, not days of training or physical robot operation. Use the current coding +agent directly; do not launch subagents without operator authorization. + +Run `./scripts/uh doctor` and `list`. Inspect the factory's source and relevant mjlab reward +functions before editing. Follow `docs/SETUP.md` to verify the installed vendored backends, +locked dependencies and bundled assets. Use `verify-vendor` to check the source inventory. +Do not install a second mjlab/rsl-rl-lib distribution. Record actual dependency provenance. + +## 2. Baseline and reward hypothesis + +Copy a baseline experiment into a NEW JSON file with an explicit task and seed. Write the +hypothesis next to the experiment or in a run note. Inspect `uh rewards --config FILE`. + +- Weights/parameters: edit JSON; misspelled term names must fail. +- New mechanisms: add vectorized functions to `src/universal_humanoid/rewards.py` and the + `FUNCTIONS` allowlist; add tests for shape `[num_envs]`, finiteness, sign and limiting cases. +- Do not change observations/control/mechanics while claiming a reward-only comparison. +- Never `eval` code supplied in experiment JSON. + +Run baseline evaluation/export before editing Python. Source hashes are strict; archived sources +may be restored only into a separate trusted workspace, never over a shared reference checkout. + +## 3. Cheap checks before expensive runs + +Run `./scripts/check`, `uh plan`, `uh rewards`, and `uh check-robot --config FILE --output NEW_REPORT`, +then `uh smoke` with 4 environments and 16 steps. +For each new robot, verify free root, DoF, mass/inertia, mesh resolution, contacts, foot sites, +IMU frame, torque limits, gains, action ordering, reset posture and observation terms. Model +compilation alone cannot validate these semantics. Zero-action success is not balance training. + +## 4. Bounded training + +Use `uh train --config FILE --output NEW_DIRECTORY`, or `pipeline` for a smoke-sized end-to-end +run. Read `docs/TRAINING_OPERATIONS.md` for lifecycle and statistics semantics. Use +`train --background` for a requested long-running Linux job, `status RUN` to query progress, +`checkpoints RUN` to inspect committed checkpoints, and `stop RUN` for a guarded graceful stop. +Resume with `train --resume CHECKPOINT --iterations ADDITIONAL --output NEW_DIRECTORY`; +optimizer continuation is not bitwise simulator/RNG replay. Never guess checkpoints by mtime. + +Keep TensorBoard local; no W&B authentication or automatic upload. Use `curves RUN --output NEW_DIR` +to inspect tracking, action distribution std and episode-length trends DURING training. Report +missing tags, non-finite samples and short curves. Rising total reward alone is not learning. +Inspect `run.json`, `progress.json`, `resolved_env.yaml`, `resolved_agent.json` and `sources.zip`. + +Long runs require an explicit operator-approved budget. Do not automatically increase parallelism +past measured VRAM capacity. This backend requires CUDA; do not pretend Warp CPU is fast CPU RL. + +## 5. Evaluate and replay + +Use `benchmark --checkpoints ... --seeds ... --profiles randomized stress --output NEW_DIR` +for stand `(0,0,0)`, forward `(0.3,0,0)`, lateral `(0,0.2,0)`, turn `(0,0,0.3)`. Keep identical +seeds, profiles, world counts and horizons for candidates. These are evaluation seeds, NOT +independent training seeds; separately train multiple seeds when investigating training variance. + +Default evaluation randomizes each world's reset state. `stress` also enables actor observation +noise and per-world interval velocity kicks; inspect actual initial states and push counts. +`nominal` intentionally repeats fixed initial states. Startup mass/friction randomization is not +part of these profiles. None constitutes full sim-to-real stress coverage. + +Fall/time-limit terminations remain active; curriculum is disabled. Report RMSE, completed-only +failure fraction, failure events per simulated world-second, first-episode failures AND censoring. +No completed episodes means completed-only means/fractions are null. Variance is across seed-level +reports, not repeated worlds, and is null for fewer than two values. Prefer horizons longer than +the episode duration. `compare` supports paired matrices and refuses protocol mismatches. +Use `select-checkpoint` with an explicit metric, inspect trade-offs, then evaluate held-out seeds; +never equate a selection-set winner with generalization or automatically promote it to hardware. + +Record MP4 with `evaluate --video FILE`. Use `play --measure FILE.csv --steps N` for bounded +headless joint/velocity/actuator-torque measurement before terminal-world reset, or +`play --viewer viser` for interactive inspection. CSV samples are policy-rate endpoints, not +substep torque peaks. Refer to the CSV sidecar for units and completion status. +Look for reward hacking: standing instead of tracking, sliding, knee collapse, asymmetric contacts, +large actions, unrealistic high-frequency gait. Videos are complementary to metrics, not proof. + +## 6. Package, never auto-enable hardware + +`uh export --checkpoint ... --evaluation ... --output NEW_DIRECTORY` checks report identity and +ONNX parity on real observations. `uh verify-bundle DIRECTORY --inference` rechecks checksums +and the saved reference. Output is a research bundle even when rollout succeeds. Inspect the +resolved action joint order, offsets/scales, observation ordering and simulation motor gains. + +Physical deployment requires a separate validated adapter for transport, estimator, bus order, +calibration, bounds, timeout watchdog, torque ramp, E-stop and operator authorization. There is +no motor-enable command in this repository. Never change `hardware_approved` to bypass this. + +## 7. Handoff + +Report checkpoint, backend, env/iteration counts, tracking/std/episode curves, evaluation matrix, +CSV/video paths and measured evidence—not only wall time or command exit codes. State whether learning was +actually demonstrated. List limitations, failed attempts and next validation requirements. Avoid +claiming general morphology support from the fact that two task adapters share a CLI. diff --git a/.agents/skills/setup-env/SKILL.md b/.agents/skills/setup-env/SKILL.md new file mode 100644 index 0000000..0cad449 --- /dev/null +++ b/.agents/skills/setup-env/SKILL.md @@ -0,0 +1,37 @@ +--- +name: setup-env +description: Detect the host, install the locked vendored runtime in a fresh environment, and verify dependencies, backend origins, CUDA and robot gates. +--- + +# Set up and verify an environment + +Use for first installation, another machine, missing imports, wrong backend, or CUDA failures. +Read [Setup](../../../docs/SETUP.md) and [Vendor provenance](../../../docs/VENDOR.md). + +## Procedure + +1. Run `python3 .agents/skills/setup-env/scripts/detect.py`. Record Python/platform, GPU/driver, + installer availability and existing import origins. Detection is not a simulation pass. +2. Confirm Linux x86-64 and compatible NVIDIA CUDA support before choosing the reference lock. + Do not substitute CPU training or silently change Torch/Warp versions. +3. Run `uv sync --locked --extra dev --python 3.13` in a fresh project environment. The ordinary + `pip install -e .` path and hashed pip alternative are documented in Setup. Never install + separate `mjlab`/`rsl-rl-lib` copies into this environment or repair another project's venv. +4. Run `./scripts/uh doctor`, `./scripts/uh verify-vendor`, and `./scripts/check`. +5. Run `.venv/bin/python .agents/skills/setup-env/scripts/check.py --config experiments/g1_smoke.json --output runs/setup-g1`. + Use the T1 smoke config for T1. Read every gate log and `summary.json`, not just a GPU name. + +## Failure handling + +- A source mismatch means wrong installation or an unreviewed vendor change. Inspect it; do not + refresh hashes or unset errors to manufacture success. +- A locked wheel download can be slow. Report installation as pending until the installer exits + successfully and dependency checks pass. +- A robot check failure belongs to the new-task procedure; changing rewards cannot repair an IMU frame. +- An unavailable GPU/driver is an external prerequisite, not permission to claim CPU equivalence. + +## Handoff + +Report interpreter, installed backend origins, lock/inventory status, GPU/driver, gate output +paths and remaining failures. Only then authorize a bounded training smoke within the operator's +budget. No credential collection, remote installation or hardware operations are part of setup. diff --git a/.agents/skills/setup-env/scripts/check.py b/.agents/skills/setup-env/scripts/check.py new file mode 100644 index 0000000..878e929 --- /dev/null +++ b/.agents/skills/setup-env/scripts/check.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Installed-environment gates with durable command logs; never installs or repairs packages.""" + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--config", type=Path, help="Also check this robot adapter and run a bounded GPU smoke" + ) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + base = [sys.executable, "-m", "universal_humanoid"] + commands = [("vendor", [*base, "verify-vendor"]), ("runtime", [*base, "doctor"])] + if shutil.which("uv"): + commands.append(("dependencies", ["uv", "pip", "check", "--python", sys.executable])) + else: + commands.append(("dependencies", [sys.executable, "-m", "pip", "check"])) + if args.config: + commands += [ + ( + "robot", + [ + *base, + "check-robot", + "--config", + str(args.config), + "--output", + str(args.output / "robot.json"), + ], + ), + ( + "smoke", + [*base, "smoke", "--config", str(args.config), "--steps", "8", "--num-envs", "4"], + ), + ] + results = [] + for name, command in commands: + with (args.output / f"{name}.log").open("x") as stream: + result = subprocess.run(command, stdout=stream, stderr=subprocess.STDOUT, check=False) + results.append({"gate": name, "command": command, "returncode": result.returncode}) + report = {"ready": all(r["returncode"] == 0 for r in results), "gates": results} + (args.output / "summary.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + raise SystemExit(0 if report["ready"] else 1) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/setup-env/scripts/detect.py b/.agents/skills/setup-env/scripts/detect.py new file mode 100644 index 0000000..6ab9dba --- /dev/null +++ b/.agents/skills/setup-env/scripts/detect.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Standard-library-only host report; safe to run before installing the project.""" + +import argparse +import importlib.util +import json +import platform +import shutil +import subprocess +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + nvidia = shutil.which("nvidia-smi") + gpu = ( + subprocess.run( + [nvidia, "--query-gpu=name,driver_version,memory.total", "--format=csv,noheader"], + capture_output=True, + text=True, + timeout=15, + ) + if nvidia + else None + ) + origins = {} + for name in ("universal_humanoid", "mjlab", "rsl_rl", "torch"): + spec = importlib.util.find_spec(name) + origins[name] = spec.origin if spec else None + report = { + "python": sys.executable, + "version": platform.python_version(), + "platform": platform.platform(), + "machine": platform.machine(), + "uv": shutil.which("uv"), + "nvidia_smi": nvidia, + "gpu_query": gpu.stdout.strip() if gpu else None, + "gpu_query_error": gpu.stderr.strip() if gpu and gpu.returncode else None, + "import_origins": origins, + "note": ( + "Discovery only, not a dependency/CUDA/simulation pass. " + "Run setup-env check.py after installation." + ), + } + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/train/SKILL.md b/.agents/skills/train/SKILL.md new file mode 100644 index 0000000..8c298d0 --- /dev/null +++ b/.agents/skills/train/SKILL.md @@ -0,0 +1,39 @@ +--- +name: train +description: Operate bounded PPO experiments, inspect learning curves, resume committed checkpoints and compare measured multi-seed evaluations rather than elapsed time. +--- + +# Train and measure a policy + +A finished run is not a learned policy. Read [Training operations](../../../docs/TRAINING_OPERATIONS.md) +and the reward-experiment sections of [rl-engineer](../rl-engineer/SKILL.md). + +## Procedure + +1. Confirm setup-env and the target robot gate passed. Agree on task, resource budget and acceptance + metrics. Measure GPU capacity before increasing worlds; do not launch the long default automatically. +2. Inspect `./scripts/uh plan --config FILE` and `./scripts/uh rewards --config FILE`; make a named + reward hypothesis without silently changing observations, controls or robot mechanics. +3. Smoke first, then `./scripts/uh train --config FILE --output NEW_RUN --background` when requested. + Query `./scripts/uh status RUN` and read `./scripts/uh curves RUN --output NEW_ANALYSIS` during training. + Inspect tracking, action std, episode length and losses together. Report missing or short curves. +4. Use `./scripts/uh checkpoints RUN`; only committed manifest entries are eligible. Stop with + `./scripts/uh stop RUN`. Resume explicitly with `train --resume CHECKPOINT --iterations ADDITIONAL` + and a new output directory. Never rewrite a failed run to completed or promise bitwise replay. +5. Run `./scripts/uh benchmark --checkpoints ... --seeds ... --profiles randomized stress --output NEW_DIR`. + Evaluate standing/forward/lateral/turning, keep protocols identical, and inspect censoring and actual pushes. +6. Use explicit-metric `select-checkpoint` and held-out evaluation seeds, then the controls manual + for measured joint replay and deploy for research packaging. + +## Interpret evidence + +Rising total reward with flat tracking can indicate reward exploitation. More time or updates do +not guarantee improvement. Evaluation seeds are not independent training replicas; a training-seed +study needs separately trained policies. Sample variance with one seed is undefined, not zero. +Compare task metrics, not raw totals from different reward functions. + +## Handoff + +Report checkpoint/backend/env/update counts, tracking/std/episode curves, evaluation table, +failure-rate definitions, CSV/video and remaining limitations. Do not declare convergence from a +smoke run or approve physical motion from an ONNX file. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e3df9a1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +* text=auto eol=lf + +# Vendor inventories fingerprint exact upstream bytes, including original line endings. +rl/** -text +*.stl binary +*.STL binary + +# A blank context line in an archived unified diff is a literal space; preserve it. +rl/upstream/*.patch -whitespace diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..47696ca --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,46 @@ +name: Core and layout checks + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + core: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1' + steps: + # Immutable revisions also used by the Jumper reference workflow. + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.13' + - name: Install pinned resolver + run: python -m pip install uv==0.13.1 + - name: Verify dependency lock and exports + run: | + uv lock --check + uv export --locked --extra dev --no-emit-project --format requirements-txt --output-file requirements-linux-cu130.lock + uv export --locked --only-group ci --no-emit-project --format requirements-txt --output-file requirements-ci.lock + git diff --exit-code -- uv.lock requirements-linux-cu130.lock requirements-ci.lock + # Core/static tests only: this group installs no project, Torch or CUDA stack. + - name: Install locked check and build tools + run: | + uv sync --locked --only-group ci --python 3.13 + uv pip install --python .venv/bin/python --no-deps --no-build-isolation -e . + - name: Check vendor inventory and wheel resources + run: | + .venv/bin/python tools/vendor_manifest.py + .venv/bin/python -m build --wheel --no-isolation + .venv/bin/python tools/check_wheel.py dist/universal_humanoid-*.whl + - name: Lint and format + run: | + .venv/bin/python -m ruff check src tests tools .agents/skills/*/scripts + .venv/bin/python -m ruff format --check src tests tools .agents/skills/*/scripts + - name: Core and layout tests + run: .venv/bin/python -m pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3ad261 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +.venv/ +.venv-*/ +build/ +dist/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +runs/ +bundles/ +analysis/ +*.job/ +*.mp4 +*.pt +*.onnx +.env +.env.* +!.env.example +*.pem +*.key +logs/ +out/ +.coverage +htmlcov/ +.DS_Store +MUJOCO_LOG.TXT diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b734719 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,59 @@ +# Agent engineering contract + +Keep project changes within this repository. Treat external dependencies and robot asset +checkouts as read-only unless the operator explicitly authorizes changes to them. + +## Task manuals + +Read the matching manual before acting; this index serves agents that do not auto-discover skills. + +| Manual | Use when | +|---|---| +| [setup-env](.agents/skills/setup-env/SKILL.md) | Detect/install a new environment or diagnose backend/CUDA dependencies | +| [new-task](.agents/skills/new-task/SKILL.md) | Add a robot/task and run mechanical/interface conformance checks | +| [train](.agents/skills/train/SKILL.md) | Train, inspect learning curves, manage checkpoints/jobs and compare experiments | +| [controls](.agents/skills/controls/SKILL.md) | Inspect control contracts, joint/torque CSV and simulation replay | +| [deploy](.agents/skills/deploy/SKILL.md) | Export/verify research bundles and state hardware exclusions | +| [rl-engineer](.agents/skills/rl-engineer/SKILL.md) | Coordinate a reward-to-evaluation experiment across the stages | + +Detailed references: [Setup](docs/SETUP.md), [Robot checks](docs/NEW_ROBOT.md), +[Training operations](docs/TRAINING_OPERATIONS.md), [Vendor sources](docs/VENDOR.md), +[CLI reference](docs/CLI.md). `tests/test_layout.py` enforces index/link/CLI synchronization. + +## Boundaries + +- The coding agent is the orchestrator; use `scripts/uh`. No separate LLM service is required. +- Do not delegate work or start multi-agent workflows unless the operator asks for them. +- Use installed vendored backends in `rl/`; do not install separate mjlab/rsl-rl-lib distributions. +- Verify `doctor`, `verify-vendor` and the lock before changing dependencies; no personal path fallbacks. +- Vendor changes require explicit review, modification notices and inventory updates; never refresh hashes merely to silence failures. +- Public documentation must distinguish reusable features from reference-environment evidence and release blockers. +- Never change the task's robot, joint order, actuator model or observation dimensions silently. +- Keep hardware facts in robot adapters, reward mechanisms in `rewards.py`, and tuning in experiments. +- Never disable falls to manufacture a successful evaluation. Keep finite episodes and censoring counts. +- Never call a two-iteration run a converged policy, nor call an ONNX bundle hardware-ready. +- No physical robot bus, motor enable, firmware flash or autonomous real-world deployment. +- Do not run untrusted checkpoints: the upstream loader uses pickle. +- Do not delete failed runs. New experiment, new directory. Read failure status and logs. +- Evaluate and export a baseline before editing source. Provenance intentionally rejects source drift. +- Read training curves (tracking, action std, episode length); elapsed time is not evidence of learning. +- Use explicit checkpoint manifests, status/stop and new-directory resumes; never guess the newest file. +- Distinguish evaluation seeds from training replicas and completed-only failure fractions from incidence. +- Check actual randomized states, push counts and censoring before claiming stress-test coverage. + +## Checks + +`./scripts/check` runs lint and tests without inherited ROS pytest plugins. +`./scripts/uh check-robot --config experiments/_smoke.json --output NEW_REPORT` checks +sites, IMU, contacts, PD/limits, joint order and initial posture against declared expectations. +`./scripts/uh smoke --config experiments/_smoke.json` checks actual GPU dynamics. +A full acceptance check needs bounded PPO training, a fixed-command evaluation and ONNX parity. + +A task is supported only after its sensors/collisions, resolved action order, reward finiteness, +training, evaluation and export have been checked. Asset discovery alone is not support. + +## Reporting + +State exact config/run/checkpoint/report paths, GPU, iteration/env/step counts and measured metrics. +Separate plumbing validation, learning quality, export parity and physical safety. Never select a +reward revision by raw reward total alone; compare fixed-protocol tracking and failures across seeds. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..729e420 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,9 @@ +# Repository instructions + +Read [AGENTS.md](AGENTS.md) first. Its task-manual index, safety boundaries and reporting rules +apply to this repository. Use the matching manual rather than assuming a task can be solved +by editing rewards alone. + +The simulator and PPO sources are installed from this repository's `rl/`; do not select an +external backend through PYTHONPATH or modify another project's environment. Setup and robot +conformance gates precede training. No command in this project authorizes physical robot motion. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0e98c41 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,36 @@ +# Contributing + +Read [AGENTS.md](AGENTS.md) for the task-manual index and safety boundaries. Install the locked +runtime using [Setup](docs/SETUP.md), then run `./scripts/check`. The GitHub core/layout job does +not provision CUDA and is not a substitute for the GPU setup/robot/pipeline gates. + +## Where changes belong + +- `src/universal_humanoid/`: first-party CLI, task adaptation, evaluation and operations. +- `src/universal_humanoid/robot_specs/`: independently reviewed robot conformance expectations. +- `experiments/`: bounded, explicit experiment recipes; no automatically launched long training. +- `rl/mjlab/`, `rl/rsl_rl/`: reviewed vendored dependencies, not a place to scatter routine task changes. +- `.agents/skills/`: stage manuals; index every new manual in `AGENTS.md`. +- `docs/`: public setup, architecture, interface contracts and validation records. +- `tests/`: core, adapter, negative conformance, provenance, telemetry and layout regression checks. + +## Required checks + +`tests/test_layout.py` requires the CLI table to match parser commands/options, robot check IDs +to match their documentation, every skill to be indexed, links to resolve, backend packages and +licenses to be present, and vendor inventory/portable path rules to hold. Update code and its +manual together; do not weaken a test just to avoid maintaining documentation. + +For a robot change, follow [NEW_ROBOT.md](docs/NEW_ROBOT.md), including negative tests and actual +contact/control probes. For a backend change, follow [VENDOR.md](docs/VENDOR.md), retain modification +notices and licenses, review inherited patches, and regenerate the vendor inventory deliberately. +Dependency changes require a reviewed `uv.lock` and refreshed hashed pip export. + +Report the exact configuration, source/asset identity, runtime, commands, outputs and observed +failures. Distinguish static/unit success, physics/interface conformance, policy quality and ONNX +parity. Never claim hardware safety from simulation or submit credentials, environments, logs, +private asset files or unreviewed source archives. + +Original project code and documentation use [Apache-2.0](LICENSE). Intentionally submitted +contributions follow its contribution terms unless explicitly stated otherwise. Retain third-party +copyright/license notices and do not relicense bundled dependencies or assets. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000..8617938 --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,30 @@ +# Third-party components and attribution + +Copyright 2026 Universal Humanoid contributors. + +Original Universal Humanoid code and documentation are licensed under the Apache License, +Version 2.0; see [LICENSE](LICENSE). This notice is informational and does not modify that license. +Bundled third-party components retain their own copyright notices and license terms below; +the project license does not replace them. + +| Included material | Attribution / license text | +|---|---| +| `rl/mjlab/` | mjlab developers, Apache-2.0; `rl/mjlab/LICENSE` | +| mjlab Isaac Lab-derived helpers | Isaac Lab Project Developers, BSD-3-Clause; `rl/mjlab/licenses/IsaacLab.txt` | +| mjlab Unitree G1/Go1 assets | Unitree Robotics / MuJoCo Menagerie, BSD-3-Clause; `rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt` and `mujoco_menagerie-unitree_go1.txt` | +| mjlab i2rt YAM assets | i2rt robotics / MuJoCo Menagerie, MIT; `rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt`; additional d405 mesh is supplied with mjlab | +| `rl/rsl_rl/` | ETH Zurich, NVIDIA Corporation & Affiliates, BSD-3-Clause; `rl/rsl_rl/LICENSE` | +| RSL-RL pfrl-derived normalization | Preferred Networks, MIT; `rl/rsl_rl/licenses/pfrl.txt` | +| T1 XML and referenced meshes | BoosterRobotics, BSD-3-Clause; `rl/mjlab/asset_zoo/robots/booster_t1/assets/LICENSE` | + +Release metadata, source commits, retained modifications and inventories are described in +[docs/VENDOR.md](docs/VENDOR.md). Changed mjlab runtime files carry `[uh-vendor]` notices; +RSL-RL algorithm source is unchanged. License texts are included in built package resources. + +The modular workflow, vendored-package layout and stage-specific operating-manual design draw +inspiration from Jumper. No Jumper hexapod controller, native-backend fork, credentials or .env +content is included. Ancillary third-party license texts were retained from that reference +project for the corresponding embedded components. + +External asset overrides and any additional models retain their own license obligations. +ONNX research bundles contain no model/mesh redistribution and do not imply hardware approval. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b89c02 --- /dev/null +++ b/README.md @@ -0,0 +1,193 @@ +# Universal Humanoid + +**An agent-ready framework for modular humanoid reinforcement learning.** + +Universal Humanoid connects AI coding assistants with a reproducible robotics workflow: define tasks, iterate on rewards, train policies, evaluate behavior, replay results, and export inference bundles. Built on **mjlab**, **MuJoCo Warp**, and **RSL-RL**, it separates robot-specific configuration from shared training and evaluation tools. + +“Universal” describes an extensible engineering workflow—not a single policy that controls every robot without adaptation. + +[中文](README.zh.md) · [Setup](docs/SETUP.md) · [Robot checks](docs/NEW_ROBOT.md) · [Training operations](docs/TRAINING_OPERATIONS.md) · [Architecture](docs/ARCHITECTURE.md) · [Agent manuals](AGENTS.md) · [CLI](docs/CLI.md) · [Validation](docs/VALIDATION.md) + +## Overview + +```text +Developer / AI coding assistant + │ + Task & reward configuration + │ + Robot-specific adapter + │ + mjlab → MuJoCo Warp → RSL-RL PPO + │ + Checkpoint & experiment record + │ + Evaluation & visual replay + │ + Verified ONNX research bundle +``` + +Use the CLI directly or let a coding assistant operate the same commands. The framework does not require a particular model provider or a separate hosted agent service; assistants use their own configured models and tools. + +## Features + +- **Vendored runtime:** mjlab and RSL-RL copies under `rl/`, bundled G1/T1 assets, editable installation, hashed dependency lock and source inventory. +- **Robot conformance:** one-command checks for foot sites, IMU frames, actual contact response, PD/limits, joint order and reset posture. +- **Modular tasks:** lazy-loaded robot/environment factories behind a shared CLI. +- **Reward iteration:** strict experiment JSON, explicit weight/parameter overrides, and registered Python reward functions. +- **GPU training:** batched MuJoCo Warp simulation with RSL-RL PPO, committed intermediate checkpoints and optimizer continuation. +- **Training operations:** optional Linux background jobs, queryable iteration progress, guarded stop, and TensorBoard scalar curves in JSON/CSV/SVG. +- **Experiment provenance:** resolved configurations, source snapshots, dependency/asset fingerprints, and checkpoint hashes. +- **Measured evaluation:** checkpoint × evaluation-seed × four-command matrices, per-world randomized resets, noisy/push stress tests, mean/sample variance and explicitly defined failure rates. +- **Checkpoint selection:** explicit-metric ranking and paired matrix comparison; no automatic deployment promotion. +- **Replay:** MP4, per-step joint/torque CSV before reset, and mjlab's native/Viser viewer integrations. +- **Policy export:** ONNX with observation/action contracts, resolved joint order, numerical parity checks, and integrity manifests. +- **Agent guidance:** setup-env, new-task, train, controls, deploy and cross-stage RL manuals, with environment scripts and layout/documentation synchronization tests. + +## Project status + +**Early-stage research framework.** Backends, robot adapters and example model resources are now included. The reference installation targets Linux x86-64 with NVIDIA CUDA; portability does not imply every platform/GPU or arbitrary robot is supported. + +| Robot | Task | Integration status | +|---|---|---| +| Unitree G1, 29 DoF | Flat-ground velocity tracking | Adapter wired to mjlab's adapted G1 model; bounded pipeline validation recorded | +| Booster T1, 23 DoF | Flat-ground velocity tracking | Included adapter and BSD-licensed model/meshes; bounded validation recorded | +| Other humanoids | — | Add and validate a robot/task adapter; asset discovery alone does not imply support | + +**Backend policy:** install this project, not separate `mjlab`/`rsl-rl-lib` distributions. The retained backend changes—including simulation warning handling and T1 velocity-command support—are recorded in [VENDOR.md](docs/VENDOR.md), not hidden in an external working tree. See [Setup](docs/SETUP.md) and the [release checklist](docs/RELEASING.md). + +The vendored build has passed [relocated installation, robot-gate and wheel checks](docs/PORTABILITY_VALIDATION.md) on the reference GPU host. + +The included experiments are starting recipes, not pretrained walking policies. Initial validation used short training runs whose policies still fell. G1 and T1 use separate policies. See [recorded evidence and limitations](docs/VALIDATION.md). + +## Getting started + +### Explore without simulation dependencies + +From the repository root, with Python 3.11 or newer: + +```bash +PYTHONPATH=src python -m universal_humanoid list +PYTHONPATH=src python -m universal_humanoid plan --config experiments/g1_smoke.json +``` + +### Configure a simulation environment + +Training requires a supported NVIDIA GPU/driver. Install the locked environment; no external backend or asset path is needed for G1/T1: + +```bash +python3 .agents/skills/setup-env/scripts/detect.py +uv sync --locked --extra dev --python 3.13 +./scripts/uh doctor +./scripts/uh verify-vendor +./scripts/uh check-robot --config experiments/g1_smoke.json --output runs/g1-check.json +./scripts/uh smoke --config experiments/g1_smoke.json --steps 16 +``` + +Standard `pip install -e '.[dev]'` is also supported; [Setup](docs/SETUP.md) explains the hashed pip lock alternative. `doctor` checks installation/CUDA, while `check-robot` checks the actual adapter and measured contacts. Use the setup-env scripts for a logged multi-gate report. Legacy external `MJLAB_ROOT` overrides are rejected. + +### Run a bounded end-to-end experiment + +```bash +# Two PPO iterations, evaluation, then a research-only ONNX bundle. +# Use a NEW output directory for every run. +./scripts/uh pipeline --config experiments/g1_smoke.json \ + --output runs/g1-first --eval-steps 250 --eval-envs 4 +./scripts/uh verify-bundle runs/g1-first/bundle --inference +``` + +This validates the pipeline, not locomotion quality. Select `experiments/t1_smoke.json` for the included T1 adapter and assets. + +## Reward → train → evaluate → replay → export + +```bash +./scripts/uh rewards --config experiments/g1_flat.json +./scripts/uh plan --config experiments/g1_reward_candidate.json + +# Longer training: choose a resource budget before launching. +./scripts/uh train --config experiments/g1_flat.json --output runs/g1-baseline + +./scripts/uh evaluate --checkpoint runs/g1-baseline/final.pt \ + --output runs/g1-baseline/evaluation.json --steps 1500 --num-envs 8 \ + --command 0.3 0 0 --video runs/g1-baseline/replay.mp4 + +./scripts/uh play --checkpoint runs/g1-baseline/final.pt --viewer viser +# Desktop viewer: --viewer native (requires a display). + +./scripts/uh export --checkpoint runs/g1-baseline/final.pt \ + --evaluation runs/g1-baseline/evaluation.json --output bundles/g1-baseline +./scripts/uh verify-bundle bundles/g1-baseline --inference +``` + +Change tuning in a new experiment JSON. Add reward mechanisms in `src/universal_humanoid/rewards.py` and register them in `FUNCTIONS`. Unknown term names, parameters, and function references are rejected. + +Evaluate/export a baseline **before** editing source. Replay and export check source, asset, and dependency identity; historical sources can be restored from a trusted run archive into a separate environment. Never overwrite shared dependencies to replay an old experiment. + +Automation should read `run.json`, `pipeline.json`, and `evaluation.json`. Console output also contains backend diagnostics and is not a single JSON document. + +## Monitor learning and manage experiments + +```bash +./scripts/uh train --config experiments/g1_flat.json --output runs/g1-long --background +./scripts/uh status runs/g1-long +./scripts/uh curves runs/g1-long --output analysis/g1-curves +./scripts/uh checkpoints runs/g1-long +# Resume an explicitly selected checkpoint into a NEW run: +./scripts/uh train --resume runs/g1-long/model_0.pt --iterations 200 \ + --output runs/g1-continued --background + +./scripts/uh benchmark --checkpoints runs/g1-long/final.pt \ + --seeds 101 202 303 --profiles randomized stress \ + --steps 1000 --num-envs 8 --output analysis/g1-matrix +./scripts/uh play --checkpoint runs/g1-long/final.pt \ + --measure analysis/g1-joints.csv --steps 1000 --profile stress +``` + +These are separate lifecycle examples, not a script to launch all jobs simultaneously. Wait for +checkpoints to be committed and reserve GPU capacity before evaluating beside a running trainer. +Matrix seeds control evaluation randomness, not independent training replicas. Read tracking, +action-noise and episode-length curves—not just elapsed time. See [Training operations](docs/TRAINING_OPERATIONS.md) +for stopping, continuation semantics, statistics, checkpoint selection and CSV timing. + +## Working with a coding assistant + +Start with [AGENTS.md](AGENTS.md) and the [RL engineer playbook](.agents/skills/rl-engineer/SKILL.md). For example: + +> Establish a G1 baseline, propose a reward revision, and test both within an agreed training budget. Compare tracking error and failure events under the same evaluation protocol, record a replay, and export a research bundle. Do not connect to physical hardware. + +Reward totals alone are not a valid comparison across different reward definitions. Use task metrics, visual inspection, multiple seeds, and appropriate evaluation horizons. + +## Assets and extensions + +```bash +# Optional inventory of additional models, not needed for the bundled G1/T1 tasks: +export UNITREE_ROBOTS_ROOT=/path/to/unitree_ros/robots +./scripts/uh assets +./scripts/uh inspect-asset "$UNITREE_ROBOTS_ROOT/g1_description/g1_29dof.xml" +``` + +G1 training uses the bundled sensor/collision-adapted asset. T1 uses its bundled `T1_23dof.xml` and meshes; `BOOSTER_ASSETS_ROOT` is an optional explicit replacement, not a requirement. Raw robot descriptions are not silently substituted into a trained task. New morphologies need validated sensors, contacts, control gains, joint ordering, and observation/action definitions. See [extension seams](docs/ARCHITECTURE.md). + +## Development + +Git, license, dependency-lock and CI details are recorded in the [repository baseline](docs/REPOSITORY_BASELINE.md). + +See [CONTRIBUTING.md](CONTRIBUTING.md) for change boundaries and required checks. +With the configured runtime and test dependencies installed: + +```bash +./scripts/check +``` + +Core tests can run without a simulator; adapter tests require mjlab/torch. Unit tests and GPU validation are separate forms of evidence. Before publishing a release, follow [RELEASING.md](docs/RELEASING.md). + +## Scope and safety + +Not currently provided: physical robot transport, state estimation, watchdog/E-stop supervision, ROS/DDS mappings, automatic morphology conversion, a universal zero-shot policy, rough-terrain task coverage, motion imitation, multi-GPU training, or native-CPU RL. + +Exported bundles are for **research/inference**, not robot executables. Every bundle has `hardware_approved: false`; no CLI command enables motors. Load only trusted PyTorch checkpoints—the upstream loader uses pickle. Hashes detect accidental changes; they are not signatures or a sandbox. + +## Acknowledgments and licensing + +The workflow design draws inspiration from Jumper's modular task and engineering conventions. Simulation and learning are provided by mjlab, MuJoCo Warp, and RSL-RL. See [NOTICE.md](NOTICE.md) for component and asset attribution. + +Original project code and documentation are licensed under [Apache-2.0](LICENSE), following Jumper's licensing approach. Bundled dependencies and robot assets retain their original licenses; see [NOTICE.md](NOTICE.md). diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..ee82155 --- /dev/null +++ b/README.zh.md @@ -0,0 +1,155 @@ +# Universal Humanoid + +**面向 AI 代码智能体的模块化人形机器人强化学习框架。** + +Universal Humanoid 将代码智能体与可复现的机器人工程流程连接起来,支持任务定义、奖励迭代、策略训练、定量评估、可视化回放和推理策略包导出。框架基于 **mjlab、MuJoCo Warp 和 RSL-RL**,将机器人专属配置与共享训练工具分离。 + +“Universal”指的是**可扩展的工程架构**,而不是一个无需适配就能控制所有机器人的策略。 + +[English](README.md) · [环境配置](docs/SETUP.md) · [训练运维](docs/TRAINING_OPERATIONS.md) · [架构设计](docs/ARCHITECTURE.md) · [Agent 手册](AGENTS.md) · [验证记录](docs/VALIDATION.md) + +## 工作流程 + +```text +开发者 / AI Coding Agent + ↓ +任务定义与奖励配置 + ↓ +机器人适配 → mjlab → MuJoCo Warp → RSL-RL PPO + ↓ +检查点与实验记录 + ↓ +定量评估 + 可视化回放 + ↓ +经过校验的 ONNX 研究用策略包 +``` + +可以直接使用 CLI,也可以让支持文件编辑与命令执行的代码智能体操作相同工具。项目不绑定某个模型厂商,不要求额外部署专用 Agent 服务;智能体使用各自配置的模型和工具。 + +## 核心能力 + +- **仓库内后端**:`rl/mjlab`、`rl/rsl_rl`、G1/T1 资产随项目安装,提供带哈希的依赖锁和源码清单。 +- **机器人自动检查**:一次检查脚部 site、IMU 参考系、实际接触反馈、PD/限幅、关节顺序及初始姿态。 +- **模块化任务**:机器人、环境和训练配置通过统一入口组合。 +- **奖励迭代**:严格校验实验 JSON,支持权重、参数和 Python 奖励函数扩展。 +- **GPU 强化学习**:MuJoCo Warp 批量仿真与 RSL-RL PPO 训练,支持中间检查点及优化器续训。 +- **训练运维**:Linux 后台任务、进度查询、安全停止,以及 TensorBoard 曲线读取与 JSON/CSV/SVG 输出。 +- **实验可追溯**:保存配置、源码快照、依赖与资产指纹、检查点哈希。 +- **批量评估**:多个检查点 × 评估 seed × 站立/前进/横移/转向,汇总均值、样本方差与定义明确的失败指标。 +- **压力评估**:各世界分别随机初始化,可叠加观测噪声和随机速度扰动,记录实际扰动次数。 +- **检查点选择**:按显式指标排序,支持同协议矩阵的配对比较,不自动批准部署。 +- **可视化回放**:MP4、重置前逐步关节/力矩 CSV,集成 mjlab 原生与 Viser 查看器。 +- **策略导出**:ONNX、观测/动作契约、关节顺序、推理一致性和完整性校验。 +- **Agent 手册体系**:setup-env、new-task、train、controls、deploy 及跨阶段 RL 手册;附环境脚本和文档/接口一致性测试。 + +## 项目状态 + +项目处于**早期研究阶段**。后端源码、适配器和示例模型已纳入仓库;参考锁定环境为 Linux x86-64 + NVIDIA CUDA,不代表支持所有平台、GPU 或任意机器人。 + +| 机器人 | 示例任务 | 接入状态 | +|---|---|---| +| Unitree G1,29 自由度 | 平地速度跟踪 | 使用 mjlab 已适配模型,已记录短流程验证结果 | +| Booster T1,23 自由度 | 平地速度跟踪 | 已包含适配器及 BSD 许可模型/网格,已记录短流程验证结果 | +| 其他人形机器人 | 待扩展 | 需新增并验证适配器;发现模型文件不等于完成支持 | + +**安装规则:**直接安装本项目,不要再单独安装 `mjlab` 或 `rsl-rl-lib`。包括 `sim.py`、速度命令与 T1 适配在内的修改均已纳入仓库,并在[后端来源说明](docs/VENDOR.md)中记录;不再依赖外部未提交的工作区。 + +实验配置是起点,不是预训练行走策略。初始短训练策略仍会跌倒;G1 和 T1 分别训练,不共享一个通用策略。具体证据与限制见 [VALIDATION.md](docs/VALIDATION.md)。 + +## 快速开始 + +### 无仿真依赖地查看任务 + +在仓库根目录,使用 Python 3.11 或更新版本: + +```bash +PYTHONPATH=src python -m universal_humanoid list +PYTHONPATH=src python -m universal_humanoid plan --config experiments/g1_smoke.json +``` + +### 配置运行环境 + +训练需要兼容的 NVIDIA GPU/驱动。G1/T1 无需外部后端或资产路径,按锁文件安装即可: + +```bash +python3 .agents/skills/setup-env/scripts/detect.py +uv sync --locked --extra dev --python 3.13 +./scripts/uh doctor +./scripts/uh verify-vendor +./scripts/uh check-robot --config experiments/g1_smoke.json --output runs/g1-check.json +./scripts/uh smoke --config experiments/g1_smoke.json --steps 16 +``` + +也支持标准 `pip install -e '.[dev]'`;[环境配置](docs/SETUP.md)提供带哈希的 pip 锁定安装方式。`doctor` 检查环境,`check-robot` 检查适配器和实际接触;setup-env 脚本可统一执行并保存各项日志。旧的 `MJLAB_ROOT` 外部后端覆盖已被拒绝。 + +### 运行一个小规模完整流程 + +```bash +./scripts/uh pipeline --config experiments/g1_smoke.json \ + --output runs/g1-first --eval-steps 250 --eval-envs 4 +./scripts/uh verify-bundle runs/g1-first/bundle --inference +``` + +每次使用新的输出目录。`*_smoke.json` 仅训练两轮,用于验证工程流程,**不用于行走能力验收**。使用已包含的 T1 适配器时,改用 `experiments/t1_smoke.json`。完整训练、评估、视频和导出命令见 [English README](README.md)。 + +## 看训练过程,而不只看耗时 + +```bash +./scripts/uh train --config experiments/g1_flat.json --output runs/g1-long --background +./scripts/uh status runs/g1-long +./scripts/uh curves runs/g1-long --output analysis/g1-curves +./scripts/uh checkpoints runs/g1-long +./scripts/uh stop runs/g1-long + +# 从已有的、已提交的检查点再训练 200 轮;始终使用新目录。 +./scripts/uh train --resume runs/g1-long/model_0.pt --iterations 200 \ + --output runs/g1-continued --background +``` + +这些是生命周期操作示例,不应无等待地全部连跑。先确认检查点已提交;训练与评估共用 GPU 时需预留显存。`curves` 同时读取跟踪项、动作分布标准差和回合长度,缺失曲线会明确报告。 + +`benchmark` 默认的 seed 是**评估 seed**;如需研究训练种子差异,应分别训练不同 seed 的配置。默认评估采用随机初始状态,`--profile stress` 额外开启任务观测噪声与随机速度扰动。续训恢复优化器等训练状态,但不是中断轨迹的逐位复现。 + +完整的矩阵评估、检查点选择、失败率口径与 `play --measure` 示例见[训练运维手册](docs/TRAINING_OPERATIONS.md)。 + +## 让代码智能体参与实验 + +可向代码智能体提出: + +> 按 AGENTS.md 和 rl-engineer 手册,为 G1 建立 baseline 并提出奖励改进。在约定预算内训练,用相同评估协议比较跟踪误差和失败事件,记录视频并导出研究用策略包,不连接真机。 + +规范入口:[AGENTS.md](AGENTS.md)、[RL 工程手册](.agents/skills/rl-engineer/SKILL.md)。不同奖励定义下的总奖励不能直接比较;需要结合任务指标、视频、多种随机种子和足够长的评估时间。 + +## 项目结构 + +| 位置 | 用途 | +|---|---| +| `rl/mjlab/`、`rl/rsl_rl/` | 仓库内固定后端源码与许可证 | +| `uv.lock` | 完整依赖解析与制品哈希 | +| `src/universal_humanoid/robot_specs/` | 独立声明的机器人检查契约 | +| `src/universal_humanoid/registry.py` | 机器人/任务注册与工厂引用 | +| `src/universal_humanoid/tasks.py` | 配置适配、奖励注入、评估协议 | +| `src/universal_humanoid/rewards.py` | 可扩展的奖励机制 | +| `experiments/` | 独立实验配方 | +| `src/universal_humanoid/runtime.py` | 训练、评估、视频与交互回放 | +| `src/universal_humanoid/deployment.py` | 策略导出、契约校验与纯推理消费者 | +| `docs/` | 环境、架构、验证和发布说明 | +| `runs/`、`bundles/` | 运行生成的产物,不纳入版本控制 | + +迁移安装、独立环境、机器人门禁、wheel 资源及完整短流程的实测结果见[可移植性验证](docs/PORTABILITY_VALIDATION.md)。 + +新增机器人需要验证模型、传感器、碰撞、控制参数及关节顺序。原始机器人描述不会被自动替换进训练任务;详细扩展方式见[架构文档](docs/ARCHITECTURE.md)。 + +## 边界与安全 + +当前不包含真机通信、状态估计、安全监督器、急停、ROS/DDS 映射、自动形态转换、复杂地形任务、动作模仿、多 GPU 训练或原生 CPU 强化学习后端。 + +导出产物是**研究/推理用策略包**,不是机器人可执行程序。所有包均标记 `hardware_approved: false`,没有启用电机的命令。仅加载可信检查点;文件哈希不是数字签名或安全沙箱。 + +## 致谢与发布 + +Git、许可证、依赖锁与 CI 的配置和本地检查结果见[仓库基线](docs/REPOSITORY_BASELINE.md)。 + +工程流程设计参考 Jumper 的模块化任务与操作规范,仿真和学习依托 mjlab、MuJoCo Warp 与 RSL-RL。组件和资产归属见 [NOTICE.md](NOTICE.md)。 + +参考 Jumper,本项目原创代码和文档采用 [Apache-2.0](LICENSE);内置依赖和机器人资产保留原许可证,见 [NOTICE.md](NOTICE.md)。对外发布前仍需完成[发布检查清单](docs/RELEASING.md)中的跨机器验证和来源审核。 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..120cff6 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,102 @@ +# Architecture and extension seams + +## Layers + +```text +Coding agent / operator + AGENTS.md + rl-engineer playbook + | +Unified CLI (no robot-specific branches) + strict experiment JSON, bounded commands, durable run/pipeline status + | +Metadata registry → lazy task factory → config adapter + robot facts mjlab baseline reward overrides / evaluation protocol + | +mjlab ManagerBasedRlEnv + observation | action | command | reward | termination | event | curriculum + | +MuJoCo Warp (CUDA) → RSL-RL PPO + | +checkpoint + source/asset identity + | +fixed-command evaluation + video → ONNX parity → research bundle +``` + +## Relationship to Jumper + +Reuse the architectural principles: independent task modules, centralized commands, explicit +agent playbooks, and a deployment contract resolved from the actual environment. Do not copy +its hexapod control assumptions, native backend patches, RKNN conversion or Rust FSM. This +project uses the existing mjlab `Simulation` and manager API without monkey-patching it. Native +MuJoCo is used only to inspect MJCF and by the optional viewer, not as a hidden RL fallback. + +## Backend and adapter contract + +Registry entries reference environment, training and asset factories through importable Python +paths. A compatible backend must provide those interfaces; the registry does not download or +convert missing adapters. Fingerprints include **all Python files**, not merely git HEAD. + +The validated mjlab 1.6.0 source, its two inherited runtime patches, T1 extension and RSL-RL 5.5.1 +are now vendored under `rl/`, installed as top-level packages with the project. Runtime guards +reject an external shadowing backend. `uv.lock` fixes transitive dependencies; `rl/VENDOR.json` +checks source and asset inventory. See [Vendor provenance](VENDOR.md) and [Setup](SETUP.md). + +G1 and T1 use bundled adapted models with original asset licenses. `BOOSTER_ASSETS_ROOT` is an +optional explicit T1 override, not a hidden HOME default. Raw Unitree ROS XML can be inspected +but is not mechanically or semantically interchangeable with those models. `check-robot` checks +resolved interfaces against independent declarations in `robot_specs/`, then measures short +contact/PD response. See [Robot checks](NEW_ROBOT.md). + +## Add a robot/task without changing the CLI + +1. Provide a robot `EntityCfg` factory: MJCF `spec_fn`, PD/motor definitions, initial posture, + collision rules, foot sites and required sensors. Do not infer motor ratings from joint names. +2. Provide an environment factory `factory(play=False)` returning an independent config and an + RSL-RL config factory. The initial contract is a flat velocity task: entity `robot`, command + `twist`, `joint_pos` position action, actor/critic observation groups, reset events. +3. Add metadata in `registry.py`: ID, robot, expected action DoF, lazy factory paths, asset + reference (Path or callable returning Path), and number of parent levels covering its assets. +4. Add an experiment. No train/evaluate/export command needs robot-specific changes. +5. Validate all task seams with the playbook. The current exporter deliberately requires one + built-in position actuator per controlled joint. Tendons, coupled drives, recurrent networks, + multiple action groups and history stacks need an explicitly versioned contract extension. + +Task registry extensibility is not automatic adaptation to arbitrary morphology. A future +tracking/manipulation task needs a separate task protocol rather than abusing the twist interface. + +## Reproducibility and evaluation + +Runs contain the input experiment, resolved environment/agent configuration, checkpoint hashes, +Python/config source archive, both backend source identities, dependency lock/versions and +source/asset fingerprints. Bundled and optional external model assets retain their original +licenses; meshes are not copied into training source archives. Replay/export refuse changed source/assets/runtime +versions. Archives are for manual trusted restoration, never automatic execution. + +Comparison intentionally permits different reward/source hashes but requires the same task and +evaluation protocol. Inspect provenance to ensure differences really are reward-only; the CLI does +not prove scientific causal isolation. Use multiple seeds, more than one command and adequate +horizons. `status=completed` means the operation finished, not the policy passed a quality gate. + +## Resource and trust model + +No remote shell, multi-agent scheduler, cloud logger or motor transport is started. Training is +bounded by the experiment's iterations. `train --background` optionally launches a detached local +Linux worker with progress, logs and PID-guarded stop; foreground execution remains the default. +`pipeline` records the current stage and failure. Graceful interruption records `interrupted`; +SIGKILL/power loss can leave stale status, which `status` reports as lost when the recorded process +is dead. Never rewrite a failed/interrupted run to completed. + +`training.py` atomically publishes intermediate checkpoints, records completed-update counts and +observes the backend logger without editing external packages. `jobs.py` / `worker.py` manage +local process lifecycle; `curves.py` reads scalar logs without a server. `evaluation.py` defines +versioned randomized/stress profiles; `benchmark.py` aggregates seed-level reports and selects +explicit candidates. `measurement.py` records terminal joint states before manual reset. + +Evaluation/resume may explicitly load a committed checkpoint from an incomplete run after +checking its hash and provenance. Export still requires a completed run. Resume uses a new output +branch and restores optimizer/model/curriculum-counter state, not simulator or RNG trajectories. +See [Training operations](TRAINING_OPERATIONS.md) for the statistical and lifecycle contracts. + +Hash manifests are integrity evidence, not cryptographic signatures. Checkpoints and Python task +factories are executable trusted inputs. Source paths and git status are recorded; `.env` and +credentials are not copied. Binary policies cannot establish physical safety. diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..5f9798e --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,46 @@ +# CLI reference + +Run `./scripts/uh COMMAND --help` or the installed `uh COMMAND --help`. All paths are explicit; +run/output directories must be new where the command creates an artifact. This table lists every +public command and its named options (excluding automatic `--help`). Layout tests compare it to +the actual argparse parser; change the table with the interface. + +| Command | Named options | Purpose / positional inputs | +|---|---|---| +| `list` | — | Registered task metadata | +| `doctor` | — | Installed backend, locked core versions and CUDA | +| `verify-vendor` | — | Checked-in source/asset inventory | +| `check-robot` | `--config --output --steps --spec` | Robot interface/mechanics/contact gate | +| `assets` | — | Discover explicitly configured optional external assets | +| `inspect-asset` | — | Native MJCF compile; positional model path | +| `plan` | `--config` | Resolve experiment metadata without simulation | +| `rewards` | `--config` | Resolve reward functions/weights/parameters | +| `smoke` | `--config --steps --num-envs` | Bounded zero-action GPU probe | +| `train` | `--config --resume --iterations --background --output` | PPO or additional updates from a committed checkpoint | +| `pipeline` | `--config --output --eval-steps --eval-envs` | Bounded train/evaluate/export sequence | +| `evaluate` | `--checkpoint --output --steps --num-envs --command --video --seed --profile --measure` | Fixed-command evaluation with optional measurement/video | +| `play` | `--checkpoint --viewer --measure --steps --seed --profile --command --video` | Interactive viewer, or bounded headless measured replay | +| `status` | — | Positional run directory; progress and process state | +| `stop` | — | Positional background run directory; guarded graceful stop | +| `checkpoints` | — | Positional run directory; committed checkpoint inventory | +| `curves` | `--output --match --window` | Positional run directory; TensorBoard JSON/CSV/SVG | +| `benchmark` | `--checkpoints --output --seeds --profiles --steps --num-envs` | Checkpoint × evaluation-seed × command × profile matrix | +| `select-checkpoint` | `--output --metric` | Positional completed matrix summary; explicit-metric ranking | +| `export` | `--checkpoint --evaluation --output` | Research-only ONNX bundle | +| `verify-bundle` | `--inference` | Positional bundle; integrity and optional inference parity | +| `compare` | — | Positional baseline/candidate reports or single-checkpoint matrix summaries | + +## Important semantics + +- `doctor`, `verify-vendor` and `check-robot` failures give a nonzero exit. A failed robot gate + writes its JSON report; a model construction failure cannot provide downstream dynamic checks. +- `--iterations` on resume is additional updates; it is not the target total update index. +- `benchmark --seeds` sets evaluation randomness, not independent training runs. +- `play --measure` selects bounded headless replay. Its seed/profile/command/steps/video options + describe that mode; ordinary `play --viewer` is the interactive viewer path. +- Default evaluation profile is randomized; stress adds configured observation noise and velocity + kicks. Nominal fixed replicas do not become independent samples merely by increasing worlds. +- Backend diagnostics may accompany JSON on stdout. Read generated JSON artifacts for automation. + +References: [Setup](SETUP.md), [Robot checks](NEW_ROBOT.md), [Training operations](TRAINING_OPERATIONS.md), +[Bundle contract](DEPLOYMENT.md). Hardware approval is not a CLI operation. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..0e744a7 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,61 @@ +# Research bundle schema v1 + +An export is an immutable directory: + +```text +policy.onnx deterministic actor, with trained observation normalization +contract.json observation/action ordering, affine transforms, simulation PD settings +reference.json one raw observation and its expected unclipped policy action +evaluation.json fixed-protocol rollout evidence bound to the checkpoint/source hashes +experiment.json original experiment +agent.json runner architecture/configuration +provenance.json source/assets/runtime versions and hashes +manifest.json SHA-256 of every payload file, checkpoint hash, ONNX parity results +``` + +The bundle contains no robot meshes, checkpoint pickle, board executable or motor transport. +It is portable for **CPU ONNX inference**, not a self-contained simulation environment. Replay +still needs the recorded task code, asset checkout and compatible mjlab environment. + +## Consumer contract + +`PolicyRuntime(Path(bundle)).predict(observation)` accepts a finite float32 `[1, observation_size]` +array. Observation terms must be concatenated in the exact `start:stop` order. Read each term's +function identity in `contract.json`; implementation semantics are in the recorded sources. + +For current flat tasks, terms are base linear/angular velocity from the configured IMU sensors, +projected gravity, joint position relative to default, joint velocity relative to default, +previous clipped raw action, and `(vx, vy, yaw_rate)` commands. Sensor frame/site and filtering +matter; a base estimator cannot be substituted without validating parity. Joint observation +ordering is explicitly separate from action ordering. Units are SI; angles in radians. + +Normalization is inside ONNX. Do **not** normalize twice. On reset, previous action is zero. +During inference: + +```text +raw_action = policy(observation) +clipped_action = clip(raw_action, -raw_clip, raw_clip) +joint_target_rad = offset + scale * clipped_action +``` + +The consumer returns all three arrays and does not send them anywhere. Host PD/control frequency +must match the recorded policy/physics timestep semantics. Simulation gain and torque data are +metadata, not a certified motor configuration. Joint-limit clamping is not silently added to +inference because it would change training semantics; a physical safety controller must be +separately designed, validated and may refuse the policy. + +## Export checks and limits + +Export requires a completed local run and matching evaluation report. It tests ONNX checker +validity and action parity at 16 actual rollout observations (`atol=1e-5`, `rtol=1e-4`), then +writes file hashes. `verify-bundle --inference` checks integrity and one saved reference. + +This detects common swapped-joint/normalizer/version mistakes, but is not a complete behavioral +proof. The current exporter requires MLP, one joint-position action group, no per-joint action +clip, one position actuator per joint and batch size one. It does not generate a robot driver. + +A report can show poor tracking and many falls and still be packaged for research. All bundles +are `hardware_approved: false`. There is no release-quality gate or hardware approval command. +Before physical deployment, implement and validate bus index mapping, estimator/calibration, +IMU frame, latency, torque/position bounds, thermal limits, watchdog, controlled startup/shutdown, +E-stop and supervised sim-to-real tests. Require explicit operator authorization for actual motion. diff --git a/docs/NEW_ROBOT.md b/docs/NEW_ROBOT.md new file mode 100644 index 0000000..692d9f8 --- /dev/null +++ b/docs/NEW_ROBOT.md @@ -0,0 +1,72 @@ +# Adding and checking a robot/task + +Asset compilation is only the first step. A robot adapter must explicitly define model mechanics, +sensors, controls, observations, reset posture and task semantics. There is no automatic +conversion of an arbitrary MJCF into a validated walking task. + +## One-command conformance check + +```bash +./scripts/uh check-robot --config experiments/g1_smoke.json --output runs/g1-check.json +./scripts/uh check-robot --config experiments/t1_smoke.json --output runs/t1-check.json +``` + +The command builds the **actual nominal task**, then performs a bounded zero-action PD probe +(default 32 control steps, one CUDA world). No policy or hardware is loaded. It produces JSON +and exits nonzero if any check fails. The report contains these stable check IDs: + +| Check ID | What is actually checked | +|---|---| +| `free_root` | Exactly one floating root joint | +| `foot_sites` | Named foot sites exist on declared bodies, with declared local positions/orientations and plausible initial height above flat ground | +| `imu` | Declared sensor names, types and dimensions; exact reference site, parent body, local position and orientation | +| `joint_order` | Resolved action joint order and observation joint order match the separately declared lists; no duplicate action names | +| `pd_and_limits` | One joint-transmission actuator per controlled hinge; enabled torque/joint limits, finite positive stiffness/nonnegative damping, expected gains, gear and ranges | +| `action_transform` | Resolved action scales/offsets and physics/policy timestep match the declaration | +| `initial_pose` | Default and actual root/joint pose, nominal +0.02 m reset height, zero initial velocities, hard-limit compliance and upright orientation | +| `initial_penetration` | Native MuJoCo contact distance at the initialized task pose stays within the declared penetration threshold | +| `contact_configuration` | Contact sensor resolves the declared foot bodies, filters terrain, requests found/force fields and has enabled collision geometry | +| `contact_response` | Each declared foot actually reports contact and positive force during the probe | +| `finite_probe` | Observations/rewards/joints/torques/contact forces stay finite, with no task termination during the bounded probe | + +Foot force magnitude, initial local site placement, gains, limits, names and timing are retained +as evidence. A construction failure is reported rather than converted into a pass; checks that +require a successfully constructed environment cannot run after construction fails. + +This checks interface consistency and a short contact/control response—not motor identification, +long-term balance, mesh realism, convergence, force accuracy or hardware safety. Initial penetration +uses native MuJoCo on the compiled model; runtime contacts use the actual Warp task. These are +complementary checks, not claims of bitwise physics parity. + +## Adapter procedure + +1. **Inspect assets and licensing.** Use `inspect-asset PATH`. Confirm mesh references, floating + root, units, mass/inertia, hinges, limits, actuator transmissions and model variant. Keep new + adapters in this project's package; do not silently edit a shared external repository. +2. **Define robot mechanics.** Implement an `EntityCfg` factory, explicit PD/effort settings, + reset pose, collisions, foot sites and IMU sensors. Record the source of hardware facts. +3. **Define the task.** Provide independent environment/PPO factories following + [ARCHITECTURE.md](ARCHITECTURE.md). Existing flat-velocity integration expects entity `robot`, + command `twist`, `joint_pos` actions, actor/critic groups, and base/joint reset events. +4. **Register metadata.** Add a `Task` in `src/universal_humanoid/registry.py` and a bounded + experiment in `experiments/`. The CLI itself should not gain robot-specific branches. +5. **Declare independent expectations.** Add `src/universal_humanoid/robot_specs/.json`, + using an existing declaration as a schema example. Review every joint order, gain, limit, + site, IMU frame and posture against intended mechanics. Do not auto-generate expected values + from the very environment being checked—that would validate a mistake against itself. +6. **Run the gate.** Use `check-robot --config ... --output NEW.json`. During development an + explicit `--spec FILE.json` can be used; it must match the task and schema. Unknown/missing + sites, changed PD gains, swapped order or absent contact must remain failures. +7. **Test behavior.** Add unit/negative tests, then bounded PPO, randomized/stress evaluation, + measured CSV/video replay and ONNX parity. A passing robot gate is not the final acceptance. + +The supplied declarations encode the existing adapted G1 and T1 baselines; they are **simulation +contracts**, not certified hardware specifications. Updating an intended control setting requires +review of both the adapter and declaration, followed by new tests and run provenance. + +## Nonstandard robots/tasks + +This checker and exporter intentionally target flat-ground humanoids with one position actuator +per controlled hinge. Tendons, coupled actuators, prismatic joints, multiple action groups, +non-flat terrain or non-velocity tasks need an explicit extension of the contract and tests. +Do not relax a check until it passes while continuing to claim the old contract. diff --git a/docs/OPERATIONS_VALIDATION.md b/docs/OPERATIONS_VALIDATION.md new file mode 100644 index 0000000..a76d9b8 --- /dev/null +++ b/docs/OPERATIONS_VALIDATION.md @@ -0,0 +1,102 @@ +# Training-operations validation + +Reference-environment checks on **2026-11-03**, using the Linux/RTX 5080 backend described in +[VALIDATION.md](VALIDATION.md). The files below are development evidence under ignored `runs/`, +not artifacts shipped with a fresh repository clone. Usage and statistical definitions are in +[TRAINING_OPERATIONS.md](TRAINING_OPERATIONS.md). + +## Automated checks + +**58 tests passed**; Ruff lint/format passed. One upstream TorchScript deprecation warning remains. +New checks cover seed-level sample variance and null statistics, duplicate/mismatched protocols, +paired comparisons, selection coverage, TensorBoard scalar reading and `/time` axis handling, +CSV joint ordering and projected torque, atomic checkpoint publication and update indexing, +committed-vs-uncommitted partial runs, Linux PID identity/stop, and both robots' evaluation profiles. + +## Background training and real interruption/resume + +- `runs/ops_g1_validation`: 16 environments × 24 steps × **4 updates**, launched with + `train --background`; state progressed to completed, with live progress and committed + `model_0.pt`, `model_3.pt`, `final.pt` visible through the CLI. +- `runs/ops_interrupt_validation`: a bounded 16-update job was deliberately stopped after its + first committed checkpoint. `status` observed it alive/training; `stop` sent SIGTERM; + worker and run both recorded **interrupted**, preserving `model_0.pt`. +- `runs/ops_resume_validation`: resumed that interrupted checkpoint for **2 additional updates**. + Final metadata reports **3 completed updates**, not 2 and not 4. Direct inspection of these + trusted generated checkpoint files confirmed Adam step counters **20 → 60** and the backend + common-step counter **24 → 72**. The optimizer was continued, not reinitialized. + +No long convergence run, scheduler restart/reboot recovery, or multi-GPU orchestration was tested. +Process-loss/PID-reuse guards are covered by process-level tests, not a power-loss experiment. + +## Actual matrix and checkpoint selection + +`runs/ops_g1_matrix` contains **32 completed cells**: + +- 2 checkpoints: after update 1 and update 4; +- 2 evaluation seeds: 101 and 202; +- 4 commands: standing, forward, lateral, turn; +- 2 profiles: randomized and stress; +- each cell: **2 worlds × 150 control steps**, dt 0.02 s, episode limit 5 s. + +Every cell recorded **2 distinct initial states out of 2 worlds**. Stress cells recorded **71 total +velocity-kick events**. Repeating the same stress seed/command produced exactly matching initial +state arrays; XY RMSE differed by about `1.88e-5 m/s`, demonstrating why seeded draws are not a +promise of bitwise deterministic CUDA trajectories. + +`table.csv`, `table.md`, and `summary.json` contain seed-level means/sample variances, failure +metrics and censoring. This 3 s observation horizon is intentionally short for engineering checks, +not sufficient policy acceptance evidence. + +`runs/ops_g1_selection.json` ranked the earlier checkpoint first by equal-weight mean failure +incidence: **0.3333 vs 0.3750 events per simulated world-second**. This is not a general quality +ranking. Both policies fall; more updates did not automatically produce a better measured result. + +`runs/ops_g1_baseline_matrix` and `runs/ops_g1_candidate_matrix` are single-checkpoint views copied +from the same 32 real cells, not additional independent experiments. `compare` successfully +produced paired-seed differences in `runs/ops_g1_comparison.json`. + +## Training curves, not time spent + +`runs/ops_g1_curves` contains JSON, CSV and parseable SVG extracted from actual TensorBoard events: + +| Scalar | First | Last | Samples | +|---|---:|---:|---:| +| Linear tracking reward | 0.01648 | 0.05844 | 4 | +| Angular tracking reward | 0.004257 | 0.001208 | 4 | +| Mean action std | 1.00023 | 1.00013 | 4 | +| Mean episode length (control steps) | 12.0 | 51.65 | 4 | + +Linear tracking/episode length increased, angular tracking decreased, and exploration std stayed +near 1. Four noisy points, random initial episode lengths and falling replays do **not** establish +learning or convergence. `/time` companion tags are no longer mislabeled as iteration curves. + +## Measured replay and T1 coverage + +| Evidence | Sampling | CSV rows | Failure events | Recorded kicks | +|---|---|---:|---:|---:| +| `runs/ops_g1_joints.csv` | 1 world × 150 steps × 29 joints | 4,350 | 1 | 2 | +| `runs/ops_t1_joints.csv` | 4 worlds × 150 steps × 23 joints | 13,800 | 6 | 7 | + +Both used the stress profile and finite projected joint torques. CSV terminal-row counts matched +failure events × joint count (29 and 138), confirming terminal samples were retained before reset. +T1 recorded 4 distinct initial states out of 4 worlds. Its policy came from a separate two-update +run at `runs/ops_t1_validation`. + +`runs/ops_g1_replay.mp4` decoded successfully: **150 frames, 50 FPS, 3 seconds**. A decoded frame +was visually inspected; robot and ground are visible. The matching measured report records a fall. +Interactive native/Viser viewing was not manually exercised in this revision. + +A separate four-step evaluation, `runs/ops_g1_censored.json`, had no completed episodes: +completed-only failure fraction/return/duration were **null**, and both worlds were explicitly +censored rather than reported as successful episodes. + +## Export regression and provenance + +The updated evaluation report was accepted by ONNX export; `runs/ops_g1_bundle` passed +`verify-bundle --inference`. All outputs remain `hardware_approved: false`. + +Earlier `ops_*_v1` development runs are retained. They predate the TensorBoard time-axis correction +and other final source changes and intentionally fail current source-identity checks. Historical +`g1_validation` / `t1_validation` runs also require their recorded sources for replay. Do not +rewrite provenance hashes to make old runs appear current. diff --git a/docs/PORTABILITY_VALIDATION.md b/docs/PORTABILITY_VALIDATION.md new file mode 100644 index 0000000..292935a --- /dev/null +++ b/docs/PORTABILITY_VALIDATION.md @@ -0,0 +1,119 @@ +# Vendoring, installation and robot-gate validation + +Measured on **2026-11-03**, Linux x86-64 / RTX 5080. This revision replaces the external-backend +bootstrap with installed vendored packages. Evidence paths below are ignored development +artifacts, not files shipped in a fresh clone. Different hardware/platforms still need validation. + +These measurements predate Git initialization and the subsequent license/CI-tool-lock changes. +Historical checkpoints retain their recorded lock and source identities; do not rewrite those +identities to replay them against a changed lock. Repository/CI checks are recorded separately in +[Repository baseline](REPOSITORY_BASELINE.md). + +## Source and dependency boundary + +- `rl/mjlab`: baseline commit `8bb2bc9e187797689ec82c78c35019afd2fd0376`, two captured runtime + patches and the included T1 adapter; details in [VENDOR.md](VENDOR.md). +- `rl/rsl_rl`: 5.5.1 Python files verified against installed release RECORD hashes before copying. +- Booster T1 asset source: commit `12a97516f57469078707afb587a5f84c1cb110d1`; only the selected + XML and referenced meshes are bundled, with their BSD license. +- Current vendor inventory: **355 non-cache files**, no missing/changed/extra files. +- `uv.lock`: **154 package records** including this project. The hashed pip export contains the + matching **153 external name/version pairs**; layout tests enforce this correspondence. +- New project `.venv` installed with `uv sync --locked --extra dev --python 3.13`: + Python **3.13.16**, Torch **2.14.0+cu130**, MuJoCo/MuJoCo Warp **3.16.0**, Warp **1.18.0**. + Dependency consistency and installed backend-origin checks passed; no separate mjlab/RSL-RL + distributions are installed in it. + +The first large Torch download attempt timed out; a subsequent download/installation completed. +The external reference environment and source repositories were not modified. The validated +runtime does not import backend code from them. + +## Automated and core-only checks + +`./scripts/check`: **93 tests passed**, Ruff lint/format passed; one upstream TorchScript +warning remains. New coverage includes manual index/link/CLI synchronization, private-path and +sys.path rules, vendor drift, lock/export correspondence, installed origins outside the repo, +bad explicit interpreter/backend overrides, and negative robot-contract cases. + +A separate lightweight environment installed the project with `--no-deps` plus pytest: +**76 passed, 3 skipped**. GPU/backend-dependent modules were deliberately skipped; this verifies +the core/layout CI path, not simulator functionality. The GitHub workflow is configured but has +not been executed on hosted GitHub infrastructure here. + +## Actual robot gates + +The setup-env check script completed all five gates—vendor, runtime, dependencies, robot and +GPU smoke—for both example configurations: + +| Report | Robot checks | Probe | Peak left/right contact force | Initial penetration | +|---|---:|---|---|---:| +| `runs/setup_vendor_g1/robot.json` | 11/11 passed | 32 steps, no termination | 535.05 / 535.13 N | 0 m | +| `runs/setup_vendor_t1/robot.json` | 11/11 passed | 32 steps, no termination | 460.06 / 460.04 N | 0 m | + +Forces are transient simulated contact measurements during the nominal drop/PD probe, not certified +load ratings or calibrated hardware force estimates. Sites, local frames, IMU parent/type, joint +orders, gains/limits, initial pose/velocity and timing were checked against separate declarations. + +An intentionally wrong G1 declaration required a nonexistent foot site. The real CUDA gate returned +**exit 1**, `ready: false`, with **only `foot_sites` failing**; the other ten checks still ran. +Evidence: `runs/negative_robot_spec.json`, `runs/negative_robot_check.json`. This was not just a +mocked negative test. Additional unit cases corrupt IMU frames, positions, PD, order, collisions, +contact force, initial velocity, penetration and finite-state/termination behavior. + +## Relocated checkout, new environment, different working directory + +A copy excluding `.venv`, runs, caches and build outputs was placed under a temporary directory +whose checkout name contained a space. A **separate environment** was installed there using +`uv sync --locked --offline --extra dev --python 3.13`, using downloaded public wheel cache entries. +No original backend checkout or virtual environment was on its import path. + +The subprocess environment removed `PYTHONPATH`, `MJLAB_ROOT`, `BOOSTER_ASSETS_ROOT`, +`UNITREE_ROBOTS_ROOT`, `UH_PYTHON` and `UH_EXTRA_PYTHONPATH`. From **outside that checkout**: + +1. Backend origins, lock versions and CUDA passed. +2. Vendor inventory passed. +3. Bundled T1 robot gate passed. +4. T1 PPO → randomized evaluation → ONNX export completed. +5. Bundle inference verification passed. + +Evidence: `runs/portability_relocation_v2.json`, `runs/portability_relocation_v2.log`, +`runs/relocated_t1_check_v2.json`, and a retained copy at `runs/relocated_t1_pipeline_v2/`. +This proves independence from the prior workspace layout on the tested host. It is not a test +on a second GPU/OS or a claim that an existing editable virtualenv can be moved without reinstalling. + +## Wheel resource and installed-package checks + +An initial wheel parity check caught missing `mjlab/scripts/train.py`: namespace discovery had +been disabled. Packaging now includes namespaces; a layout assertion, `tools/check_wheel.py` +and the CI build/parity step guard the regression. + +The corrected wheel passed **370 exact file comparisons**, including backend Python modules, +T1/G1 model resources, license files and robot-spec JSONs. It was then installed in the relocated +environment, replacing only the editable project distribution. Imports resolved to the wheel's +`site-packages`, not either checkout. T1's actual robot gate passed again from outside the repo: +`runs/wheel_validation_v2.log`, `runs/wheel_t1_check_v2.json`. + +## Training/export regression + +Both final pipelines used **16 environments × 2 PPO updates × 24 steps**, followed by +**2 evaluation worlds × 150 steps**: + +| Run | Failures in evaluation | ONNX max absolute error (16 actual observations) | +|---|---:|---:| +| `runs/vendored_g1_pipeline_v2` | 3 | 5.59e-8 | +| `runs/relocated_t1_pipeline_v2` | 2 | 4.66e-8 | + +Both bundles passed inference verification. `runs/vendored_resume_v2` also resumed G1 for one +additional update as an installed background worker; progress reached **3 total updates**. +No validation workers were intentionally left running. + +These short policies still fall. Adapter conformance, installation portability and export parity +are engineering checks—not proof of learned walking or real-robot safety. Older validation +runs retain their original source/dependency identities and require their recorded environment +for replay. Do not rewrite historical hashes after vendoring. + +## Remaining release work + +Original code now has an Apache-2.0 license. Review downstream attribution, set actual hosting +metadata, and validate additional hardware/driver/platform combinations. The dependency lock +cannot provide a driver, certify dynamics or substitute for a hardware safety controller. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..a9bc15d --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,50 @@ +# Public release checklist + +A public repository description should explain the reusable framework, not assume access to a +maintainer's workstation. Documentation portability and executable portability are separate tasks. +This checklist records what is still needed before describing the project as a reproducible, +ready-to-install release. + +## Packaging and dependencies + +- [x] Include backend copies, required G1/T1 adapters/assets, source inventories and license notices; see `VENDOR.md`. +- [x] Publish `uv.lock`, its hashed pip export and reference CUDA/platform guidance. +- [x] Remove personal workspace defaults from launchers, T1 and asset discovery. +- [x] Reject bad explicit interpreter paths and legacy external-backend overrides. +- [x] Install the lock in a new project environment without the external backend environment. +- [x] Test a relocated checkout/new environment from another working directory, plus wheel resource parity and installed T1 conformance; see `PORTABILITY_VALIDATION.md`. +- [ ] Independently validate additional machines/GPUs and any additional platform-specific lock. +- [x] Document bundled model variants/licenses and explicit optional external overrides. + +## Repository presentation + +- [x] Use audience-neutral English and Chinese introductions and configurable-path command examples. +- [x] Distinguish architectural goals, implemented features, reference validation and unsupported capabilities. +- [x] Document setup requirements and current distribution limitations. +- [x] Add Apache-2.0 for original code, retain third-party licenses, and include license texts in package metadata. `NOTICE.md` remains attribution, not a replacement license. +- [ ] Set the actual repository URL and package metadata when hosting is established; do not publish placeholder links as real project URLs. +- [x] Add contribution guidance, stage manuals, synchronization tests and core/layout CI configuration. +- [ ] Configure a GPU CI runner and hosted issue templates when the repository is published. + +Suggested GitHub repository description: + +> An agent-ready framework for modular humanoid reinforcement learning with mjlab and MuJoCo Warp: reward iteration, training, evaluation, replay, and ONNX export. + +Suggested topics: `humanoid-robotics`, `reinforcement-learning`, `mujoco`, `mjlab`, `robot-learning`, +`ai-agents`, `sim-to-real` (research direction, not a claim of hardware deployment support). + +## Artifacts, privacy and claims + +- [ ] Inspect the actual files staged for publication. Ignore rules do not remove files already tracked by Git. +- [ ] Exclude credentials, `.env`, virtual environments, proprietary meshes and unlicensed third-party files. +- [ ] Review run manifests, logs and archives for usernames, absolute paths, git status and source licensing before sharing them as release assets. +- [ ] Do not present development-only `runs/` paths as downloadable artifacts unless those artifacts have actually been published. +- [ ] Separate successful pipeline execution from policy convergence; include failed episodes and evaluation protocol details. +- [ ] Preserve `hardware_approved: false` and the absence of a physical robot execution interface. + +## Release validation + +Run lint/unit tests, selected adapter tests, GPU smoke, bounded training, fixed-protocol evaluation, +video recording and ONNX parity in the documented release environment. Record commit and dependency +identities alongside measured evidence. Existing historical validation is useful context, not a +substitute for verifying the released revision on a clean installation. diff --git a/docs/REPOSITORY_BASELINE.md b/docs/REPOSITORY_BASELINE.md new file mode 100644 index 0000000..762aba4 --- /dev/null +++ b/docs/REPOSITORY_BASELINE.md @@ -0,0 +1,43 @@ +# Repository baseline + +This directory is a standalone Git repository on `main`. The initial snapshot includes source, +vendored backends/assets, licenses, experiment recipes, manuals, tests, CI and dependency locks. +No remote is configured or pushed automatically. Configure your actual hosting URL before pushing; +do not reuse the reference project's remote. + +## License + +Original code and documentation use Apache-2.0, following Jumper. The unmodified license text +is in [LICENSE](../LICENSE); project attribution and third-party obligations are in +[NOTICE.md](../NOTICE.md). Existing Apache/BSD/MIT vendor licenses remain intact. Package metadata +uses the combined `Apache-2.0 AND BSD-3-Clause AND MIT` expression for the distribution, with +license texts included in the wheel. + +## Locks and CI + +- `uv.lock`: complete reference Linux CUDA dependency resolution, plus the lightweight CI group. +- `requirements-linux-cu130.lock`: hashed pip export for the development/training environment. +- `requirements-ci.lock`: hashed pip export for check/build tools only, without Torch/CUDA. +- `.github/workflows/check.yml`: immutable action revisions, pinned uv, locked check/build tools, + lock/export synchronization, vendor inventory, wheel parity, Ruff and core/layout tests. + +The CI job does not run GPU simulation. Hosted Actions only run after publication on GitHub; +local reproduction is not a claim of a hosted green build. See [Setup](SETUP.md) for commands. + +## Local verification + +The full configured environment passed **94 tests** and Ruff lint/format checks. A separate +locked CI-only environment passed **77 tests, 3 skipped**; GPU/backend-dependent tests were +intentionally skipped. Wheel parity checked **370 package files** plus **7 distribution license +files** and the SPDX metadata expression. Both requirements exports were regenerated from the lock without drift. + +The staged snapshot was reviewed for generated files, symlinks, oversized files, personal +workspace paths and common secret formats. `.venv*`, runs, logs, bundles, build output, caches, +checkpoints, `.env` variants and private-key files are excluded. This screening is not a guarantee +that no possible sensitive information exists; review changes before publishing. + +## Provenance boundary + +Existing runs remain untouched. Git initialization and lock updates are not retraining or renewed +GPU acceptance evidence. Historical runs keep their original source/lock identities; restore their +trusted recorded environment when needed rather than modifying provenance to force reuse. diff --git a/docs/SETUP.md b/docs/SETUP.md new file mode 100644 index 0000000..be34bdd --- /dev/null +++ b/docs/SETUP.md @@ -0,0 +1,162 @@ +# Environment setup + +The simulation/learning sources are **vendored in this repository**: +`rl/mjlab` and `rl/rsl_rl`. G1 and T1 models required by the registered examples are also bundled, +with their licenses. No external mjlab checkout, personal directory layout or asset download is +needed for these examples. See [VENDOR.md](VENDOR.md) for exact origins and retained modifications. + +## Supported reference target + +The locked training target is **Linux x86-64, Python 3.11–3.13, NVIDIA CUDA 13.0 PyTorch**. +Python 3.13 is the tested interpreter. A compatible NVIDIA driver is required; the Python lock +cannot install a driver or guarantee support for every GPU. EGL is the default offscreen renderer. +Other OS/architectures and CUDA builds need a separately validated lock; there is no native-CPU +training fallback. + +Before installing, the standard-library detector works without simulation packages: + +```bash +python3 .agents/skills/setup-env/scripts/detect.py +``` + +## Recommended: locked installation in a new environment + +Install `uv`, then run from the repository root: + +```bash +uv sync --locked --extra dev --python 3.13 +./scripts/uh doctor +./scripts/uh verify-vendor +./scripts/check +``` + +`uv.lock` pins transitive dependencies and artifact hashes, including the PyTorch CUDA index and +NVIDIA Warp index. `--locked` refuses to silently resolve a changed project. `uv sync` installs +this project as editable and creates `.venv`; it does not reuse another project's environment. +Large CUDA wheels may take time to download. Do not interpret a partial download as an installed +runtime. + +## Standard pip editable installation + +The same packaging layout supports `pip install -e .`, as in Jumper. **mjlab and rsl_rl are included +in this distribution**, not separately installed PyPI packages. Always use a fresh virtual environment: + +```bash +python3.13 -m venv .venv +.venv/bin/python -m pip install -e '.[dev]' +``` + +This convenient command resolves the declared dependencies but does not enforce the entire +transitive lock. For the locked pip alternative, install the exported requirements first: + +```bash +.venv/bin/python -m pip install --require-hashes -r requirements-linux-cu130.lock \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + --extra-index-url https://pypi.nvidia.com/ +.venv/bin/python -m pip install --no-deps -e . +``` + +The exported file contains exact versions and hashes. It targets the same Linux CUDA environment, +not all platforms. Regenerate it from `uv.lock`, never edit it by hand: + +```bash +uv export --locked --extra dev --no-emit-project --format requirements-txt \ + --output-file requirements-linux-cu130.lock +``` + +Do not install `mjlab` or `rsl-rl-lib` separately. `doctor` checks package origins and detects +conflicting distributions. Repair a polluted environment by creating a new one, not by uninstalling +packages from another project's shared runtime. Installed wheels include the same model resources +and licenses; the source inventory check itself requires a repository checkout. + +## Lightweight CI environment + +`.github/workflows/check.yml` checks locks, licenses/layout, source inventory, wheel resource +parity, lint/format and core tests on hosted Linux/Python 3.13. Actions use immutable revisions; +the resolver is version-pinned. No GPU runtime is installed by this job, so backend-dependent +tests skip rather than claiming a simulator pass. The GPU gates below remain separate. + +The `ci` dependency group is locked in `uv.lock`. Its small hashed pip export is +`requirements-ci.lock`; regenerate it with: + +```bash +uv export --locked --only-group ci --no-emit-project --format requirements-txt \ + --output-file requirements-ci.lock +``` + +CI regenerates both exports and rejects Git diffs. To reproduce its environment without replacing +your training `.venv`, use another directory: + +```bash +UV_PROJECT_ENVIRONMENT=.venv-ci uv sync --locked --only-group ci --python 3.13 +uv pip install --python .venv-ci/bin/python --no-deps --no-build-isolation -e . +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 .venv-ci/bin/python -m pytest -q +``` + +The wheel is built with `--no-isolation`, using the locked build tools, rather than downloading +an unpinned build environment. CI is triggered by pushes, pull requests and manual dispatch after +this repository is hosted on GitHub; a local Git commit does not itself execute GitHub Actions. + +## Run the setup gates + +```bash +.venv/bin/python .agents/skills/setup-env/scripts/check.py \ + --config experiments/g1_smoke.json --output runs/setup-g1 +.venv/bin/python .agents/skills/setup-env/scripts/check.py \ + --config experiments/t1_smoke.json --output runs/setup-t1 +``` + +The script runs vendor inventory, backend/CUDA/version checks, dependency consistency, +`check-robot`, and a bounded GPU smoke. It preserves a log for every gate and a JSON summary; +any failed gate gives a nonzero exit. It does not install packages, change gains, repair assets +or weaken checks. Use a new output directory each time. + +`check-robot` validates declared interfaces AND measured contact response; it does not prove +locomotion or physical safety. See [NEW_ROBOT.md](NEW_ROBOT.md). + +## Interpreter and optional asset configuration + +`scripts/uh` chooses an explicit executable `UH_PYTHON`, otherwise the project `.venv/bin/python`, +otherwise `python3` from PATH. The project must be installed in that interpreter. A bad explicit +path fails; it never falls back to another project's environment. The launcher clears inherited +`PYTHONPATH`/user-site leakage and does not inject source directories. An installed `uh` entry +point also works from outside the repository. + +`scripts/uh` rejects legacy `MJLAB_ROOT` and `UH_EXTRA_PYTHONPATH` overrides. Unset them when migrating. +The installed Python entry point also enforces vendored import origins; those old variables cannot select a backend. +Use `UH_PYTHON` only to select another **installed, compatible** environment. + +Optional external asset discovery/override: + +```bash +export UNITREE_ROBOTS_ROOT=/path/to/unitree_ros/robots +./scripts/uh assets +./scripts/uh inspect-asset "$UNITREE_ROBOTS_ROOT/g1_description/g1_29dof.xml" + +# Optional T1 replacement, not required for the bundled example: +export BOOSTER_ASSETS_ROOT=/path/to/booster_assets +``` + +An explicit Booster override expects `robots/T1/T1_23dof.xml` and referenced meshes. An invalid +override fails rather than silently using the bundled asset. G1 training uses the bundled adapted +G1 model, not Unitree ROS XML. External model changes require a new conformance report and source/ +asset provenance; an inventory entry is not a registered task. + +## Lightweight configuration inspection + +`list` and `plan` do not import the simulation stack. If intentionally inspecting source without +an install, `PYTHONPATH=src python -m universal_humanoid list` remains a metadata-only convenience; +it is **not** a supported way to select a training backend. + +## Troubleshooting + +- **Backend shadowing:** inspect `doctor` origins. Both backends must resolve to this checkout's + `rl/` (editable install), or the packages installed by the same built wheel. +- **Core version mismatch:** restore the documented lock in a new environment. +- **No CUDA:** check driver compatibility and GPU access; CPU discovery is not a training pass. +- **Vendor mismatch:** inspect the diff against `rl/VENDOR.json` / `rl/upstream/import.json`; + do not regenerate the inventory just to hide an unexplained change. +- **Robot gate failed:** read the failed check IDs and actual/declared values. Do not widen a + tolerance or remove a foot/sensor requirement merely to turn the report green. +- **Historical run rejected:** restore its trusted recorded source/assets/dependencies separately; + vendoring intentionally changes provenance. Do not rewrite old hashes. diff --git a/docs/TRAINING_OPERATIONS.md b/docs/TRAINING_OPERATIONS.md new file mode 100644 index 0000000..892e0f5 --- /dev/null +++ b/docs/TRAINING_OPERATIONS.md @@ -0,0 +1,205 @@ +# Training operations and experimental evidence + +Training duration is not evidence of learning. Read tracking curves, exploration noise and +completed episode lengths during training; then measure the policy under explicit evaluation +conditions. These commands use the same configured runtime as the main README. + +## 1. Train in the foreground or background + +```bash +./scripts/uh train --config experiments/g1_flat.json --output runs/g1-baseline --background +./scripts/uh status runs/g1-baseline +./scripts/uh checkpoints runs/g1-baseline +./scripts/uh stop runs/g1-baseline +``` + +Omit `--background` for foreground training. Background execution is a **local Linux process**, +not a remote scheduler or reboot-persistent service. It uses the selected Python environment, +redirects stdout/stderr to `runs/g1-baseline.job/stdout.log`, and disconnects stdin/session from +the launching terminal. No TensorBoard HTTP server is started. + +`status` is machine-readable JSON: process liveness, effective state, latest committed checkpoint, +completed/target iterations, session environment steps, iteration duration, learning rate, +action standard deviation, losses, last-update age and an approximate ETA. ETA includes setup +cost and is not an acceptance metric. Progress updates once per completed optimizer iteration; +a long compilation/rollout may be alive without a new progress record. A dead worker whose +manifest still says running is reported as `lost`, not completed. + +`stop` sends SIGTERM to the recorded process, guarded by Linux process identity and pidfd to +avoid signaling a reused PID. Poll `status` until termination. Graceful interruption preserves +previous committed checkpoints; an incomplete optimizer update is not saved as a usable model. +SIGKILL, OOM or power loss may leave temporary files; only checkpoints published in the manifest +are usable. Failed runs and job logs are retained. + +Set `checkpoint_interval` in an experiment JSON to control periodic saves (default 50 optimizer +iterations). The backend also saves after its first update and at completion. Each checkpoint +is written to a temporary file, renamed, hashed and committed to `run.json`. Published files +are immutable; new experiments and resumes always use **new output directories**. + +## 2. Resume a specific checkpoint + +```bash +./scripts/uh train --resume runs/g1-baseline/model_49.pt \ + --iterations 200 --output runs/g1-continued --background +``` + +`--iterations 200` means **200 additional updates**, not a total of 200. Choose a filename from +`checkpoints`; do not assume `model_49.pt` exists. `model_N.pt` stores the zero-based last update: +`model_49.pt` represents 50 completed updates, so its continuation starts with update index 50. +The manifest records `completed_iterations` to remove this ambiguity. + +Resume restores actor, critic, observation normalization, optimizer state (including learning +rate), and the backend's common-step curriculum counter. Simulator state, pending rollouts, +random-number state and logger history restart: this is **learning continuation, not bitwise +trajectory replay**. The new run records parent checkpoint and experiment hashes. Code, assets, +dependencies and experiment settings must match; only the additional iteration budget and +checkpoint interval may change. Older checkpoints lacking explicit completed-update metadata +are not accepted for continuation. + +A committed intermediate checkpoint may be evaluated or resumed even if its original run is +still training or was interrupted. Export still requires a completed training run. Uncommitted +files are never recovered by guessing their modification time. + +## 3. Inspect TensorBoard curves without a server + +```bash +./scripts/uh curves runs/g1-baseline --output analysis/g1-curves --window 20 +# Inspect another set of scalar tags: +./scripts/uh curves runs/g1-baseline --output analysis/g1-losses --match 'Loss/|Metrics/' +``` + +Outputs: + +- `curves.json`: all selected scalar samples, available tags, missing categories, first/last + values, min/max, recent-window mean and slope per iteration. +- `curves.csv`: tag, scalar step, step-axis type, wall-clock timestamp, value. +- `curves.svg`: independently labelled panels; no GUI or plotting dependency required. + +By default, select tracking terms (`Episode_Reward/track_*`, velocity-error metrics), +`Policy/mean_std`, `Train/mean_episode_length`, total reward and curriculum terms. The command +excludes RSL-RL's duplicate `/time` tags by default; if explicitly matched, these are labelled +elapsed seconds, not iterations, and their slopes are per second. It reads scalar TensorBoard +events directly in the specified run directory, including while a +training process is active. It does not recursively mix different runs. Duplicate steps use the +latest wall-time sample; non-finite points are counted and excluded, not disguised as zeros. +A missing curve is reported as missing. Episode metrics may not exist until episodes finish. + +Interpret together: + +- Rising total reward with flat tracking may reflect regularizer exploitation rather than command following. +- Inspect action standard deviation together with learning rate/losses; a plateau alone does not diagnose its cause. +- Check episode length, failure events, command coverage and video—not just a smooth reward curve. +- Tracking rewards are weighted training terms, not physical RMSE or calibrated success probabilities. + +No curve slope automatically declares convergence or approves a policy. + +## 4. Evaluate independent initial-state draws and disturbances + +```bash +./scripts/uh evaluate --checkpoint runs/g1-baseline/final.pt \ + --output analysis/g1-stress.json --seed 101 --profile stress \ + --command 0.3 0 0 --num-envs 8 --steps 1000 +``` + +Profiles are versioned in the recorded `fixed-twist-v2` protocol: + +| Profile | Initial state on every reset | Actor observation noise | Interval disturbances | +|---|---|---|---| +| `nominal` | Fixed pose/joints/velocity | Off | None | +| `randomized` (evaluation default) | Separate uniform samples for each world | Off | None | +| `stress` | Same reset distribution as randomized | Configured task noise enabled | Per-world random velocity kicks | + +Randomized reset offsets: root x/y ±0.1 m, z +0.01…0.03 m, roll/pitch ±0.03 rad, yaw ±0.2 rad; +root x/y velocity ±0.05 m/s, yaw rate ±0.1 rad/s; joint position ±0.03 rad and velocity ±0.1 rad/s. +Joint positions are clamped to the adapter's soft limits. Root offsets are relative to the robot's +default root state and world origin. These are modest test distributions, not physical validity +guarantees for future adapters. + +Stress kicks add x/y velocity increments within ±0.4 m/s and yaw-rate increments within +±0.5 rad/s, at independently sampled 0.5–1.5 s intervals per world (timers restart on reset). +These are **mass-independent instantaneous velocity changes**, not force-calibrated pushes. +Training startup mass/friction/COM randomization is not enabled by these profiles; this is not a +complete sim-to-real robustness suite. + +Reports include all initial states relative to world origins, unique-state counts, exact reset +ranges, actor noise definitions, per-world kick counts, failure counts and first-episode censoring. +A short rollout may experience no kick; inspect the actual counts. Repeating a seed reproduces +initial-state draws, but CUDA trajectories are not guaranteed bitwise deterministic. Distinct world samples share +a policy and are not independent training replicas. Nominal replicas remain intentionally fixed. + +## 5. Run a checkpoint × seed × four-command matrix + +```bash +./scripts/uh benchmark \ + --checkpoints runs/g1-baseline/model_0.pt runs/g1-baseline/final.pt \ + --seeds 101 202 303 --profiles randomized stress \ + --steps 1000 --num-envs 8 --output analysis/g1-matrix +``` + +The four commands are standing `(0,0,0)`, forward `(0.3,0,0)`, lateral `(0,0.2,0)`, and turn +`(0,0,0.3)`. This example has 2 × 3 × 2 × 4 = **48 evaluation cells**, executed serially to bound +GPU memory. `summary.json` updates after every cell and records failures; completed cells are +retained, but automatic continuation of an interrupted matrix is not implemented. + +Outputs include per-cell reports and aggregate `table.csv` / `table.md`: + +- Mean and **sample variance across evaluation seeds**, with the number of defined values. +- XY/yaw RMSE, completed-episode duration and failure metrics. +- **Failure fraction among completed episodes** = failures / completed episodes; null if none ended. +- **Failure incidence** = failure events / simulated world-seconds, including repeated episodes. +- **First-episode failure fraction** = worlds whose first episode failed within the observation + horizon / worlds. Report censoring alongside it; unfinished episodes are not proven successes. + +Variance is null with fewer than two values. Never treat repeated world resets as independent +training seeds. If a termination and timeout coincide, failure takes precedence. Means weight +seed-level reports equally; the table also supplies raw completion/failure counts. Long-lived +censored episodes can bias completed-only statistics, so inspect the entire set of metrics. + +`--seeds` changes **evaluation randomness**, not training seeds. For training-seed robustness, +train separate experiment JSONs with different `seed` values, then pass their checkpoints to +this command. Results remain separated by checkpoint, with each training seed recorded. + +### Select, compare, then validate on held-out seeds + +```bash +./scripts/uh select-checkpoint analysis/g1-matrix/summary.json \ + --metric failure_events_per_sim_second --output analysis/g1-selection.json + +# Each input here must be a completed matrix containing one checkpoint: +./scripts/uh compare analysis/baseline-matrix/summary.json analysis/candidate-matrix/summary.json +``` + +Selection ranks the **explicit candidate list**, equally weighting command/profile mean values +for a named lower-is-better metric. No undefined cells are accepted. Ties retain input order; +there is no hidden composite quality score or hardware promotion. The selection file references +the checkpoint; it does not overwrite `final.pt`. Evaluate the winner on held-out seeds and +inspect tracking/video, since low failure incidence alone can favor standing still. + +Matrix comparison reports paired candidate-minus-baseline means and sample variances by seed, +command and profile. It refuses mismatched protocols; it does not claim statistical significance. +The original two-single-report comparison remains available for quick diagnostics. + +## 6. Replay with joint measurements + +```bash +./scripts/uh play --checkpoint runs/g1-baseline/final.pt \ + --measure analysis/g1-joints.csv --steps 1000 --seed 101 \ + --profile stress --command 0.3 0 0 --video analysis/g1-replay.mp4 +``` + +`play --measure` is a **bounded headless replay**, not an interactive viewer session. Without +`--measure`, `play --viewer native|viser` retains the interactive viewer. For multiple worlds, +use `evaluate --measure FILE --num-envs N`. + +The long-form CSV has one row per controlled joint × world × control step: global and episode +time, world/episode IDs, termination/timeout flags, joint name, position (rad), velocity (rad/s), +actuator contribution to joint torque (Nm), raw/applied action and actual joint target. + +Simulation uses manual resets during evaluation. CSV and video are sampled **after the control +step but before resetting ended worlds**. Torque is `qfrc_actuator` projected into joint DOFs, +not motor-space force, a desired torque, or a decimation-averaged value. Sampling is at policy +rate; substep peaks may be missed. A `.csv.json` sidecar records schema semantics, row count and +completion/failure status; `.evaluation.json` contains task metrics. + +All commands remain simulation-only. These measurements do not replace hardware sensing, +transport adapters, safety supervision or sim-to-real validation. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md new file mode 100644 index 0000000..770e88f --- /dev/null +++ b/docs/VALIDATION.md @@ -0,0 +1,103 @@ +# Reference-environment validation + +For current vendored installation, relocated-checkout/wheel tests and automatic robot gates, +see [Portability validation](PORTABILITY_VALIDATION.md). + +For the subsequent background/resume, randomized matrix evaluation, TensorBoard and CSV work, +see [Training-operations validation](OPERATIONS_VALIDATION.md). The sections below preserve the +initial pipeline validation and its original source/protocol version. + +This is a historical validation record for one development configuration, not a minimum +hardware specification or a guarantee of compatibility with every installation. See +[Setup](SETUP.md) for environment requirements and [Releasing](RELEASING.md) for distribution +limitations. Paths under `runs/` below identify development artifacts; they are ignored by Git +and are not bundled with a repository checkout or offered here as downloads. + +Measured on **2026-11-03**, NVIDIA GeForce **RTX 5080, 16 GB**, in a reference mjlab +Python 3.13 environment. These results validate engineering plumbing, **not learned locomotion**. + +## Runtime + +| Component | Observed version | +|---|---| +| mjlab | 1.6.0, customized checkout | +| mjlab git HEAD | `8bb2bc9e187797689ec82c78c35019afd2fd0376` (dirty/untracked changes retained) | +| MuJoCo / MuJoCo Warp | 3.16.0 / 3.16.0 | +| Warp | 1.18.0 | +| PyTorch | 2.14.0+cu130 | +| RSL-RL | 5.5.1 | + +The exact Python/asset hashes and dirty git status are in each `train/run.json`. The reference +repositories were not modified. `sources.zip` preserves 230 Python/license entries per final run. +Do not replace the customized dependency with a public version just because its version number matches. + +## Automated checks + +`./scripts/check`: **35 tests passed**, Ruff lint and format checks passed. One upstream +`torch.jit.script` deprecation warning remains. Tests cover strict configs, lightweight discovery, +independent factories, reward shape/sign, unknown overrides, retained evaluation terminations, +source/experiment/checkpoint drift, partial-run rejection, bundle tampering/path traversal and +protocol mismatch. + +An initial direct pytest invocation inherited ROS's Python 3.12 plugin path and failed on a missing +`lark` import. The project check script excludes inherited ROS paths and disables unrelated pytest +plugin autoload; no external Python environment was modified. + +## Actual CUDA simulation and PPO + +Zero-action checks ran for G1 and T1 with 4 environments and 8 steps; observations and rewards +were finite. Shapes: G1 actor/critic/action = **99 / 111 / 29**; T1 = **81 / 93 / 23**. + +Final end-to-end runs each used **16 environments, 2 PPO iterations, 24 rollout steps per iteration** +(768 collected environment transitions), then **4 evaluation worlds × 500 steps**, dt 0.02 s, +5 s episode limit, fixed command `(0.3, 0, 0)`, seed 42. + +| Run directory | XY tracking RMSE (m/s) | Yaw RMSE (rad/s) | Failure events | Mean completed episode (s) | ONNX max abs error | +|---|---:|---:|---:|---:|---:| +| `runs/g1_validation` | 0.709230 | 0.322807 | 24 | 1.500 | 5.59e-8 | +| `runs/t1_validation` | 0.480955 | 0.047538 | 24 | 1.560 | 5.96e-8 | +| `runs/g1_reward_validation` | 0.675789 | 0.313275 | 24 | 1.480 | 5.22e-8 | + +**All three policies fall. None is a deployment candidate.** Failure events count repeated +episode failures across resets, not a probability. The current nominal evaluation deliberately +uses identical resets in parallel worlds; those worlds are not independent statistical trials. +The small candidate RMSE difference is not evidence of an improved learned policy after two updates. + +Each run has `pipeline.json`, `train/final.pt`, `evaluation.json`, and `bundle/`. +`verify-bundle --inference` passed for G1 and T1. ONNX parity was checked against the PyTorch actor +at 16 actual rollout observations for every export. The reward candidate exercises local reward +function injection, training, evaluation, export and protocol-matched comparison. + +## Replay evidence + +`runs/t1_validation/replay.mp4`: H.264, **100 frames / 2 s / 50 FPS / 320×240**; +`video_evaluation.json` is the corresponding one-world report. Encoding and decoding succeeded, +and a decoded frame was inspected: the T1 and ground are visible, not an empty render. +This replay also records an episode failure. Desktop native and interactive Viser sessions were +not manually exercised; their CLI paths use mjlab's existing viewer classes. + +## External Unitree asset checks + +Native MuJoCo compilation succeeded for: + +- `unitree_ros/robots/g1_description/g1_29dof.xml`: nq=36, nv=35, nu=29. +- `unitree_ros/robots/h1_description/mjcf/h1.xml`: nq=26, nv=25, nu=19. + +These are **asset compilation checks only**. They are not registered RL task adapters; the +validated G1 training run uses mjlab's adapted G1 asset. T1 training uses an external Booster +T1 asset through the registered adapter. + +## Failed development runs retained + +`runs/g1_smoke_v1` trained but evaluation failed because the upstream command rejected +`heading_command=False` with a non-null heading range. The adapter now clears the heading range +and regression tests check evaluation configuration. `runs/g1_smoke_v2` passed an earlier short +pipeline before the final provenance/archive changes. Its old checkpoints intentionally fail +current source identity checks; use the final `*_validation` runs above. + +## Remaining validation + +Long-run convergence, gait quality, domain-randomized/stress evaluation, independent seeds/reset +conditions, contact realism, raw Unitree asset adaptation, rough terrain, motion imitation, +sim-to-sim dynamics parity and all physical deployment safety remain unverified/unimplemented. +No long 3,000-iteration recipe, robot transport or motor-enable operation was executed. diff --git a/docs/VENDOR.md b/docs/VENDOR.md new file mode 100644 index 0000000..8f6fb20 --- /dev/null +++ b/docs/VENDOR.md @@ -0,0 +1,79 @@ +# Vendored backend sources and assets + +Like Jumper's `rl/` layout, this project installs backend copies as **top-level packages** through +`pip install -e .` / `uv sync`, not through working-directory-dependent import tricks. +`rl/` deliberately has no `__init__.py`. Package discovery includes namespaces: upstream's +`mjlab/scripts` has no initializer but still contains real source that must be included in wheels. + +| Path | Source | Baseline | License | +|---|---|---|---| +| `rl/mjlab/` | [mujocolab/mjlab](https://github.com/mujocolab/mjlab) plus recorded local changes | 1.6.0, commit in `rl/upstream/import.json` | Apache-2.0; embedded components retain their licenses | +| `rl/rsl_rl/` | Installed `rsl-rl-lib` release, checked against its distribution RECORD hashes | 5.5.1 | BSD-3-Clause | +| `rl/mjlab/asset_zoo/robots/booster_t1/assets/` | [BoosterRobotics/booster_assets](https://github.com/BoosterRobotics/booster_assets) | `T1_23dof.xml` and its referenced meshes; commit recorded | BSD-3-Clause | +| `rl/mjlab/asset_zoo/robots/unitree_g1/` | mjlab's adapted MuJoCo Menagerie model | Preserved with upstream mjlab assets | Unitree BSD-3-Clause | + +## What was imported + +The mjlab copy contains tracked package files from the validated checkout, **including its two +modified runtime files**, plus the T1 robot/task Python modules. Unrelated private/untracked +Apollo/R1 experiments, prompts, logs, environments and keyboard applications are not imported. +Other tracked upstream example assets remain intact; their presence does not register them as +supported Universal Humanoid tasks. + +RSL-RL Python files were verified against the installed release's RECORD before copying. No +algorithm changes were applied. Its release METADATA and upstream mjlab pyproject are preserved +under `rl/upstream/`, without creating separately installed backend distributions. + +## Explicit differences + +| File | Origin / purpose | +|---|---| +| `rl/mjlab/sim/sim.py` | Retains the pre-existing local patch suppressing only LS_ITERATIONS/ITERATIONS warnings. This is **not** evidence of solver convergence; other overflow warnings remain. | +| `rl/mjlab/tasks/velocity/mdp/velocity_command.py` | Retains configurable `min_forward_velocity`; T1 uses a smaller minimum than G1. | +| `rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py` | The imported T1 adapter now defaults to bundled assets, with an explicit optional `BOOSTER_ASSETS_ROOT` override. No personal HOME fallback. | +| `rl/mjlab/tasks/velocity/config/booster_t1/` | T1 environment, PPO configuration and reward extension, previously external/untracked, now included. | + +The first two inherited changes are preserved as the actual upstream-relative diff in +`rl/upstream/mjlab-local.patch`; their regression tests and the imported T1 test are retained in +`rl/upstream/tests/` as reference material. They are not automatically mixed into this project's +pytest suite. Our adapter/conformance/behavior tests exercise the supported integration. +Changed runtime files carry `[uh-vendor]` notices. The initial source inventory is +`rl/upstream/import.json`; the current distributable inventory is `rl/VENDOR.json`. + +## Verify and update deliberately + +```bash +python3 tools/vendor_manifest.py +./scripts/uh verify-vendor +``` + +Built-wheel resource parity can also be checked with +`python3 tools/check_wheel.py dist/universal_humanoid-0.2.0-py3-none-any.whl` after `uv build --wheel`. +It checks all package sources, model resources and licenses against the checkout, catching missing +namespace modules that an editable install could conceal. + +Both inventory commands compare every non-cache file under `rl/` against recorded SHA-256 hashes; extra, missing and +modified files fail. No updater runs automatically. For a reviewed vendor revision: + +1. Record the upstream release/commit, licenses and inherited patches. +2. Make changes in this repository only, adding a clear modification notice to changed source. +3. Update this document and review the diff against the initial import inventory. +4. Run the adapter, negative conformance, setup and GPU pipeline checks. +5. Explicitly record the accepted files with `python3 tools/vendor_manifest.py --record`. +6. If dependencies changed, regenerate `uv.lock` and its pip export, and repeat clean-install checks. + +A hash inventory detects accidental drift; it does not prove safety or authenticate arbitrary code. +A deliberate hash refresh without review defeats its purpose. Source snapshots/provenance now +include RSL-RL source identity, robot declarations and the dependency lock, not just mjlab. + +## Licensing and distribution + +License texts travel inside both package directories and are included as wheel resources. The +mjlab assets and Isaac Lab-derived helpers carry their corresponding license texts in +`rl/mjlab/licenses/`; RSL-RL's dependencies and pfrl attribution are under `rl/rsl_rl/licenses/`. +T1 assets carry their own `assets/LICENSE`. See [NOTICE.md](../NOTICE.md) for attribution. + +The upstream licenses do not automatically choose a license for this project's original code or +settle authorship of downstream additions. Select that license and review downstream attribution +before announcing an open-source release. No credentials or proprietary robot transport code is +included, and no physical deployment support is implied. diff --git a/experiments/g1_flat.json b/experiments/g1_flat.json new file mode 100644 index 0000000..4b762cb --- /dev/null +++ b/experiments/g1_flat.json @@ -0,0 +1,15 @@ +{ + "task": "g1.flat", + "seed": 42, + "num_envs": 1024, + "iterations": 3000, + "steps_per_env": 24, + "episode_seconds": 20.0, + "device": "cuda:0", + "action_clip": 5.0, + "reward_weights": { + "track_linear_velocity": 2.0, + "track_angular_velocity": 2.0, + "action_rate_l2": -0.1 + } +} diff --git a/experiments/g1_reward_candidate.json b/experiments/g1_reward_candidate.json new file mode 100644 index 0000000..1636372 --- /dev/null +++ b/experiments/g1_reward_candidate.json @@ -0,0 +1,19 @@ +{ + "task": "g1.flat", + "seed": 42, + "num_envs": 1024, + "iterations": 3000, + "steps_per_env": 24, + "episode_seconds": 20.0, + "device": "cuda:0", + "action_clip": 5.0, + "reward_weights": {"track_linear_velocity": 0.0}, + "extra_rewards": { + "track_planar": { + "function": "planar_velocity_exp", + "weight": 2.0, + "params": {"std": 0.5} + }, + "joint_speed": {"function": "joint_speed_l2", "weight": -0.0001} + } +} diff --git a/experiments/g1_reward_smoke.json b/experiments/g1_reward_smoke.json new file mode 100644 index 0000000..1e875a0 --- /dev/null +++ b/experiments/g1_reward_smoke.json @@ -0,0 +1,16 @@ +{ + "task": "g1.flat", + "seed": 42, + "num_envs": 16, + "iterations": 2, + "steps_per_env": 24, + "episode_seconds": 5.0, + "device": "cuda:0", + "reward_weights": {"track_linear_velocity": 0.0}, + "extra_rewards": { + "track_planar": { + "function": "planar_velocity_exp", "weight": 2.0, "params": {"std": 0.5} + }, + "joint_speed": {"function": "joint_speed_l2", "weight": -0.0001} + } +} diff --git a/experiments/g1_smoke.json b/experiments/g1_smoke.json new file mode 100644 index 0000000..9cb9002 --- /dev/null +++ b/experiments/g1_smoke.json @@ -0,0 +1,9 @@ +{ + "task": "g1.flat", + "seed": 42, + "num_envs": 16, + "iterations": 2, + "steps_per_env": 24, + "episode_seconds": 5.0, + "device": "cuda:0" +} diff --git a/experiments/t1_flat.json b/experiments/t1_flat.json new file mode 100644 index 0000000..0568448 --- /dev/null +++ b/experiments/t1_flat.json @@ -0,0 +1,15 @@ +{ + "task": "t1.flat", + "seed": 42, + "num_envs": 1024, + "iterations": 3000, + "steps_per_env": 24, + "episode_seconds": 20.0, + "device": "cuda:0", + "action_clip": 5.0, + "reward_weights": { + "track_linear_velocity": 3.0, + "track_angular_velocity": 1.5, + "action_rate_l2": -0.03 + } +} diff --git a/experiments/t1_smoke.json b/experiments/t1_smoke.json new file mode 100644 index 0000000..6ca8225 --- /dev/null +++ b/experiments/t1_smoke.json @@ -0,0 +1,9 @@ +{ + "task": "t1.flat", + "seed": 42, + "num_envs": 16, + "iterations": 2, + "steps_per_env": 24, + "episode_seconds": 5.0, + "device": "cuda:0" +} diff --git a/licenses/IsaacLab.txt b/licenses/IsaacLab.txt new file mode 100644 index 0000000..dee9ba5 --- /dev/null +++ b/licenses/IsaacLab.txt @@ -0,0 +1,30 @@ +Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). + +All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/mujoco_menagerie-i2rt_yam.txt b/licenses/mujoco_menagerie-i2rt_yam.txt new file mode 100644 index 0000000..12db770 --- /dev/null +++ b/licenses/mujoco_menagerie-i2rt_yam.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 i2rt robotics + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/mujoco_menagerie-unitree_g1.txt b/licenses/mujoco_menagerie-unitree_g1.txt new file mode 100644 index 0000000..62637c8 --- /dev/null +++ b/licenses/mujoco_menagerie-unitree_g1.txt @@ -0,0 +1,27 @@ +Copyright (c) 2016-2023 HangZhou YuShu TECHNOLOGY CO.,LTD. ("Unitree Robotics") +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/mujoco_menagerie-unitree_go1.txt b/licenses/mujoco_menagerie-unitree_go1.txt new file mode 100644 index 0000000..e472bbd --- /dev/null +++ b/licenses/mujoco_menagerie-unitree_go1.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2016-2022 HangZhou YuShu TECHNOLOGY CO.,LTD. ("Unitree Robotics") +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/pfrl.txt b/licenses/pfrl.txt new file mode 100644 index 0000000..4493c49 --- /dev/null +++ b/licenses/pfrl.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Preferred Networks, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..577139c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=77,<83"] +build-backend = "setuptools.build_meta" + +[project] +name = "universal-humanoid" +version = "0.2.0" +description = "An agent-ready framework for modular humanoid reinforcement learning with mjlab and MuJoCo Warp" +readme = "README.md" +# Original code: Apache-2.0. The wheel also redistributes BSD/MIT components. +license = "Apache-2.0 AND BSD-3-Clause AND MIT" +license-files = ["LICENSE", "NOTICE.md", "licenses/*.txt"] +requires-python = ">=3.11,<3.14" +# mjlab and rsl_rl are vendored packages, NOT separately installed distributions. +dependencies = [ + "torch==2.14.0", "mujoco==3.16.0", "mujoco-warp==3.16.0", "warp-lang==1.18.0", + "numpy", "tensordict>=0.7", "torchrunx>=0.3.4", "tyro>=1.0.1", "trimesh>=4.8.3", + "scipy>=1.15", "viser>=1.0.27", "mjviser>=0.0.14", "mediapy>=1.2.7", + "imageio", "imageio-ffmpeg", "prettytable", "tqdm", "PyYAML", "GitPython>=3.1.59", + "tensorboard>=2.20", "onnx", "onnxscript>=0.5.4", "onnxruntime>=1.19", + "wandb>=0.22.3", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "ruff>=0.9", "build", "setuptools>=77,<83"] +# Compatibility extras; export/analysis are now included in the default installation. +export = [] +analysis = [] + +[project.scripts] +uh = "universal_humanoid.cli:main" + +# Same tools as dev, without the project/CUDA dependencies on hosted CI runners. +[dependency-groups] +ci = ["pytest>=8", "ruff>=0.9", "build", "setuptools>=77,<83"] + +[tool.setuptools.packages.find] +where = ["src", "rl"] +include = ["universal_humanoid*", "mjlab*", "rsl_rl*"] +namespaces = true + +[tool.setuptools.package-data] +universal_humanoid = ["robot_specs/*.json"] +mjlab = ["**/*.xml", "**/*.stl", "**/*.STL", "**/*.obj", "**/*.png", "**/*.jpg", "**/*.jpeg", "**/*.json", "**/*.txt", "**/*.md", "**/LICENSE*", "LICENSE", "py.typed"] +rsl_rl = ["LICENSE", "**/*.txt", "**/LICENSE*", "*.md"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +markers = ["integration: requires the installed vendored mjlab environment and assets"] + +[tool.ruff] +line-length = 100 +extend-exclude = ["rl/mjlab", "rl/rsl_rl", "rl/upstream"] + +[tool.ruff.lint] +select = ["E", "F", "I"] + +[tool.uv] +environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"] + +[[tool.uv.index]] +name = "pytorch-cu130" +url = "https://download.pytorch.org/whl/cu130" +explicit = true + +[[tool.uv.index]] +name = "nvidia" +url = "https://pypi.nvidia.com/" +explicit = true + +[tool.uv.sources] +torch = { index = "pytorch-cu130" } +warp-lang = { index = "nvidia" } diff --git a/requirements-ci.lock b/requirements-ci.lock new file mode 100644 index 0000000..d89e016 --- /dev/null +++ b/requirements-ci.lock @@ -0,0 +1,37 @@ +# This file was autogenerated by uv via the following command: +# uv export --locked --only-group ci --no-emit-project --format requirements-txt --output-file requirements-ci.lock +build==1.6.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53 \ + --hash=sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7 +iniconfig==2.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ + --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 + # via pytest +packaging==26.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # build + # pytest +pluggy==1.7.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec \ + --hash=sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8 + # via pytest +pygments==2.21.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via pytest +pyproject-hooks==1.3.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b \ + --hash=sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43 + # via build +pytest==9.2.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588 \ + --hash=sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b +ruff==0.17.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:692fdb46d0fd9c71e380947dc5fc72f2c3f881073bc1f739e4579376231affbe \ + --hash=sha256:7fa4b0f931cdbcb8a52044f370a3eb429fb42eb88807b0e800fcaa98dd42b24e \ + --hash=sha256:a57049c8addb3662073859bb55a7ad35aa54f8a7c9ed268911d1333c5aadaaa9 +setuptools==82.0.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb diff --git a/requirements-linux-cu130.lock b/requirements-linux-cu130.lock new file mode 100644 index 0000000..7188302 --- /dev/null +++ b/requirements-linux-cu130.lock @@ -0,0 +1,952 @@ +# This file was autogenerated by uv via the following command: +# uv export --locked --extra dev --no-emit-project --format requirements-txt --output-file requirements-linux-cu130.lock +absl-py==2.5.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:286e71c82c1a38e75bbcf185f9b37d0305ad7786535107cb49bf4df9ff2e1f95 \ + --hash=sha256:721200f2f0e9960f2ca9dc3a2a706b201f5f75d812c158f059cbbe29eeafbdb8 + # via + # mujoco + # mujoco-warp + # tensorboard +annotated-types==0.8.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.16.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:59bb03251907983d1603d27c3087ddb1e93299ea2163adcdba37899a4fb12ffb \ + --hash=sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026 + # via + # cwevals + # httpx +asttokens==3.0.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:3ecdbd8f2cc195f53ccada3a613538bb5f9ef6f6869129f13e03c30a677b8fe2 \ + --hash=sha256:9da13157f5b28becde0bd374fc677dcd3c290614264eff096f167c469cd9f933 + # via stack-data +bcrypt==5.0.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:046ad6db88edb3c5ece4369af997938fb1c19d6a699b9c1b27b0db432faae4c4 \ + --hash=sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464 \ + --hash=sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746 \ + --hash=sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41 \ + --hash=sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb \ + --hash=sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef \ + --hash=sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42 \ + --hash=sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a \ + --hash=sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172 \ + --hash=sha256:6b8f520b61e8781efee73cba14e3e8c9556ccfb375623f4f97429544734545b4 \ + --hash=sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254 \ + --hash=sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff \ + --hash=sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f \ + --hash=sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d \ + --hash=sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2 \ + --hash=sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb \ + --hash=sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd \ + --hash=sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822 + # via paramiko +build==1.6.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53 \ + --hash=sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7 + # via universal-humanoid +certifi==2026.10.23 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 + # via + # httpcore + # httpx + # requests +cffi==2.1.1 ; platform_machine == 'x86_64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa + # via + # cryptography + # pynacl +charset-normalizer==3.5.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d + # via requests +click==8.5.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via wandb +cloudpickle==3.1.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414 \ + --hash=sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a + # via + # tensordict + # torchrunx +contourpy==1.3.3 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880 \ + --hash=sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e \ + --hash=sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f \ + --hash=sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411 \ + --hash=sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1 \ + --hash=sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9 \ + --hash=sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b \ + --hash=sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db \ + --hash=sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989 \ + --hash=sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7 + # via matplotlib +contourpy==1.4.0 ; python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1 \ + --hash=sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f \ + --hash=sha256:6ced1670fafee703b8277ab1645072a226e74f167166eae146fb743916119b67 \ + --hash=sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591 \ + --hash=sha256:abd0f7e51ecc52bf8c95713bd80b1c17c516a71fa391c54475308a4377903c2b \ + --hash=sha256:f1219a8898523cba821085f2da8a1b962cc696325763f09d7f93538ba43b2d70 + # via matplotlib +cryptography==50.0.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 + # via paramiko +cuda-bindings==13.4.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:433efa31e33c2868621b5f6036bd3bf5de9c45c5546cb1361a5f6ff3415b8bb5 \ + --hash=sha256:5d6adc44f098472d8ac2d6211923eea6081830521852dc77f8c9237bf9320a59 \ + --hash=sha256:e1cef1eaf5a08cc3bf3675ca9d3f97f3fc7085998556c848b4053e74ba4ba392 + # via torch +cuda-pathfinder==1.8.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:16d1026fdd78ac0222b966441de53dec3f0dbe6b1e2c43d19d92eb29bfd81e67 + # via cuda-bindings +cuda-toolkit==13.0.3.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f + # via torch +cwevals==0.0.1b1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2ae997369ff44941925e4db1bd07eb4e13c05de412e615636fd12881d63f8a13 \ + --hash=sha256:6936baf0ccdc5d9092abd5c2ab628cedcdae9bb72edfdf730f7aaac19118855a + # via wandb +cycler==0.12.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c + # via matplotlib +decorator==5.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c + # via fabric +deprecated==1.3.1 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f \ + --hash=sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223 + # via fabric +deprecated==3.0.1 ; python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2f5a9fecc9961e26462846cf1e57f73a612b1d0b79c27dceeed8de647beadcee \ + --hash=sha256:eda314c115e8bc0fe52b90a6869e19240ac3cedb7449adfdba5981cf9ef693e1 + # via fabric +distro==1.9.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 + # via cwevals +docstring-parser==0.18.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015 \ + --hash=sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b + # via tyro +etils==1.15.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7a7e242b299813f454d3c2910b8428fd5758ef3c6df58af4808fd0a511f0a032 \ + --hash=sha256:b9e7f3d7ab980c564245cd54e2b928e6abcb0521d6042ad3fdb13eea725309f0 + # via + # mujoco + # mujoco-warp +executing==2.3.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:15919cb5d667e5cb4e099511971d00d659573fff2dd5c4e6cd8b71636c7858d2 \ + --hash=sha256:736e859c9f8701f11fcf516856f26f562e04776387824b43a35a1dfe21c84122 + # via stack-data +fabric==3.2.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:ce61917f4f398018337ce279b357650a3a74baecf3fdd53a5839013944af965e \ + --hash=sha256:dcbd2c47ad87688facaef5cc11aab6d1ec9ed05645fed97a5de7204d5d17cc44 + # via torchrunx +filelock==4.2.7 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:31f6f7e3a8b6f0bd89114d30c5d5218ea1cfb63e5c4aa695b7559314a209bc7f \ + --hash=sha256:d08972bbca2b8767ee552aff877b091dd111ffb1d22de7de885766ea7d719d5e + # via torch +flatbuffers==25.12.19 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4 + # via onnxruntime +fonttools==4.68.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:08851f7d2b983412f490b73a7c56f17cbda1d4d85d2ad3f857564bf15b5d9475 \ + --hash=sha256:3edce69ecdefa73ab1041901ca22eefed6ebb93a77bc77e889d940479f25bf7d \ + --hash=sha256:4b9c89b5876f1a74b1341632dfaf4c4c2be565d76c060445df401ebe7a4ceefa \ + --hash=sha256:73294e233070f71e2b3653b6bd8564af676d57f26d075761fb06d1ab2e4a7a5d \ + --hash=sha256:7ab8097e7e191784974c5608eb95fb31e1243fe3054cb66778ac82558e9b36f8 \ + --hash=sha256:b5a015b322275d3d745b06d5e69cf74ef4b1ce2dae02afbe4826cb186d8527a3 \ + --hash=sha256:d76f8fdcb6936f5e1045a0c9dd721a4bf11112e60d1e243bfa1d39f06a8b3104 \ + --hash=sha256:e3ecdb0b248cd0bf43100a8b3cb40f7b45d52ea4be8616edea5dea8fefe90229 + # via matplotlib +fsspec==2026.9.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via + # etils + # torch +gitdb==4.0.12 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ + --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf + # via gitpython +gitpython==3.2.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0f1229ec5c057bdabdeeda8bda4f17bda679ab33dbccc00f24e6a480553c8613 \ + --hash=sha256:d66b0832aa8755c1a84892937282df0befb16c97e5a64aeebd68be9c5e323f53 + # via universal-humanoid +glfw==2.11.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4bd3a462ed02ae25455dceaeb76d154faf08b48962c0cb2cf685419a34a71305 \ + --hash=sha256:76484fae4a4ef0f778a05e02422ecbe6960ae23039087fe2e0ad9742ab8c7bbe \ + --hash=sha256:a08e1135d02c9408d7622db3d391e3e33e4187754ec45633e9a482c775f4846c + # via mujoco +googleapis-common-protos==1.76.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac \ + --hash=sha256:71db0e44a1d4c2dd024ebad44ca67129cc1e2892e4861df466764749d5ef8642 + # via opentelemetry-exporter-otlp-proto-http +grpcio==1.85.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:000999ea9bba80217b1625782d638e7a688790f6154e845e18826f7da859bfc7 \ + --hash=sha256:5f530287d5de5f7f66089ead81de44f6787162944dab3922efa65dfdff108a1d \ + --hash=sha256:83aced2a1c05648cee0de34e531351118b14c58238f0a61b43559ac8851647a8 \ + --hash=sha256:83b09ca3f9602caf4f1b4295b4bb002b10477789ff9797feeedc811b9a6be0b7 \ + --hash=sha256:9998f03ac0dfed226eead6e93cee1b547d7a32e47e1915dee94eff0e113b167e \ + --hash=sha256:a1a72d19c0ab84b4b73b6c47eeaade76d84b1973a51593ff1160a73df33d03d9 \ + --hash=sha256:de520b1555ae445cdc4f8f1cc7ccba12a19992bfafa7c292ebe7fe1fe5da6c53 + # via tensorboard +h11==0.16.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +httpcore==1.0.9 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via cwevals +idna==3.20 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # anyio + # httpx + # requests +imageio==2.38.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:36d23eb7423d2fb63f637098758edb3d2df3687f7125e0a5ce8596572022dcf9 \ + --hash=sha256:6769f1f01c4dd46448307863a787c9a22fa4dbe11c0c88f525c6e410fdfd7983 + # via + # universal-humanoid + # viser +imageio-ffmpeg==0.8.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0385d9bab7dc7d7141af7d0f160059e05f0d874116faf9ce91f6fcfd66e938f2 \ + --hash=sha256:543c8c5d316fc48575dfb5d44adba84a473d9c5a0e482b7fac5988f24a9cb17b + # via universal-humanoid +iniconfig==2.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ + --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 + # via pytest +invoke==2.2.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2413bc441b376e5cd3f55bb5d364f973ad8bdd7bf87e53c79de3c11bf3feecc8 \ + --hash=sha256:515bf49b4a48932b79b024590348da22f39c4942dff991ad1fb8b8baea1be707 + # via + # fabric + # paramiko +ipython==9.18.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:6319c0f5372c196397899479e6db1783dd7937abae06815d1317e8d5d51aaaaf \ + --hash=sha256:997ce9140d374531a61069c70c0f9ef299d6dd3559ea0f94b8af589726e3c1d5 + # via mediapy +ipython-pygments-lexers==1.1.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81 \ + --hash=sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c + # via ipython +jedi==0.20.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0fb16d86c4a4c73c37ba518c77419975e30fcc620658a8d14fbb5720cdd34142 \ + --hash=sha256:2f71208c3f9c1bca057c0e90d3f272aba44ace88fc4067d7587e9e069331b7e5 + # via ipython +jinja2==3.1.6 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +kiwisolver==1.5.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0 \ + --hash=sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38 \ + --hash=sha256:5daa1f19e097050b9c4d9a78fcc9263cb96c9dfae08037ddc1b7c4ad1889f2a2 \ + --hash=sha256:74ad5c3dad54a4641b4c28cd15ded70899d04459c6c7aeacafea716be97cce6d \ + --hash=sha256:95a02752aa032eef4aed01cda6d9b687c669bd0396bf4519eef8bba22a286720 \ + --hash=sha256:aefe930d113798330e9462f7874542977869c0613cba3262e2de3a8d5dee8f3a \ + --hash=sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444 \ + --hash=sha256:d09037ca068d784ebc4aec290ef952ca27ac15dd9c0b5801a88c6e1096b83e6b \ + --hash=sha256:e12dfea7f5fc2a34a9080efbf79c4c44eb380ec5b9c6fea09407e08f0d1e941d \ + --hash=sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a + # via matplotlib +markdown==3.11.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:87bf5d90f2d2882e2c02fcda182c2800456b985bff38913867c49706481514b0 \ + --hash=sha256:f80390747a2a0ff2ba54da074ce4f862f72a6685f608cd2637b646a6d83993f5 + # via tensorboard +markdown-it-py==4.2.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e + # via + # jinja2 + # werkzeug +matplotlib==3.11.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:07d9b9fa60cd4c393692f50d0bb03123242ddf61c99bb0e95e75feb354e7c1a8 \ + --hash=sha256:116cdb0eb0eb5644fc98eb2975d4b4dd4ad35e5c8e6b851c22e6976f371f7ab5 \ + --hash=sha256:1944895967f87c84c9b4bad29a707b31f5b36df4a3a2339ea1f8ea3ce5105539 \ + --hash=sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2 \ + --hash=sha256:7e5a90f8a707ebb6004a713b1cff091a40cc5df4c7c1cb165e5a505ebc11c292 \ + --hash=sha256:7ef7a53b66780e5d942923724f08577fbc5be1f7322da0f0f3f9dcaa45dd803d \ + --hash=sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4 \ + --hash=sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27 + # via mediapy +matplotlib-inline==0.2.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6 \ + --hash=sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79 + # via ipython +mdurl==0.1.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mediapy==1.2.7 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:fc81a9190d9724b02286efac4e7a67f384024e078fda0a3cac9fdf25e2f8878f \ + --hash=sha256:fd3826ca156f9165cd369f44d4d345d5609df1da3caf867df7ce304e19dd7fde + # via universal-humanoid +mjviser==0.0.16 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:f50ea07ec4f35749eb9fd0470e921bc5ce6b8ae449926888c35fe8800ef3c8a0 \ + --hash=sha256:fe8042eb1e97edbb310ac46e72e61d986b4dbe92048ef3c7a4598d2efe60ca53 + # via universal-humanoid +ml-dtypes==0.6.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:26b1f1fa4f0435a2946859823f6e2bf06796f1e9f10f5a05b08a5e3c8f46ff69 \ + --hash=sha256:3b4a480aa8fd54a1805b8ac10f3f91763926a74f73c0c364c10f9231854f4170 \ + --hash=sha256:5e60251d32ced5598972e4d5e06a2f044341f9291402551a3f6f0ec44f9299b0 \ + --hash=sha256:6c8e39b53e90afda8ce52859c93de4dba3e02b76d85dcf091cc469f9184c6dae + # via + # onnx + # onnx-ir + # onnxscript +mpmath==1.3.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +msgspec==0.22.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38 \ + --hash=sha256:0b25dcbc108783cb72503ed705b9fbb8c3cb02ee5801923f44b5f038c91cc365 \ + --hash=sha256:6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019 \ + --hash=sha256:99c401861c5bb3a57f7d6423ea7ed4352cd57aa3f04f4fbe9f3e3e4564a10f08 \ + --hash=sha256:9a696f23f7c1ffb31fae308502e01a3965c3891d5c400f01d0d1096dbe77519e \ + --hash=sha256:a6db3806b3b76ca78064255eac6fa101a8a64fe6f698d80fbaf81fdfa21217d4 \ + --hash=sha256:c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032 + # via viser +mujoco==3.16.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:76a5e87f89c68e6616519914cd8ac06dc0b60cd9108ef4aef92e3e506f59dc53 \ + --hash=sha256:9990aecfe442a516d0595e5762f89da83431ed46fd353bee1e135e3a7140a551 \ + --hash=sha256:a9ea38656fc75d9a6575b5fa7350882dc39461cb655e4ae8027ec58fda6245e1 \ + --hash=sha256:bb8cfcf69cc0f3f2d07d988a3d8ac554bfea9b4a2331d9448a4c66c20722bfe4 + # via + # mjviser + # mujoco-warp + # universal-humanoid +mujoco-warp==3.16.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:00708e267fc2b2723c151fa4acf9f96f39765e43b845c28f5c81fb32eb534097 \ + --hash=sha256:0852a1facaa97ddc9697e9818cf0e16cf4bbc893f95dab7b61b01882b1c79ee0 + # via universal-humanoid +networkx==3.6.1 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via torch +networkx==3.7 ; python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0 \ + --hash=sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a + # via torch +numpy==2.4.6 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 + # via + # contourpy + # imageio + # matplotlib + # mediapy + # mjviser + # ml-dtypes + # mujoco + # mujoco-warp + # onnx + # onnx-ir + # onnxruntime + # onnxscript + # scipy + # tensorboard + # tensordict + # torchrunx + # trimesh + # universal-humanoid + # viser + # warp-lang +numpy==2.5.4 ; python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0 \ + --hash=sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a \ + --hash=sha256:b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b \ + --hash=sha256:b89d0aaae2fe498c648f4c4795c084db535af5bd98ef942b2a3681fb74ce8645 \ + --hash=sha256:fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a + # via + # contourpy + # imageio + # matplotlib + # mediapy + # mjviser + # ml-dtypes + # mujoco + # mujoco-warp + # onnx + # onnx-ir + # onnxruntime + # onnxscript + # scipy + # tensorboard + # tensordict + # torchrunx + # trimesh + # universal-humanoid + # viser + # warp-lang +nvidia-cublas==13.1.1.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436 + # via + # cuda-toolkit + # nvidia-cudnn-cu13 + # nvidia-cusolver +nvidia-cuda-cupti==13.0.85 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8 + # via cuda-toolkit +nvidia-cuda-nvrtc==13.0.88 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575 + # via + # cuda-toolkit + # nvidia-cublas +nvidia-cuda-runtime==13.0.96 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548 + # via cuda-toolkit +nvidia-cudnn-cu13==9.24.0.43 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:71f181cd810e90f9b6023b01186fe82d13d65f0ec098581ee201d39fad769e4b + # via torch +nvidia-cufft==12.0.0.61 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3 + # via cuda-toolkit +nvidia-cufile==1.15.1.6 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44 + # via cuda-toolkit +nvidia-curand==10.4.0.35 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc + # via cuda-toolkit +nvidia-cusolver==12.0.4.66 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112 + # via cuda-toolkit +nvidia-cusparse==12.6.3.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b + # via + # cuda-toolkit + # nvidia-cusolver +nvidia-cusparselt-cu13==0.8.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0 + # via torch +nvidia-nccl-cu13==2.30.7 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50 + # via torch +nvidia-nvjitlink==13.4.92 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:e0391f24ed94ec879b84e3da4d4ec320c879aff681f2c7a638462f7199284323 + # via + # cuda-toolkit + # nvidia-cufft + # nvidia-cusolver + # nvidia-cusparse +nvidia-nvshmem-cu13==3.4.5 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80 + # via torch +nvidia-nvtx==13.0.85 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4 + # via cuda-toolkit +onnx==1.23.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:008cb0467b2bbee41448acc7da8b6f4e704624cb0d327a2d5adafc7ce19bc5b8 \ + --hash=sha256:7abf381d278f31ac62487fddedc9dd42da842dce94d5d43536836ee3efdf4a2b \ + --hash=sha256:c03ecf6b835d136108eeaeeafbd0026fc7b3cf98661409fbc6b63d5a29361348 \ + --hash=sha256:e79e35e152d3095c6910ae81013bbc68679e32bfc0ca76f840968d4b6fdfb864 + # via + # onnx-ir + # onnxscript + # universal-humanoid +onnx-ir==1.1.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:76cc313a788c073b75271c1b6abb172aaf90fcc6335ed40f9f3238b2064a2fa3 \ + --hash=sha256:b72176de575d820ec26864d17270d58ef886a7cf64db7649d15fcc3d5ae2a8ee + # via onnxscript +onnxruntime==1.31.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:1f3dea47f2f0e5117c2404cea7d21fd253ae7836a3bf167a501cdfbb2600d536 \ + --hash=sha256:5752bd508126ae7c49894d3b9d2e83f0c3d487319246d04c444b393308d7ad7f \ + --hash=sha256:6711108bcddf8b401145fb31ac93776b8ca6e7e7e56acdfa9d478575b8df9a2f \ + --hash=sha256:74a6a52994616c4d18b33db1e13c226333a70a6ba07480a5cab9c2a50dff1430 + # via universal-humanoid +onnxscript==0.7.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2c664f6383d10f332a4d47b2876dcab16dba84909fe703656b19abc281fda165 \ + --hash=sha256:d0e7121c6a1eefd608058928e111cbdb76709f70d269ff0d07aee493bd1d13c9 + # via universal-humanoid +opentelemetry-api==1.45.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34 \ + --hash=sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4 + # via + # opentelemetry-exporter-http-transport + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # wandb +opentelemetry-exporter-http-transport==0.66b2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:66d6ed810a515432c8891da9a834464332fd70badeca45bdeeae81b4280c903b \ + --hash=sha256:fd5f5b490c099acda1cd7b909d5e8981436bf18cc8c6098d302d114c271720e9 + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-common==0.66b2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:06f5a9b88638e310efa1f1d206cdf6b75393e112aba4e174a963416192bd1028 \ + --hash=sha256:ace86efb75489f925f1ef809856ae5f0c7db534e4afdc624530222228b1236a5 + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-common==1.45.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:af483b24217028892f017096731b794b196b785025d2e6808874720fd0b05a80 \ + --hash=sha256:f1f2806f3f28ce7938033823e50934fcd91595a39de38e69a28a51e19f40fc6d + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-http==1.45.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:5984a3fb46bc4628e4e2b85d4a6e72637e03962aae6cf372791137768a4ad939 \ + --hash=sha256:6e4d025138829d0ee57946930c2ccad6491ad4d92a958e9c7eea69278fe60cfc + # via wandb +opentelemetry-proto==1.45.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:03a7ca1b43f89fd5b965d5975902bb2515c0c86e2428b456836e9a6149c6235a \ + --hash=sha256:bd8d23cf770605b0af9b7be2071d26a464fea289d35429abfd2a66226114ad64 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.45.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7eec4de13dd21ff3a1670dcb0e02079e692736f94d733dfdce5af461d884248d \ + --hash=sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6 + # via + # opentelemetry-exporter-otlp-common + # opentelemetry-exporter-otlp-proto-http + # wandb +opentelemetry-semantic-conventions==0.66b2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8 \ + --hash=sha256:7da5f1a5dc4383eb66711d4d3d3e71824d9403ac1559fea021cc103c1a4c4976 + # via opentelemetry-sdk +packaging==26.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # build + # matplotlib + # onnx-ir + # onnxruntime + # onnxscript + # pytest + # tensorboard + # wandb +paramiko==5.1.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:239b5f9b2a53006cd54f83cf8495ee742083708ead947882c8dd2786de79f39b \ + --hash=sha256:d9fe6d67a9d0bbf3eb85b051d26ab850b678c02d060f32915b944e5ca6d0d9e1 + # via fabric +parso==0.8.7 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c \ + --hash=sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1 + # via jedi +pexpect==4.9.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \ + --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f + # via ipython +pillow==13.0.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 + # via + # imageio + # matplotlib + # mediapy + # mjviser + # tensorboard +platformdirs==4.14.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:021add1ab113483edbcd4694f5fbb2d688b893ea30d35e1155bb371cf4be0be8 \ + --hash=sha256:cba2d05fe5e18b26b1fa6f376b8e6234c012abe1f8696c486fface7d0054e931 + # via wandb +pluggy==1.7.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec \ + --hash=sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8 + # via pytest +prettytable==3.18.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680 \ + --hash=sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a + # via universal-humanoid +prompt-toolkit==3.0.53 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2 \ + --hash=sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6 + # via ipython +protobuf==7.36.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \ + --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e + # via + # googleapis-common-protos + # onnx + # onnxruntime + # opentelemetry-proto + # tensorboard + # wandb +psutil==7.2.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc + # via ipython +ptyprocess==0.7.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ + --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 + # via pexpect +pure-eval==0.2.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:260c2774686e651b79f8b8e7fc9d80b3599ea6a66334b47d5f4abb69fc2c0ea1 \ + --hash=sha256:96cae060a313cfaad51bb761278bfb0e62dc0248d9315a81173752dc546cd37a + # via stack-data +pycparser==3.12 ; implementation_name != 'PyPy' and platform_machine == 'x86_64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' \ + --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e \ + --hash=sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6 + # via cffi +pydantic==2.14.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118 \ + --hash=sha256:fb7b2db6cefe8414dd612edeede49c4c433acb0432b0f92b95024e45f66c505d + # via + # cwevals + # wandb +pydantic-core==2.50.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:16ec8e07c37f02dcf68939cc7f202ba9a148cc155ee2acf22d337e10b24c2cb7 \ + --hash=sha256:1b0e8dc3932cf4e9ed5727570325cacc716b9409ffc794a221c0ccc72e8eb775 \ + --hash=sha256:31a1fb3ffbd1195b24490f671d12623dd4c9bfa56d6af0a299fa43b516134f5b \ + --hash=sha256:86e7abd24a67f61a4bfe8df059b67a2bdd8bf33705178e98463195d140bd3780 \ + --hash=sha256:8772c2c0091e6fc2d0499151793d31bf811826986b0727052935a95104845544 \ + --hash=sha256:91238117cd296352f54f24117ff5a227bdea428865dba1244aa0cdd1424c0bc2 \ + --hash=sha256:928a44ad1e21a7e923eb924d601605f0694cd187d2ce5d40bf7ad76650f4627d \ + --hash=sha256:92ada606285dab054e40848a951f583c537cde1e54701ea1305ec0096877012d \ + --hash=sha256:9667637665acba166a7dbbc786f14c50512d3a01f7199e0e1a13b929e92d7c7a \ + --hash=sha256:9e7460abce637f90e37f8c443559fbd687c38884d05cb69ef265e634945f4324 \ + --hash=sha256:b5675a12a7b67063fe3ca187e735b0463cf13fabc10166ae4d4b2f775fd0632d \ + --hash=sha256:cb8d2d7b07cbe2d7488de7e53671f836c4f4a8032dcda079df33dc1b7e2397c0 + # via pydantic +pygments==2.21.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # ipython + # ipython-pygments-lexers + # pytest + # rich +pynacl==1.6.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c \ + --hash=sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4 \ + --hash=sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c \ + --hash=sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577 \ + --hash=sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6 + # via paramiko +pyopengl==3.1.10 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:794a943daced39300879e4e47bd94525280685f42dbb5a998d336cfff151d74f \ + --hash=sha256:c4a02d6866b54eb119c8e9b3fb04fa835a95ab802dd96607ab4cdb0012df8335 + # via mujoco +pyparsing==3.3.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36 \ + --hash=sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4 + # via matplotlib +pyproject-hooks==1.3.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b \ + --hash=sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43 + # via build +pytest==9.2.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588 \ + --hash=sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b + # via universal-humanoid +python-dateutil==4.0.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:a5e246f79eb80475e691bf93862cead98decd8a498cf53ba0516fcdc0f50dc1c \ + --hash=sha256:d7295ff14fdc0f92f662100bfb6c551d160ca090f2939f1ec5b9794265c03f06 + # via matplotlib +pyvers==0.2.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:6f5b5612f2f4bd08caa49baa70fc5f875fc7da701a5385c13e244eea6b8114dd \ + --hash=sha256:c4b81c3a033963245e124cdecb052783c9c4cea3bb08c051833af1c44faa6283 + # via tensordict +pyyaml==6.0.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be + # via + # universal-humanoid + # wandb +requests==2.34.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # opentelemetry-exporter-http-transport + # opentelemetry-exporter-otlp-proto-http + # viser + # wandb +rich==14.3.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952 \ + --hash=sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9 + # via viser +ruff==0.17.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:692fdb46d0fd9c71e380947dc5fc72f2c3f881073bc1f739e4579376231affbe \ + --hash=sha256:7fa4b0f931cdbcb8a52044f370a3eb429fb42eb88807b0e800fcaa98dd42b24e \ + --hash=sha256:a57049c8addb3662073859bb55a7ad35aa54f8a7c9ed268911d1333c5aadaaa9 + # via universal-humanoid +scipy==1.17.1 ; python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea + # via universal-humanoid +scipy==1.18.1 ; python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307 \ + --hash=sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1 \ + --hash=sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07 \ + --hash=sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218 \ + --hash=sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9 + # via universal-humanoid +setuptools==82.0.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb + # via + # tensorboard + # torch + # universal-humanoid +smmap==5.0.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ + --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f + # via gitdb +sniffio==1.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via cwevals +stack-data==0.6.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ + --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 + # via ipython +sympy==1.14.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via + # onnx-ir + # torch +tensorboard==2.21.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7279316dcb6bd5bc391d623dea841531299cde1887310e8133bc34a996d32255 + # via universal-humanoid +tensorboard-data-server==0.7.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb \ + --hash=sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530 + # via tensorboard +tensordict==0.15.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:e3bacbbb739b2c5aa535dc73c72a5cf6afd8843c1681225946f5aae5c5ffd5f2 + # via universal-humanoid +torch==2.14.0+cu130 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:07686af6e00801b04fe0feb9e9aa22ff7a58112fb60e0b29a4146f5ac553bf6b \ + --hash=sha256:745010695e0458d6f28accb697b5371b6fd245aa8696530165f972cd126bbd8d \ + --hash=sha256:9c5693d3ca91743202034094ad3b3df20b5eb1373d5e0bc8a5d976be12c35302 + # via + # tensordict + # torchrunx + # universal-humanoid +torchrunx==0.4.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:a8aa42292941ddb7d68fe42a5eabe25800176a12250d417192bd7501b839e52f \ + --hash=sha256:adcdbfd82ef633e549ba6e7694bc6eb258f55a6a64c8c59776794b922ae7b3ff + # via universal-humanoid +tqdm==4.70.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # universal-humanoid + # viser +traitlets==5.16.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:ed900c2b631aa3a112811139fa97b8d2c3bad5e989656bba4b7e52c7852c18c1 \ + --hash=sha256:f775618166caa0396c8e337099240f2bd3e5e917d203b2e6fbe21a58d3cb1f6b + # via + # ipython + # matplotlib-inline +trimesh==5.2.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:04db2c7ec62afdd7763fb04c82bb5f257ddd984993a19ee83697f475fc58fd29 \ + --hash=sha256:d0a845d058e5a4ff145d0f0082d80c58d761687ce389c79d422dbe774966957b + # via + # mjviser + # universal-humanoid + # viser +triton==3.8.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:1f0497218e26b7d79773ad9c2a3fa3b539ee69f587a13fac2e552b1d322a8015 \ + --hash=sha256:68988ac85d5e7086baeda0ddc175af9667db7529b3c5e11a5c0601b8bef2200a \ + --hash=sha256:e91ffa46d095b252248297292dd22bcbacd53a125a0c2eefbbbf74925a320bc3 + # via torch +typeguard==4.6.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:79878165bb86f2cf5d41d159a0ff1792a796cf496882d2fe1b1c6c7049b9cdd7 \ + --hash=sha256:e7414f09111317de3e335de92cd397c5c0ca00b1cc1676de12e1d444a79b3f21 + # via tyro +typing-extensions==4.16.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # cwevals + # etils + # grpcio + # ipython + # onnx + # onnx-ir + # onnxscript + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # torch + # typeguard + # typing-inspection + # tyro + # viser + # wandb +typing-inspection==0.4.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via pydantic +tyro==1.0.16 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:2f9087196113699647190459a12b6b4f2784dd0703c50ab1f7bf0f85a3a794ea \ + --hash=sha256:edf23c9c3eecba5a2b5af152df8f34a723b689a001ff5b7311f93eff1893a636 + # via universal-humanoid +tzdata==2026.6 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec \ + --hash=sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d + # via python-dateutil +urllib3==2.8.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests +viser==1.1.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0a7ba97dab32b8a7106a2dd216fe1373f2e66e17529b5b43502d33d9b9ac9352 \ + --hash=sha256:f528877553c56e1c01d43b861a1434920a2e7ba4fdc7b2e7ff03721e3c3f5b3d + # via + # mjviser + # universal-humanoid +wandb==0.31.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:009bcf370598d5e7a3d3bb66e33510f78c17fcc0505f8aa7ecf51dbc4378f979 \ + --hash=sha256:dcdaa635576ff17b18a757b39fc22b0db3d1c6e01d5ce090fe80f6adc776028d \ + --hash=sha256:df16a46cd6468e6580783b02560c79991c181f63a4691949020fff1b4ac87072 + # via universal-humanoid +warp-lang==1.18.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:090962bfae7dd22ade0d8e0258f41635e0e58980f15bcc9e7a7d312819dfe440 + # via + # mujoco-warp + # universal-humanoid +wcwidth==0.9.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:901eace6d2dd25ebb917d0ce5db18e5c537e1504c905cf32f0d006c12098b05d \ + --hash=sha256:b185c8b927b23af2721be1783499d1cfa5c028b098b0858951b4ae224477f1c2 \ + --hash=sha256:d8905f1390d828d745fdd1904374c90386d902d3cfa9ee88e9875099f75a4e17 \ + --hash=sha256:fd8d57d3cdfe570e3265c8fab1885c24e34ef40889c0759c0c304a19e9835954 + # via + # prettytable + # prompt-toolkit +websockets==16.1.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1 \ + --hash=sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e \ + --hash=sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3 \ + --hash=sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3 \ + --hash=sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81 \ + --hash=sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2 \ + --hash=sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57 \ + --hash=sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87 \ + --hash=sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8 + # via + # mujoco + # viser +werkzeug==3.1.9 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:55ca7c70a75689be937aa27f8ff4b018f06ff4838fc73045560bf0f5a1291060 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab + # via tensorboard +wrapt==2.5.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:0743950d3dda9940e56f81f5334ba051c4fe2e1421e4e4a4ef8eb8fb86a90894 \ + --hash=sha256:092472b7b1494deef40dac5e9f6295013a9d51b112b5fbc4350f64dc2574e46a \ + --hash=sha256:40e03506f09dbf4854fd69848e843dadd9b7fbbdd217837cb27b37fb8f0b5bc3 \ + --hash=sha256:551cb49e88b3722faad780303833bc4fa70685fb1773270fb98b40d39d9fddbf \ + --hash=sha256:61eb54e9540ea522c2c4752a57549a83d3c67e6704ed708403d28e067aaf90f0 \ + --hash=sha256:7a7efde469ad904b8e05f98c76d63b437f4756c30ec7b9136bef02bdb5d1b14c \ + --hash=sha256:8b039b3a296762db1908d6108365225393c4fb33c1dbc5f00aa36f8302eaeaa0 \ + --hash=sha256:eff91af619ad5d587f8855f6c02164e12d33d205c702eb73d487deb499520265 + # via deprecated +xxhash==4.0.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:050c1d66587e8f1ce70ccb7f661ec117e01497d48da817c14543beba0b8c677e \ + --hash=sha256:180b349bbc89af61240adab8519426e0cb6546c2487e4b26b464364bf59dcbed \ + --hash=sha256:42fcaec8c705d1ea362d4cf011aea0dad1ebe13096ed227f9cfecfcac396c72c \ + --hash=sha256:56ee43c284daffc12d269b3fb36b1fc624c7ab007064a24d54099049ee71ce80 \ + --hash=sha256:61ded7dee4f95da69295c228fc18a65447f5741c45a315aa3f9a3adda6038611 \ + --hash=sha256:63522739ce94e4e73c1605aa60a52b3863e77e835f6292cc8a5e7e7b626ff00c \ + --hash=sha256:79cfd336c2ecaceef1061e7a7b3da079eab5920ba1352a28be690e857f67f2ea \ + --hash=sha256:7c7c95dba490be3b4e5a2fa5595b4d4f088fe7b8726e5998b8c1d83b1f9eb3e4 \ + --hash=sha256:87255720aa1a1be35256b688c5ee4151d3273894186cda5c8a5b82f1dc05debf \ + --hash=sha256:892a66641657cf9aa0ae5c2c88258bc281221a33bd4af34bed786eea23e06d83 \ + --hash=sha256:af3d1b3cfa5280e84d8968e9f30c9eca75781db4915eb6d6acb0dd4fc93e0f47 \ + --hash=sha256:c37fcacb7f5b8590c822bbf3bc41690e40a7581d8c4bcea7b15148f0743e716d \ + --hash=sha256:e359ff2f38be3c856a129c1c5c4cbd70e219188456c1bf266fd4f94e2baae278 \ + --hash=sha256:f8e86a0b5e28418162dfa68d6dc4bee873ea2efbd6ee67789903a0ee37a81c14 + # via wandb +zipp==4.1.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d \ + --hash=sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f + # via etils +zstandard==0.25.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ + --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ + --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ + --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ + --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ + --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ + --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ + --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ + --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ + --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ + --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 + # via viser diff --git a/rl/VENDOR.json b/rl/VENDOR.json new file mode 100644 index 0000000..c18e829 --- /dev/null +++ b/rl/VENDOR.json @@ -0,0 +1,395 @@ +{ + "added_since_import": [ + "rl/mjlab/licenses/IsaacLab.txt", + "rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt", + "rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt", + "rl/mjlab/licenses/mujoco_menagerie-unitree_go1.txt", + "rl/rsl_rl/licenses/pfrl.txt", + "rl/upstream/import.json" + ], + "changed_since_import": [ + "rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py", + "rl/mjlab/sim/sim.py", + "rl/mjlab/tasks/velocity/mdp/velocity_command.py" + ], + "files": { + "rl/mjlab/LICENSE": "8ca04d9dbbccd20f0f1842fb33e13949f9e57064b150dad36af8e7a751ca3321", + "rl/mjlab/__init__.py": "7a447eb4e744ff67a0557c32c3aa1a27c352b7eaccc6df80ebb1b9cad71a646b", + "rl/mjlab/actuator/__init__.py": "ae89d8f9cab9ab238a881597ec94f729feee7058b0d9773dca83000783a790df", + "rl/mjlab/actuator/actuator.py": "85c5744fe149d7a7886450f15e30317f4b01a3d2f1807730946fdd808e91716e", + "rl/mjlab/actuator/builtin_actuator.py": "4bd44d70c59a150313a0e8321c4700d8ab85c856c2554d55507d5d99e3544c03", + "rl/mjlab/actuator/builtin_group.py": "f4429b846fc645c443734e45435618958d9f24502b2c627156e7d5a69cbbd9e3", + "rl/mjlab/actuator/dc_actuator.py": "a24fc574b1606d66338bde4905e83d24ca2e66a2f7b11e0235fa441a81d253f6", + "rl/mjlab/actuator/fused_group.py": "21a5d5ca68c0b886ba5d5abf89665bbc8c63991eb0bad9f4cb204f5300df46e1", + "rl/mjlab/actuator/learned_actuator.py": "aefb27cc12881e67c87171cb94e38a5aee0171299ffd952e07053fbbd7fac487", + "rl/mjlab/actuator/pd_actuator.py": "b2c36f1759827890055cb3549aa7b441a50abd914c883fde8bbfd631e6f2f101", + "rl/mjlab/actuator/xml_actuator.py": "a8df6bbf07dda9e182d5ef248b0022816d1aa211ba321fa00ab676286b9c9548", + "rl/mjlab/asset_zoo/README.md": "e6bed2af0c36ad4768c4cdba7710ff07354a76f5e0573ec60070baa69212b9db", + "rl/mjlab/asset_zoo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/asset_zoo/robots/__init__.py": "7b64b50e70acf42fca4be9fe23b4793c725876847a81bd0fa540a7e55e60e449", + "rl/mjlab/asset_zoo/robots/booster_t1/__init__.py": "6630c30df59a46d1f341f8dfc350cb99a723a839ec643745a1fa0790d60bfdf4", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/LICENSE": "056db8cc7dcb21feae042d97185d4689c21bb1ba7fe59cefdc7ef5917c0ea77f", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/T1_23dof.xml": "f00e3410362b1e2267e6db05d25c33d6240bbf2436d62a409f07042bad4f7fb2", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL1.STL": "43f5d268152b3c169d34731545ddfa2a510a7e16f209a2a80f8ba9ae0e1a40aa", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL2.STL": "4bca60adb29c73f701350abd23e8a8d660dc61e87c5163a80157650dd3ad88c3", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL3.STL": "bfdf5611dfe90f8e02b07b037c36a244f4ce7184b45113371ef46e624fb30854", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR1.STL": "6dbbd3dab4486600747be9d0e7f8523d67bf00d24d0dc9c23db1766c4d808fcf", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR2.STL": "0499136785fe1bc0289d9e503e1e0ea48f5a63420a2e45b0fcd9e6b3bcf8f999", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR3.STL": "92330bd8286fe420fb76949ee4c5d5bdec2e6bb30426acdc4d6fae9257464631", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Left.STL": "e6ec38741838475834c147f2ad18196c0254741e602c2b676f888fd12b5d7981", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Right.STL": "b15d91f0f2470fc4d5634db906a3d7efaf1f700302e6671824494da8faf50308", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H1.STL": "527a2e426db4e3d476a195754764b7f9565a950b809a34e47618f177b139c3f2", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H2.STL": "9cbf4fe83a721c809de65b1f0d0a379314f4523f7e2f8af7f0edc27c2ce385aa", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Left.STL": "7bcb8be569aeb5379cb75a0827d682bc1d73a52a7f515d3aeffece43842c33c9", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Right.STL": "054d579c3c6ccc1f97aaf1461d802dec748ce1bf26c6fdd289283a2d0dbb2032", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Left.STL": "72dbd65bdb4e87444c8e538474f631d63bf8c0960922517456b8cf2be9e1d0f0", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Right.STL": "d715a7649ea4bf20d7ee17b3a57bae31732e467b9da1053fee2242c90f50980f", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Left.STL": "8d7de83ab4034639db3d1be1e6f43ae16e004dc2f1c2bec170b615686d187918", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Right.STL": "1a7d98e021884e313c55780a3c7c24ef2aa322807d8f9f8b18b2a63c7e2b9bf6", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Logo.STL": "221b00d090fdc9e684892805db75265c9688b152b3ccf641084f950f220596c8", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Left.STL": "6ae8dd7f88cb70a4a3cb3972cad4035a76f1604fd73ea66e285da553b3703c40", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Right.STL": "dc66a8a0e2e31e413884a053e427d9b5007bef4508986587916a638a8a448599", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Trunk.STL": "39d9610b23ef8ac7ea4343231a499f9c485903bede62df9729a2cd1254fede8f", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Waist.STL": "2fe0a9ccfc284beca402e554db38e83022592192afa00f02db13cb374ffe9fbc", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_foot_link.STL": "2de026f2269fc55f80d4a8abd2e33dfd152adf9eb2c240d7d7e1dc40c13b55aa", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_hand_link.STL": "34df992aed438446481cfa20e78327810cd5cc5ecbb4a6f32ba2bf625462f53d", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_foot_link.STL": "313a9ff2f95da5727dce2b98c50b60226bf0a3affc72050ee251ea503e4b370a", + "rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_hand_link.STL": "c5a73e493e17db7611a81785f80c01167f51018271b4042f72fa8d2351d8b0eb", + "rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py": "04c351bff19f6d63d860f5eebfb41eb783a616e58298254d140dadf9d68c2cc7", + "rl/mjlab/asset_zoo/robots/i2rt_yam/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/d405.stl": "6617f3fb22b604a66c27e16d2397f305fcdca8a781248abdbabbf69c9fe6a3c5", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2.stl": "5ac61c0313ed00c38655e41681af1b8b307cf8d5ef57209a75f215cea02df754", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__10.stl": "b879cc07f547b7a2ac52e428dd206e80a0d4b0e805cb56b9a8230df04a690923", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__11.stl": "af220d2a51e8444392aef36e2fe4a6f69aed22bc292a7f38e54da3b4f5253434", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__12.stl": "e79614869bd2283f70fafa150ce748d5b6c03377a651448c6fe9893a22f6d1de", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__13.stl": "32b62f14f00ea82bfc79b935f7dac32c7cb276a3abc0aae8362cbba3fb1326de", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__14.stl": "72d2178789dd5e0358c2caf9dad90b8881e505f4c52a8d6d18c71b5e93e8c84b", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__15.stl": "ca8788184d54f30c18482cb9b48dffb010a93dbc029054b6cc515083a740d324", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__16.stl": "18bb99691eed524393e8f3a0ff29e79d97ab75689bc92144bbdfafa2cc79d37b", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__17.stl": "6513dbd192d2bf3a4f7016231f1ba5eca93916c9ebfdc334e9b435ca1d302ff6", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__2.stl": "30dab41f14bae95d737abca3c934eb5d570ab107dff0b40952fbdfb9d07eab79", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__3.stl": "9fd96f5ad387fe7dd96eaa65e386637800e2f5637b6167fbad4ecb37d8598f47", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__4.stl": "33c0e7ef538e2c4473efa9c4ef2a8441b6094b65090b4dec71abe82001d974df", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__5.stl": "09f08e7dead2d6363618471adadb1b94af39e8056676670f3fd21ec06c197b20", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__6.stl": "888b5919d10dbfb59b964c3cafc2ddbdac3b5426e20dd13bcaf4baedd5ee15c9", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__7.stl": "2d64b18f1e70b0460eb53335d340b7c879976a3093ec67b07bcb7245e371a91d", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__8.stl": "db3708da6b14daf3fc9a1909a8325b68f65ff69146cdd20b69a9ddc4ce51f50d", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__9.stl": "74969d09b8b7f75cc984c5c7e5526d2941c6724f48b3efa1fa8730cc7820f4f5", + "rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/yam.xml": "79078fe009fd31c90343d5240b6f7df475ea30848aa994108b90d342e5b37d06", + "rl/mjlab/asset_zoo/robots/i2rt_yam/yam_constants.py": "3aa54282545d9c42534e9410ceedfda08e0f199addf14a64d7322aafa7843c2e", + "rl/mjlab/asset_zoo/robots/unitree_g1/__init__.py": "c11e649d6d5209becaf6dbca2d09dee83c17844b5d941c2e1cc6f16ba618bf14", + "rl/mjlab/asset_zoo/robots/unitree_g1/g1_constants.py": "30805e420c1dd03518ee582465fbce9cf097f4dc2eb0c1c2fce55c0332040c07", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/head_link.STL": "005fb67fbd3eff94aa8bf4a6e83238174e9f91b6721f7111594322f223724411", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_pitch_link.STL": "d49e3abc6f5b12e532062cd575b87b5ef40cd2a3fc18f54a1ca5bba4f773d51d", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_roll_link.STL": "c4092af943141d4d9f74232f3cfa345afc6565f46a077793b8ae0e68b39dc33f", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_elbow_link.STL": "fa752198accd104d5c4c3a01382e45165b944fbbc5acce085059223324e5bed3", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_pitch_link.STL": "4725168105ee768ee31638ef22b53f6be2d7641bfd7cfefe803488d884776fa4", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_roll_link.STL": "91f25922ee8a7c3152790051bebad17b4d9cd243569c38fe340285ff93a97acf", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_yaw_link.STL": "a16d88aa6ddac8083aa7ad55ed317bea44b1fa003d314fba88965b7ed0f3b55b", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_knee_link.STL": "8d92b9e3d3a636761150bb8025e32514c4602b91c7028d523ee42b3e632de477", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_rubber_hand.STL": "cff2221a690fa69303f61fce68f2d155c1517b52efb6ca9262dd56e0bc6e70fe", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_pitch_link.STL": "f0d1cfd02fcf0d42f95e678eeca33da3afbcc366ffba5c052847773ec4f31d52", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_roll_link.STL": "fb9df21687773522598dc384f1a2945c7519f11cbc8bd372a49170316d6eee88", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_yaw_link.STL": "1aa97e9748e924336567992181f78c7cd0652fd52a4afcca3df6b2ef6f9e712e", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_pitch_link.STL": "b251d8e05047f695d0f536cd78c11973cfa4e78d08cfe82759336cc3471de3a9", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_roll_link.STL": "edc387c9a0ba8c2237e9b296d32531426fabeb6f53e58df45c76106bca74148c", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_yaw_link.STL": "83f8fb3a726bf9613d65dd14f0f447cb918c3c95b3938042a0c9c09749267d3b", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/logo_link.STL": "8571a0a19bc4916fa55f91449f51d5fdefd751000054865a842449429d5f155b", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis.STL": "5ba6bbc888e630550140d3c26763f10206da8c8bd30ed886b8ede41c61f57a31", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis_contour_link.STL": "5cc5c2c7a312329e3feeb2b03d3fc09fc29705bd01864f6767e51be959662420", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_pitch_link.STL": "15be426539ec1be70246d4d82a168806db64a41301af8b35c197a33348c787a9", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_roll_link.STL": "4b66222ea56653e627711b56d0a8949b4920da5df091da0ceb343f54e884e3a5", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_elbow_link.STL": "1be925d7aa268bb8fddf5362b9173066890c7d32092c05638608126e59d1e2ab", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_pitch_link.STL": "e4f3c99d7f4a7d34eadbef9461fc66e3486cb5442db1ec50c86317d459f1a9c6", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_roll_link.STL": "4c254ef66a356f492947f360dd931965477b631e3fcc841f91ccc46d945d54f6", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_yaw_link.STL": "e479c2936ca2057e9eb2f7dff6c189b7419d7b8484dea0b298cbb36a2a6aa668", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_knee_link.STL": "63c4008449c9bbe701a6e2b557b7a252e90cf3a5abcf54cee46862b9a69f8ec8", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_rubber_hand.STL": "99533b778bca6246144fa511bb9d4e555e075c641f2a0251e04372869cd99d67", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_pitch_link.STL": "24cdb387e0128dfe602770a81c56cdce3a0181d34d039a11d1aaf8819b7b8c02", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_roll_link.STL": "962b97c48f9ce9e8399f45dd9522e0865d19aa9fd299406b2d475a8fc4a53e81", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_yaw_link.STL": "a0b76489271da0c72461a344c9ffb0f0c6e64f019ea5014c1624886c442a2fe5", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_pitch_link.STL": "d22f8f3b3127f15a63e5be1ee273cd5075786c3142f1c3d9f76cbf43d2a26477", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_roll_link.STL": "a7ee9212ff5b94d6cb7f52bb1bbf3f352194d5b598acff74f4c77d340c5b344f", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_yaw_link.STL": "bc9dece2d12509707e0057ba2e48df8f3d56db0c79410212963a25e8a50f61a6", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/torso_link_rev_1_0.STL": "11ddb46f2098efbbd8816b1d65893632d8e78be936376c7cdcd6771899ccc723", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_roll_link_rev_1_0.STL": "b67c347a05abc3e8ddae600d98a082bee273bb39f8e651647708b0a7140a8a97", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_yaw_link_rev_1_0.STL": "ec6db442b11f25eed898b5add07940c85d804f300de24dcbd264ccd8be7d554c", + "rl/mjlab/asset_zoo/robots/unitree_g1/xmls/g1.xml": "febdcbeffbbf84051556ae41a5ac1b43fb479a5d76bdb3f54824dbc2721c20aa", + "rl/mjlab/asset_zoo/robots/unitree_go1/__init__.py": "976aec5cf5633d29630ae1ef74d1139504a22cdffc2ef18ba1617ae679144f2f", + "rl/mjlab/asset_zoo/robots/unitree_go1/go1_constants.py": "fca9478fa292b4a0d7c473be31d6537ae613df7490f4941a885ad31d9723f565", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/calf.stl": "ad3685e132454fd64e4ca3e5089f0d812fb53958e95c14913a22e3193e9e9ed8", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/hip.stl": "b6599b9dd2751ebd87449203223f5683a6c235268b76fcc443e00e0d974a06c0", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh.stl": "bbf7c0a441323088c59276dbec8a32e708a1c61f42d30a6f6775176366ea23f8", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh_mirror.stl": "1a90b89de47a57cb12a2d703f6b008a6ace8a1890a381c27723296fd0d00b9d8", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/trunk.stl": "051a57995dc6f892fc9bc53cd94e9a894f3dc209fd646e94d55d764150013970", + "rl/mjlab/asset_zoo/robots/unitree_go1/xmls/go1.xml": "2f1856b0c83994c009cafbad1f5eac38d09efa1a64db73681ff72c9237822db3", + "rl/mjlab/entity/__init__.py": "2718cbce89ff2c6549d855d096329a72a87d5d77e92daa18b257ee946489cba6", + "rl/mjlab/entity/data.py": "03deabf1afc122192f031ef71a328e1443d602801d70da5cefe4b754980cfb71", + "rl/mjlab/entity/entity.py": "f0353cfc1e689d6e4cf0dc8d59d26847367cc90842ecebd9bc1a5a3044613eda", + "rl/mjlab/entity/variants.py": "f3c3ca5c2712f87a1647b3e9a27bf5d538a6e28a60f3e871ce2d8be65474e41d", + "rl/mjlab/envs/__init__.py": "5c41a9f8cfb59196dbd322e81a54df719d74fd30ff4e1e1958b5778ebc9831a1", + "rl/mjlab/envs/manager_based_rl_env.py": "e05034969baaa6ca38d7587190eef95d468fd413f182edd2c8d387c2950701f1", + "rl/mjlab/envs/mdp/__init__.py": "69d5a26435e69f8636093ef71a977bcb4f45ded4a4277c80c60defda41be1da9", + "rl/mjlab/envs/mdp/actions/__init__.py": "af023e77e5e624d86121742acab67a3eae2d67a7d2f1c595c9a59ba168386859", + "rl/mjlab/envs/mdp/actions/actions.py": "9cae83a2b0006320a8e15e040a0edbf5b4ae76d023832563f1ac36630f3b28d1", + "rl/mjlab/envs/mdp/actions/differential_ik.py": "b8b9654921afb7a46abb57719e2429c2274653296ea345469df7b92bc101124d", + "rl/mjlab/envs/mdp/curriculums.py": "844336b87cf71dfc6d6e60ba8001fdae42736deb9c8cd3e2cb81c1af0f7cb652", + "rl/mjlab/envs/mdp/dr/__init__.py": "633547b48443bffb3e0a04db289559e684e7c75b6c7becb069a82886930b04e1", + "rl/mjlab/envs/mdp/dr/_core.py": "166781592b42e07845acb9e836036b57d377c9b2ef8bc062a7059f6e3b5da41e", + "rl/mjlab/envs/mdp/dr/_types.py": "0dadb1cbf88bdf97cd92f636742715520cc5d27c7a6b0a2863a1be703c8c044b", + "rl/mjlab/envs/mdp/dr/actuator.py": "20ecdff1cb0d590726b1f5ed9d4e9d1de5a6380ca51ec7b05c0cf2ce34812b9f", + "rl/mjlab/envs/mdp/dr/body.py": "87a1135a03648880438270467301ff78b47297d1ec74755734b0859c12f6e15d", + "rl/mjlab/envs/mdp/dr/camera.py": "ee102f4a08fe984765cd6744c44b9a9a50f99229abc498002338b992835b38db", + "rl/mjlab/envs/mdp/dr/geom.py": "634dfe88eb5118d48708b9851cb42e9c9b38bf83b2f11b194686b644806f66a1", + "rl/mjlab/envs/mdp/dr/joint.py": "8ca0f9626b5c8b5fedb57013d3ed229ca00fa397378cfde329ebbfe1b2617e19", + "rl/mjlab/envs/mdp/dr/light.py": "26bf5a2209bd381319bff624b572918b5966e18c876851cccb0f7b0032710b24", + "rl/mjlab/envs/mdp/dr/material.py": "582ce46c775deb750648970c279fc0f7a3c0e8d021d99286fa27bb571b5c4b45", + "rl/mjlab/envs/mdp/dr/pair.py": "d1f99a5f200ff29da98761330a00a6e4cda9cc8073191697fc85b4ce0b46115d", + "rl/mjlab/envs/mdp/dr/site.py": "a3ed4a1d072a7417653d37f5ddcd8474278901b57f5309d0955d335985bec694", + "rl/mjlab/envs/mdp/dr/tendon.py": "eaf357196128be8c06af0e79715981509bd08ae427c367cdb0c6564c8f5ce42e", + "rl/mjlab/envs/mdp/events.py": "922a2475e58fe5ef15bb8a4dbae43608b874b9fb25cf12c0d794e6682ef646ad", + "rl/mjlab/envs/mdp/metrics.py": "5a0d5582aa7fe8c2d530dc5afdf151eace17d529429176a2fbff003d7d60e1ba", + "rl/mjlab/envs/mdp/observations.py": "610b2682fada5fbd7784416e53ba3375cd9e480542737ee4bd5b1f5af79b4b72", + "rl/mjlab/envs/mdp/rewards.py": "9a5908d02ee9ab05390961f70d021fb5762c03db53413b2bbe1fe8ce0699697a", + "rl/mjlab/envs/mdp/terminations.py": "3c0d64bdc9aa69d8fd372f343e47ab6f9da59a77d2efd8e3d860eff2c1ebf68a", + "rl/mjlab/envs/types.py": "9508db1d92998c2465bd24ff24b6966f5634a7b0ad7ecb0fcdb9df6e6fdf93d0", + "rl/mjlab/licenses/IsaacLab.txt": "df351ebfa8af317bf5a9ad83df63a8b68e4f5c04b71f30767cdb27aa434eac62", + "rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt": "dc46c3a7dc48221aa783e634d15db0f816ba83ba5cdb77bbaa91525c61c7552f", + "rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt": "98335465f43a20b5850e4651db6e74c4aa1e9fc8e8813d38f345178045c0da50", + "rl/mjlab/licenses/mujoco_menagerie-unitree_go1.txt": "2b794acf1e250f5545ae5a47d81e5685ea5488ce077d1087ed8be5f81e9ba5c0", + "rl/mjlab/managers/__init__.py": "6fd6c581b1f283290e1e03d8a9feb5516a4e505763885e870ea47af38ddf20ae", + "rl/mjlab/managers/action_manager.py": "fec79f933fd9437dc9fcb784e4f214b9bae4aad2252c2d2d1387b11b605a5332", + "rl/mjlab/managers/command_manager.py": "e9cf2e6697722af77fa32060844819287e3a12fffb091234514b74c18762342a", + "rl/mjlab/managers/curriculum_manager.py": "cc34f5b0897ac148d1a51503a63d987e8b549549fcb1713cd35c12e20147cf0a", + "rl/mjlab/managers/event_manager.py": "c8d74e87aab629238ee6b4d0cb458e97a6938670fe492f76ce74bc0488401689", + "rl/mjlab/managers/manager_base.py": "90dedd602785e8a439a2eca5789137bb128674d181709a1be6ec42901a0326cd", + "rl/mjlab/managers/metrics_manager.py": "5c1f24e75fd21220b5c96f974127568150ce6e9ee16d0abb97dcd933173f6723", + "rl/mjlab/managers/observation_manager.py": "1be6b23b36b595e359f4b7d055dc51f59885579fc904767b5aa499f08e6d14fe", + "rl/mjlab/managers/recorder_manager.py": "9c30862b75b2c2a77b49f331125be382912099147c6ae982aa682a58ce7c3e66", + "rl/mjlab/managers/reward_manager.py": "c43705a88730a16b5dc9031367cfe423259fa1d752bf85aa20165d5e10174f04", + "rl/mjlab/managers/scene_entity_config.py": "12f0ac1baf8894a6435b2733f148cb0282dab561e481e657e7324a40c1e7fe92", + "rl/mjlab/managers/termination_manager.py": "87cc51aa9738e6d137ee936317d76d0b7472c24c58776b4f99dd1bde81d29a6a", + "rl/mjlab/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/rl/__init__.py": "d67a01c3df70b1d481498bf8f258c3c7f589e9d163605a7dc70fee83e9189597", + "rl/mjlab/rl/config.py": "7d391cc0d7a37a4f86c3c4f4a0adc6d75c546273085feed0ddf597aedba6134a", + "rl/mjlab/rl/exporter_utils.py": "4841e3c17518e51286670d6e5e7ff11ec0faec12a6555216312d442f9c1c33be", + "rl/mjlab/rl/runner.py": "61e9200bf304e3df71096cd69f28f4d5973157e61cdc549fc21b6f156326f77e", + "rl/mjlab/rl/spatial_softmax.py": "413922be88fe797c528f59409f9975979cd1886d8ed51b17e28b9377ceadec7e", + "rl/mjlab/rl/vecenv_wrapper.py": "d458aa421d72c979d0269a24e51a4b49a02e4ec8cb5a9d10e47290aa5d86a3e2", + "rl/mjlab/scene/__init__.py": "ff411f5da07d192bd797040453398eb68f5577c35e41b1e8f41aef58a02b9610", + "rl/mjlab/scene/scene.py": "42893e03dc3cc3ad2199831f5b70331bfe8be56cd0b785e07b8dad6b8fb73897", + "rl/mjlab/scene/scene.xml": "0fa9355112e117328986cda47e4fbf4f5d47904b7fccb805d70dccaccdd038a4", + "rl/mjlab/scripts/_cli.py": "bcb1f4394126ee112578c905fb4ba89d8f8e67c9e7df55205e0c425ab3759219", + "rl/mjlab/scripts/csv_to_npz.py": "ea412e75427b61150eb9a8bb72bdbc0ecb0c42b062538195cf360cd53cfc2d7a", + "rl/mjlab/scripts/demo.py": "b20dec8c1e9c71fec5929294d242ea34a9e2fc7f81fb384346b13a5dcfa6917a", + "rl/mjlab/scripts/export_scene.py": "da4519902a6e31a7883105d596521fd4db0bb1a14f55764c603e944a2ee2469b", + "rl/mjlab/scripts/gcs.py": "521bfdbd95156026ec568a11e3da24caab3826d9ece8852d3a8d015fc6d82143", + "rl/mjlab/scripts/list_envs.py": "0fd1214a8be3e0e757a620f4d547b6b733dee1203aac053c3e4bcf31bdc32e91", + "rl/mjlab/scripts/nan_viz.py": "2766d372e6ae5f2250b2b781e1307a3ec26f87d7884a1a28a56b016a10aa0826", + "rl/mjlab/scripts/play.py": "606f537c568247c5e9ba2e94ac1a197675411b360fb1db3231f1b46710539456", + "rl/mjlab/scripts/train.py": "25da18c990b2267dfa14db97b10397caa089fe9fafa071a614e166cbb75af0f7", + "rl/mjlab/scripts/visualize_terrain.py": "8e260e01a2abf5acb1ca73f26e9e9aef668132a4bb2b934730ea44b95317c588", + "rl/mjlab/sensor/__init__.py": "7961da9847ef0fe23e107bccf0b795cda90008e456e7dc85dfed000f86bfb332", + "rl/mjlab/sensor/builtin_sensor.py": "401e495124fa96a89c87cd510da8f43c693db3f9110b93000dcb515171e57adf", + "rl/mjlab/sensor/camera_sensor.py": "52e9761f838a224ea71c276aeddc11af47bee7050664fe0a88ede724471f1597", + "rl/mjlab/sensor/contact_sensor.py": "8b55ab0d6a21d0256e2fc29187c00cbfe4412cd08993050386c6a1f08197fac7", + "rl/mjlab/sensor/raycast_sensor.py": "9e714ac82a7916d1c9bc0f7c15fb209834a5c2b3c6198e4dc3f4c172127263d6", + "rl/mjlab/sensor/sensor.py": "c700983150dbd74fbceec7486b842af4dc8917a251af0277436aee09a69dd077", + "rl/mjlab/sensor/sensor_context.py": "9cb90de6639bcaf38756a584ea9c34f6b35626a5ccaa4a44a3db435f8a7600ca", + "rl/mjlab/sensor/terrain_height_sensor.py": "18a7b190a01fe57cf1e2dcf597dedc3da410eef1d571f4ae000edb6a68e5d692", + "rl/mjlab/sim/__init__.py": "6c7e8c6d5bd848c06b243d54522ca446a94e023235c0c8cb8dd53e13551cf3e2", + "rl/mjlab/sim/randomization.py": "96762d28662fea35c410245df8281fd45794ace4b69a2765f873d4c285dece94", + "rl/mjlab/sim/sim.py": "73ba9898c14cbac674e1e08cec0bdac8e22289c84d17c783e67daefcbc7709b8", + "rl/mjlab/sim/sim_data.py": "fe0191280f56e6a51c39f94363c1306d9dda65588a05a96177f6cf2ff8578667", + "rl/mjlab/tasks/__init__.py": "310ff3bbeed45f4792394e36ff98eb4b5b70e0657c2e5ef67dd363d510133d9b", + "rl/mjlab/tasks/cartpole/__init__.py": "161dd6cccab68333cd14fa206aba646b78ea2fb0ed191e59bdc943f9e761bd5a", + "rl/mjlab/tasks/cartpole/cartpole.xml": "c4adb6759cea1f380d8b20c067339976ca21a5d824a515a3cd5f75e0969bdbe1", + "rl/mjlab/tasks/cartpole/cartpole_env_cfg.py": "4523135d867e3ba3492147d056c3916f1db427d57866f57e9ac0a8f3bdaec7bc", + "rl/mjlab/tasks/manipulation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/tasks/manipulation/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/tasks/manipulation/config/yam/__init__.py": "9f5978bf241f6890a2b64459c8b365818ab22068c1100b0f97d1de67a64af36c", + "rl/mjlab/tasks/manipulation/config/yam/env_cfgs.py": "d6358310e5a97e3d0254b249b6e19be9a1e30e42ed8ef348f3a0d6c11fdd7b5b", + "rl/mjlab/tasks/manipulation/config/yam/rl_cfg.py": "c013e8f318d51c3cacf6198986dcd5c3781d10a618165d9681b54e9975d25cbe", + "rl/mjlab/tasks/manipulation/lift_cube_env_cfg.py": "093b235694fb99d8c2bea79f9bf1ad0bc3078f0c97b981b46b8c10488206b390", + "rl/mjlab/tasks/manipulation/mdp/__init__.py": "02b2ea12a27078ba3ab5c444da0cdb461a765b0fec17f06d9120c4f767568802", + "rl/mjlab/tasks/manipulation/mdp/commands.py": "a8f9ccb2040f8e82452d4c82c1d7caedc41b4e1cbdac740da16202c2d0e26c26", + "rl/mjlab/tasks/manipulation/mdp/observations.py": "ef3a2dae26b4a626990474881d2daef485c9c7c26b78b0e873430767a2f04065", + "rl/mjlab/tasks/manipulation/mdp/rewards.py": "716b20339ffbdfdd839b28f855da257ff3cada2db587bfed995fe483f3674345", + "rl/mjlab/tasks/manipulation/mdp/terminations.py": "8da09eb99b1365c7e6bd5f7cf68e6b77a57da2c93975b265d6f211c757186186", + "rl/mjlab/tasks/manipulation/rl/__init__.py": "52e7cb371a86b6f62d3d0df8ad4fe6ac08875858b2e0ae11d33ac401935628f6", + "rl/mjlab/tasks/manipulation/rl/runner.py": "c9c43b3b8aebc444d655f40a5caa1ef145a8ce48745b7f08ce53eae1bd09ec37", + "rl/mjlab/tasks/registry.py": "f1c82c8f1ff5d24263787f4cfc2805b7b4103253fe59c054bf649d24c7cc9476", + "rl/mjlab/tasks/tracking/__init__.py": "269e19fac2c24adc6064d7c868741fa265f12d78ca07dd12ef7333febde21adf", + "rl/mjlab/tasks/tracking/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/tasks/tracking/config/g1/__init__.py": "fe1f3b8b0fb27d8a353f7d5875b1624c3bdbe04802e2f65c56231b3ccb72e590", + "rl/mjlab/tasks/tracking/config/g1/env_cfgs.py": "ce67f0a0e4f8246aa13e7a1e855b30cbd5f5c40fec7ee8ee1c92ddf72ac44d50", + "rl/mjlab/tasks/tracking/config/g1/rl_cfg.py": "88a8a0acc8052a4bb64957a93ab2a88fa345ee84f63e405ffc6fa70148c6e104", + "rl/mjlab/tasks/tracking/mdp/__init__.py": "1c2637ed117786d35fc7a8b72fa36e11134268d8a2aaefa7109ed826c7eb75bf", + "rl/mjlab/tasks/tracking/mdp/commands.py": "b8700e65bb519b5e68962216ed78a107f6a3c02bab2ff75854e62b1983d1ad97", + "rl/mjlab/tasks/tracking/mdp/metrics.py": "d13a3af779454b911eea976e01115d11f35da64f14c8cf3a83c8a9b934f4c23a", + "rl/mjlab/tasks/tracking/mdp/observations.py": "af2cf6613d657529455eb254c7b13892ab67a0cf8ea5aec2d4df5fad6b4a5330", + "rl/mjlab/tasks/tracking/mdp/rewards.py": "fd7ef49a8015ca19c12586c6b87afca8c8ee7c03c3bb4b3f0a4b58c1dde6c7d4", + "rl/mjlab/tasks/tracking/mdp/terminations.py": "5f88576b5995408a744cde451c7e43982d2f03284cfeebb8856969ca3dc76019", + "rl/mjlab/tasks/tracking/rl/__init__.py": "95fe9fdc0cf223f6415d67607fbd4b2c6653df184cbb15f36403d0cb5ff7052b", + "rl/mjlab/tasks/tracking/rl/runner.py": "978b6f1c783d67bafd942d8790f1021108054441cdd6dd59be3f1c14262d705f", + "rl/mjlab/tasks/tracking/scripts/evaluate.py": "6ca116e3d5273f39acac3117e68871952c55a000c236e4dbcd762035cc99d742", + "rl/mjlab/tasks/tracking/tracking_env_cfg.py": "631b446ffc478c170ce5ef2a1a74b766e963f13a660f78b168060cd425cac243", + "rl/mjlab/tasks/velocity/__init__.py": "479a0ae48d56e95521d9cfd6ccb3bbcebc2bf985355d96879ce3a50d20a142ea", + "rl/mjlab/tasks/velocity/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/tasks/velocity/config/booster_t1/__init__.py": "1a612197876f9187ace170cfd2e7c84f0e23232d5bfb578350185ca8e5e6eb40", + "rl/mjlab/tasks/velocity/config/booster_t1/env_cfgs.py": "eb2d8707af01380de8f718585b92912861f9e2172dc06fcfbaddd0e51baa1aa3", + "rl/mjlab/tasks/velocity/config/booster_t1/mdp.py": "14342b21816900df66c61223b9752f60ce383219abb411ce439ff24481e1d690", + "rl/mjlab/tasks/velocity/config/booster_t1/rl_cfg.py": "eafeaa715631bebb0cfbaf46e91fe299c66f62d3671726135911379010706b90", + "rl/mjlab/tasks/velocity/config/g1/__init__.py": "f4482cf1f1bb27989b0de3f1f45c0c1568185a632c9eb36cc18595cf95ac79cf", + "rl/mjlab/tasks/velocity/config/g1/env_cfgs.py": "5d891ef40987118d1bb2d4bb8791922387d8fdae22752f1e1420dc31f4ddeabf", + "rl/mjlab/tasks/velocity/config/g1/rl_cfg.py": "dcf4a23e031064085e081798c4d3b5bd1cda6ec58ad0081907f2e72c9142a131", + "rl/mjlab/tasks/velocity/config/go1/__init__.py": "9688ac3d4f925a59fcdfe5870753fb52ffc62c6944bb259bae6a3b7143f908cf", + "rl/mjlab/tasks/velocity/config/go1/env_cfgs.py": "3fe9a3f2882d0bb627c791fd009447911bf02dcb977d739ee87c26355909a7af", + "rl/mjlab/tasks/velocity/config/go1/rl_cfg.py": "ac9744e0a2c11db9e60f819c0e8423e6479e094c4aaebe189a6965b06bb90f63", + "rl/mjlab/tasks/velocity/mdp/__init__.py": "231f35d88929e89ae17dbf403b6d049dd9f1f8bba21bfaf74a087bb9435e7dfe", + "rl/mjlab/tasks/velocity/mdp/curriculums.py": "60a857b7e3456f48ba0ddcc33a70c154998a04ef215dbd40200cbdcdad4cea8b", + "rl/mjlab/tasks/velocity/mdp/observations.py": "4e56cefa6481c42e641902cc124cfdd02c3c92b45d8b294253aad0fc87ec9134", + "rl/mjlab/tasks/velocity/mdp/rewards.py": "da3a7b51d5d5c8c67493714e0a26ad29adc7c159e8df875f88bd4bcf5e1de4d4", + "rl/mjlab/tasks/velocity/mdp/terminations.py": "afebaf4170ed51e44e45626ed32e295d29dc43e96f93ca82ccb7b5c0c2c873cf", + "rl/mjlab/tasks/velocity/mdp/terrain_utils.py": "cb03bece16e1848746738404a4070454c4f753261b1c6537dfb1549c3ef7e627", + "rl/mjlab/tasks/velocity/mdp/velocity_command.py": "f20680098726b4f2811a02cbfe3391c946a493ef4536432762b63778e8a81231", + "rl/mjlab/tasks/velocity/rl/__init__.py": "998d67bb2c79979ed2999b14c74231e39a6d018f97c5429405b2991c36cdffa5", + "rl/mjlab/tasks/velocity/rl/runner.py": "05c396737a75720c09ea4495b6b500a5a21e66d9bf60162a2353245d54b5a91e", + "rl/mjlab/tasks/velocity/velocity_env_cfg.py": "8a272a8b3c81c4a0321027e404645c99a4143076522ed3521636bf293dd3137b", + "rl/mjlab/terrains/README.md": "2cf265575a9b76418306a1b4a3c053c3a4c04e181a6b37130b610efa22113021", + "rl/mjlab/terrains/__init__.py": "76c3c30dcc3027e9f8fc7929d9f473db22022eb2fc6cf68dcda556b74e30389a", + "rl/mjlab/terrains/config.py": "3005acad2ae9420be19c0c3c6a8954a190d70cbf19952e86e51c1ce7ba5d06ab", + "rl/mjlab/terrains/heightfield_terrains.py": "98ae7a098ac55a59c0958e3f040e1603395a87c643d93df89b7a44c1105eb08e", + "rl/mjlab/terrains/primitive_terrains.py": "8c526781a1e757fa7af1e165283d9b03f059298fc6704f4f147fdafaf60fd4b7", + "rl/mjlab/terrains/terrain_entity.py": "272a6e1c39f4221ded391f75792982c31f6ede8f44393f8a705d6acf83be9ca1", + "rl/mjlab/terrains/terrain_generator.py": "7f390739f8d074ce9b02b3e2df3d79780899b83096f9efa339d987aa533a4e65", + "rl/mjlab/terrains/utils.py": "a2a1650176f084b447d7c3a6a704dae17bc3a7c050f0cf4aa800c8430f64aa94", + "rl/mjlab/utils/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/utils/actuator.py": "064435b65ac6f6c117c351882652ebe00a9cb36523d82f94a6aa9502276b1fbd", + "rl/mjlab/utils/buffers/__init__.py": "bfe46ad1bdc8c29555f2f6d675e703d91b4698d1cc735d90ffb86d23b120a929", + "rl/mjlab/utils/buffers/circular_buffer.py": "90713e0d41615b627337bfd245761e47f5650fa7373011cc6bfe0eaeb8b71809", + "rl/mjlab/utils/buffers/delay_buffer.py": "c9ab3fca0d384dce6d28478c8b0c1e48fc2a85b2a5180c4a392aa6937717b4d7", + "rl/mjlab/utils/color.py": "ac8c048df0c21ba48558f048d31a4298595105d02fefa1940b226d86f2188ef9", + "rl/mjlab/utils/gpu.py": "73995a8a2f38323463101e2db2621173682709d4347d610a94f62cdccf3c6e7a", + "rl/mjlab/utils/lab_api/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "rl/mjlab/utils/lab_api/math.py": "85e267ee202dc8dc2b340be07ec7abe9b10da9b82c3bf33e16889f422fcf9ca8", + "rl/mjlab/utils/lab_api/string.py": "d0edaedbb1c0a6fed7dd4a72e1b0d8e7f2aa149bbc1ebc52547405e6f669371d", + "rl/mjlab/utils/lab_api/tasks/importer.py": "350f36e86224bcbec60bf427cb53c5fa1de111b34e3c489f3ab67ba766c7839d", + "rl/mjlab/utils/logging.py": "cbd7d5dceb22b84edc27a782925d080a0df2ee776df69d9df3d30c99af1a2200", + "rl/mjlab/utils/mujoco.py": "879b8109e03c4a32143899be4648ef02aa57125a40b36658d1fb361dd74216de", + "rl/mjlab/utils/nan_guard.py": "b3ba9d2c3f34c5e8f1c5249e28699141bd389971eed41cf8d05e0bf5f3199943", + "rl/mjlab/utils/noise/__init__.py": "cd43d801881ad37bc9d34c5263cf1f0c4aaad12355b1a565dfba5d2a722fd958", + "rl/mjlab/utils/noise/noise_cfg.py": "1132aa0bc88f498e5641504e9d79a26545513b123a526c91f61d5823693d54c8", + "rl/mjlab/utils/noise/noise_model.py": "1b6caa33ad612e512027072365ac2ecd3e761ae470cfff38ea945a7763da76bb", + "rl/mjlab/utils/os.py": "8364587683fd9c062e85a40302ed792b8ace389f64b6e0ebd5ec4dde1a9f6241", + "rl/mjlab/utils/random.py": "5ba445435efa5f064aac4b572d035c6a6e98ca3ce459faf1f00dc6f73cbabbc8", + "rl/mjlab/utils/spaces.py": "42eb28b6a48b06886df1c6b385243d340b14f89e71c0e407fda54e0907a53a60", + "rl/mjlab/utils/spec.py": "585b3fc90df15a12984b58e9fbc657950344a3cd45438fd73e35544c6d983719", + "rl/mjlab/utils/spec_config.py": "ac8cd1c7ec64a079d91a27b9f51c0dd4ded4cb7abf0fb85df1cca5acf5d6abcb", + "rl/mjlab/utils/string.py": "702c5ee5bc6b9a3fc5b27a1419daa0653301494bda9f7e5981eda898a601f8bc", + "rl/mjlab/utils/torch.py": "4ba058efa94cc779d2de54db969465338f365372b59943417bbce9193bf33536", + "rl/mjlab/utils/wandb.py": "f70e9108282c6fe115218ec0f5b6fcd0dc9f69b5a945ac2fe43f2e73fb47f155", + "rl/mjlab/utils/wrappers/__init__.py": "bd180f557c3fff4267fab0a7cc838c792a274b4f5d5e6590a3b694d2cad55fe6", + "rl/mjlab/utils/wrappers/video_recorder.py": "1e1dbf62726c711c0b4bd51c523ff0d89435a909a4a28f151c6a9ba959cfd29d", + "rl/mjlab/utils/xml.py": "5976ce00b501a39b42ac5605c2ed9bd05ea4feb5fbf1a2d778a1656ec0caaa3b", + "rl/mjlab/viewer/__init__.py": "6215875f50dd7fd1459682b707ce82da953f70dbcaedf0d363d55a49679b7d1f", + "rl/mjlab/viewer/base.py": "26da5b876828fd72387c351ef74ebe308c83beccae9b006d856780670a7dea29", + "rl/mjlab/viewer/debug_visualizer.py": "d96616c4d9f0ace6921350da19fb6d2d011675e67cba32d42510a6b4ba4d1b3a", + "rl/mjlab/viewer/model_sync.py": "bb23b1c23192a6c4f81364a45e8e0fc32c846d98fc1ec1b438fb1646748afd5c", + "rl/mjlab/viewer/native/__init__.py": "1cc4c390f38e36f2f708330fb39695a789d6ff5557503e5af64862ab2c29f928", + "rl/mjlab/viewer/native/keys.py": "30cf4d7872f7af74d9c8238289d22c6ffca82a56767debe54f4074ec41db9cf9", + "rl/mjlab/viewer/native/viewer.py": "95149f0266a546f9a9ba11215a6f13dcdafbe8324f38d07bbca49234d96a928c", + "rl/mjlab/viewer/native/visualizer.py": "4d91ee004f502740e726d44469073b1a2ab5dc5b94927ead0ab5a7cb9f6fd85a", + "rl/mjlab/viewer/offscreen_renderer.py": "4f8ecc4552413529b4559ea8112211f23603f78a091dd6491e75e324258660a6", + "rl/mjlab/viewer/viewer_config.py": "8040b01ebf951c59f56ec73f965cb9555b03dd11f711de49210c38109bb23154", + "rl/mjlab/viewer/viser/__init__.py": "ea76203023f96327501a320fe3012632d0bee61174152beb98fe3cb556ed71c0", + "rl/mjlab/viewer/viser/camera_viewer.py": "b60f0ed335e40f018847f172ce7e0f70c8ba152d61e2f25eda7092ac1e7fbca4", + "rl/mjlab/viewer/viser/overlays.py": "f6184f1571964fbc94c81a9a607548b4a7d691e54c72ca74e06cf978f9a2eb56", + "rl/mjlab/viewer/viser/reward_bar_panel.py": "bff4194a48f6deaf929ee4945d670d73946bb302e13bddca782179f6ec6a4872", + "rl/mjlab/viewer/viser/scene.py": "107bbbf30da399289161c9815ed6dd53d4e2aa889f56eb32dde822f734bea0b2", + "rl/mjlab/viewer/viser/term_plotter.py": "4b2694fb5633fae6c01e71cf5e2c05a68ad2e5bbe6579f9339296572b6c5371d", + "rl/mjlab/viewer/viser/viewer.py": "8f11f8a13f6240178dcd5ea00df7ca387fe9e136cef839393d8ebfd93138b17e", + "rl/rsl_rl/LICENSE": "3f58bb6b9272e416b8ccbe65889463f75a2fdf66e0e5722513f8622fb23dc778", + "rl/rsl_rl/__init__.py": "9b4c420fef3ae39683544833b6d2efbb26aecfc0583ec59be7f4eeb748fecf66", + "rl/rsl_rl/algorithms/__init__.py": "e370271e689bada5cdf2b4623e0d16d75e8439864a2ae913b22215f7b2985843", + "rl/rsl_rl/algorithms/distillation.py": "3d9fb8e010346ab42f3f4a84584e7e175a7632b39700e7778d0a01746a6317a7", + "rl/rsl_rl/algorithms/ppo.py": "01a82cea45fa9d8ccb161bd75e07d8623d25450ff92982e9c9d59ca8a9b78393", + "rl/rsl_rl/env/__init__.py": "33bfc9562580377c2ce6388c37b0df2b8e992c22ce46923443681cbbf1960d60", + "rl/rsl_rl/env/vec_env.py": "4a72df2fbf9b3e162e0e4b10e0b473511f02ad8c3ebaab0b7993ddfd63a9fa43", + "rl/rsl_rl/extensions/__init__.py": "20dc79170fb9777b7267abe4296b5c22f4b417d5a18c4346fb8159f9d4aea62f", + "rl/rsl_rl/extensions/rnd.py": "7ff5a34f54bcfe8e9494cf5c27155102b6f50c9b10dff49f7a1d98f32e6e16ca", + "rl/rsl_rl/extensions/symmetry.py": "4121d8804c9555118f031f604bc5232c85db0b41ce8d8295a5a22b85190b4048", + "rl/rsl_rl/licenses/LICENSE": "3f58bb6b9272e416b8ccbe65889463f75a2fdf66e0e5722513f8622fb23dc778", + "rl/rsl_rl/licenses/licenses/dependencies/codespell-license.txt": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643", + "rl/rsl_rl/licenses/licenses/dependencies/furo-license.txt": "f68043bb5f4701df244347c82cd2fe24741a56c56a767a1d02bea053670a1ec4", + "rl/rsl_rl/licenses/licenses/dependencies/gitpython-license.txt": "b63a4621e0151014fea8bf668b04c9fd633ea16745d5041a29594ebb12a97ba4", + "rl/rsl_rl/licenses/licenses/dependencies/lucas-c-pre-commit-hooks-license.txt": "6465878993b3f6bbec852eceb213d8163731a637f3e422707377aadf557590f2", + "rl/rsl_rl/licenses/licenses/dependencies/neptune-license.txt": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", + "rl/rsl_rl/licenses/licenses/dependencies/numpy-license.txt": "c17011467b4fe6e20fbfbeda4df09e58c2bca5480a063e55735a4000a379aad9", + "rl/rsl_rl/licenses/licenses/dependencies/onnx-license.txt": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "rl/rsl_rl/licenses/licenses/dependencies/onnxscript-license.txt": "2f07c72751aed99790b8a4869cf2311df85a860b22ded05fa22803587a48922c", + "rl/rsl_rl/licenses/licenses/dependencies/pre-commit-hooks-license.txt": "ea2ca27cba7cc35822d95a46d59bcd3cc88e196592e6390d1949a359ffc990e8", + "rl/rsl_rl/licenses/licenses/dependencies/pre-commit-license.txt": "ea2ca27cba7cc35822d95a46d59bcd3cc88e196592e6390d1949a359ffc990e8", + "rl/rsl_rl/licenses/licenses/dependencies/pytest-license.txt": "ca836a5f9ecca3b2f350230faa20a48fb8b145653b5568d784862df864706b9b", + "rl/rsl_rl/licenses/licenses/dependencies/ruff-license.txt": "aaa18222a10861f3210e4c843794bf87cc56ce62f6732caa52b9525c2117ab25", + "rl/rsl_rl/licenses/licenses/dependencies/setuptools-license.txt": "dc0589ce4ab317782b1bfe5b24d1a3528eed3c028be8f197c516997c777cdd0c", + "rl/rsl_rl/licenses/licenses/dependencies/sphinx-license.txt": "98eac23a35c127d59613f757846b6af90cd4137bbdde8bc3feaf38e28e92f913", + "rl/rsl_rl/licenses/licenses/dependencies/tensorboard-license.txt": "edba09b5279dd73272b22c60479221736076dc3866352712db760cc8179061a4", + "rl/rsl_rl/licenses/licenses/dependencies/tensordict-license.txt": "c5d8d2e3fc64f88c272ee2050af4d84e5f58f1a5d17a3aa99a47b77466a6fe72", + "rl/rsl_rl/licenses/licenses/dependencies/torch-license.txt": "6f598239866f2794b1170c870217a604b6acc98d4aeb4694d1a2f855da1f289c", + "rl/rsl_rl/licenses/licenses/dependencies/wandb-license.txt": "8b338a449a463b1d4fac988638a4741ec35d941e49747d9dd19e0fedaeecb137", + "rl/rsl_rl/licenses/licenses/dependencies/wheel-license.txt": "30c23618679108f3e8ea1d2a658c7ca417bdfc891c98ef1a89fa4ff0c9828654", + "rl/rsl_rl/licenses/pfrl.txt": "1d68cbdc99c9967f63a588ff73fb59461882b21508fe2f91a22f5cf3a4d7ef65", + "rl/rsl_rl/models/__init__.py": "6444fc1df8bc94d40ab1911f0e8e6b3f777459986e7f7944d25b7e5ca9b3e206", + "rl/rsl_rl/models/cnn_model.py": "bc8119274d2058f0abe3482563e6e391aba312fa19c59d961413cfe09f4b0d93", + "rl/rsl_rl/models/mlp_model.py": "86a8ea5c77fdae8167b84d817e5280fe226bf15d2aafab8455c6366155ab2eee", + "rl/rsl_rl/models/rnn_model.py": "80c1d6452ef021e4d7b174231ed7abac1ed0db8de9604561f8060148d4333ccc", + "rl/rsl_rl/modules/__init__.py": "336085b45fb1e4a807f29cb2703b63f933cdf2c2ff00279cef3cfed14bdccc49", + "rl/rsl_rl/modules/cnn.py": "4f5bab5f192c42180125f32a4af350108dd1c247e1e84c81f6cf4ea560037ccf", + "rl/rsl_rl/modules/distribution.py": "ef490fac4baa8a6e6c2b6541cddd4c17f8838097c4298fc7c7efdfefa119941f", + "rl/rsl_rl/modules/mlp.py": "bad934b364b26cd47b6d1612e00ace107a425ae4d272c0cd0c82cbfc57bbcdbc", + "rl/rsl_rl/modules/normalization.py": "67a70c88b540f212759c6e6c023aed41321ea9ad949e5faf428670fafbdf73bd", + "rl/rsl_rl/modules/rnn.py": "07f0eec06c7f2638edbdbc3ba25f56122a7941cb0543138f4a6b04675b2a97e6", + "rl/rsl_rl/runners/__init__.py": "26f4130a403414e13041b62a848d04da3fb0166757603950ea758b151d4b5aaa", + "rl/rsl_rl/runners/distillation_runner.py": "7445cbaf3e1fa241d742447ab17d941a659c92b6f8365ef22c5d818923d9e275", + "rl/rsl_rl/runners/on_policy_runner.py": "c2dbe47974b047e6992e37ff39ac0a236c8fa5cb595cb9e1715e1344e81dc852", + "rl/rsl_rl/storage/__init__.py": "7c9d96df3cad10aa27f34e77f7105a29f74e2d7dd10467b2919f99867a69298e", + "rl/rsl_rl/storage/rollout_storage.py": "d2728ab2cbc280c163a3d33289216d807ac60e1edf41fb85dd09560e966c9b04", + "rl/rsl_rl/utils/__init__.py": "b04e7ce7ccbbaefa275930e5ffc33edd84e77364a14d7e5717e6a5cb02338066", + "rl/rsl_rl/utils/log_writer.py": "53e9447d4337ab7e2d9f48f6ee8ee07ab865789ff1c722e2bc8b3f44ee247693", + "rl/rsl_rl/utils/logger.py": "ff9a592899c6eb803d4a434cef8712aa0424575ac01b13575f3efe2596d4116e", + "rl/rsl_rl/utils/neptune_log_writer.py": "971784c3291d7d7dc8e93202fb8d0102fa67214227440569719cbad278c2cf77", + "rl/rsl_rl/utils/utils.py": "37e1cf2961d91204633e3d3e46951c3abd695c9c4b48a269cbc97846ac155751", + "rl/rsl_rl/utils/wandb_log_writer.py": "69effd6926c8eb7ca4de96d3311e33ccef9cfe8556ca13fec9421e9c8f17ad8d", + "rl/upstream/import.json": "4d2a13b00f229418012b5acd9afa26f3d4fb7f0ab611905dce111392f19671a0", + "rl/upstream/mjlab-local.patch": "178cb15db535a057fd1a064b58f4658f9b0a35e119b1c08d7e37dbb833e4938c", + "rl/upstream/mjlab-pyproject.toml": "2ef132b218db99f26aef59e9df221b520003a83bd7fc4ad0112d9cada2ca9b44", + "rl/upstream/rsl_rl-METADATA": "de4a41da7206f3310ea8f1c9149a0a17859e72cdf4ec8e53888e7a89ef7bc9f3", + "rl/upstream/tests/test_booster_t1.py": "6c8c2549db932150fb87edf58ff8240fd539f4f90fdb9fb074ae5c8c9349a088", + "rl/upstream/tests/test_sim.py": "d6caca50e5e5d86556cd713f3497cd3dcb39a32556738883ea1e5fd08c8ab737", + "rl/upstream/tests/test_velocity_command.py": "4ed39d6345958a48d76db11562f64f4d84d5d2099ed391255f9d0dd78173ff99" + }, + "schema_version": 1, + "upstream": { + "booster_assets": { + "commit": "12a97516f57469078707afb587a5f84c1cb110d1", + "model": "T1_23dof.xml", + "upstream": "https://github.com/BoosterRobotics/booster_assets.git" + }, + "mjlab": { + "added_modules": [ + "asset_zoo/robots/booster_t1", + "tasks/velocity/config/booster_t1" + ], + "commit": "8bb2bc9e187797689ec82c78c35019afd2fd0376", + "inherited_patch": "rl/upstream/mjlab-local.patch", + "upstream": "https://github.com/mujocolab/mjlab", + "version": "1.6.0" + }, + "rsl_rl": { + "installed_RECORD_verified": true, + "upstream": "https://github.com/leggedrobotics/rsl_rl", + "version": "5.5.1" + } + } +} diff --git a/rl/mjlab/LICENSE b/rl/mjlab/LICENSE new file mode 100644 index 0000000..7eb574f --- /dev/null +++ b/rl/mjlab/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025, The mjlab Developers + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/rl/mjlab/__init__.py b/rl/mjlab/__init__.py new file mode 100644 index 0000000..d1d56a3 --- /dev/null +++ b/rl/mjlab/__init__.py @@ -0,0 +1,75 @@ +import os +import sys + +# Default to EGL for GPU-accelerated offscreen rendering on Linux. Must be set +# before any mujoco import: mujoco's gl_context module captures MUJOCO_GL once +# at load time. Override with e.g. MUJOCO_GL=osmesa on clusters without EGL. +# Linux-only because mujoco's gl_context rejects "egl" on macOS/Windows and +# raises at import. On those platforms we leave MUJOCO_GL alone so mujoco +# defaults to GLFW. +if sys.platform.startswith("linux"): + os.environ.setdefault("MUJOCO_GL", "egl") + +import traceback +from importlib.metadata import entry_points +from pathlib import Path + +import tyro +import warp as wp + +MJLAB_SRC_PATH: Path = Path(__file__).parent + +TYRO_FLAGS = ( + # Don't let users switch between types in unions. This produces a simpler CLI + # with flatter helptext, at the cost of some flexibility. Type changes can + # just be done in code. + tyro.conf.AvoidSubcommands, + # Disable automatic flag conversion (e.g., use `--flag False` instead of + # `--no-flag` for booleans). + tyro.conf.FlagConversionOff, + # Use Python syntax for collections: --tuple (1,2,3) instead of --tuple 1 2 3. + # Helps with wandb sweep compatibility: https://brentyi.github.io/tyro/wandb_sweeps/ + tyro.conf.UsePythonSyntaxForLiteralCollections, +) + + +def _configure_warp() -> None: + """Configure Warp globally for mjlab.""" + wp.config.enable_backward = False + + # Keep warp verbose by default to show kernel compilation progress. + # Override with MJLAB_WARP_QUIET=1 environment variable if needed. + quiet = os.environ.get("MJLAB_WARP_QUIET", "0").lower() in ("1", "true", "yes") + wp.config.log_level = wp.LOG_WARNING if quiet else wp.LOG_INFO + + +def _import_registered_packages() -> None: + """Auto-discover and import packages registered via entry points. + + Looks for packages registered under the 'mjlab.tasks' entry point group. + Each discovered package is imported, which allows it to register custom + environments with gymnasium. + """ + mjlab_tasks = entry_points().select(group="mjlab.tasks") + for entry_point in mjlab_tasks: + try: + entry_point.load() + except Exception: + print( + f"[WARN] Failed to load task package '{entry_point.name}' ({entry_point.value}):", + file=sys.stderr, + ) + traceback.print_exc(file=sys.stderr) + + +def _configure_mediapy() -> None: + """Point mediapy at the bundled imageio-ffmpeg binary.""" + import imageio_ffmpeg + import mediapy + + mediapy.set_ffmpeg(imageio_ffmpeg.get_ffmpeg_exe()) + + +_configure_warp() +_configure_mediapy() +_import_registered_packages() diff --git a/rl/mjlab/actuator/__init__.py b/rl/mjlab/actuator/__init__.py new file mode 100644 index 0000000..e61cea9 --- /dev/null +++ b/rl/mjlab/actuator/__init__.py @@ -0,0 +1,63 @@ +"""Actuator implementations for mjlab.""" + +from mjlab.actuator.actuator import Actuator as Actuator +from mjlab.actuator.actuator import ActuatorCfg as ActuatorCfg +from mjlab.actuator.actuator import ActuatorCmd as ActuatorCmd +from mjlab.actuator.actuator import CommandField as CommandField +from mjlab.actuator.builtin_actuator import ( + BuiltinDcMotorActuator as BuiltinDcMotorActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinDcMotorActuatorCfg as BuiltinDcMotorActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinMotorActuator as BuiltinMotorActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinMotorActuatorCfg as BuiltinMotorActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinMuscleActuator as BuiltinMuscleActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinMuscleActuatorCfg as BuiltinMuscleActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinPdActuator as BuiltinPdActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinPdActuatorCfg as BuiltinPdActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinPositionActuator as BuiltinPositionActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinPositionActuatorCfg as BuiltinPositionActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinVelocityActuator as BuiltinVelocityActuator, +) +from mjlab.actuator.builtin_actuator import ( + BuiltinVelocityActuatorCfg as BuiltinVelocityActuatorCfg, +) +from mjlab.actuator.builtin_actuator import ( + DcMotorDatasheetParams as DcMotorDatasheetParams, +) +from mjlab.actuator.builtin_actuator import ( + DcMotorInputMode as DcMotorInputMode, +) +from mjlab.actuator.builtin_actuator import ( + DcMotorPhysicalParams as DcMotorPhysicalParams, +) +from mjlab.actuator.builtin_group import BuiltinActuatorGroup as BuiltinActuatorGroup +from mjlab.actuator.dc_actuator import DcMotorActuator as DcMotorActuator +from mjlab.actuator.dc_actuator import DcMotorActuatorCfg as DcMotorActuatorCfg +from mjlab.actuator.fused_group import FusedActuatorGroup as FusedActuatorGroup +from mjlab.actuator.learned_actuator import LearnedMlpActuator as LearnedMlpActuator +from mjlab.actuator.learned_actuator import ( + LearnedMlpActuatorCfg as LearnedMlpActuatorCfg, +) +from mjlab.actuator.pd_actuator import IdealPdActuator as IdealPdActuator +from mjlab.actuator.pd_actuator import IdealPdActuatorCfg as IdealPdActuatorCfg +from mjlab.actuator.xml_actuator import XmlActuator as XmlActuator +from mjlab.actuator.xml_actuator import XmlActuatorCfg as XmlActuatorCfg diff --git a/rl/mjlab/actuator/actuator.py b/rl/mjlab/actuator/actuator.py new file mode 100644 index 0000000..756d450 --- /dev/null +++ b/rl/mjlab/actuator/actuator.py @@ -0,0 +1,436 @@ +"""Base actuator interface.""" + +from __future__ import annotations + +import dataclasses +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, ClassVar, Generic, Literal, TypeVar + +import mujoco +import mujoco_warp as mjwarp +import torch + +from mjlab.utils.buffers import DelayBuffer + +if TYPE_CHECKING: + from mjlab.entity import Entity + from mjlab.entity.data import EntityData + +ActuatorCfgT = TypeVar("ActuatorCfgT", bound="ActuatorCfg") + +CommandField = Literal["position", "velocity", "effort"] + + +class TransmissionType(str, Enum): + """Transmission types for actuators.""" + + JOINT = "joint" + TENDON = "tendon" + SITE = "site" + + +@dataclass(kw_only=True) +class ActuatorCfg(ABC): + target_names_expr: tuple[str, ...] + """Targets that are part of this actuator group. + + Can be a tuple of names or tuple of regex expressions. Interpreted based on + transmission_type. + """ + + transmission_type: TransmissionType = TransmissionType.JOINT + """Transmission type. Defaults to JOINT.""" + + armature: float | None = None + """Reflected rotor inertia. None preserves the XML value.""" + + frictionloss: float | None = None + """Friction loss force limit. None preserves the XML value. + + Applies a constant friction force opposing motion, independent of load or velocity. + Also known as dry friction or load-independent friction. + """ + + viscous_damping: float | None = None + """Passive viscous damping coefficient. None preserves the XML value. + + Produces a dissipative force f(v) = -b·v proportional to velocity. Always present + regardless of actuator activity. Unlike ``damping`` (the PD derivative gain kv, which + is active control), this is a passive property. + + Maps to ```` for JOINT transmission and ```` + for TENDON transmission. Ignored for SITE. + """ + + delay_min_lag: int = 0 + """Minimum command delay in physics timesteps. + + Each step, a lag is sampled uniformly from [min, max]. The command target arrives + that many steps late at the actuator's control law. Models communication and bus + latency between the policy and the motor (as opposed to observation delay, which + models sensor pipeline latency). + """ + + delay_max_lag: int = 0 + """Maximum command delay in physics timesteps. Set > 0 to enable delay.""" + + delay_hold_prob: float = 0.0 + """Probability of keeping the current lag instead of resampling.""" + + delay_update_period: int = 0 + """How often to resample the lag, in physics timesteps (0 = every step).""" + + delay_per_env_phase: bool = True + """Stagger lag resampling across environments so they don't all update + on the same step.""" + + def __post_init__(self) -> None: + if self.armature is not None: + assert self.armature >= 0.0, "armature must be non-negative." + if self.frictionloss is not None: + assert self.frictionloss >= 0.0, "frictionloss must be non-negative." + if self.viscous_damping is not None: + assert self.viscous_damping >= 0.0, "viscous_damping must be non-negative." + if self.transmission_type == TransmissionType.SITE: + if ( + (self.armature is not None and self.armature > 0.0) + or (self.frictionloss is not None and self.frictionloss > 0.0) + or (self.viscous_damping is not None and self.viscous_damping > 0.0) + ): + raise ValueError( + f"{self.__class__.__name__}: armature, frictionloss, and viscous_damping are " + "not supported for SITE transmission type." + ) + assert self.delay_min_lag >= 0, "delay_min_lag must be non-negative." + assert self.delay_max_lag >= 0, "delay_max_lag must be non-negative." + assert self.delay_min_lag <= self.delay_max_lag, ( + "delay_min_lag must be <= delay_max_lag." + ) + assert 0.0 <= self.delay_hold_prob <= 1.0, "delay_hold_prob must be in [0, 1]." + assert self.delay_update_period >= 0, "delay_update_period must be non-negative." + + @abstractmethod + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> Actuator: + """Build actuator instance. + + Args: + entity: Entity this actuator belongs to. + target_ids: Local target indices (for indexing entity arrays). + target_names: Target names corresponding to target_ids. + + Returns: + Actuator instance. + """ + raise NotImplementedError + + +@dataclass +class ActuatorCmd: + """High-level actuator command with targets and current state. + + Passed to actuator's `compute()` method to generate low-level control signals. + All tensors have shape (num_envs, num_targets). + """ + + position_target: torch.Tensor + """Desired positions (joint positions, tendon lengths, or site positions).""" + velocity_target: torch.Tensor + """Desired velocities (joint velocities, tendon velocities, or site velocities).""" + effort_target: torch.Tensor + """Feedforward effort (torques or forces).""" + pos: torch.Tensor + """Current positions (joint positions, tendon lengths, or site positions).""" + vel: torch.Tensor + """Current velocities (joint velocities, tendon velocities, or site velocities).""" + + +def delay_command( + buffer: DelayBuffer, + position_target: torch.Tensor, + velocity_target: torch.Tensor, + effort_target: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Delay the three command-target fields through one shared channel. + + Every target the policy issues (position, velocity, effort) travels the same + command bus and incurs the same latency, so they are stacked and delayed + together by a single DelayBuffer. Feedback (pos, vel) is never delayed. Shared + by Actuator.apply_delay and the fused group so the command-delay semantics + live in exactly one place. + + Args: + buffer: The delay buffer to append to and read the delayed command from. + position_target: Position targets, shape (num_envs, num_targets). + velocity_target: Velocity targets, shape (num_envs, num_targets). + effort_target: Feedforward effort targets, shape (num_envs, num_targets). + + Returns: + The delayed (position_target, velocity_target, effort_target) tuple. + """ + stacked = torch.stack((position_target, velocity_target, effort_target), dim=-1) + buffer.append(stacked) + delayed = buffer.compute() + return delayed[..., 0], delayed[..., 1], delayed[..., 2] + + +class Actuator(ABC, Generic[ActuatorCfgT]): + """Base actuator interface.""" + + # Fusion contract. An actuator whose control output is a stateless function of + # per-target parameters and the command implements control_law over the + # parameter tensors named in param_names, and applies it through the shared + # compute (see IdealPdActuator). Such actuators are fused automatically (see + # mjlab.actuator.fused_group). Actuators with a custom compute (built-ins, XML, + # learned networks) are not fused; they need no opt-out flag, since overriding + # compute is itself the signal. + param_names: ClassVar[tuple[str, ...]] = () + + @staticmethod + def control_law(params: dict[str, torch.Tensor], cmd: ActuatorCmd) -> torch.Tensor: + """Stateless control law over per-target parameters. + + params maps each name in param_names to its tensor of shape + (num_envs, num_targets); for a fused group these are the concatenated + tensors. Returns the control signal of the same trailing shape. Implemented + by stateless-law actuator types. + """ + raise NotImplementedError + + def __init__( + self, + cfg: ActuatorCfgT, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + self.cfg = cfg + self.entity = entity + self._target_ids_list = target_ids + self._target_names = target_names + self._target_ids: torch.Tensor | None = None + self._ctrl_ids: torch.Tensor | None = None + self._global_ctrl_ids: torch.Tensor | None = None + self._mjs_actuators: list[mujoco.MjsActuator] = [] + self._site_zeros: torch.Tensor | None = None + self._delay_buffer: DelayBuffer | None = None + + @property + def has_delay(self) -> bool: + """Whether this actuator has delay configured.""" + return self.cfg.delay_max_lag > 0 + + @property + def target_ids(self) -> torch.Tensor: + """Local indices of targets controlled by this actuator.""" + assert self._target_ids is not None + return self._target_ids + + @property + def target_names(self) -> list[str]: + """Names of targets controlled by this actuator.""" + return self._target_names + + @property + def transmission_type(self) -> TransmissionType: + """Transmission type of this actuator.""" + return self.cfg.transmission_type + + @property + def ctrl_ids(self) -> torch.Tensor: + """Local indices of control inputs within the entity.""" + assert self._ctrl_ids is not None + return self._ctrl_ids + + @property + def global_ctrl_ids(self) -> torch.Tensor: + """Global indices of control inputs in the MuJoCo model.""" + assert self._global_ctrl_ids is not None + return self._global_ctrl_ids + + @abstractmethod + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + """Edit the MjSpec to add actuators. + + This is called during entity construction, before the model is compiled. + + Args: + spec: The entity's MjSpec to edit. + target_names: Names of targets (joints, tendons, or sites) as they + appear in the spec. When the entity's ``spec_fn`` uses internal + ``MjSpec.attach(prefix=...)``, these will include the prefix + (e.g., ``"left/elbow"`` rather than ``"elbow"``). + """ + raise NotImplementedError + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + device: str, + ) -> None: + """Initialize the actuator after model compilation. + + This is called after the MjSpec is compiled into an MjModel. + + Args: + mj_model: The compiled MuJoCo model. + model: The compiled mjwarp model. + data: The mjwarp data arrays. + device: Device for tensor operations (e.g., "cuda", "cpu"). + """ + del mj_model, model # Unused. + self._target_ids = torch.tensor( + self._target_ids_list, dtype=torch.long, device=device + ) + global_ctrl_ids_list = [act.id for act in self._mjs_actuators] + self._global_ctrl_ids = torch.tensor( + global_ctrl_ids_list, dtype=torch.long, device=device + ) + entity_ctrl_ids = self.entity.indexing.ctrl_ids + global_to_local = {gid.item(): i for i, gid in enumerate(entity_ctrl_ids)} + self._ctrl_ids = torch.tensor( + [global_to_local[gid] for gid in global_ctrl_ids_list], + dtype=torch.long, + device=device, + ) + + # Pre-allocate zeros for SITE transmission type to avoid repeated allocations. + if self.transmission_type == TransmissionType.SITE: + nenvs = data.nworld + ntargets = len(self._target_ids_list) + self._site_zeros = torch.zeros((nenvs, ntargets), device=device) + + self._init_delay_buffer(data.nworld, device) + + def _init_delay_buffer(self, num_envs: int, device: str) -> None: + """Create delay buffer. Called during initialize().""" + if not self.has_delay: + return + self._delay_buffer = DelayBuffer( + min_lag=self.cfg.delay_min_lag, + max_lag=self.cfg.delay_max_lag, + batch_size=num_envs, + device=device, + hold_prob=self.cfg.delay_hold_prob, + update_period=self.cfg.delay_update_period, + per_env_phase=self.cfg.delay_per_env_phase, + ) + + def apply_delay(self, cmd: ActuatorCmd) -> ActuatorCmd: + """Delay all command targets with one shared lag. No-op without delay. + + Every target the policy issues (position, velocity, effort) travels the same + command channel and experiences the same latency, so they are stacked and + delayed together. Feedback fields (``pos``, ``vel``) are never delayed. + """ + if self._delay_buffer is None: + return cmd + position_target, velocity_target, effort_target = delay_command( + self._delay_buffer, + cmd.position_target, + cmd.velocity_target, + cmd.effort_target, + ) + return dataclasses.replace( + cmd, + position_target=position_target, + velocity_target=velocity_target, + effort_target=effort_target, + ) + + def set_lags( + self, + lags: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set delay lag values for specified environments. + + Built-in actuators with the same delay config share a fused delay + buffer for performance. Calling ``set_lags`` on any one of them + affects the entire fused group. + + Args: + lags: Lag values in physics timesteps. Shape: (num_env_ids,) or scalar. + env_ids: Environment indices to set. If None, sets all environments. + """ + if self._delay_buffer is not None: + self._delay_buffer.set_lags(lags, env_ids) + + def get_command(self, data: EntityData) -> ActuatorCmd: + """Extract command data for this actuator from entity data. + + Args: + data: The entity data containing all state and target information. + + Returns: + ActuatorCmd with appropriate data based on transmission type. + """ + if self.transmission_type == TransmissionType.JOINT: + return ActuatorCmd( + position_target=data.joint_pos_target[:, self.target_ids], + velocity_target=data.joint_vel_target[:, self.target_ids], + effort_target=data.joint_effort_target[:, self.target_ids], + pos=data.joint_pos[:, self.target_ids], + vel=data.joint_vel[:, self.target_ids], + ) + elif self.transmission_type == TransmissionType.TENDON: + return ActuatorCmd( + position_target=data.tendon_len_target[:, self.target_ids], + velocity_target=data.tendon_vel_target[:, self.target_ids], + effort_target=data.tendon_effort_target[:, self.target_ids], + pos=data.tendon_len[:, self.target_ids], + vel=data.tendon_vel[:, self.target_ids], + ) + elif self.transmission_type == TransmissionType.SITE: + assert self._site_zeros is not None + return ActuatorCmd( + position_target=self._site_zeros, + velocity_target=self._site_zeros, + effort_target=data.site_effort_target[:, self.target_ids], + pos=self._site_zeros, + vel=self._site_zeros, + ) + else: + raise ValueError(f"Unknown transmission type: {self.transmission_type}") + + @abstractmethod + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + """Compute low-level actuator control signal from high-level commands. + + Args: + cmd: High-level actuator command. + + Returns: + Control signal tensor of shape (num_envs, num_actuators). + """ + raise NotImplementedError + + # Optional methods. + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + """Reset actuator state for specified environments. + + Resets delay buffers if present. Subclasses that override this should + call ``super().reset(env_ids)``. + + Args: + env_ids: Environment indices to reset. If None, reset all environments. + """ + if self._delay_buffer is not None: + self._delay_buffer.reset(env_ids) + + def update(self, dt: float) -> None: + """Update actuator state after a simulation step. + + Base implementation does nothing. Override in subclasses that need + per-step updates. + + Args: + dt: Time step in seconds. + """ + del dt # Unused. diff --git a/rl/mjlab/actuator/builtin_actuator.py b/rl/mjlab/actuator/builtin_actuator.py new file mode 100644 index 0000000..6abf4bb --- /dev/null +++ b/rl/mjlab/actuator/builtin_actuator.py @@ -0,0 +1,631 @@ +"""MuJoCo built-in actuators. + +This module provides actuators that use MuJoCo's native actuator implementations, +created programmatically via the MjSpec API. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum +from typing import TYPE_CHECKING + +import mujoco +import numpy as np +import torch + +from mjlab.actuator.actuator import ( + Actuator, + ActuatorCfg, + ActuatorCmd, + TransmissionType, +) +from mjlab.utils.spec import ( + apply_target_overrides, + create_motor_actuator, + create_muscle_actuator, + create_position_actuator, + create_velocity_actuator, +) + +if TYPE_CHECKING: + from mjlab.entity import Entity + + +@dataclass(kw_only=True) +class BuiltinPositionActuatorCfg(ActuatorCfg): + """Configuration for MuJoCo built-in position actuator. + + Under the hood, this creates a actuator for each target and sets the + stiffness, damping and effort limits accordingly. If armature or frictionloss are + set, they override the corresponding joint/tendon properties from XML. + """ + + stiffness: float + """PD proportional gain.""" + damping: float + """PD derivative gain.""" + effort_limit: float | None = None + """Maximum actuator force/torque. If None, no limit is applied.""" + + def __post_init__(self) -> None: + super().__post_init__() + if self.transmission_type == TransmissionType.SITE: + raise ValueError( + "BuiltinPositionActuatorCfg does not support SITE transmission. " + "Use BuiltinMotorActuatorCfg for site transmission." + ) + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinPositionActuator: + return BuiltinPositionActuator(self, entity, target_ids, target_names) + + +class BuiltinPositionActuator(Actuator[BuiltinPositionActuatorCfg]): + """MuJoCo built-in position actuator.""" + + def __init__( + self, + cfg: BuiltinPositionActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Add actuator to spec, one per target. + for target_name in target_names: + actuator = create_position_actuator( + spec, + target_name, + stiffness=self.cfg.stiffness, + damping=self.cfg.damping, + effort_limit=self.cfg.effort_limit, + armature=self.cfg.armature, + frictionloss=self.cfg.frictionloss, + viscous_damping=self.cfg.viscous_damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(actuator) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + return cmd.position_target + + +@dataclass(kw_only=True) +class BuiltinPdActuatorCfg(ActuatorCfg): + """Implicit-integration version of IdealPdActuator. + + Both consume a position target and a velocity target with kp/kd gains. The + difference is in how the PD is delivered to MuJoCo: IdealPdActuator computes + the PD force in Python and feeds it to a ```` element, which MuJoCo + sees as an opaque external force. This actuator expresses the PD as native + MuJoCo elements (a ```` carrying kp, a ```` carrying kd), + so the implicit and implicitfast integrators include the kp/kd derivatives + in their velocity update. That makes the actuator numerically stable at + gain/timestep combinations where explicit Python PD would diverge, which + matters when you want to run a real motor's stiff on-board PD gains in sim. + """ + + stiffness: float + """Proportional gain (kp).""" + damping: float + """Derivative gain (kd).""" + effort_limit: float | None = None + """Maximum total torque applied to the joint or tendon. Enforced as a + sum-clamp on the two PD terms via jnt_actfrcrange (JOINT) or + tendon_actfrcrange (TENDON). None leaves the limit unset.""" + + def __post_init__(self) -> None: + super().__post_init__() + if self.transmission_type == TransmissionType.SITE: + raise ValueError( + "BuiltinPdActuatorCfg does not support SITE transmission. " + "Use BuiltinMotorActuatorCfg for site transmission." + ) + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinPdActuator: + return BuiltinPdActuator(self, entity, target_ids, target_names) + + +class BuiltinPdActuator(Actuator[BuiltinPdActuatorCfg]): + """MuJoCo native PD: paired + elements per target.""" + + def __init__( + self, + cfg: BuiltinPdActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + + @property + def num_targets(self) -> int: + """Number of targets. ``ctrl_ids`` is laid out as ``[pos..., vel...]``, + each block of length ``num_targets``.""" + return len(self._target_ids_list) + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Position elements first, then velocity elements, so ctrl_ids is laid out + # as [pos_0..pos_{N-1}, vel_0..vel_{N-1}]. + for target_name in target_names: + pos_act = create_position_actuator( + spec, + target_name, + actuator_name=f"{target_name}_pd_pos", + stiffness=self.cfg.stiffness, + damping=0.0, # damping lives on the element. + armature=self.cfg.armature, + frictionloss=self.cfg.frictionloss, + viscous_damping=self.cfg.viscous_damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(pos_act) + for target_name in target_names: + vel_act = create_velocity_actuator( + spec, + target_name, + actuator_name=f"{target_name}_pd_vel", + damping=self.cfg.damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(vel_act) + # Effort limit: sum-clamp on the joint/tendon, not on each element. + if self.cfg.effort_limit is not None: + lim = self.cfg.effort_limit + for target_name in target_names: + if self.cfg.transmission_type == TransmissionType.JOINT: + target = spec.joint(target_name) + else: + target = spec.tendon(target_name) + target.actfrclimited = mujoco.mjtLimited.mjLIMITED_TRUE + target.actfrcrange[:] = np.array([-lim, lim]) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + return torch.cat((cmd.position_target, cmd.velocity_target), dim=1) + + +@dataclass(kw_only=True) +class BuiltinMotorActuatorCfg(ActuatorCfg): + """Configuration for MuJoCo built-in motor actuator. + + Under the hood, this creates a actuator for each target and sets its effort + limit and gear ratio accordingly. If armature or frictionloss are set, they override + the corresponding joint/tendon properties from XML. + """ + + effort_limit: float + """Maximum actuator effort.""" + gear: float = 1.0 + """Actuator gear ratio.""" + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinMotorActuator: + return BuiltinMotorActuator(self, entity, target_ids, target_names) + + +class BuiltinMotorActuator(Actuator[BuiltinMotorActuatorCfg]): + """MuJoCo built-in motor actuator.""" + + def __init__( + self, + cfg: BuiltinMotorActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Add actuator to spec, one per target. + for target_name in target_names: + actuator = create_motor_actuator( + spec, + target_name, + effort_limit=self.cfg.effort_limit, + gear=self.cfg.gear, + armature=self.cfg.armature, + frictionloss=self.cfg.frictionloss, + viscous_damping=self.cfg.viscous_damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(actuator) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + return cmd.effort_target + + +def _or_zeros(t: tuple[float, ...] | None, n: int) -> list[float]: + return list(t) if t is not None else [0.0] * n + + +class DcMotorInputMode(IntEnum): + """What the ``ctrl`` signal of a ```` represents. + + Values match mjtCtrlInput, consumed by mjs_setToDCMotor and stored in + actuator_ctrlspec. + """ + + VOLTAGE = mujoco.mjtCtrlInput.mjINPUT_VOLTAGE.value + POSITION = mujoco.mjtCtrlInput.mjINPUT_POS.value + VELOCITY = mujoco.mjtCtrlInput.mjINPUT_VEL.value + + +@dataclass(frozen=True) +class DcMotorDatasheetParams: + """Datasheet characterization of a DC motor.""" + + nominal_voltage: float + """Nominal (rated) voltage V_n [V].""" + stall_torque: float + """Stall torque tau_stall at V_n [N*m].""" + no_load_speed: float + """No-load angular velocity omega_no_load at V_n [rad/s].""" + + def _pack(self) -> tuple[list[float], float, list[float]]: + """Returns (motorconst, resistance, nominal) for set_to_dcmotor.""" + return ( + [0.0, 0.0], + 0.0, + [self.nominal_voltage, self.stall_torque, self.no_load_speed], + ) + + +@dataclass(frozen=True) +class DcMotorPhysicalParams: + """Physical characterization of a DC motor.""" + + kt: float + """Torque constant [N*m/A].""" + ke: float + """Back-EMF constant [V*s/rad].""" + resistance: float + """Terminal resistance R [Ohm].""" + + def _pack(self) -> tuple[list[float], float, list[float]]: + """Returns (motorconst, resistance, nominal) for set_to_dcmotor.""" + return [self.kt, self.ke], self.resistance, [0.0, 0.0, 0.0] + + +@dataclass(kw_only=True) +class BuiltinDcMotorActuatorCfg(ActuatorCfg): + """Native MuJoCo ```` wrapper. + + Models a DC motor: torque is derived from voltage via the motor constant K and + back-EMF, tau = K * (V - K * omega) / R. The back-EMF term lives in biasprm, so + MuJoCo's implicit / implicitfast integrators pick up its velocity derivative as + effective damping. + + Three input modes select what ctrl carries: + + * VOLTAGE: ctrl is the drive voltage. cmd.effort_target carries volts, not torque. + * POSITION / VELOCITY: an internal PID closes on the setpoint with torque-space + gains, stiffness on the position error and damping on the velocity error. The + driver compensates back-EMF, so the commanded torque is delivered until the + drive voltage reaches Vmax. + + Motor characterization: pass either DcMotorDatasheetParams or DcMotorPhysicalParams + as motor_params. mjs_setToDCMotor derives K and R (including the viscous-damping + correction) and packs the generic gainprm / biasprm / dynprm slots. + + Optional extensions, off by default: integral_gain / integral_limit, slew_rate, + inductance / electrical_time_constant, thermal, lugre, cogging. + + dr.pd_gains randomizes only kp and kd; for DR over the extensions, write directly to + actuator_gainprm or actuator_dynprm. + """ + + motor_params: DcMotorDatasheetParams | DcMotorPhysicalParams + """Motor characterization. Datasheet form: (V_n, tau_stall, omega_no_load). + Physical form: (Kt, Ke, R).""" + + mode: DcMotorInputMode = DcMotorInputMode.POSITION + """ctrl input semantics. See class docstring.""" + + stiffness: float = 0.0 + """PID gain kp on the position error, in torque space. Required in POSITION + mode; must be 0 otherwise.""" + + damping: float = 0.0 + """PID gain kd on the velocity error, in torque space. Required in VELOCITY + mode, optional in POSITION mode; must be 0 in VOLTAGE mode.""" + + voltage_limit: float = 0.0 + """Max drive voltage Vmax. Required in POSITION / VELOCITY mode (clamps the + PID's drive voltage); must be 0 in VOLTAGE mode.""" + + integral_gain: float = 0.0 + """PID integral gain ki on the accumulated position error. POSITION mode + only.""" + + integral_limit: float = 0.0 + """Anti-windup clamp Imax on the integrator state. 0 disables (the + integrator can run away).""" + + slew_rate: float = 0.0 + """Max rate of change of the setpoint per second. Must be 0 in VOLTAGE mode. + 0 disables.""" + + effort_limit: float | None = None + """Continuous torque cap [N*m]. Sets actuator_forcerange. None leaves the + per-element forcerange unset.""" + + gear: float = 1.0 + """Mechanical gear ratio.""" + + inductance: float = 0.0 + """Winding inductance L [H]. Enables first-order electrical dynamics on the + motor current. MuJoCo internally uses te = L / R; pass + electrical_time_constant directly to skip the divide. 0 disables.""" + + electrical_time_constant: float = 0.0 + """Alternative to inductance: specify te [s] directly. Ignored if + inductance > 0. 0 disables.""" + + thermal: tuple[float, float, float, float, float, float] | None = None + """Thermal model (R_thermal, C_thermal, tau_thermal, alpha, T0, T_ambient). + See MuJoCo's ```` reference for units and which of the + first three may be underspecified. Effective resistance becomes + R * (1 + alpha * (T + T_ambient - T0)). None disables.""" + + cogging: tuple[float, float, float] | None = None + """Cogging torque (amplitude, periodicity, phase) in (N*m, cycles per unit + length, rad). Models magnetic torque ripple from rotor-stator interaction; + at joint angle q the contribution is amplitude * sin(periodicity * q + phase). + + Added *after* effort_limit is enforced, matching MuJoCo's physical model: + effort_limit bounds the electromagnetic torque (the current limit), not the + mechanical torque. Total joint torque can exceed effort_limit by up to + amplitude. None disables.""" + + lugre: tuple[float, float, float, float, float] | None = None + """LuGre friction (sigma0, sigma1, F_Coulomb, F_Stribeck, v_Stribeck). + Stick-slip friction with bristle-deflection state. Subtracted from joint + torque after the effort_limit clamp (mechanical, like cogging). None + disables.""" + + def __post_init__(self) -> None: + super().__post_init__() + if self.transmission_type == TransmissionType.SITE: + raise ValueError( + "BuiltinDcMotorActuatorCfg does not support SITE transmission. " + "Use BuiltinMotorActuatorCfg for site transmission." + ) + + if self.mode in (DcMotorInputMode.POSITION, DcMotorInputMode.VELOCITY): + if self.mode == DcMotorInputMode.POSITION and self.stiffness <= 0.0: + raise ValueError("POSITION mode requires stiffness > 0.") + if self.mode == DcMotorInputMode.VELOCITY and ( + self.stiffness != 0.0 or self.damping <= 0.0 + ): + raise ValueError("VELOCITY mode requires damping > 0 and stiffness = 0.") + if self.voltage_limit <= 0.0: + raise ValueError(f"{self.mode.name} mode requires voltage_limit > 0.") + elif any((self.stiffness, self.damping, self.voltage_limit, self.slew_rate)): + raise ValueError( + "stiffness, damping, voltage_limit, and slew_rate are unused in VOLTAGE mode." + ) + if self.integral_gain != 0.0 and self.mode != DcMotorInputMode.POSITION: + raise ValueError("integral_gain requires POSITION mode.") + + for name in ( + "integral_gain", + "integral_limit", + "slew_rate", + "inductance", + "electrical_time_constant", + ): + if getattr(self, name) < 0.0: + raise ValueError(f"{name} must be non-negative.") + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinDcMotorActuator: + return BuiltinDcMotorActuator(self, entity, target_ids, target_names) + + +class BuiltinDcMotorActuator(Actuator[BuiltinDcMotorActuatorCfg]): + """MuJoCo native ````: one actuator per target.""" + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + cfg = self.cfg + motorconst, resistance, nominal = cfg.motor_params._pack() + saturation = ( + [cfg.effort_limit, 0.0, 0.0] if cfg.effort_limit is not None else [0.0] * 3 + ) + controller = [ + cfg.stiffness, # kp + cfg.integral_gain, # ki + cfg.damping, # kd + cfg.slew_rate, # slewmax + cfg.integral_limit, # Imax (anti-windup) + cfg.voltage_limit, # v_max + ] + # SITE is rejected in __post_init__, so only JOINT and TENDON remain. + trntype = ( + mujoco.mjtTrn.mjTRN_JOINT + if cfg.transmission_type == TransmissionType.JOINT + else mujoco.mjtTrn.mjTRN_TENDON + ) + + for target_name in target_names: + actuator = spec.add_actuator(name=target_name, target=target_name) + actuator.trntype = trntype + actuator.gear[0] = cfg.gear + actuator.set_to_dcmotor( + motorconst=motorconst, + resistance=resistance, + nominal=nominal, + saturation=saturation, + controller=controller, + cogging=_or_zeros(cfg.cogging, 3), + inductance=[cfg.inductance, cfg.electrical_time_constant], + thermal=_or_zeros(cfg.thermal, 6), + lugre=_or_zeros(cfg.lugre, 5), + ctrlspec=cfg.mode, + ) + + apply_target_overrides( + spec, + target_name, + cfg.transmission_type, + armature=cfg.armature, + frictionloss=cfg.frictionloss, + viscous_damping=cfg.viscous_damping, + ) + + self._mjs_actuators.append(actuator) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + if self.cfg.mode == DcMotorInputMode.POSITION: + return cmd.position_target + if self.cfg.mode == DcMotorInputMode.VELOCITY: + return cmd.velocity_target + # voltage mode: ctrl is the drive voltage carried in effort_target. + return cmd.effort_target + + +@dataclass(kw_only=True) +class BuiltinVelocityActuatorCfg(ActuatorCfg): + """Configuration for MuJoCo built-in velocity actuator. + + Under the hood, this creates a actuator for each target and sets the + damping gain. If armature or frictionloss are set, they override the corresponding + joint/tendon properties from XML. + """ + + damping: float + """Damping gain.""" + effort_limit: float | None = None + """Maximum actuator force/torque. If None, no limit is applied.""" + + def __post_init__(self) -> None: + super().__post_init__() + if self.transmission_type == TransmissionType.SITE: + raise ValueError( + "BuiltinVelocityActuatorCfg does not support SITE transmission. " + "Use BuiltinMotorActuatorCfg for site transmission." + ) + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinVelocityActuator: + return BuiltinVelocityActuator(self, entity, target_ids, target_names) + + +class BuiltinVelocityActuator(Actuator[BuiltinVelocityActuatorCfg]): + """MuJoCo built-in velocity actuator.""" + + def __init__( + self, + cfg: BuiltinVelocityActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Add actuator to spec, one per target. + for target_name in target_names: + actuator = create_velocity_actuator( + spec, + target_name, + damping=self.cfg.damping, + effort_limit=self.cfg.effort_limit, + armature=self.cfg.armature, + frictionloss=self.cfg.frictionloss, + viscous_damping=self.cfg.viscous_damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(actuator) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + return cmd.velocity_target + + +@dataclass(kw_only=True) +class BuiltinMuscleActuatorCfg(ActuatorCfg): + """Configuration for MuJoCo built-in muscle actuator.""" + + length_range: tuple[float, float] = (0.0, 0.0) + """Length range for muscle actuator.""" + gear: float = 1.0 + """Gear ratio.""" + timeconst: tuple[float, float] = (0.01, 0.04) + """Activation and deactivation time constants.""" + tausmooth: float = 0.0 + """Smoothing time constant.""" + range: tuple[float, float] = (0.75, 1.05) + """Operating range of normalized muscle length.""" + force: float = -1.0 + """Peak force (if -1, defaults to scale * FLV).""" + scale: float = 200.0 + """Force scaling factor.""" + lmin: float = 0.5 + """Minimum normalized muscle length.""" + lmax: float = 1.6 + """Maximum normalized muscle length.""" + vmax: float = 1.5 + """Maximum normalized muscle velocity.""" + fpmax: float = 1.3 + """Passive force at lmax.""" + fvmax: float = 1.2 + """Active force at -vmax.""" + + def __post_init__(self) -> None: + super().__post_init__() + if self.transmission_type == TransmissionType.SITE: + raise ValueError( + "BuiltinMuscleActuatorCfg does not support SITE transmission. " + "Use BuiltinMotorActuatorCfg for site transmission." + ) + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> BuiltinMuscleActuator: + return BuiltinMuscleActuator(self, entity, target_ids, target_names) + + +class BuiltinMuscleActuator(Actuator[BuiltinMuscleActuatorCfg]): + """MuJoCo built-in muscle actuator.""" + + def __init__( + self, + cfg: BuiltinMuscleActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Add actuator to spec, one per target. + for target_name in target_names: + actuator = create_muscle_actuator( + spec, + target_name, + length_range=self.cfg.length_range, + gear=self.cfg.gear, + timeconst=self.cfg.timeconst, + tausmooth=self.cfg.tausmooth, + range=self.cfg.range, + force=self.cfg.force, + scale=self.cfg.scale, + lmin=self.cfg.lmin, + lmax=self.cfg.lmax, + vmax=self.cfg.vmax, + fpmax=self.cfg.fpmax, + fvmax=self.cfg.fvmax, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(actuator) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + return cmd.effort_target diff --git a/rl/mjlab/actuator/builtin_group.py b/rl/mjlab/actuator/builtin_group.py new file mode 100644 index 0000000..39b791f --- /dev/null +++ b/rl/mjlab/actuator/builtin_group.py @@ -0,0 +1,192 @@ +"""Batch ctrl writes for builtin actuators. + +Builtin actuators have identity compute(): they just pass the target through to ctrl. +Instead of looping over each actuator individually, this module reads the target tensor +and writes ctrl in one batched operation per actuator type. Delayed builtins go through +a shared DelayBuffer first. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import torch + +from mjlab.actuator.actuator import TransmissionType +from mjlab.actuator.builtin_actuator import ( + BuiltinMotorActuator, + BuiltinMuscleActuator, + BuiltinPositionActuator, + BuiltinVelocityActuator, +) +from mjlab.utils.buffers import DelayBuffer + +if TYPE_CHECKING: + from mjlab.actuator.actuator import Actuator + from mjlab.entity.data import EntityData + +BuiltinActuatorType = ( + BuiltinMotorActuator + | BuiltinMuscleActuator + | BuiltinPositionActuator + | BuiltinVelocityActuator +) + +# Maps (actuator_type, transmission_type) to EntityData target tensor attribute name. +_TARGET_TENSOR_MAP: dict[tuple[type[BuiltinActuatorType], TransmissionType], str] = { + (BuiltinPositionActuator, TransmissionType.JOINT): "joint_pos_target", + (BuiltinVelocityActuator, TransmissionType.JOINT): "joint_vel_target", + (BuiltinMotorActuator, TransmissionType.JOINT): "joint_effort_target", + (BuiltinPositionActuator, TransmissionType.TENDON): "tendon_len_target", + (BuiltinVelocityActuator, TransmissionType.TENDON): "tendon_vel_target", + (BuiltinMotorActuator, TransmissionType.TENDON): "tendon_effort_target", + (BuiltinMotorActuator, TransmissionType.SITE): "site_effort_target", + (BuiltinMuscleActuator, TransmissionType.JOINT): "joint_effort_target", + (BuiltinMuscleActuator, TransmissionType.TENDON): "tendon_effort_target", +} + + +@dataclass +class _FusedDelayGroup: + """A fused group of delayed builtin actuators sharing delay config.""" + + target_attr: str + target_ids: torch.Tensor + ctrl_ids: torch.Tensor + min_lag: int + max_lag: int + hold_prob: float + update_period: int + per_env_phase: bool + absorbed_actuators: list[Actuator] = field(default_factory=list) + delay_buffer: DelayBuffer | None = field(default=None, init=False) + + +@dataclass +class BuiltinActuatorGroup: + """Groups builtin actuators for batch processing.""" + + _index_groups: dict[ + tuple[type[BuiltinActuatorType], TransmissionType], + tuple[torch.Tensor, torch.Tensor], + ] + _delayed_groups: list[_FusedDelayGroup] + + @staticmethod + def process( + actuators: list[Actuator], + ) -> tuple[BuiltinActuatorGroup, tuple[Actuator, ...]]: + """Classify actuators into builtin groups and custom actuators. + + Non-delayed builtins go into direct index groups. Delayed builtins with matching + delay config are fused into shared delay buffer groups. Everything else is returned + as custom actuators. + + Args: + actuators: List of initialized actuators to process. + + Returns: + A tuple of (builtin group, custom actuators). + """ + builtin_groups: dict[ + tuple[type[BuiltinActuatorType], TransmissionType], list[Actuator] + ] = {} + delayed_grouped: dict[tuple, list[Actuator]] = {} + custom_actuators: list[Actuator] = [] + + for act in actuators: + if not isinstance(act, BuiltinActuatorType): + custom_actuators.append(act) + continue + + if not act.has_delay: + key: tuple[type[BuiltinActuatorType], TransmissionType] = ( + type(act), + act.cfg.transmission_type, + ) + builtin_groups.setdefault(key, []).append(act) + continue + + # Delayed builtin: fuse into shared delay buffer group. + delay_key = ( + type(act), + act.cfg.transmission_type, + act.cfg.delay_min_lag, + act.cfg.delay_max_lag, + act.cfg.delay_hold_prob, + act.cfg.delay_update_period, + act.cfg.delay_per_env_phase, + ) + delayed_grouped.setdefault(delay_key, []).append(act) + + # Build non-delayed index groups. + index_groups: dict[ + tuple[type[BuiltinActuatorType], TransmissionType], + tuple[torch.Tensor, torch.Tensor], + ] = { + key: ( + torch.cat([a.target_ids for a in acts], dim=0), + torch.cat([a.ctrl_ids for a in acts], dim=0), + ) + for key, acts in builtin_groups.items() + } + + # Build delayed fused groups. + delayed_groups: list[_FusedDelayGroup] = [] + for (base_type, transmission_type, *_), acts in delayed_grouped.items(): + target_attr = _TARGET_TENSOR_MAP[(base_type, transmission_type)] + target_ids = torch.cat([a.target_ids for a in acts], dim=0) + ctrl_ids = torch.cat([a.ctrl_ids for a in acts], dim=0) + cfg = acts[0].cfg + delayed_groups.append( + _FusedDelayGroup( + target_attr=target_attr, + target_ids=target_ids, + ctrl_ids=ctrl_ids, + min_lag=cfg.delay_min_lag, + max_lag=cfg.delay_max_lag, + hold_prob=cfg.delay_hold_prob, + update_period=cfg.delay_update_period, + per_env_phase=cfg.delay_per_env_phase, + absorbed_actuators=list(acts), + ) + ) + + return BuiltinActuatorGroup(index_groups, delayed_groups), tuple(custom_actuators) + + def initialize(self, num_envs: int, device: str) -> None: + """Create fused delay buffers for delayed builtin groups.""" + for group in self._delayed_groups: + group.delay_buffer = DelayBuffer( + min_lag=group.min_lag, + max_lag=group.max_lag, + batch_size=num_envs, + device=device, + hold_prob=group.hold_prob, + update_period=group.update_period, + per_env_phase=group.per_env_phase, + ) + # Alias the fused buffer into each absorbed actuator so that per-actuator reset + # and set_lags operate on the shared buffer. + for act in group.absorbed_actuators: + act._delay_buffer = group.delay_buffer + + def apply_controls(self, data: EntityData) -> None: + """Write builtin actuator controls to simulation data.""" + # Non-delayed: direct write. + for (actuator_type, transmission_type), ( + target_ids, + ctrl_ids, + ) in self._index_groups.items(): + attr_name = _TARGET_TENSOR_MAP[(actuator_type, transmission_type)] + target_tensor = getattr(data, attr_name) + data.write_ctrl(target_tensor[:, target_ids], ctrl_ids) + + # Delayed: append to buffer, compute delayed value, write. + for group in self._delayed_groups: + assert group.delay_buffer is not None + target_tensor = getattr(data, group.target_attr) + targets = target_tensor[:, group.target_ids] + group.delay_buffer.append(targets) + data.write_ctrl(group.delay_buffer.compute(), group.ctrl_ids) diff --git a/rl/mjlab/actuator/dc_actuator.py b/rl/mjlab/actuator/dc_actuator.py new file mode 100644 index 0000000..b1396c9 --- /dev/null +++ b/rl/mjlab/actuator/dc_actuator.py @@ -0,0 +1,189 @@ +"""DC motor actuator with velocity-based saturation model. + +This module provides a DC motor actuator that implements a realistic torque-speed +curve for more accurate motor behavior simulation. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import mujoco +import mujoco_warp as mjwarp +import torch + +from mjlab.actuator.actuator import ActuatorCmd +from mjlab.actuator.pd_actuator import IdealPdActuator, IdealPdActuatorCfg, pd_torque + +if TYPE_CHECKING: + from mjlab.entity import Entity + +DcMotorCfgT = TypeVar("DcMotorCfgT", bound="DcMotorActuatorCfg") + + +def dc_motor_clip( + effort: torch.Tensor, + saturation_effort: torch.Tensor, + velocity_limit: torch.Tensor, + force_limit: torch.Tensor, + vel: torch.Tensor, +) -> torch.Tensor: + """Clip effort to the DC motor torque-speed curve. + + Linear torque-speed curve: full saturation_effort at zero velocity falling to + zero at velocity_limit, further bounded by the continuous force_limit. Shared + by DcMotorActuator._clip_effort and the fused control law so the curve lives in + one place. + + The corner velocity where the curve intersects force_limit is recomputed from + force_limit every call rather than cached, since force_limit can be + domain-randomized after initialize() (e.g. via set_effort_limit); caching it + would silently go stale. + """ + vel_at_effort_lim = velocity_limit * (1 + force_limit / saturation_effort) + vel_clipped = torch.clamp(vel, min=-vel_at_effort_lim, max=vel_at_effort_lim) + torque_speed_top = saturation_effort * (1.0 - vel_clipped / velocity_limit) + torque_speed_bottom = saturation_effort * (-1.0 - vel_clipped / velocity_limit) + max_effort = torch.clamp(torque_speed_top, max=force_limit) + min_effort = torch.clamp(torque_speed_bottom, min=-force_limit) + return torch.clamp(effort, min=min_effort, max=max_effort) + + +@dataclass(kw_only=True) +class DcMotorActuatorCfg(IdealPdActuatorCfg): + """Configuration for DC motor actuator with velocity-based saturation. + + This actuator implements a DC motor torque-speed curve for more realistic + actuator behavior. The motor produces maximum torque (saturation_effort) at + zero velocity and reduces linearly to zero torque at maximum velocity. + + Note: effort_limit should be explicitly set to a realistic value for proper + motor modeling. Using the default (inf) will trigger a warning. Use + IdealPdActuator if unlimited torque is desired. + + For a native MuJoCo ```` with back-EMF, voltage saturation, and + configurable ``Kt`` / ``Ke`` / ``R``, see ``BuiltinDcMotorActuator``. + """ + + saturation_effort: float + """Peak motor torque at zero velocity (stall torque).""" + + velocity_limit: float + """Maximum motor velocity (no-load speed).""" + + def __post_init__(self) -> None: + """Validate DC motor parameters.""" + super().__post_init__() + import warnings + + if self.effort_limit == float("inf"): + warnings.warn( + "effort_limit is set to inf for DcMotorActuator, which creates an " + "unrealistic motor with unlimited continuous torque. Consider setting " + "effort_limit to your motor's continuous rating (<= saturation_effort). " + "Use IdealPdActuator if you truly want unlimited torque.", + UserWarning, + stacklevel=2, + ) + + if self.effort_limit > self.saturation_effort: + warnings.warn( + f"effort_limit ({self.effort_limit}) exceeds saturation_effort " + f"({self.saturation_effort}). For realistic motors, continuous torque " + "should be <= peak torque.", + UserWarning, + stacklevel=2, + ) + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> DcMotorActuator: + return DcMotorActuator(self, entity, target_ids, target_names) + + +class DcMotorActuator(IdealPdActuator[DcMotorCfgT], Generic[DcMotorCfgT]): + """DC motor actuator with velocity-based saturation model. + + This actuator extends IdealPdActuator with a realistic DC motor model + that limits torque based on current joint velocity. The model implements + a linear torque-speed curve where: + - At zero velocity: can produce full saturation_effort (stall torque) + - At max velocity: can produce zero torque + - Between: torque limit varies linearly + + The continuous torque limit (effort_limit) further constrains the output. + """ + + # Same stateless-law compute as IdealPd, with the torque-speed curve added to + # the law. The velocity feedback the curve needs comes straight from cmd.vel. + param_names = ( + *IdealPdActuator.param_names, + "saturation_effort", + "velocity_limit_motor", + ) + + @staticmethod + def control_law(params: dict[str, torch.Tensor], cmd: ActuatorCmd) -> torch.Tensor: + torque = pd_torque(params["stiffness"], params["damping"], cmd) + return dc_motor_clip( + torque, + params["saturation_effort"], + params["velocity_limit_motor"], + params["force_limit"], + cmd.vel, + ) + + def __init__( + self, + cfg: DcMotorCfgT, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + self.saturation_effort: torch.Tensor | None = None + self.velocity_limit_motor: torch.Tensor | None = None + self._joint_vel_clipped: torch.Tensor | None = None + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + device: str, + ) -> None: + super().initialize(mj_model, model, data, device) + + num_envs = data.nworld + num_joints = len(self._target_names) + + self.saturation_effort = torch.full( + (num_envs, num_joints), + self.cfg.saturation_effort, + dtype=torch.float, + device=device, + ) + self.velocity_limit_motor = torch.full( + (num_envs, num_joints), + self.cfg.velocity_limit, + dtype=torch.float, + device=device, + ) + self._joint_vel_clipped = torch.zeros(num_envs, num_joints, device=device) + + def _clip_effort(self, effort: torch.Tensor) -> torch.Tensor: + # Retained for LearnedMlpActuator, which has its own (stateful) compute and + # reuses this torque-speed clip. DcMotorActuator itself clips inside + # control_law via dc_motor_clip. + assert self.saturation_effort is not None + assert self.velocity_limit_motor is not None + assert self.force_limit is not None + assert self._joint_vel_clipped is not None + return dc_motor_clip( + effort, + self.saturation_effort, + self.velocity_limit_motor, + self.force_limit, + self._joint_vel_clipped, + ) diff --git a/rl/mjlab/actuator/fused_group.py b/rl/mjlab/actuator/fused_group.py new file mode 100644 index 0000000..6ecc446 --- /dev/null +++ b/rl/mjlab/actuator/fused_group.py @@ -0,0 +1,223 @@ +"""Batch compute for stateless-control-law actuators. + +Actuators like IdealPdActuator and DcMotorActuator evaluate their control law in +PyTorch every physics substep. Done per actuator, a robot split into several +groups (e.g. one per motor type) issues that many gathers, delay-buffer updates, +control-law evaluations, and ctrl writes each substep. At small env counts (play +mode with a synchronizing viewer) this is host-launch-bound and noticeably slow, +whereas built-in actuators are already fused into one batched write. + +This module brings the same fusion to actuators whose control output is a +stateless function of per-target parameters and the command. Such actuators apply +their control law through the shared compute inherited from IdealPdActuator; +keeping that compute is exactly what marks an actuator as fusable. Actuators with +a custom compute (built-ins, XML, learned networks) are skipped, with no opt-out +flag required. + +Fusable actuators sharing an exact type, transmission type, and delay config are +fused into one group: their per-target parameters and indices are concatenated so +the gather, optional delay, control law, and ctrl write each happen once for the +whole group, behind a single shared DelayBuffer. SITE transmission is not fused. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import torch + +from mjlab.actuator.actuator import ActuatorCmd, TransmissionType, delay_command +from mjlab.actuator.pd_actuator import IdealPdActuator +from mjlab.utils.buffers import DelayBuffer + +if TYPE_CHECKING: + from mjlab.actuator.actuator import Actuator + from mjlab.entity.data import EntityData + +# Per transmission type, the EntityData attribute names for the five fields the +# control law reads: (position_target, velocity_target, effort_target, position, +# velocity). +_FIELD_MAP: dict[TransmissionType, tuple[str, str, str, str, str]] = { + TransmissionType.JOINT: ( + "joint_pos_target", + "joint_vel_target", + "joint_effort_target", + "joint_pos", + "joint_vel", + ), + TransmissionType.TENDON: ( + "tendon_len_target", + "tendon_vel_target", + "tendon_effort_target", + "tendon_len", + "tendon_vel", + ), +} + + +def _is_fusable(act: Actuator) -> bool: + """An actuator is fusable iff it keeps the shared stateless-law compute. + + IdealPd and DcMotor apply their control law through that inherited compute; + anything with a custom compute (built-ins, XML, and learned actuators, + including DcMotor subclasses that override it) is not fusable. Comparing the + bound function is immune to the inheritance trap an isinstance check would hit. + """ + return ( + type(act).compute is IdealPdActuator.compute + and act.cfg.transmission_type in _FIELD_MAP + ) + + +def _require(t: torch.Tensor | None) -> torch.Tensor: + assert t is not None + return t + + +@dataclass +class _FusedGroup: + """A fused group of actuators sharing type, transmission, and delay config.""" + + actuator_type: type[Actuator] + transmission_type: TransmissionType + target_ids: torch.Tensor + ctrl_ids: torch.Tensor + min_lag: int + max_lag: int + hold_prob: float + update_period: int + per_env_phase: bool + absorbed_actuators: list[Actuator] = field(default_factory=list) + params: dict[str, torch.Tensor] = field(default_factory=dict, init=False) + delay_buffer: DelayBuffer | None = field(default=None, init=False) + + @property + def has_delay(self) -> bool: + return self.max_lag > 0 + + +@dataclass +class FusedActuatorGroup: + """Groups stateless-control-law actuators for batch processing.""" + + _groups: list[_FusedGroup] + + @staticmethod + def process( + actuators: tuple[Actuator, ...] | list[Actuator], + ) -> tuple[FusedActuatorGroup, tuple[Actuator, ...]]: + """Classify actuators into fused groups and remaining custom actuators. + + Fusable actuators (see _is_fusable) with JOINT or TENDON transmission are + fused by (exact type, transmission type, delay config). Everything else is + returned unchanged for the per-actuator path. + + Args: + actuators: List of initialized actuators to process. + + Returns: + A tuple of (fused group, remaining custom actuators). + """ + grouped: dict[tuple, list[Actuator]] = {} + remaining: list[Actuator] = [] + + for act in actuators: + if not _is_fusable(act): + remaining.append(act) + continue + key = ( + type(act), + act.cfg.transmission_type, + act.cfg.delay_min_lag, + act.cfg.delay_max_lag, + act.cfg.delay_hold_prob, + act.cfg.delay_update_period, + act.cfg.delay_per_env_phase, + ) + grouped.setdefault(key, []).append(act) + + groups: list[_FusedGroup] = [] + for (actuator_type, transmission_type, *_), acts in grouped.items(): + cfg = acts[0].cfg + groups.append( + _FusedGroup( + actuator_type=actuator_type, + transmission_type=transmission_type, + target_ids=torch.cat([a.target_ids for a in acts], dim=0), + ctrl_ids=torch.cat([a.ctrl_ids for a in acts], dim=0), + min_lag=cfg.delay_min_lag, + max_lag=cfg.delay_max_lag, + hold_prob=cfg.delay_hold_prob, + update_period=cfg.delay_update_period, + per_env_phase=cfg.delay_per_env_phase, + absorbed_actuators=list(acts), + ) + ) + + return FusedActuatorGroup(groups), tuple(remaining) + + def initialize(self, num_envs: int, device: str) -> None: + """Concatenate parameters and create shared delay buffers for fused groups.""" + for group in self._groups: + # Concatenate each control-law parameter across absorbed actuators, then + # alias every actuator's parameter back to a slice view of the fused tensor. + # + # Invariant: after this an absorbed actuator's parameter tensors are views + # into the group tensors. Mutations must be in-place (e.g. set_gains and + # set_effort_limit assign into a slice) so they write through to what the + # group computes with. Rebinding the attribute would silently detach it; the + # only rebind site is the actuator's initialize(), which runs before fusion. + for name in group.actuator_type.param_names: + group.params[name] = torch.cat( + [_require(getattr(a, name)) for a in group.absorbed_actuators], dim=1 + ) + offset = 0 + for act in group.absorbed_actuators: + n = act.target_ids.numel() + for name in group.actuator_type.param_names: + setattr(act, name, group.params[name][:, offset : offset + n]) + offset += n + + if group.has_delay: + group.delay_buffer = DelayBuffer( + min_lag=group.min_lag, + max_lag=group.max_lag, + batch_size=num_envs, + device=device, + hold_prob=group.hold_prob, + update_period=group.update_period, + per_env_phase=group.per_env_phase, + ) + # Alias the shared buffer into each absorbed actuator so per-actuator + # reset and set_lags operate on it. + for act in group.absorbed_actuators: + act._delay_buffer = group.delay_buffer + + def apply_controls(self, data: EntityData) -> None: + """Compute and write fused actuator controls to simulation data.""" + for group in self._groups: + pos_attr, vel_attr, eff_attr, cur_pos_attr, cur_vel_attr = _FIELD_MAP[ + group.transmission_type + ] + ids = group.target_ids + pos_target = getattr(data, pos_attr)[:, ids] + vel_target = getattr(data, vel_attr)[:, ids] + effort_target = getattr(data, eff_attr)[:, ids] + pos = getattr(data, cur_pos_attr)[:, ids] + vel = getattr(data, cur_vel_attr)[:, ids] + + if group.delay_buffer is not None: + pos_target, vel_target, effort_target = delay_command( + group.delay_buffer, pos_target, vel_target, effort_target + ) + + cmd = ActuatorCmd( + position_target=pos_target, + velocity_target=vel_target, + effort_target=effort_target, + pos=pos, + vel=vel, + ) + torques = group.actuator_type.control_law(group.params, cmd) + data.write_ctrl(torques, group.ctrl_ids) diff --git a/rl/mjlab/actuator/learned_actuator.py b/rl/mjlab/actuator/learned_actuator.py new file mode 100644 index 0000000..f954933 --- /dev/null +++ b/rl/mjlab/actuator/learned_actuator.py @@ -0,0 +1,208 @@ +"""Learned actuator models.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import mujoco +import mujoco_warp as mjwarp +import torch + +from mjlab.actuator.actuator import ActuatorCmd +from mjlab.actuator.dc_actuator import DcMotorActuator, DcMotorActuatorCfg +from mjlab.utils.buffers import CircularBuffer + +if TYPE_CHECKING: + from mjlab.entity import Entity + + +@dataclass(kw_only=True) +class LearnedMlpActuatorCfg(DcMotorActuatorCfg): + """Configuration for MLP-based learned actuator model. + + This actuator learns the mapping from joint commands and state history to + actual torque output. It's useful for capturing actuator dynamics that are + difficult to model analytically, such as delays, non-linearities, and + friction effects. + + The network is trained offline using data from the real system and loaded + as a TorchScript file. The model uses a sliding window of historical joint + position errors and velocities as inputs. + """ + + network_file: str + """Path to the TorchScript file containing the trained MLP model.""" + + pos_scale: float + """Scaling factor for joint position error inputs to the network.""" + + vel_scale: float + """Scaling factor for joint velocity inputs to the network.""" + + torque_scale: float + """Scaling factor for torque outputs from the network.""" + + input_order: Literal["pos_vel", "vel_pos"] = "pos_vel" + """Order of inputs to the network. + + - "pos_vel": position errors followed by velocities + - "vel_pos": velocities followed by position errors + """ + + history_length: int = 3 + """Number of timesteps of history to use as network inputs. + + For example, history_length=3 uses the current timestep plus the previous + 2 timesteps (total of 3 frames). + """ + + # Learned actuators don't use stiffness/damping from PD controller. + stiffness: float = 0.0 + damping: float = 0.0 + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> LearnedMlpActuator: + return LearnedMlpActuator(self, entity, target_ids, target_names) + + +class LearnedMlpActuator(DcMotorActuator[LearnedMlpActuatorCfg]): + """MLP-based learned actuator with joint history. + + This actuator uses a trained neural network to map from joint commands and + state history to torque outputs. The network captures complex actuator + dynamics that are difficult to model analytically. + + The actuator maintains circular buffers of joint position errors and + velocities, which are used as inputs to the MLP. The network outputs are + then scaled and clipped using the DC motor limits from the parent class. + """ + + def __init__( + self, + cfg: LearnedMlpActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + self.network: torch.jit.ScriptModule | None = None + self._pos_error_history: CircularBuffer | None = None + self._vel_history: CircularBuffer | None = None + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + device: str, + ) -> None: + super().initialize(mj_model, model, data, device) + + # Load the trained network from TorchScript file. + self.network = torch.jit.load(self.cfg.network_file, map_location=device) + assert self.network is not None + self.network.eval() + + # Create history buffers. + num_envs = data.nworld + + self._pos_error_history = CircularBuffer( + max_len=self.cfg.history_length, + batch_size=num_envs, + device=device, + ) + self._vel_history = CircularBuffer( + max_len=self.cfg.history_length, + batch_size=num_envs, + device=device, + ) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + """Reset history buffers for specified environments. + + Args: + env_ids: Environment indices to reset. If None, reset all environments. + """ + super().reset(env_ids) + assert self._pos_error_history is not None + assert self._vel_history is not None + + if env_ids is None: + self._pos_error_history.reset() + self._vel_history.reset() + elif isinstance(env_ids, slice): + # Convert slice to indices for CircularBuffer. + batch_size = self._pos_error_history.batch_size + indices = list(range(*env_ids.indices(batch_size))) + self._pos_error_history.reset(indices) + self._vel_history.reset(indices) + else: + self._pos_error_history.reset(env_ids) + self._vel_history.reset(env_ids) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + """Compute actuator torques using the learned MLP model. + + Args: + cmd: High-level actuator command containing targets and current state. + + Returns: + Computed torque tensor of shape (num_envs, num_joints). + """ + assert self.network is not None + assert self._pos_error_history is not None + assert self._vel_history is not None + assert self._joint_vel_clipped is not None + + # Update history buffers with current state. + pos_error = cmd.position_target - cmd.pos + self._pos_error_history.append(pos_error) + self._vel_history.append(cmd.vel) + + # Save velocity for DC motor clipping in parent class. + self._joint_vel_clipped[:] = cmd.vel + + num_envs = cmd.pos.shape[0] + num_joints = cmd.pos.shape[1] + + # Extract history from current to history_length-1 steps back. + # Each returns shape: (num_envs, num_joints). + pos_inputs = [ + self._pos_error_history[lag] for lag in range(self.cfg.history_length) + ] + vel_inputs = [self._vel_history[lag] for lag in range(self.cfg.history_length)] + + # Stack along feature dimension: (num_envs, num_joints, history_length). + pos_stacked = torch.stack(pos_inputs, dim=2) + vel_stacked = torch.stack(vel_inputs, dim=2) + + # Reshape to (num_envs * num_joints, num_history_steps) for network. + pos_flat = pos_stacked.reshape(num_envs * num_joints, -1) + vel_flat = vel_stacked.reshape(num_envs * num_joints, -1) + + # Scale and concatenate inputs based on specified order. + if self.cfg.input_order == "pos_vel": + network_input = torch.cat( + [pos_flat * self.cfg.pos_scale, vel_flat * self.cfg.vel_scale], dim=1 + ) + elif self.cfg.input_order == "vel_pos": + network_input = torch.cat( + [vel_flat * self.cfg.vel_scale, pos_flat * self.cfg.pos_scale], dim=1 + ) + else: + raise ValueError( + f"Invalid input order: {self.cfg.input_order}. Must be 'pos_vel' or 'vel_pos'." + ) + + # Run network inference. + with torch.inference_mode(): + torques_flat = self.network(network_input) + + # Reshape and scale output torques. + computed_torques = torques_flat.reshape(num_envs, num_joints) + computed_torques = computed_torques * self.cfg.torque_scale + + # Clip using DC motor limits from parent class. + return self._clip_effort(computed_torques) diff --git a/rl/mjlab/actuator/pd_actuator.py b/rl/mjlab/actuator/pd_actuator.py new file mode 100644 index 0000000..c783c55 --- /dev/null +++ b/rl/mjlab/actuator/pd_actuator.py @@ -0,0 +1,163 @@ +"""An ideal PD control actuator.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import mujoco +import mujoco_warp as mjwarp +import torch + +from mjlab.actuator.actuator import Actuator, ActuatorCfg, ActuatorCmd +from mjlab.utils.spec import create_motor_actuator + +if TYPE_CHECKING: + from mjlab.entity import Entity + +IdealPdCfgT = TypeVar("IdealPdCfgT", bound="IdealPdActuatorCfg") + + +def pd_torque( + stiffness: torch.Tensor, damping: torch.Tensor, cmd: ActuatorCmd +) -> torch.Tensor: + """Ideal PD control torque, unclamped. + + kp * (pos_target - pos) + kd * (vel_target - vel) + effort_target. Shared by + the ideal PD and DC motor control laws so the PD sum lives in one place. + """ + torque = stiffness * (cmd.position_target - cmd.pos) + torque += damping * (cmd.velocity_target - cmd.vel) + torque += cmd.effort_target + return torque + + +@dataclass(kw_only=True) +class IdealPdActuatorCfg(ActuatorCfg): + """Configuration for ideal PD actuator.""" + + stiffness: float + """PD stiffness (proportional gain).""" + damping: float + """PD damping (derivative gain).""" + effort_limit: float = float("inf") + """Maximum force/torque limit.""" + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> IdealPdActuator: + return IdealPdActuator(self, entity, target_ids, target_names) + + +class IdealPdActuator(Actuator, Generic[IdealPdCfgT]): + """Ideal PD control actuator.""" + + param_names = ("stiffness", "damping", "force_limit") + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + # Shared stateless-law compute. Keeping this (rather than writing a custom + # compute) is what marks an actuator as fusable; subclasses with a different + # control law (e.g. learned networks) override compute instead. + params = {name: getattr(self, name) for name in self.param_names} + return type(self).control_law(params, cmd) + + @staticmethod + def control_law(params: dict[str, torch.Tensor], cmd: ActuatorCmd) -> torch.Tensor: + torque = pd_torque(params["stiffness"], params["damping"], cmd) + force_limit = params["force_limit"] + return torch.clamp(torque, -force_limit, force_limit) + + def __init__( + self, + cfg: IdealPdCfgT, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + self.stiffness: torch.Tensor | None = None + self.damping: torch.Tensor | None = None + self.force_limit: torch.Tensor | None = None + self.default_stiffness: torch.Tensor | None = None + self.default_damping: torch.Tensor | None = None + self.default_force_limit: torch.Tensor | None = None + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Add actuator to spec, one per target. + for target_name in target_names: + actuator = create_motor_actuator( + spec, + target_name, + effort_limit=self.cfg.effort_limit, + armature=self.cfg.armature, + frictionloss=self.cfg.frictionloss, + viscous_damping=self.cfg.viscous_damping, + transmission_type=self.cfg.transmission_type, + ) + self._mjs_actuators.append(actuator) + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + device: str, + ) -> None: + super().initialize(mj_model, model, data, device) + + num_envs = data.nworld + num_joints = len(self._target_names) + self.stiffness = torch.full( + (num_envs, num_joints), self.cfg.stiffness, dtype=torch.float, device=device + ) + self.damping = torch.full( + (num_envs, num_joints), self.cfg.damping, dtype=torch.float, device=device + ) + self.force_limit = torch.full( + (num_envs, num_joints), self.cfg.effort_limit, dtype=torch.float, device=device + ) + + self.default_stiffness = self.stiffness.clone() + self.default_damping = self.damping.clone() + self.default_force_limit = self.force_limit.clone() + + def set_gains( + self, + env_ids: torch.Tensor | slice, + kp: torch.Tensor | None = None, + kd: torch.Tensor | None = None, + ) -> None: + """Set PD gains for specified environments. + + Args: + env_ids: Environment indices to update. + kp: New proportional gains. Shape: (num_envs, num_actuators) or (num_envs,). + kd: New derivative gains. Shape: (num_envs, num_actuators) or (num_envs,). + """ + assert self.stiffness is not None + assert self.damping is not None + + if kp is not None: + if kp.ndim == 1: + kp = kp.unsqueeze(-1) + self.stiffness[env_ids] = kp + + if kd is not None: + if kd.ndim == 1: + kd = kd.unsqueeze(-1) + self.damping[env_ids] = kd + + def set_effort_limit( + self, env_ids: torch.Tensor | slice, effort_limit: torch.Tensor + ) -> None: + """Set effort limits for specified environments. + + Args: + env_ids: Environment indices to update. + effort_limit: New effort limits. Shape: (num_envs, num_actuators) or (num_envs,). + """ + assert self.force_limit is not None + + if effort_limit.ndim == 1: + effort_limit = effort_limit.unsqueeze(-1) + self.force_limit[env_ids] = effort_limit diff --git a/rl/mjlab/actuator/xml_actuator.py b/rl/mjlab/actuator/xml_actuator.py new file mode 100644 index 0000000..98a04ef --- /dev/null +++ b/rl/mjlab/actuator/xml_actuator.py @@ -0,0 +1,132 @@ +"""Use actuators already defined in your robot XML. + +When your MJCF already has ````, ````, ````, or ```` +elements, ``XmlActuatorCfg`` wraps them without re-specifying gains or limits. The +actuator type is auto-detected from the spec element. Delay and other ``ActuatorCfg`` +fields work as usual. + +If your XML has actuators on both joints and tendons, use two configs with different +``transmission_type`` values; a single config can only target one namespace. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast + +import mujoco +import torch + +from mjlab.actuator.actuator import Actuator, ActuatorCfg, ActuatorCmd, CommandField +from mjlab.utils.mujoco import detect_command_field + +if TYPE_CHECKING: + from mjlab.entity import Entity + + +@dataclass(kw_only=True) +class XmlActuatorCfg(ActuatorCfg): + """Wrap existing XML-defined actuators. + + Detects the actuator type (position, velocity, motor, muscle) from the MuJoCo spec + element during ``edit_spec``, or validates against an explicit ``command_field`` if + provided. + """ + + command_field: CommandField | None = None + """If provided, require XML actuators to match this command field. + If None, auto-detect strictly from the spec element.""" + + def build( + self, entity: Entity, target_ids: list[int], target_names: list[str] + ) -> XmlActuator: + return XmlActuator(self, entity, target_ids, target_names) + + +class XmlActuator(Actuator[XmlActuatorCfg]): + """Wrapper for XML-defined actuators.""" + + def __init__( + self, + cfg: XmlActuatorCfg, + entity: Entity, + target_ids: list[int], + target_names: list[str], + ) -> None: + super().__init__(cfg, entity, target_ids, target_names) + self._command_field: CommandField | None = None + + @property + def command_field(self) -> CommandField: + """The resolved command field (detected or explicit), not the raw config value. + + Set during ``edit_spec``. Use ``self.cfg.command_field`` to access the user's + original input (which may be None for auto-detection). + """ + assert self._command_field is not None + return self._command_field + + def edit_spec(self, spec: mujoco.MjSpec, target_names: list[str]) -> None: + # Filter to only targets that have corresponding XML actuators. + filtered_target_ids = [] + filtered_target_names = [] + for i, target_name in enumerate(target_names): + actuator = self._find_actuator_for_target(spec, target_name) + if actuator is not None: + self._mjs_actuators.append(actuator) + filtered_target_ids.append(self._target_ids_list[i]) + # Store the user-facing (stripped) name, not the spec name. + filtered_target_names.append(self._target_names[i]) + + if len(filtered_target_names) == 0: + raise ValueError( + f"No XML actuators found for any targets matching the patterns. " + f"Searched targets: {target_names}. " + f"XML actuator config expects actuators to already exist in the XML." + ) + + # Update target IDs and names to only include those with actuators. + self._target_ids_list = filtered_target_ids + self._target_names = filtered_target_names + + if self.cfg.command_field is not None: + # Validate that every matched actuator is compatible with the + # requested command field. + for act, name in zip(self._mjs_actuators, self._target_names, strict=True): + try: + detected = detect_command_field(act) + except ValueError: + continue # Unrecognized type; trust the explicit override. + if detected != self.cfg.command_field: + raise ValueError( + f"XML actuator for '{name}' is type '{detected}', but " + f"command_field='{self.cfg.command_field}' was requested." + ) + self._command_field = self.cfg.command_field + else: + # Auto-detect from spec elements. Raises for unrecognized types. + fields = [detect_command_field(a) for a in self._mjs_actuators] + unique = set(fields) + if len(unique) > 1: + breakdown = dict(zip(self._target_names, fields, strict=True)) + raise ValueError( + f"Mixed XML actuator types for targets: {breakdown}. " + f"Split into separate XmlActuatorCfg groups." + ) + self._command_field = cast(CommandField, fields[0]) + + def compute(self, cmd: ActuatorCmd) -> torch.Tensor: + if self._command_field == "position": + return cmd.position_target + if self._command_field == "velocity": + return cmd.velocity_target + return cmd.effort_target + + def _find_actuator_for_target( + self, spec: mujoco.MjSpec, target_name: str + ) -> mujoco.MjsActuator | None: + """Find an actuator that targets the given target.""" + for actuator in spec.actuators: + if actuator.target == target_name: + return actuator + return None diff --git a/rl/mjlab/asset_zoo/README.md b/rl/mjlab/asset_zoo/README.md new file mode 100644 index 0000000..24663cf --- /dev/null +++ b/rl/mjlab/asset_zoo/README.md @@ -0,0 +1,7 @@ +# Asset Zoo + +| Robot | Image | +|-------|-------| +| **Unitree G1** | Unitree G1 Humanoid | +| **Unitree Go1** | Unitree Go1 Quadruped | +| **i2rt YAM** | i2rt YAM Robot Arm | diff --git a/rl/mjlab/asset_zoo/__init__.py b/rl/mjlab/asset_zoo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rl/mjlab/asset_zoo/robots/__init__.py b/rl/mjlab/asset_zoo/robots/__init__.py new file mode 100644 index 0000000..f855d96 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/__init__.py @@ -0,0 +1,18 @@ +from mjlab.asset_zoo.robots.i2rt_yam.yam_constants import ( + YAM_ACTION_SCALE as YAM_ACTION_SCALE, +) +from mjlab.asset_zoo.robots.i2rt_yam.yam_constants import ( + get_yam_robot_cfg as get_yam_robot_cfg, +) +from mjlab.asset_zoo.robots.unitree_g1.g1_constants import ( + G1_ACTION_SCALE as G1_ACTION_SCALE, +) +from mjlab.asset_zoo.robots.unitree_g1.g1_constants import ( + get_g1_robot_cfg as get_g1_robot_cfg, +) +from mjlab.asset_zoo.robots.unitree_go1.go1_constants import ( + GO1_ACTION_SCALE as GO1_ACTION_SCALE, +) +from mjlab.asset_zoo.robots.unitree_go1.go1_constants import ( + get_go1_robot_cfg as get_go1_robot_cfg, +) diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/__init__.py b/rl/mjlab/asset_zoo/robots/booster_t1/__init__.py new file mode 100644 index 0000000..788be4d --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/booster_t1/__init__.py @@ -0,0 +1 @@ +"""Booster T1 external asset adapter.""" diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/LICENSE b/rl/mjlab/asset_zoo/robots/booster_t1/assets/LICENSE new file mode 100644 index 0000000..d7a14af --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/booster_t1/assets/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2025, BoosterRobotics + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/T1_23dof.xml b/rl/mjlab/asset_zoo/robots/booster_t1/assets/T1_23dof.xml new file mode 100644 index 0000000..c0e069b --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/booster_t1/assets/T1_23dof.xml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL1.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL1.STL new file mode 100644 index 0000000..0693be3 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL1.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL2.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL2.STL new file mode 100644 index 0000000..3be09ba Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL2.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL3.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL3.STL new file mode 100644 index 0000000..b605c89 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AL3.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR1.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR1.STL new file mode 100644 index 0000000..2be2cec Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR1.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR2.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR2.STL new file mode 100644 index 0000000..1994df8 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR2.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR3.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR3.STL new file mode 100644 index 0000000..e2e4f9f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/AR3.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Left.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Left.STL new file mode 100644 index 0000000..1be4549 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Left.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Right.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Right.STL new file mode 100644 index 0000000..6846acc Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Ankle_Cross_Right.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H1.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H1.STL new file mode 100644 index 0000000..24fd45d Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H1.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H2.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H2.STL new file mode 100644 index 0000000..b14dfb1 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/H2.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Left.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Left.STL new file mode 100644 index 0000000..575936e Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Left.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Right.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Right.STL new file mode 100644 index 0000000..6a24039 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Pitch_Right.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Left.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Left.STL new file mode 100644 index 0000000..e335084 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Left.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Right.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Right.STL new file mode 100644 index 0000000..dbb7e59 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Roll_Right.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Left.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Left.STL new file mode 100644 index 0000000..1330a29 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Left.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Right.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Right.STL new file mode 100644 index 0000000..c8cb541 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Hip_Yaw_Right.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Logo.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Logo.STL new file mode 100644 index 0000000..3fc7dec Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Logo.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Left.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Left.STL new file mode 100644 index 0000000..61dec29 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Left.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Right.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Right.STL new file mode 100644 index 0000000..69d3d6b Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Shank_Right.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Trunk.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Trunk.STL new file mode 100644 index 0000000..d27dfe2 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Trunk.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Waist.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Waist.STL new file mode 100644 index 0000000..ceb1198 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/Waist.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_foot_link.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_foot_link.STL new file mode 100644 index 0000000..b62d41a Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_foot_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_hand_link.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_hand_link.STL new file mode 100644 index 0000000..d6fe8a0 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/left_hand_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_foot_link.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_foot_link.STL new file mode 100644 index 0000000..6b31993 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_foot_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_hand_link.STL b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_hand_link.STL new file mode 100644 index 0000000..850c110 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/booster_t1/assets/meshes/right_hand_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py b/rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py new file mode 100644 index 0000000..a952eb5 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/booster_t1/t1_constants.py @@ -0,0 +1,144 @@ +"""Booster T1 (23 DoF), using bundled BSD-licensed assets by default. + +[uh-vendor] External BOOSTER_ASSETS_ROOT is an explicit optional model override. + +Inertias, limits, armatures and torque ratings come from T1_23dof.xml. +PD gains are simulation training defaults, not a hardware deployment recipe. +""" + +import os +from pathlib import Path + +import mujoco + +from mjlab.actuator import BuiltinPositionActuatorCfg +from mjlab.entity import EntityArticulationInfoCfg, EntityCfg +from mjlab.utils.spec_config import CollisionCfg + + +def get_t1_xml() -> Path: + root = os.environ.get("BOOSTER_ASSETS_ROOT") + if root: + return Path(root).expanduser() / "robots/T1/T1_23dof.xml" + return Path(__file__).parent / "assets/T1_23dof.xml" + + +def get_spec() -> mujoco.MjSpec: + path = get_t1_xml() + if not path.is_file(): + raise FileNotFoundError( + f"T1 asset not found: {path}. Check the bundled assets or BOOSTER_ASSETS_ROOT override." + ) + spec = mujoco.MjSpec.from_file(str(path)) + # The scene owns the ground. Replace torque motors, rather than duplicating them. + spec.delete(spec.geom("ground")) + for actuator in list(spec.actuators): + spec.delete(actuator) + for sensor in list(spec.sensors): + spec.delete(sensor) + for light in list(spec.lights): + spec.delete(light) + + for body in spec.bodies: + for i, geom in enumerate(body.geoms): + if geom.type == mujoco.mjtGeom.mjGEOM_MESH: + geom.name = f"{body.name}_visual_{i}" + else: + geom.name = f"{body.name}_collision_{i}" + geom.group = 3 + + for side in ("left", "right"): + foot = spec.body(f"{side}_ankle_roll_link") + foot.add_site( + name=f"{side}_foot", + pos=(0.0101079, 0.0, -0.0432508), + size=(0.01, 0.01, 0.01), + ) + for name, sensor_type in ( + ("imu_lin_vel", mujoco.mjtSensor.mjSENS_VELOCIMETER), + ("imu_ang_vel", mujoco.mjtSensor.mjSENS_GYRO), + ): + spec.add_sensor( + name=name, type=sensor_type, objtype=mujoco.mjtObj.mjOBJ_SITE, objname="imu" + ) + spec.add_sensor( + name="root_angmom", + type=mujoco.mjtSensor.mjSENS_SUBTREEANGMOM, + objtype=mujoco.mjtObj.mjOBJ_BODY, + objname="trunk", + ) + return spec + + +# XML armatures are preserved (no armature override). +# Each tuple is (joint expression, kp, kd, XML torque limit). +_MOTORS = ( + ("aahead_.*_joint", 5.0, 0.5, 7.0), + (".*_shoulder_.*_joint", 40.0, 2.0, 38.3), + (".*_elbow_.*_joint", 40.0, 2.0, 38.3), + ("waist_yaw_joint", 100.0, 3.0, 68.0), + (".*_hip_pitch_joint", 200.0, 5.0, 98.8), + (".*_hip_roll_joint", 150.0, 4.0, 68.0), + (".*_hip_yaw_joint", 100.0, 3.0, 68.0), + (".*_knee_pitch_joint", 200.0, 5.0, 130.5), + (".*_ankle_pitch_joint", 50.0, 2.0, 73.1), + (".*_ankle_roll_joint", 20.0, 1.0, 17.2), +) +T1_ACTION_SCALE = {pattern: 0.25 * effort / kp for pattern, kp, _, effort in _MOTORS} +T1_FOOT_GEOMS = tuple( + f"{side}_ankle_roll_link_collision_1" for side in ("left", "right") +) + + +def get_t1_robot_cfg() -> EntityCfg: + """Return independent train/play configs; defer external asset IO until build.""" + return EntityCfg( + spec_fn=get_spec, + sort_actuators=True, + init_state=EntityCfg.InitialStateCfg( + pos=(0.0, 0.0, 0.6811), + joint_pos={ + ".*_hip_pitch_joint": -0.2, + ".*_knee_pitch_joint": 0.4, + ".*_ankle_pitch_joint": -0.2, + "left_shoulder_roll_joint": -1.3, + "right_shoulder_roll_joint": 1.3, + ".*_shoulder_pitch_joint": 0.2, + "left_elbow_yaw_joint": -0.4, + "right_elbow_yaw_joint": 0.4, + }, + joint_vel={".*": 0.0}, + ), + articulation=EntityArticulationInfoCfg( + actuators=tuple( + BuiltinPositionActuatorCfg( + target_names_expr=(pattern,), + stiffness=kp, + damping=kd, + effort_limit=effort, + ) + for pattern, kp, kd, effort in _MOTORS + ), + soft_joint_pos_limit_factor=0.9, + ), + # Terrain contacts everywhere; opposite legs also collide. Same-chain CAD + # overlaps and arm/torso self contacts are deliberately excluded. + collisions=( + CollisionCfg( + geom_names_expr=(".*_collision_.*",), + contype={ + "left_(hip|knee|ankle).*_collision_.*": 2, + "right_(hip|knee|ankle).*_collision_.*": 4, + ".*": 0, + }, + conaffinity={ + "left_(hip|knee|ankle).*_collision_.*": 5, + "right_(hip|knee|ankle).*_collision_.*": 3, + ".*": 1, + }, + condim={".*_ankle_roll_link_collision_.*": 3, ".*": 1}, + priority={".*_ankle_roll_link_collision_.*": 1, ".*": 0}, + friction=(0.8,), + ), + ), + ) diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/__init__.py b/rl/mjlab/asset_zoo/robots/i2rt_yam/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/d405.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/d405.stl new file mode 100644 index 0000000..7504e2a Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/d405.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2.stl new file mode 100644 index 0000000..d597a05 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__10.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__10.stl new file mode 100644 index 0000000..b065de1 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__10.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__11.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__11.stl new file mode 100644 index 0000000..736495b Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__11.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__12.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__12.stl new file mode 100644 index 0000000..356322f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__12.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__13.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__13.stl new file mode 100644 index 0000000..41dcd97 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__13.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__14.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__14.stl new file mode 100644 index 0000000..d5f7aa0 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__14.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__15.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__15.stl new file mode 100644 index 0000000..28cabf6 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__15.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__16.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__16.stl new file mode 100644 index 0000000..1b397ba Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__16.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__17.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__17.stl new file mode 100644 index 0000000..c685929 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__17.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__2.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__2.stl new file mode 100644 index 0000000..2d180f0 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__2.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__3.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__3.stl new file mode 100644 index 0000000..89c512c Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__3.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__4.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__4.stl new file mode 100644 index 0000000..7186d0f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__4.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__5.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__5.stl new file mode 100644 index 0000000..afd1996 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__5.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__6.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__6.stl new file mode 100644 index 0000000..71fc6c2 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__6.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__7.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__7.stl new file mode 100644 index 0000000..1aef9c1 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__7.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__8.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__8.stl new file mode 100644 index 0000000..ad2475d Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__8.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__9.stl b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__9.stl new file mode 100644 index 0000000..e6287e0 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/assets/model2__9.stl differ diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/yam.xml b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/yam.xml new file mode 100644 index 0000000..08051bb --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/i2rt_yam/xmls/yam.xml @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rl/mjlab/asset_zoo/robots/i2rt_yam/yam_constants.py b/rl/mjlab/asset_zoo/robots/i2rt_yam/yam_constants.py new file mode 100644 index 0000000..b9872f1 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/i2rt_yam/yam_constants.py @@ -0,0 +1,236 @@ +"""YAM constants.""" + +from pathlib import Path + +import mujoco + +from mjlab import MJLAB_SRC_PATH +from mjlab.actuator import BuiltinPositionActuatorCfg +from mjlab.entity import EntityArticulationInfoCfg, EntityCfg +from mjlab.utils.actuator import ( + ElectricActuator, + reflect_rotary_to_linear, +) +from mjlab.utils.spec_config import CollisionCfg + +## +# MJCF and assets. +## + +YAM_XML: Path = ( + MJLAB_SRC_PATH / "asset_zoo" / "robots" / "i2rt_yam" / "xmls" / "yam.xml" +) +assert YAM_XML.exists() + + +def get_spec() -> mujoco.MjSpec: + return mujoco.MjSpec.from_file(str(YAM_XML)) + + +## +# Actuator config. +## + +EFFECTIVE_INERTIAS = { + "joint1": 0.123153, + "joint2": 0.277411, + "joint3": 0.232763, + "joint4": 0.030154, + "joint5": 0.009126, + "joint6": 0.002868, + "left_finger": 2.781624, +} + +ARMATURE_DM_4340 = 0.032 +ARMATURE_DM_4310 = 0.0018 + +# Reference: https://github.com/i2rt-robotics/i2rt/blob/cbe48976b44aae45af856c62545be00ea2feed11/i2rt/motor_drivers/utils.py#L159-L169 +DM_4340 = ElectricActuator( + reflected_inertia=ARMATURE_DM_4340, + velocity_limit=10.0, + effort_limit=28.0, +) +# Reference: https://github.com/i2rt-robotics/i2rt/blob/cbe48976b44aae45af856c62545be00ea2feed11/i2rt/motor_drivers/utils.py#L139-L149 +DM_4310 = ElectricActuator( + reflected_inertia=ARMATURE_DM_4310, + velocity_limit=30.0, + effort_limit=10.0, +) + +NATURAL_FREQ = 2 * 2.0 * 3.1415926535 # 2Hz +DAMPING_RATIO = 2.0 + +# Per-joint PD gains using effective inertia, and actuator configs. +_ARM_JOINTS: dict[str, ElectricActuator] = { + "joint1": DM_4340, + "joint2": DM_4340, + "joint3": DM_4340, + "joint4": DM_4310, + "joint5": DM_4310, + "joint6": DM_4310, +} +ARM_ACTUATORS = tuple( + BuiltinPositionActuatorCfg( + target_names_expr=(name,), + stiffness=EFFECTIVE_INERTIAS[name] * NATURAL_FREQ**2, + damping=2.0 * DAMPING_RATIO * EFFECTIVE_INERTIAS[name] * NATURAL_FREQ, + effort_limit=motor.effort_limit, + armature=motor.reflected_inertia, + ) + for name, motor in _ARM_JOINTS.items() +) + +## +# Gripper transmission parameters. +## + +# Reference: https://github.com/i2rt-robotics/i2rt/blob/cbe48976b44aae45af856c62545be00ea2feed11/i2rt/robots/utils.py#L106-L118 + +# Crank gripper: DM 4310 drives a crank arm that converts rotation to linear motion. +# The mechanism geometry (8° to 170°) provides 71mm stroke, but we operate over +# 10° to 165° (2.7 rad motor range) for safety, giving ~70mm usable stroke. +# Note: Transmission ratio varies with position as r(θ) = r_crank*sin(θ); we use +# an effective average (dx/dθ) for simulation. +GRIPPER_MOTOR_STROKE_CRANK = 2.7 # [rad]: operational motor range (from limits) +GRIPPER_LINEAR_STROKE_CRANK = 0.071 # [m]: design stroke (full mechanism range) +GRIPPER_TRANSMISSION_RATIO_CRANK = ( + GRIPPER_LINEAR_STROKE_CRANK / GRIPPER_MOTOR_STROKE_CRANK +) + +# Reflect motor properties to linear gripper joint. +( + ARMATURE_DM_4310_LINEAR_CRANK, + VELOCITY_LIMIT_DM_4310_LINEAR_CRANK, + EFFORT_LIMIT_DM_4310_LINEAR_CRANK, +) = reflect_rotary_to_linear( + armature_rotary=ARMATURE_DM_4310, + velocity_limit_rotary=DM_4310.velocity_limit, + effort_limit_rotary=DM_4310.effort_limit, + transmission_ratio=GRIPPER_TRANSMISSION_RATIO_CRANK, +) + +# PD controller gains using effective inertia. +NATURAL_FREQ_GRIPPER = 1.0 * 2.0 * 3.1415926535 # 1Hz +STIFFNESS_GRIPPER = EFFECTIVE_INERTIAS["left_finger"] * NATURAL_FREQ_GRIPPER**2 +DAMPING_GRIPPER = ( + 2.0 * DAMPING_RATIO * EFFECTIVE_INERTIAS["left_finger"] * NATURAL_FREQ_GRIPPER +) + +# Artificially limit gripper force for sim stability (must also be done on hardware). +EFFORT_LIMIT_DM_4310_LINEAR_CRANK_SAFE = EFFORT_LIMIT_DM_4310_LINEAR_CRANK * 0.1 + +# Only actuate left_finger; right_finger is coupled via equality constraint. +ACTUATOR_DM_4310_LINEAR_CRANK = BuiltinPositionActuatorCfg( + target_names_expr=("left_finger",), + stiffness=STIFFNESS_GRIPPER, + damping=DAMPING_GRIPPER, + effort_limit=EFFORT_LIMIT_DM_4310_LINEAR_CRANK_SAFE, + armature=ARMATURE_DM_4310_LINEAR_CRANK, +) + +## +# Keyframe config. +## + +HOME_KEYFRAME = EntityCfg.InitialStateCfg( + pos=(0.0, 0.0, 0.01), + joint_pos={ + "joint2": 1.047, + "joint3": 1.05, + "joint4": -0.9, + "left_finger": 0.0375 / 2, + "right_finger": -0.0375 / 2, + }, + joint_vel={".*": 0.0}, +) + +## +# Collision config. +## + +FULL_COLLISION = CollisionCfg( + geom_names_expr=(".*_collision",), + contype=1, + conaffinity=1, + condim={ + "[lr]f_down(6|7|8|9|10|11)_collision": 6, + ".*_collision": 3, + }, + friction={ + "[lr]f_down(6|7|8|9|10|11)_collision": (1, 5e-3, 5e-4), + ".*_collision": (0.6,), + }, + solref={ + "[lr]f_down(6|7|8|9|10|11)_collision": (0.01, 1), + }, + priority={ + "[lr]f_down(6|7|8|9|10|11)_collision": 1, + ".*": 0, + }, +) + +GRIPPER_ONLY_COLLISION = CollisionCfg( + geom_names_expr=(".*_collision",), + contype={ + "(link6|[lr]f)_.*_collision": 1, + ".*_collision": 0, + }, + conaffinity={ + "(link6|[lr]f)_.*_collision": 1, + ".*_collision": 0, + }, + condim={ + "[lr]f_down(6|7|8|9|10|11)_collision": 6, + ".*_collision": 3, + }, + friction={ + "[lr]f_down(6|7|8|9|10|11)_collision": (1, 5e-3, 5e-4), + ".*_collision": (0.6,), + }, + solref={ + "[lr]f_down(6|7|8|9|10|11)_collision": (0.01, 1), + }, + priority={ + "[lr]f_down(6|7|8|9|10|11)_collision": 1, + ".*": 0, + }, +) + +## +# Final config. +## + +ARTICULATION = EntityArticulationInfoCfg( + actuators=(*ARM_ACTUATORS, ACTUATOR_DM_4310_LINEAR_CRANK), + soft_joint_pos_limit_factor=0.9, +) + + +def get_yam_robot_cfg() -> EntityCfg: + return EntityCfg( + init_state=HOME_KEYFRAME, + collisions=(GRIPPER_ONLY_COLLISION,), + spec_fn=get_spec, + articulation=ARTICULATION, + ) + + +YAM_ACTION_SCALE: dict[str, float] = {} +for a in ARTICULATION.actuators: + assert isinstance(a, BuiltinPositionActuatorCfg) + e = a.effort_limit + s = a.stiffness + names = a.target_names_expr + assert e is not None + for n in names: + YAM_ACTION_SCALE[n] = 0.25 * e / s + + +if __name__ == "__main__": + import mujoco.viewer as viewer + + from mjlab.entity.entity import Entity + + robot = Entity(get_yam_robot_cfg()) + + viewer.launch(robot.spec.compile()) diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/__init__.py b/rl/mjlab/asset_zoo/robots/unitree_g1/__init__.py new file mode 100644 index 0000000..3ebf37d --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_g1/__init__.py @@ -0,0 +1 @@ +"""Unitree G1 humanoid.""" diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/g1_constants.py b/rl/mjlab/asset_zoo/robots/unitree_g1/g1_constants.py new file mode 100644 index 0000000..76aaef4 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_g1/g1_constants.py @@ -0,0 +1,298 @@ +"""Unitree G1 constants.""" + +from pathlib import Path + +import mujoco + +from mjlab import MJLAB_SRC_PATH +from mjlab.actuator import BuiltinPositionActuatorCfg +from mjlab.entity import EntityArticulationInfoCfg, EntityCfg +from mjlab.utils.actuator import ( + ElectricActuator, + reflected_inertia_from_two_stage_planetary, +) +from mjlab.utils.spec_config import CollisionCfg + +## +# MJCF and assets. +## + +G1_XML: Path = ( + MJLAB_SRC_PATH / "asset_zoo" / "robots" / "unitree_g1" / "xmls" / "g1.xml" +) +assert G1_XML.exists() + + +def get_spec() -> mujoco.MjSpec: + return mujoco.MjSpec.from_file(str(G1_XML)) + + +## +# Actuator config. +## + +# Motor specs (from Unitree). +ROTOR_INERTIAS_5020 = ( + 0.139e-4, + 0.017e-4, + 0.169e-4, +) +GEARS_5020 = ( + 1, + 1 + (46 / 18), + 1 + (56 / 16), +) +ARMATURE_5020 = reflected_inertia_from_two_stage_planetary( + ROTOR_INERTIAS_5020, GEARS_5020 +) + +ROTOR_INERTIAS_7520_14 = ( + 0.489e-4, + 0.098e-4, + 0.533e-4, +) +GEARS_7520_14 = ( + 1, + 4.5, + 1 + (48 / 22), +) +ARMATURE_7520_14 = reflected_inertia_from_two_stage_planetary( + ROTOR_INERTIAS_7520_14, GEARS_7520_14 +) + +ROTOR_INERTIAS_7520_22 = ( + 0.489e-4, + 0.109e-4, + 0.738e-4, +) +GEARS_7520_22 = ( + 1, + 4.5, + 5, +) +ARMATURE_7520_22 = reflected_inertia_from_two_stage_planetary( + ROTOR_INERTIAS_7520_22, GEARS_7520_22 +) + +ROTOR_INERTIAS_4010 = ( + 0.068e-4, + 0.0, + 0.0, +) +GEARS_4010 = ( + 1, + 5, + 5, +) +ARMATURE_4010 = reflected_inertia_from_two_stage_planetary( + ROTOR_INERTIAS_4010, GEARS_4010 +) + +ACTUATOR_5020 = ElectricActuator( + reflected_inertia=ARMATURE_5020, + velocity_limit=37.0, + effort_limit=25.0, +) +ACTUATOR_7520_14 = ElectricActuator( + reflected_inertia=ARMATURE_7520_14, + velocity_limit=32.0, + effort_limit=88.0, +) +ACTUATOR_7520_22 = ElectricActuator( + reflected_inertia=ARMATURE_7520_22, + velocity_limit=20.0, + effort_limit=139.0, +) +ACTUATOR_4010 = ElectricActuator( + reflected_inertia=ARMATURE_4010, + velocity_limit=22.0, + effort_limit=5.0, +) + +NATURAL_FREQ = 10 * 2.0 * 3.1415926535 # 10Hz +DAMPING_RATIO = 2.0 + +STIFFNESS_5020 = ARMATURE_5020 * NATURAL_FREQ**2 +STIFFNESS_7520_14 = ARMATURE_7520_14 * NATURAL_FREQ**2 +STIFFNESS_7520_22 = ARMATURE_7520_22 * NATURAL_FREQ**2 +STIFFNESS_4010 = ARMATURE_4010 * NATURAL_FREQ**2 + +DAMPING_5020 = 2.0 * DAMPING_RATIO * ARMATURE_5020 * NATURAL_FREQ +DAMPING_7520_14 = 2.0 * DAMPING_RATIO * ARMATURE_7520_14 * NATURAL_FREQ +DAMPING_7520_22 = 2.0 * DAMPING_RATIO * ARMATURE_7520_22 * NATURAL_FREQ +DAMPING_4010 = 2.0 * DAMPING_RATIO * ARMATURE_4010 * NATURAL_FREQ + +G1_ACTUATOR_5020 = BuiltinPositionActuatorCfg( + target_names_expr=( + ".*_elbow_joint", + ".*_shoulder_pitch_joint", + ".*_shoulder_roll_joint", + ".*_shoulder_yaw_joint", + ".*_wrist_roll_joint", + ), + stiffness=STIFFNESS_5020, + damping=DAMPING_5020, + effort_limit=ACTUATOR_5020.effort_limit, + armature=ACTUATOR_5020.reflected_inertia, +) +G1_ACTUATOR_7520_14 = BuiltinPositionActuatorCfg( + target_names_expr=(".*_hip_pitch_joint", ".*_hip_yaw_joint", "waist_yaw_joint"), + stiffness=STIFFNESS_7520_14, + damping=DAMPING_7520_14, + effort_limit=ACTUATOR_7520_14.effort_limit, + armature=ACTUATOR_7520_14.reflected_inertia, +) +G1_ACTUATOR_7520_22 = BuiltinPositionActuatorCfg( + target_names_expr=(".*_hip_roll_joint", ".*_knee_joint"), + stiffness=STIFFNESS_7520_22, + damping=DAMPING_7520_22, + effort_limit=ACTUATOR_7520_22.effort_limit, + armature=ACTUATOR_7520_22.reflected_inertia, +) +G1_ACTUATOR_4010 = BuiltinPositionActuatorCfg( + target_names_expr=(".*_wrist_pitch_joint", ".*_wrist_yaw_joint"), + stiffness=STIFFNESS_4010, + damping=DAMPING_4010, + effort_limit=ACTUATOR_4010.effort_limit, + armature=ACTUATOR_4010.reflected_inertia, +) + +# Waist pitch/roll and ankles are 4-bar linkages with 2 5020 actuators. +# Due to the parallel linkage, the effective armature at the ankle and waist joints +# is configuration dependent. Since the exact geometry of the linkage is unknown, we +# assume a nominal 1:1 gear ratio. Under this assumption, the joint armature in the +# nominal configuration is approximated as the sum of the 2 actuators' armatures. +G1_ACTUATOR_WAIST = BuiltinPositionActuatorCfg( + target_names_expr=("waist_pitch_joint", "waist_roll_joint"), + stiffness=STIFFNESS_5020 * 2, + damping=DAMPING_5020 * 2, + effort_limit=ACTUATOR_5020.effort_limit * 2, + armature=ACTUATOR_5020.reflected_inertia * 2, +) +G1_ACTUATOR_ANKLE = BuiltinPositionActuatorCfg( + target_names_expr=(".*_ankle_pitch_joint", ".*_ankle_roll_joint"), + stiffness=STIFFNESS_5020 * 2, + damping=DAMPING_5020 * 2, + effort_limit=ACTUATOR_5020.effort_limit * 2, + armature=ACTUATOR_5020.reflected_inertia * 2, +) + +## +# Keyframe config. +## + +HOME_KEYFRAME = EntityCfg.InitialStateCfg( + pos=(0, 0, 0.783675), + joint_pos={ + ".*_hip_pitch_joint": -0.1, + ".*_knee_joint": 0.3, + ".*_ankle_pitch_joint": -0.2, + ".*_shoulder_pitch_joint": 0.2, + ".*_elbow_joint": 1.28, + "left_shoulder_roll_joint": 0.2, + "right_shoulder_roll_joint": -0.2, + }, + joint_vel={".*": 0.0}, +) + +KNEES_BENT_KEYFRAME = EntityCfg.InitialStateCfg( + pos=(0, 0, 0.76), + joint_pos={ + ".*_hip_pitch_joint": -0.312, + ".*_knee_joint": 0.669, + ".*_ankle_pitch_joint": -0.363, + ".*_elbow_joint": 0.6, + "left_shoulder_roll_joint": 0.2, + "left_shoulder_pitch_joint": 0.2, + "right_shoulder_roll_joint": -0.2, + "right_shoulder_pitch_joint": 0.2, + }, + joint_vel={".*": 0.0}, +) + +## +# Collision config. +## + +# This enables all collisions, including self collisions. +# Self-collisions are given condim=1 while foot collisions +# are given condim=3. +FULL_COLLISION = CollisionCfg( + geom_names_expr=(".*_collision",), + contype=1, + conaffinity=1, + condim={r"^(left|right)_foot[1-7]_collision$": 3, ".*_collision": 1}, + priority={r"^(left|right)_foot[1-7]_collision$": 1, ".*": 0}, + friction={r"^(left|right)_foot[1-7]_collision$": (0.6,)}, +) + +FULL_COLLISION_WITHOUT_SELF = CollisionCfg( + geom_names_expr=(".*_collision",), + contype=0, + conaffinity=1, + condim={r"^(left|right)_foot[1-7]_collision$": 3, ".*_collision": 1}, + priority={r"^(left|right)_foot[1-7]_collision$": 1, ".*": 0}, + friction={r"^(left|right)_foot[1-7]_collision$": (0.6,)}, +) + +# This disables all collisions except the feet. +# Feet get condim=3, all other geoms are disabled. +FEET_ONLY_COLLISION = CollisionCfg( + geom_names_expr=(r"^(left|right)_foot[1-7]_collision$",), + contype=0, + conaffinity=1, + condim=3, + priority=1, + friction=(0.6,), +) + +## +# Final config. +## + +G1_ARTICULATION = EntityArticulationInfoCfg( + actuators=( + G1_ACTUATOR_5020, + G1_ACTUATOR_7520_14, + G1_ACTUATOR_7520_22, + G1_ACTUATOR_4010, + G1_ACTUATOR_WAIST, + G1_ACTUATOR_ANKLE, + ), + soft_joint_pos_limit_factor=0.9, +) + + +def get_g1_robot_cfg() -> EntityCfg: + """Get a fresh G1 robot configuration instance. + + Returns a new EntityCfg instance each time to avoid mutation issues when + the config is shared across multiple places. + """ + return EntityCfg( + init_state=KNEES_BENT_KEYFRAME, + collisions=(FULL_COLLISION,), + spec_fn=get_spec, + articulation=G1_ARTICULATION, + ) + + +G1_ACTION_SCALE: dict[str, float] = {} +for a in G1_ARTICULATION.actuators: + assert isinstance(a, BuiltinPositionActuatorCfg) + e = a.effort_limit + s = a.stiffness + names = a.target_names_expr + assert e is not None + for n in names: + G1_ACTION_SCALE[n] = 0.25 * e / s + + +if __name__ == "__main__": + import mujoco.viewer as viewer + + from mjlab.entity.entity import Entity + + robot = Entity(get_g1_robot_cfg()) + + viewer.launch(robot.spec.compile()) diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/head_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/head_link.STL new file mode 100644 index 0000000..2ee5fba Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/head_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_pitch_link.STL new file mode 100644 index 0000000..69de849 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_roll_link.STL new file mode 100644 index 0000000..8864e9f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_ankle_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_elbow_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_elbow_link.STL new file mode 100644 index 0000000..1a96d99 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_elbow_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_pitch_link.STL new file mode 100644 index 0000000..5b751c7 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_roll_link.STL new file mode 100644 index 0000000..778437f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_yaw_link.STL new file mode 100644 index 0000000..383093a Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_hip_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_knee_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_knee_link.STL new file mode 100644 index 0000000..f2e98e5 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_knee_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_rubber_hand.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_rubber_hand.STL new file mode 100644 index 0000000..c44830f Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_rubber_hand.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_pitch_link.STL new file mode 100644 index 0000000..e698311 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_roll_link.STL new file mode 100644 index 0000000..80bca84 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_yaw_link.STL new file mode 100644 index 0000000..281e699 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_shoulder_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_pitch_link.STL new file mode 100644 index 0000000..82cc224 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_roll_link.STL new file mode 100644 index 0000000..f3c263a Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_yaw_link.STL new file mode 100644 index 0000000..31be4fd Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/left_wrist_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/logo_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/logo_link.STL new file mode 100644 index 0000000..e979209 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/logo_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis.STL new file mode 100644 index 0000000..691a779 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis_contour_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis_contour_link.STL new file mode 100644 index 0000000..4243433 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/pelvis_contour_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_pitch_link.STL new file mode 100644 index 0000000..e77d8a2 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_roll_link.STL new file mode 100644 index 0000000..d4261dd Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_ankle_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_elbow_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_elbow_link.STL new file mode 100644 index 0000000..f259e38 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_elbow_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_pitch_link.STL new file mode 100644 index 0000000..998a0a0 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_roll_link.STL new file mode 100644 index 0000000..47b2eeb Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_yaw_link.STL new file mode 100644 index 0000000..3718564 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_hip_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_knee_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_knee_link.STL new file mode 100644 index 0000000..76d21a3 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_knee_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_rubber_hand.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_rubber_hand.STL new file mode 100644 index 0000000..0aacffb Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_rubber_hand.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_pitch_link.STL new file mode 100644 index 0000000..3f5b4ed Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_roll_link.STL new file mode 100644 index 0000000..179d617 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_yaw_link.STL new file mode 100644 index 0000000..2ba6076 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_shoulder_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_pitch_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_pitch_link.STL new file mode 100644 index 0000000..da19454 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_pitch_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_roll_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_roll_link.STL new file mode 100644 index 0000000..26868d2 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_roll_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_yaw_link.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_yaw_link.STL new file mode 100644 index 0000000..d788902 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/right_wrist_yaw_link.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/torso_link_rev_1_0.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/torso_link_rev_1_0.STL new file mode 100644 index 0000000..8a759a7 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/torso_link_rev_1_0.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_roll_link_rev_1_0.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_roll_link_rev_1_0.STL new file mode 100644 index 0000000..a64f330 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_roll_link_rev_1_0.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_yaw_link_rev_1_0.STL b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_yaw_link_rev_1_0.STL new file mode 100644 index 0000000..0fabb63 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/assets/waist_yaw_link_rev_1_0.STL differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/g1.xml b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/g1.xml new file mode 100644 index 0000000..dda4779 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_g1/xmls/g1.xml @@ -0,0 +1,308 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/__init__.py b/rl/mjlab/asset_zoo/robots/unitree_go1/__init__.py new file mode 100644 index 0000000..0695b41 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_go1/__init__.py @@ -0,0 +1 @@ +"""Unitree Go1 quadruped.""" diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/go1_constants.py b/rl/mjlab/asset_zoo/robots/unitree_go1/go1_constants.py new file mode 100644 index 0000000..dc8b445 --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_go1/go1_constants.py @@ -0,0 +1,173 @@ +"""Unitree Go1 constants.""" + +from pathlib import Path + +import mujoco + +from mjlab import MJLAB_SRC_PATH +from mjlab.actuator import BuiltinPositionActuatorCfg +from mjlab.entity import EntityArticulationInfoCfg, EntityCfg +from mjlab.utils.actuator import ElectricActuator, reflected_inertia +from mjlab.utils.spec_config import CollisionCfg + +## +# MJCF and assets. +## + +GO1_XML: Path = ( + MJLAB_SRC_PATH / "asset_zoo" / "robots" / "unitree_go1" / "xmls" / "go1.xml" +) +assert GO1_XML.exists() + + +def get_spec() -> mujoco.MjSpec: + return mujoco.MjSpec.from_file(str(GO1_XML)) + + +## +# Actuator config. +## + +# Rotor inertia. +# Ref: https://github.com/unitreerobotics/unitree_ros/blob/master/robots/go1_description/urdf/go1.urdf#L515 +# Extracted Ixx (rotation along x-axis). +ROTOR_INERTIA = 0.000111842 + +# Gearbox. +HIP_GEAR_RATIO = 6 +KNEE_GEAR_RATIO = HIP_GEAR_RATIO * 1.5 + +HIP_ACTUATOR = ElectricActuator( + reflected_inertia=reflected_inertia(ROTOR_INERTIA, HIP_GEAR_RATIO), + velocity_limit=30.1, + effort_limit=23.7, +) +KNEE_ACTUATOR = ElectricActuator( + reflected_inertia=reflected_inertia(ROTOR_INERTIA, KNEE_GEAR_RATIO), + velocity_limit=20.06, + effort_limit=35.55, +) + +NATURAL_FREQ = 10 * 2.0 * 3.1415926535 # 10Hz +DAMPING_RATIO = 2.0 + +STIFFNESS_HIP = HIP_ACTUATOR.reflected_inertia * NATURAL_FREQ**2 +DAMPING_HIP = 2 * DAMPING_RATIO * HIP_ACTUATOR.reflected_inertia * NATURAL_FREQ + +STIFFNESS_KNEE = KNEE_ACTUATOR.reflected_inertia * NATURAL_FREQ**2 +DAMPING_KNEE = 2 * DAMPING_RATIO * KNEE_ACTUATOR.reflected_inertia * NATURAL_FREQ + +GO1_HIP_ACTUATOR_CFG = BuiltinPositionActuatorCfg( + target_names_expr=(".*_hip_joint", ".*_thigh_joint"), + stiffness=STIFFNESS_HIP, + damping=DAMPING_HIP, + effort_limit=HIP_ACTUATOR.effort_limit, + armature=HIP_ACTUATOR.reflected_inertia, +) +GO1_KNEE_ACTUATOR_CFG = BuiltinPositionActuatorCfg( + target_names_expr=(".*_calf_joint",), + stiffness=STIFFNESS_KNEE, + damping=DAMPING_KNEE, + effort_limit=KNEE_ACTUATOR.effort_limit, + armature=KNEE_ACTUATOR.reflected_inertia, +) + +## +# Keyframes. +## + + +INIT_STATE = EntityCfg.InitialStateCfg( + pos=(0.0, 0.0, 0.278), + joint_pos={ + ".*thigh_joint": 0.9, + ".*calf_joint": -1.8, + ".*R_hip_joint": 0.1, + ".*L_hip_joint": -0.1, + }, + joint_vel={".*": 0.0}, +) + +## +# Collision config. +## + +_foot_regex = "^[FR][LR]_foot_collision$" + +# This disables all collisions except the feet. +# Furthermore, feet self collisions are disabled. +FEET_ONLY_COLLISION = CollisionCfg( + geom_names_expr=(_foot_regex,), + contype=0, + conaffinity=1, + condim=3, + priority=1, + friction=(0.6,), + solimp=(0.9, 0.95, 0.023), +) + +# This enables all collisions. +# Foot collisions are given custom condim, friction. +# The go1 XML numbers its secondary collision capsules (FR_thigh_collision1, +# RL_calf_collision2, ...), so the pattern allows a trailing index. This keeps +# the match set identical whether patterns are matched by prefix or in full. +_collision_regex = r".*_collision\d*" + +FULL_COLLISION = CollisionCfg( + geom_names_expr=(_collision_regex,), + contype=1, + conaffinity=1, + # Harden all collision geoms. + solref=(0.01, 1), + # Configure feet colliders. Other colliders are frictionless (condim=1). + condim={_foot_regex: 6, _collision_regex: 1}, + priority={_foot_regex: 1, ".*": 0}, + friction={_foot_regex: (1, 5e-3, 5e-4)}, +) + +## +# Final config. +## + +GO1_ARTICULATION = EntityArticulationInfoCfg( + actuators=( + GO1_HIP_ACTUATOR_CFG, + GO1_KNEE_ACTUATOR_CFG, + ), + soft_joint_pos_limit_factor=0.9, +) + + +def get_go1_robot_cfg() -> EntityCfg: + """Get a fresh Go1 robot configuration instance. + + Returns a new EntityCfg instance each time to avoid mutation issues when + the config is shared across multiple places. + """ + return EntityCfg( + init_state=INIT_STATE, + collisions=(FULL_COLLISION,), + spec_fn=get_spec, + articulation=GO1_ARTICULATION, + ) + + +GO1_ACTION_SCALE: dict[str, float] = {} +for a in GO1_ARTICULATION.actuators: + assert isinstance(a, BuiltinPositionActuatorCfg) + e = a.effort_limit + s = a.stiffness + names = a.target_names_expr + assert e is not None + for n in names: + GO1_ACTION_SCALE[n] = 0.25 * e / s + + +if __name__ == "__main__": + import mujoco.viewer as viewer + + from mjlab.entity.entity import Entity + + robot = Entity(get_go1_robot_cfg()) + + viewer.launch(robot.spec.compile()) diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/calf.stl b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/calf.stl new file mode 100644 index 0000000..3e8ac94 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/calf.stl differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/hip.stl b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/hip.stl new file mode 100644 index 0000000..ae590bf Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/hip.stl differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh.stl b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh.stl new file mode 100644 index 0000000..353801b Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh.stl differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh_mirror.stl b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh_mirror.stl new file mode 100644 index 0000000..11ef770 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/thigh_mirror.stl differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/trunk.stl b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/trunk.stl new file mode 100644 index 0000000..2659eb1 Binary files /dev/null and b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/assets/trunk.stl differ diff --git a/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/go1.xml b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/go1.xml new file mode 100644 index 0000000..cc6cacb --- /dev/null +++ b/rl/mjlab/asset_zoo/robots/unitree_go1/xmls/go1.xml @@ -0,0 +1,171 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rl/mjlab/entity/__init__.py b/rl/mjlab/entity/__init__.py new file mode 100644 index 0000000..f6cffff --- /dev/null +++ b/rl/mjlab/entity/__init__.py @@ -0,0 +1,7 @@ +from mjlab.entity.data import EntityData as EntityData +from mjlab.entity.entity import Entity as Entity +from mjlab.entity.entity import EntityArticulationInfoCfg as EntityArticulationInfoCfg +from mjlab.entity.entity import EntityCfg as EntityCfg +from mjlab.entity.entity import EntityIndexing as EntityIndexing +from mjlab.entity.variants import VariantEntityCfg as VariantEntityCfg +from mjlab.entity.variants import VariantMetadata as VariantMetadata diff --git a/rl/mjlab/entity/data.py b/rl/mjlab/entity/data.py new file mode 100644 index 0000000..9414b77 --- /dev/null +++ b/rl/mjlab/entity/data.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Sequence + +import mujoco_warp as mjwarp +import torch + +from mjlab.utils.lab_api.math import ( + quat_apply, + quat_apply_inverse, + quat_from_matrix, + quat_mul, +) + +if TYPE_CHECKING: + from mjlab.entity.entity import EntityIndexing + + +def compute_velocity_from_cvel( + pos: torch.Tensor, + subtree_com: torch.Tensor, + cvel: torch.Tensor, +) -> torch.Tensor: + """Convert cvel quantities to world-frame velocities.""" + lin_vel_c = cvel[..., 3:6] + ang_vel_c = cvel[..., 0:3] + offset = subtree_com - pos + lin_vel_w = lin_vel_c - torch.cross(ang_vel_c, offset, dim=-1) + ang_vel_w = ang_vel_c + return torch.cat([lin_vel_w, ang_vel_w], dim=-1) + + +@dataclass +class EntityData: + """Data container for an entity. + + Note: Write methods (write_*) modify state directly. Read properties (e.g., + root_link_pose_w) require sim.forward() to be current. If you write then read, + call sim.forward() in between. Event order matters when mixing reads and writes. + All inputs/outputs use world frame. + """ + + indexing: EntityIndexing + data: mjwarp.Data + model: mjwarp.Model + device: str + + default_root_state: torch.Tensor + default_joint_pos: torch.Tensor + default_joint_vel: torch.Tensor + + default_joint_pos_limits: torch.Tensor + joint_pos_limits: torch.Tensor + soft_joint_pos_limits: torch.Tensor + + gravity_vec_w: torch.Tensor + forward_vec_b: torch.Tensor + + is_fixed_base: bool + is_articulated: bool + is_actuated: bool + + joint_pos_target: torch.Tensor + joint_vel_target: torch.Tensor + joint_effort_target: torch.Tensor + + tendon_len_target: torch.Tensor + tendon_vel_target: torch.Tensor + tendon_effort_target: torch.Tensor + + site_effort_target: torch.Tensor + + encoder_bias: torch.Tensor + + # State dimensions. + POS_DIM = 3 + QUAT_DIM = 4 + LIN_VEL_DIM = 3 + ANG_VEL_DIM = 3 + ROOT_POSE_DIM = POS_DIM + QUAT_DIM # 7 + ROOT_VEL_DIM = LIN_VEL_DIM + ANG_VEL_DIM # 6 + ROOT_STATE_DIM = ROOT_POSE_DIM + ROOT_VEL_DIM # 13 + + def write_root_state( + self, root_state: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + if self.is_fixed_base: + raise ValueError("Cannot write root state for fixed-base entity.") + assert root_state.shape[-1] == self.ROOT_STATE_DIM + + self.write_root_pose(root_state[:, : self.ROOT_POSE_DIM], env_ids) + self.write_root_velocity(root_state[:, self.ROOT_POSE_DIM :], env_ids) + + def write_root_pose( + self, pose: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + if self.is_fixed_base: + raise ValueError("Cannot write root pose for fixed-base entity.") + assert pose.shape[-1] == self.ROOT_POSE_DIM + + env_ids = self._resolve_env_ids(env_ids) + self.data.qpos[env_ids, self.indexing.free_joint_q_adr] = pose + + def write_root_velocity( + self, velocity: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + if self.is_fixed_base: + raise ValueError("Cannot write root velocity for fixed-base entity.") + assert velocity.shape[-1] == self.ROOT_VEL_DIM + + env_ids = self._resolve_env_ids(env_ids) + quat_w = self.data.qpos[env_ids, self.indexing.free_joint_q_adr[3:7]] + ang_vel_b = quat_apply_inverse(quat_w, velocity[:, 3:]) + velocity_qvel = torch.cat([velocity[:, :3], ang_vel_b], dim=-1) + self.data.qvel[env_ids, self.indexing.free_joint_v_adr] = velocity_qvel + + def write_root_velocity_b( + self, velocity_b: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + """Write the root link velocity given in the root link's body frame. + + The orientation is read from qpos, so this is safe to call during a reset + before forward() has refreshed derived kinematics. + """ + if self.is_fixed_base: + raise ValueError("Cannot write root velocity for fixed-base entity.") + assert velocity_b.shape[-1] == self.ROOT_VEL_DIM + + env_ids = self._resolve_env_ids(env_ids) + quat_w = self.data.qpos[env_ids, self.indexing.free_joint_q_adr[3:7]] + lin_vel_w = quat_apply(quat_w, velocity_b[:, :3]) + # Free joint qvel holds world-frame linear and body-frame angular velocity. + velocity_qvel = torch.cat([lin_vel_w, velocity_b[:, 3:]], dim=-1) + self.data.qvel[env_ids, self.indexing.free_joint_v_adr] = velocity_qvel + + def write_root_com_velocity( + self, velocity: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + if self.is_fixed_base: + raise ValueError("Cannot write root COM velocity for fixed-base entity.") + assert velocity.shape[-1] == self.ROOT_VEL_DIM + + env_ids = env_ids if env_ids is not None else slice(None) + com_offset_b = self.model.body_ipos[:, self.indexing.root_body_id] + quat_w = self.data.qpos[:, self.indexing.free_joint_q_adr[3:7]][env_ids] + com_offset_w = quat_apply(quat_w, com_offset_b[env_ids]) + lin_vel_com = velocity[:, :3] + ang_vel_w = velocity[:, 3:] + lin_vel_link = lin_vel_com - torch.cross(ang_vel_w, com_offset_w, dim=-1) + link_velocity = torch.cat([lin_vel_link, ang_vel_w], dim=-1) + self.write_root_velocity(link_velocity, env_ids) + + def write_joint_state( + self, + position: torch.Tensor, + velocity: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + if not self.is_articulated: + raise ValueError("Cannot write joint state for non-articulated entity.") + + self.write_joint_position(position, joint_ids, env_ids) + self.write_joint_velocity(velocity, joint_ids, env_ids) + + def write_joint_position( + self, + position: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + if not self.is_articulated: + raise ValueError("Cannot write joint position for non-articulated entity.") + + env_ids = self._resolve_env_ids(env_ids) + joint_ids = joint_ids if joint_ids is not None else slice(None) + q_slice = self.indexing.joint_q_adr[joint_ids] + self.data.qpos[env_ids, q_slice] = position + + def write_joint_velocity( + self, + velocity: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + if not self.is_articulated: + raise ValueError("Cannot write joint velocity for non-articulated entity.") + + env_ids = self._resolve_env_ids(env_ids) + joint_ids = joint_ids if joint_ids is not None else slice(None) + v_slice = self.indexing.joint_v_adr[joint_ids] + self.data.qvel[env_ids, v_slice] = velocity + + def write_external_wrench( + self, + force: torch.Tensor | None, + torque: torch.Tensor | None, + body_ids: Sequence[int] | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + env_ids = self._resolve_env_ids(env_ids) + local_body_ids = body_ids if body_ids is not None else slice(None) + global_body_ids = self.indexing.body_ids[local_body_ids] + if force is not None: + self.data.xfrc_applied[env_ids, global_body_ids, 0:3] = force + if torque is not None: + self.data.xfrc_applied[env_ids, global_body_ids, 3:6] = torque + + def write_ctrl( + self, + ctrl: torch.Tensor, + ctrl_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + if not self.is_actuated: + raise ValueError("Cannot write control for non-actuated entity.") + + env_ids = self._resolve_env_ids(env_ids) + local_ctrl_ids = ctrl_ids if ctrl_ids is not None else slice(None) + global_ctrl_ids = self.indexing.ctrl_ids[local_ctrl_ids] + self.data.ctrl[env_ids, global_ctrl_ids] = ctrl + + def write_mocap_pose( + self, pose: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + if self.indexing.mocap_id is None: + raise ValueError("Cannot write mocap pose for non-mocap entity.") + assert pose.shape[-1] == self.ROOT_POSE_DIM + + # Bypass _resolve_env_ids since mocap is always a single body. + if env_ids is None: + env_ids = slice(None) + self.data.mocap_pos[env_ids, self.indexing.mocap_id] = pose[:, 0:3] + self.data.mocap_quat[env_ids, self.indexing.mocap_id] = pose[:, 3:7] + + def clear_state(self, env_ids: torch.Tensor | slice | None = None) -> None: + if self.is_actuated: + env_ids = self._resolve_env_ids(env_ids) + self.joint_pos_target[env_ids] = 0.0 + self.joint_vel_target[env_ids] = 0.0 + self.joint_effort_target[env_ids] = 0.0 + self.tendon_len_target[env_ids] = 0.0 + self.tendon_vel_target[env_ids] = 0.0 + self.tendon_effort_target[env_ids] = 0.0 + self.site_effort_target[env_ids] = 0.0 + + def _resolve_env_ids( + self, env_ids: torch.Tensor | slice | None + ) -> torch.Tensor | slice: + """Convert env_ids to consistent indexing format.""" + if env_ids is None: + return slice(None) + if isinstance(env_ids, torch.Tensor): + return env_ids[:, None] + return env_ids + + def _joint_dof_field(self, field_name: str) -> torch.Tensor: + """Return a generalized-force field sliced to this entity's joint DoFs.""" + field = getattr(self.data, field_name) + return field[:, self.indexing.joint_v_adr] + + # Root properties + + @property + def root_link_pose_w(self) -> torch.Tensor: + """Root link pose in world frame. Shape (num_envs, 7).""" + pos_w = self.data.xpos[:, self.indexing.root_body_id] # (num_envs, 3) + quat_w = self.data.xquat[:, self.indexing.root_body_id] # (num_envs, 4) + return torch.cat([pos_w, quat_w], dim=-1) # (num_envs, 7) + + @property + def root_link_vel_w(self) -> torch.Tensor: + """Root link velocity in world frame. Shape (num_envs, 6).""" + # NOTE: Equivalently, can read this from qvel[:6] but the angular part + # will be in body frame and needs to be rotated to world frame. + # Note also that an extra forward() call might be required to make + # both values equal. + pos = self.data.xpos[:, self.indexing.root_body_id] # (num_envs, 3) + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, self.indexing.root_body_id] # (num_envs, 6) + return compute_velocity_from_cvel(pos, subtree_com, cvel) # (num_envs, 6) + + @property + def root_com_pose_w(self) -> torch.Tensor: + """Root center-of-mass pose in world frame. Shape (num_envs, 7).""" + pos_w = self.data.xipos[:, self.indexing.root_body_id] + quat = self.data.xquat[:, self.indexing.root_body_id] + body_iquat = self.model.body_iquat[:, self.indexing.root_body_id] + assert body_iquat is not None + quat_w = quat_mul(quat, body_iquat.squeeze(1)) + return torch.cat([pos_w, quat_w], dim=-1) + + @property + def root_com_vel_w(self) -> torch.Tensor: + """Root center-of-mass velocity in world frame. Shape (num_envs, 6).""" + # NOTE: Equivalent sensor is framelinvel/frameangvel with objtype="body". + pos = self.data.xipos[:, self.indexing.root_body_id] # (num_envs, 3) + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, self.indexing.root_body_id] # (num_envs, 6) + return compute_velocity_from_cvel(pos, subtree_com, cvel) # (num_envs, 6) + + # Body properties + + @property + def body_link_pose_w(self) -> torch.Tensor: + """Body link pose in world frame. Shape (num_envs, num_bodies, 7).""" + pos_w = self.data.xpos[:, self.indexing.body_ids] + quat_w = self.data.xquat[:, self.indexing.body_ids] + return torch.cat([pos_w, quat_w], dim=-1) + + @property + def body_link_vel_w(self) -> torch.Tensor: + """Body link velocity in world frame. Shape (num_envs, num_bodies, 6).""" + # NOTE: Equivalent sensor is framelinvel/frameangvel with objtype="xbody". + pos = self.data.xpos[:, self.indexing.body_ids] # (num_envs, num_bodies, 3) + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, self.indexing.body_ids] + return compute_velocity_from_cvel(pos, subtree_com.unsqueeze(1), cvel) + + @property + def body_com_pose_w(self) -> torch.Tensor: + """Body center-of-mass pose in world frame. Shape (num_envs, num_bodies, 7).""" + pos_w = self.data.xipos[:, self.indexing.body_ids] + quat = self.data.xquat[:, self.indexing.body_ids] + body_iquat = self.model.body_iquat[:, self.indexing.body_ids] + quat_w = quat_mul(quat, body_iquat) + return torch.cat([pos_w, quat_w], dim=-1) + + @property + def body_com_vel_w(self) -> torch.Tensor: + """Body center-of-mass velocity in world frame. Shape (num_envs, num_bodies, 6).""" + # NOTE: Equivalent sensor is framelinvel/frameangvel with objtype="body". + pos = self.data.xipos[:, self.indexing.body_ids] + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, self.indexing.body_ids] + return compute_velocity_from_cvel(pos, subtree_com.unsqueeze(1), cvel) + + @property + def body_external_wrench(self) -> torch.Tensor: + """Body external wrench in world frame. Shape (num_envs, num_bodies, 6).""" + return self.data.xfrc_applied[:, self.indexing.body_ids] + + # Geom properties + + @property + def geom_pose_w(self) -> torch.Tensor: + """Geom pose in world frame. Shape (num_envs, num_geoms, 7).""" + pos_w = self.data.geom_xpos[:, self.indexing.geom_ids] + xmat = self.data.geom_xmat[:, self.indexing.geom_ids] + quat_w = quat_from_matrix(xmat) + return torch.cat([pos_w, quat_w], dim=-1) + + @property + def geom_vel_w(self) -> torch.Tensor: + """Geom velocity in world frame. Shape (num_envs, num_geoms, 6).""" + pos = self.data.geom_xpos[:, self.indexing.geom_ids] + body_ids = self.model.geom_bodyid[self.indexing.geom_ids] # (num_geoms,) + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, body_ids] + return compute_velocity_from_cvel(pos, subtree_com.unsqueeze(1), cvel) + + # Site properties + + @property + def site_pose_w(self) -> torch.Tensor: + """Site pose in world frame. Shape (num_envs, num_sites, 7).""" + pos_w = self.data.site_xpos[:, self.indexing.site_ids] + mat_w = self.data.site_xmat[:, self.indexing.site_ids] + quat_w = quat_from_matrix(mat_w) + return torch.cat([pos_w, quat_w], dim=-1) + + @property + def site_vel_w(self) -> torch.Tensor: + """Site velocity in world frame. Shape (num_envs, num_sites, 6).""" + pos = self.data.site_xpos[:, self.indexing.site_ids] + body_ids = self.model.site_bodyid[self.indexing.site_ids] # (num_sites,) + subtree_com = self.data.subtree_com[:, self.indexing.root_body_id] + cvel = self.data.cvel[:, body_ids] + return compute_velocity_from_cvel(pos, subtree_com.unsqueeze(1), cvel) + + # Joint properties + + @property + def joint_pos(self) -> torch.Tensor: + """Joint positions. Shape (num_envs, num_joints).""" + return self.data.qpos[:, self.indexing.joint_q_adr] + + @property + def joint_pos_biased(self) -> torch.Tensor: + """Joint positions with encoder bias applied. Shape (num_envs, num_joints).""" + return self.joint_pos + self.encoder_bias + + @property + def joint_vel(self) -> torch.Tensor: + """Joint velocities. Shape (num_envs, nv).""" + return self.data.qvel[:, self.indexing.joint_v_adr] + + @property + def joint_acc(self) -> torch.Tensor: + """Joint accelerations. Shape (num_envs, nv).""" + return self.data.qacc[:, self.indexing.joint_v_adr] + + # Tendon properties + + @property + def tendon_len(self) -> torch.Tensor: + """Tendon lengths. Shape (num_envs, num_tendons).""" + return self.data.ten_length[:, self.indexing.tendon_ids] + + @property + def tendon_vel(self) -> torch.Tensor: + """Tendon velocities. Shape (num_envs, num_tendons).""" + return self.data.ten_velocity[:, self.indexing.tendon_ids] + + # Generalized forces + + @property + def joint_torques(self) -> torch.Tensor: + """Joint torques. Shape (num_envs, nv).""" + raise NotImplementedError( + "Joint torques are ambiguous. Use 'qfrc_actuator' for actuator forces " + "in joint space, or 'qfrc_external' for body wrench contributions." + ) + + @property + def actuator_force(self) -> torch.Tensor: + """Scalar actuator output in actuation space. Shape (num_envs, nu). + + This is not the same as joint-space generalized force. Use ``qfrc_actuator`` for + the actuator contribution projected into DoF space. + """ + return self.data.actuator_force[:, self.indexing.ctrl_ids] + + @property + def qfrc_actuator(self) -> torch.Tensor: + """Forces produced by all actuators, mapped into joint space. + + For motors this is the commanded torque times the gear ratio. For position and + velocity actuators this is the force computed by the internal PD law. When + ``actuatorgravcomp`` is enabled on a joint, the gravity compensation force is + included here. + """ + return self._joint_dof_field("qfrc_actuator") + + @property + def qfrc_external(self) -> torch.Tensor: + """Forces on joints due to Cartesian wrenches applied to bodies. + + When a force or torque is applied to a body via ``xfrc_applied``, this property + gives the equivalent joint forces (the Jacobian transpose mapping). + """ + # MuJoCo folds J^T * xfrc_applied into qfrc_smooth without storing it. + # Recover via the qfrc_smooth identity: + # qfrc_smooth = qfrc_actuator + qfrc_passive - qfrc_bias + # + qfrc_applied + J^T * xfrc_applied + f = self._joint_dof_field + return ( + f("qfrc_smooth") + - f("qfrc_actuator") + - f("qfrc_applied") + - f("qfrc_passive") + + f("qfrc_bias") + ) + + # Pose and velocity component accessors. + + @property + def root_link_pos_w(self) -> torch.Tensor: + """Root link position in world frame. Shape (num_envs, 3).""" + return self.root_link_pose_w[:, 0:3] + + @property + def root_link_quat_w(self) -> torch.Tensor: + """Root link quaternion in world frame. Shape (num_envs, 4).""" + return self.root_link_pose_w[:, 3:7] + + @property + def root_link_lin_vel_w(self) -> torch.Tensor: + """Root link linear velocity in world frame. Shape (num_envs, 3).""" + return self.root_link_vel_w[:, 0:3] + + @property + def root_link_ang_vel_w(self) -> torch.Tensor: + """Root link angular velocity in world frame. Shape (num_envs, 3).""" + return self.data.cvel[:, self.indexing.root_body_id, 0:3] + + @property + def root_com_pos_w(self) -> torch.Tensor: + """Root COM position in world frame. Shape (num_envs, 3).""" + return self.root_com_pose_w[:, 0:3] + + @property + def root_com_quat_w(self) -> torch.Tensor: + """Root COM quaternion in world frame. Shape (num_envs, 4).""" + return self.root_com_pose_w[:, 3:7] + + @property + def root_com_lin_vel_w(self) -> torch.Tensor: + """Root COM linear velocity in world frame. Shape (num_envs, 3).""" + return self.root_com_vel_w[:, 0:3] + + @property + def root_com_ang_vel_w(self) -> torch.Tensor: + """Root COM angular velocity in world frame. Shape (num_envs, 3).""" + # Angular velocity is the same for link and COM frames. + return self.data.cvel[:, self.indexing.root_body_id, 0:3] + + @property + def body_link_pos_w(self) -> torch.Tensor: + """Body link positions in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_link_pose_w[..., 0:3] + + @property + def body_link_quat_w(self) -> torch.Tensor: + """Body link quaternions in world frame. Shape (num_envs, num_bodies, 4).""" + return self.body_link_pose_w[..., 3:7] + + @property + def body_link_lin_vel_w(self) -> torch.Tensor: + """Body link linear velocities in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_link_vel_w[..., 0:3] + + @property + def body_link_ang_vel_w(self) -> torch.Tensor: + """Body link angular velocities in world frame. Shape (num_envs, num_bodies, 3).""" + return self.data.cvel[:, self.indexing.body_ids, 0:3] + + @property + def body_com_pos_w(self) -> torch.Tensor: + """Body COM positions in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_com_pose_w[..., 0:3] + + @property + def body_com_quat_w(self) -> torch.Tensor: + """Body COM quaternions in world frame. Shape (num_envs, num_bodies, 4).""" + return self.body_com_pose_w[..., 3:7] + + @property + def body_com_lin_vel_w(self) -> torch.Tensor: + """Body COM linear velocities in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_com_vel_w[..., 0:3] + + @property + def body_com_ang_vel_w(self) -> torch.Tensor: + """Body COM angular velocities in world frame. Shape (num_envs, num_bodies, 3).""" + # Angular velocity is the same for link and COM frames. + return self.data.cvel[:, self.indexing.body_ids, 0:3] + + @property + def body_external_force(self) -> torch.Tensor: + """Body external forces in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_external_wrench[..., 0:3] + + @property + def body_external_torque(self) -> torch.Tensor: + """Body external torques in world frame. Shape (num_envs, num_bodies, 3).""" + return self.body_external_wrench[..., 3:6] + + @property + def geom_pos_w(self) -> torch.Tensor: + """Geom positions in world frame. Shape (num_envs, num_geoms, 3).""" + return self.geom_pose_w[..., 0:3] + + @property + def geom_quat_w(self) -> torch.Tensor: + """Geom quaternions in world frame. Shape (num_envs, num_geoms, 4).""" + return self.geom_pose_w[..., 3:7] + + @property + def geom_lin_vel_w(self) -> torch.Tensor: + """Geom linear velocities in world frame. Shape (num_envs, num_geoms, 3).""" + return self.geom_vel_w[..., 0:3] + + @property + def geom_ang_vel_w(self) -> torch.Tensor: + """Geom angular velocities in world frame. Shape (num_envs, num_geoms, 3).""" + body_ids = self.model.geom_bodyid[self.indexing.geom_ids] + return self.data.cvel[:, body_ids, 0:3] + + @property + def site_pos_w(self) -> torch.Tensor: + """Site positions in world frame. Shape (num_envs, num_sites, 3).""" + return self.site_pose_w[..., 0:3] + + @property + def site_quat_w(self) -> torch.Tensor: + """Site quaternions in world frame. Shape (num_envs, num_sites, 4).""" + return self.site_pose_w[..., 3:7] + + @property + def site_lin_vel_w(self) -> torch.Tensor: + """Site linear velocities in world frame. Shape (num_envs, num_sites, 3).""" + return self.site_vel_w[..., 0:3] + + @property + def site_ang_vel_w(self) -> torch.Tensor: + """Site angular velocities in world frame. Shape (num_envs, num_sites, 3).""" + body_ids = self.model.site_bodyid[self.indexing.site_ids] + return self.data.cvel[:, body_ids, 0:3] + + # Derived properties. + + @property + def projected_gravity_b(self) -> torch.Tensor: + """Gravity vector projected into body frame. Shape (num_envs, 3).""" + return quat_apply_inverse(self.root_link_quat_w, self.gravity_vec_w) + + @property + def heading_w(self) -> torch.Tensor: + """Heading angle in world frame. Shape (num_envs,).""" + forward_w = quat_apply(self.root_link_quat_w, self.forward_vec_b) + return torch.atan2(forward_w[:, 1], forward_w[:, 0]) + + @property + def root_link_lin_vel_b(self) -> torch.Tensor: + """Root link linear velocity in body frame. Shape (num_envs, 3).""" + return quat_apply_inverse(self.root_link_quat_w, self.root_link_lin_vel_w) + + @property + def root_link_ang_vel_b(self) -> torch.Tensor: + """Root link angular velocity in body frame. Shape (num_envs, 3).""" + return quat_apply_inverse(self.root_link_quat_w, self.root_link_ang_vel_w) + + @property + def root_com_lin_vel_b(self) -> torch.Tensor: + """Root COM linear velocity in body frame. Shape (num_envs, 3).""" + return quat_apply_inverse(self.root_link_quat_w, self.root_com_lin_vel_w) + + @property + def root_com_ang_vel_b(self) -> torch.Tensor: + """Root COM angular velocity in body frame. Shape (num_envs, 3).""" + return quat_apply_inverse(self.root_link_quat_w, self.root_com_ang_vel_w) diff --git a/rl/mjlab/entity/entity.py b/rl/mjlab/entity/entity.py new file mode 100644 index 0000000..350c085 --- /dev/null +++ b/rl/mjlab/entity/entity.py @@ -0,0 +1,1340 @@ +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Sequence + +import mujoco +import mujoco_warp as mjwarp +import numpy as np +import torch + +from mjlab import actuator +from mjlab.actuator import BuiltinActuatorGroup, FusedActuatorGroup +from mjlab.actuator.actuator import TransmissionType +from mjlab.actuator.xml_actuator import XmlActuator +from mjlab.entity.data import EntityData +from mjlab.utils import spec_config as spec_cfg +from mjlab.utils.lab_api.string import resolve_matching_names +from mjlab.utils.mujoco import dof_width, qpos_width +from mjlab.utils.spec import auto_wrap_fixed_base_mocap +from mjlab.utils.string import resolve_expr +from mjlab.utils.xml import fix_spec_xml, strip_buffer_textures + +if TYPE_CHECKING: + from mjlab.entity.variants import VariantMetadata + + +@dataclass(frozen=False) +class EntityIndexing: + """Maps entity elements to global indices and addresses in the simulation.""" + + # Elements. + bodies: tuple[mujoco.MjsBody, ...] + joints: tuple[mujoco.MjsJoint, ...] + geoms: tuple[mujoco.MjsGeom, ...] + sites: tuple[mujoco.MjsSite, ...] + tendons: tuple[mujoco.MjsTendon, ...] + cameras: tuple[mujoco.MjsCamera, ...] + lights: tuple[mujoco.MjsLight, ...] + materials: tuple[mujoco.MjsMaterial, ...] + textures: tuple[mujoco.MjsTexture, ...] + pairs: tuple[mujoco.MjsPair, ...] + actuators: tuple[mujoco.MjsActuator, ...] | None + + # Indices. + body_ids: torch.Tensor + geom_ids: torch.Tensor + site_ids: torch.Tensor + tendon_ids: torch.Tensor + cam_ids: torch.Tensor + light_ids: torch.Tensor + mat_ids: torch.Tensor + tex_ids: torch.Tensor + pair_ids: torch.Tensor + ctrl_ids: torch.Tensor + joint_ids: torch.Tensor + mocap_id: int | None + + # Addresses. + joint_q_adr: torch.Tensor + joint_v_adr: torch.Tensor + free_joint_q_adr: torch.Tensor + free_joint_v_adr: torch.Tensor + + @property + def root_body_id(self) -> int: + return self.bodies[0].id + + +def _outer_index( + env_ids: torch.Tensor | slice, ids: torch.Tensor | slice +) -> tuple[torch.Tensor | slice, torch.Tensor | slice]: + """Make [env_ids, ids] select the outer product when both are tensors. + + Plain [tensor, tensor] indexing pairs the ids elementwise, which silently + writes a diagonal whenever the shapes happen to broadcast. + """ + if isinstance(env_ids, torch.Tensor) and isinstance(ids, torch.Tensor): + return env_ids[:, None], ids + return env_ids, ids + + +@dataclass +class EntityCfg: + @dataclass + class InitialStateCfg: + # Root position and orientation. + pos: tuple[float, float, float] = (0.0, 0.0, 0.0) + rot: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0) + # Root linear and angular velocity (only for floating base entities). + lin_vel: tuple[float, float, float] = (0.0, 0.0, 0.0) + ang_vel: tuple[float, float, float] = (0.0, 0.0, 0.0) + # Articulation (only for articulated entities). + # Set to None to use the model's existing keyframe (errors if none exists). + joint_pos: dict[str, float] | None = field(default_factory=lambda: {".*": 0.0}) + joint_vel: dict[str, float] = field(default_factory=lambda: {".*": 0.0}) + + init_state: InitialStateCfg = field(default_factory=InitialStateCfg) + spec_fn: Callable[[], mujoco.MjSpec] = field( + default_factory=lambda: lambda: mujoco.MjSpec() + ) + articulation: EntityArticulationInfoCfg | None = None + sort_actuators: bool = False + """When True, reorder actuators so that ``model.ctrl`` follows joint/tendon/site + definition order rather than the order actuators appear in the config. XML actuators + are excluded from sorting and always retain their declaration order. + """ + + # Editors. + lights: tuple[spec_cfg.LightCfg, ...] = field(default_factory=tuple) + cameras: tuple[spec_cfg.CameraCfg, ...] = field(default_factory=tuple) + textures: tuple[spec_cfg.TextureCfg, ...] = field(default_factory=tuple) + materials: tuple[spec_cfg.MaterialCfg, ...] = field(default_factory=tuple) + meshes: tuple[spec_cfg.MeshCfg, ...] = field(default_factory=tuple) + geoms: tuple[spec_cfg.GeomCfg, ...] = field(default_factory=tuple) + collisions: tuple[spec_cfg.CollisionCfg, ...] = field(default_factory=tuple) + + def build(self) -> Entity: + """Build entity instance from this config. + + Override in subclasses to return custom Entity types. + """ + return Entity(self) + + +@dataclass +class EntityArticulationInfoCfg: + actuators: tuple[actuator.ActuatorCfg, ...] = field(default_factory=tuple) + soft_joint_pos_limit_factor: float = 1.0 + + +class Entity: + """An entity represents a physical object in the simulation. + + Entity Type Matrix + ================== + MuJoCo entities can be categorized along two dimensions: + + 1. Base Type: + - Fixed Base: Entity is welded to the world (no freejoint) + - Floating Base: Entity has 6 DOF movement (has freejoint) + + 2. Articulation: + - Non-articulated: No joints other than freejoint + - Articulated: Has joints in kinematic tree (may or may not be actuated) + + Fixed non-articulated entities can optionally be mocap bodies, whereby their + position and orientation can be set directly each timestep rather than being + determined by physics. This property can be useful for creating props with + adjustable position and orientation. + + Supported Combinations: + ---------------------- + | Type | Example | is_fixed_base | is_articulated | is_actuated | + |---------------------------|---------------------|---------------|----------------|-------------| + | Fixed Non-articulated | Table, wall | True | False | False | + | Fixed Articulated | Robot arm, door | True | True | True/False | + | Floating Non-articulated | Box, ball, mug | False | False | False | + | Floating Articulated | Humanoid, quadruped | False | True | True/False | + """ + + def __init__(self, cfg: EntityCfg) -> None: + self.cfg = cfg + self._actuators: list[actuator.Actuator] = [] + self._variant_metadata: VariantMetadata | None = None + self._build_spec() + self._identify_joints() + self._apply_spec_editors() + self._add_actuators() + self._add_initial_state_keyframe() + + def _build_spec(self) -> None: + from mjlab.entity.variants import VariantEntityCfg, build_merged_variant_spec + + if isinstance(self.cfg, VariantEntityCfg): + self._spec, self._variant_metadata = build_merged_variant_spec(self.cfg) + else: + self._spec = auto_wrap_fixed_base_mocap(self.cfg.spec_fn)() + + @property + def variant_metadata(self) -> VariantMetadata | None: + return self._variant_metadata + + def _identify_joints(self) -> None: + self._all_joints = self._spec.joints + self._free_joint = None + self._non_free_joints = tuple(self._all_joints) + + free_joints = [j for j in self._all_joints if j.type == mujoco.mjtJoint.mjJNT_FREE] + if len(free_joints) > 1: + raise ValueError( + f"Entity spec has {len(free_joints)} freejoints. An Entity models a " + "single rigid- or articulated-body system with at most one freejoint, " + "which serves as its root. Model each detached floating body as its own " + "entry in SceneCfg.entities instead." + ) + + if self._all_joints and self._all_joints[0].type == mujoco.mjtJoint.mjJNT_FREE: + self._free_joint = self._all_joints[0] + if not self._free_joint.name: + self._free_joint.name = "floating_base_joint" + self._non_free_joints = tuple(self._all_joints[1:]) + + def _apply_spec_editors(self) -> None: + spec_cfg.warn_overlapping_geom_edits( + self.cfg.geoms, self.cfg.collisions, self._spec + ) + for cfg_list in [ + self.cfg.lights, + self.cfg.cameras, + self.cfg.textures, + self.cfg.materials, + self.cfg.meshes, + self.cfg.geoms, + self.cfg.collisions, + ]: + for cfg in cfg_list: + cfg.edit_spec(self._spec) + + def _add_actuators(self) -> None: + if self.cfg.articulation is None: + return + + # Collect actuator instances and their targets. + pending: list[tuple[actuator.ActuatorCfg, actuator.Actuator, list[str]]] = [] + for actuator_cfg in self.cfg.articulation.actuators: + # Find targets based on transmission type. resolve_matching_names raises + # ValueError when no regex matches; we catch that to produce a better error with + # namespace hints below. + target_ids: list[int] = [] + target_names: list[str] = [] + target_spec_names: list[str] = [] + try: + if actuator_cfg.transmission_type == TransmissionType.JOINT: + target_ids, target_names = self.find_joints(actuator_cfg.target_names_expr) + target_spec_names = [self._non_free_joints[i].name for i in target_ids] + elif actuator_cfg.transmission_type == TransmissionType.TENDON: + target_ids, target_names = self.find_tendons(actuator_cfg.target_names_expr) + target_spec_names = [self._spec.tendons[i].name for i in target_ids] + elif actuator_cfg.transmission_type == TransmissionType.SITE: + target_ids, target_names = self.find_sites(actuator_cfg.target_names_expr) + target_spec_names = [self.spec.sites[i].name for i in target_ids] + else: + raise TypeError( + f"Invalid transmission_type: {actuator_cfg.transmission_type}. " + f"Must be TransmissionType.JOINT, TransmissionType.TENDON, " + f"or TransmissionType.SITE." + ) + except ValueError: + pass # target_names stays empty, fall through to hint logic + + # Check other namespaces for matches. If we found nothing, this produces a + # helpful error. If we did find targets, it warns about unactuated matches in + # other namespaces. + current = actuator_cfg.transmission_type + other_matches: dict[TransmissionType, tuple[str, list[str]]] = {} + other_namespaces = { + TransmissionType.JOINT: ("joint", self.joint_names), + TransmissionType.TENDON: ("tendon", self.tendon_names), + TransmissionType.SITE: ("site", self.site_names), + } + for tt, (label, names) in other_namespaces.items(): + if tt == current or not names: + continue + try: + _, matched = resolve_matching_names(actuator_cfg.target_names_expr, names) + other_matches[tt] = (label, matched) + except ValueError: + pass + + if len(target_names) == 0: + msg = ( + f"No {current.value}s matched expressions: {actuator_cfg.target_names_expr}" + ) + if other_matches: + hints = [ + f"{label}s ({', '.join(matched)})" + for label, matched in other_matches.values() + ] + msg += ( + f". Matches were found in: {'; '.join(hints)}. " + f"Check that transmission_type is correct." + ) + raise ValueError(msg) + + for tt, (label, matched) in other_matches.items(): + warnings.warn( + f"Actuator config matched {len(target_names)} {current.value}(s) " + f"but the same expressions also match {len(matched)} {label}(s): " + f"{', '.join(matched)}. Add a separate config with " + f"transmission_type=TransmissionType.{tt.name} if those should " + f"be actuated too.", + stacklevel=2, + ) + + actuator_instance = actuator_cfg.build(self, target_ids, target_names) + self._actuators.append(actuator_instance) + pending.append((actuator_cfg, actuator_instance, target_spec_names)) + + if not self.cfg.sort_actuators: + for _, inst, names in pending: + inst.edit_spec(self._spec, names) + return + + # Sort actuators so ctrl order matches joint/tendon/site definition order. + # XmlActuators are added first (they wrap pre-existing XML actuators), + # then remaining actuators sorted by transmission type and target order. + order_maps = { + TransmissionType.JOINT: {name: i for i, name in enumerate(self.joint_names)}, + TransmissionType.TENDON: {name: i for i, name in enumerate(self.tendon_names)}, + TransmissionType.SITE: {name: i for i, name in enumerate(self.site_names)}, + } + # Group by transmission type (ordering is conventional, not physics-motivated). + # Within each group, actuators are sorted by their target's definition order in the + # spec. + type_priority = { + TransmissionType.JOINT: 0, + TransmissionType.TENDON: 1, + TransmissionType.SITE: 2, + } + + # XmlActuators go first in declaration order (they reference actuators already + # present in the spec). + for _, inst, names in pending: + if isinstance(inst, XmlActuator): + inst.edit_spec(self._spec, names) + + # Flatten remaining actuators to (instance, single_target) pairs and sort. + flat: list[tuple[actuator.ActuatorCfg, actuator.Actuator, str]] = [] + for cfg, inst, names in pending: + if not isinstance(inst, XmlActuator): + for name in names: + flat.append((cfg, inst, name)) + + flat.sort( + key=lambda item: ( + type_priority[item[0].transmission_type], + order_maps[item[0].transmission_type].get(item[2], float("inf")), + ) + ) + for _, inst, name in flat: + inst.edit_spec(self._spec, [name]) + + def _add_initial_state_keyframe(self) -> None: + # If joint_pos is None, use existing keyframe from the model. + if self.cfg.init_state.joint_pos is None: + if not self._spec.keys: + raise ValueError( + "joint_pos=None requires the model to have a keyframe, but none exists." + ) + # Keep the existing keyframe, just rename it. + self._spec.keys[0].name = "init_state" + if self.is_fixed_base: + self.root_body.pos[:] = self.cfg.init_state.pos + self.root_body.quat[:] = self.cfg.init_state.rot + return + + qpos_components = [] + + if self._free_joint is not None: + qpos_components.extend([self.cfg.init_state.pos, self.cfg.init_state.rot]) + + joint_pos = None + if self._non_free_joints: + joint_pos = resolve_expr(self.cfg.init_state.joint_pos, self.joint_names, 0.0) + qpos_components.append(joint_pos) + + key_qpos = np.hstack(qpos_components) if qpos_components else np.array([]) + key = self._spec.add_key(name="init_state", qpos=key_qpos.tolist()) + + if self.is_actuated and joint_pos is not None: + name_to_pos = {name: joint_pos[i] for i, name in enumerate(self.joint_names)} + ctrl = [] + for act in self._spec.actuators: + joint_name = act.target + ctrl.append(name_to_pos.get(joint_name, 0.0)) + key.ctrl = np.array(ctrl) + + if self.is_fixed_base: + self.root_body.pos[:] = self.cfg.init_state.pos + self.root_body.quat[:] = self.cfg.init_state.rot + + # Attributes. + + @property + def is_fixed_base(self) -> bool: + """Entity is welded to the world.""" + return self._free_joint is None + + @property + def is_articulated(self) -> bool: + """Entity is articulated (has fixed or actuated joints).""" + return len(self._non_free_joints) > 0 + + @property + def is_actuated(self) -> bool: + """Entity has actuated joints.""" + return len(self._actuators) > 0 + + @property + def has_tendon_actuators(self) -> bool: + """Entity has actuators using tendon transmission.""" + if self.cfg.articulation is None: + return False + return any( + act.transmission_type == TransmissionType.TENDON + for act in self.cfg.articulation.actuators + ) + + @property + def has_site_actuators(self) -> bool: + """Entity has actuators using site transmission.""" + if self.cfg.articulation is None: + return False + return any( + act.transmission_type == TransmissionType.SITE + for act in self.cfg.articulation.actuators + ) + + @property + def is_mocap(self) -> bool: + """Entity root body is a mocap body (only for fixed-base entities).""" + return bool(self.root_body.mocap) if self.is_fixed_base else False + + @property + def spec(self) -> mujoco.MjSpec: + return self._spec + + @property + def data(self) -> EntityData: + return self._data + + @property + def actuators(self) -> list[actuator.Actuator]: + return self._actuators + + # Names. + + @property + def body_names(self) -> tuple[str, ...]: + return tuple(b.name.split("/")[-1] for b in self.spec.bodies[1:]) + + @property + def all_joint_names(self) -> tuple[str, ...]: + return tuple(j.name.split("/")[-1] for j in self._all_joints) + + @property + def joint_names(self) -> tuple[str, ...]: + return tuple(j.name.split("/")[-1] for j in self._non_free_joints) + + @property + def geom_names(self) -> tuple[str, ...]: + return tuple(g.name.split("/")[-1] for g in self.spec.geoms) + + @property + def site_names(self) -> tuple[str, ...]: + return tuple(s.name.split("/")[-1] for s in self.spec.sites) + + @property + def tendon_names(self) -> tuple[str, ...]: + return tuple(t.name.split("/")[-1] for t in self._spec.tendons) + + @property + def camera_names(self) -> tuple[str, ...]: + return tuple(c.name.split("/")[-1] for c in self.spec.cameras) + + @property + def light_names(self) -> tuple[str, ...]: + return tuple(lt.name.split("/")[-1] for lt in self.spec.lights) + + @property + def material_names(self) -> tuple[str, ...]: + return tuple(m.name.split("/")[-1] for m in self.spec.materials) + + @property + def texture_names(self) -> tuple[str, ...]: + return tuple(t.name.split("/")[-1] for t in self.spec.textures) + + @property + def pair_names(self) -> tuple[str, ...]: + return tuple(p.name.split("/")[-1] for p in self.spec.pairs) + + @property + def actuator_names(self) -> tuple[str, ...]: + return tuple(a.name.split("/")[-1] for a in self.spec.actuators) + + # Counts. + + @property + def num_bodies(self) -> int: + return len(self.body_names) + + @property + def num_joints(self) -> int: + return len(self.joint_names) + + @property + def num_geoms(self) -> int: + return len(self.geom_names) + + @property + def num_sites(self) -> int: + return len(self.site_names) + + @property + def num_tendons(self) -> int: + return len(self.tendon_names) + + @property + def num_cameras(self) -> int: + return len(self.camera_names) + + @property + def num_lights(self) -> int: + return len(self.light_names) + + @property + def num_materials(self) -> int: + return len(self.material_names) + + @property + def num_textures(self) -> int: + return len(self.texture_names) + + @property + def num_pairs(self) -> int: + return len(self.pair_names) + + @property + def num_actuators(self) -> int: + return len(self.actuator_names) + + @property + def root_body(self) -> mujoco.MjsBody: + return self.spec.bodies[1] + + # Find methods. + + def find_bodies( + self, name_keys: str | Sequence[str], preserve_order: bool = False + ) -> tuple[list[int], list[str]]: + return resolve_matching_names(name_keys, self.body_names, preserve_order) + + def find_joints( + self, + name_keys: str | Sequence[str], + joint_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if joint_subset is None: + joint_subset = self.joint_names + return resolve_matching_names(name_keys, joint_subset, preserve_order) + + def find_joints_by_actuator_names( + self, + actuator_name_keys: str | Sequence[str], + ) -> tuple[list[int], list[str]]: + # Collect all actuated joint names. + actuated_joint_names_set = set() + for act in self._actuators: + actuated_joint_names_set.update(act.target_names) + + # Filter self.joint_names to only actuated joints, preserving natural order. + actuated_in_natural_order = [ + name for name in self.joint_names if name in actuated_joint_names_set + ] + + # Find joints matching the pattern within actuated joints. + _, matched_joint_names = self.find_joints( + actuator_name_keys, joint_subset=actuated_in_natural_order, preserve_order=False + ) + + # Map joint names back to entity-local indices (indices into self.joint_names). + name_to_entity_idx = {name: i for i, name in enumerate(self.joint_names)} + joint_ids = [name_to_entity_idx[name] for name in matched_joint_names] + return joint_ids, matched_joint_names + + def find_geoms( + self, + name_keys: str | Sequence[str], + geom_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if geom_subset is None: + geom_subset = self.geom_names + return resolve_matching_names(name_keys, geom_subset, preserve_order) + + def find_sites( + self, + name_keys: str | Sequence[str], + site_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if site_subset is None: + site_subset = self.site_names + return resolve_matching_names(name_keys, site_subset, preserve_order) + + def find_tendons( + self, + name_keys: str | Sequence[str], + tendon_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if tendon_subset is None: + tendon_subset = self.tendon_names + return resolve_matching_names(name_keys, tendon_subset, preserve_order) + + def find_cameras( + self, + name_keys: str | Sequence[str], + camera_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if camera_subset is None: + camera_subset = self.camera_names + return resolve_matching_names(name_keys, camera_subset, preserve_order) + + def find_lights( + self, + name_keys: str | Sequence[str], + light_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if light_subset is None: + light_subset = self.light_names + return resolve_matching_names(name_keys, light_subset, preserve_order) + + def find_materials( + self, + name_keys: str | Sequence[str], + material_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if material_subset is None: + material_subset = self.material_names + return resolve_matching_names(name_keys, material_subset, preserve_order) + + def find_textures( + self, + name_keys: str | Sequence[str], + texture_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if texture_subset is None: + texture_subset = self.texture_names + return resolve_matching_names(name_keys, texture_subset, preserve_order) + + def find_pairs( + self, + name_keys: str | Sequence[str], + pair_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if pair_subset is None: + pair_subset = self.pair_names + return resolve_matching_names(name_keys, pair_subset, preserve_order) + + def find_actuators( + self, + name_keys: str | Sequence[str], + actuator_subset: Sequence[str] | None = None, + preserve_order: bool = False, + ) -> tuple[list[int], list[str]]: + if actuator_subset is None: + actuator_subset = self.actuator_names + return resolve_matching_names(name_keys, actuator_subset, preserve_order) + + def compile(self) -> mujoco.MjModel: + """Compile the underlying MjSpec into an MjModel.""" + return self.spec.compile() + + def write_xml(self, xml_path: Path) -> None: + """Write the MjSpec to disk. + + Operates on a copy of the spec to avoid mutating the original. + """ + tmp = self.spec.copy() + strip_buffer_textures(tmp) + xml_path.write_text(fix_spec_xml(tmp.to_xml())) + + def to_zip(self, path: Path) -> None: + """Write the MjSpec to a zip file.""" + with path.open("wb") as f: + mujoco.MjSpec.to_zip(self.spec, f) + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + device: str, + ) -> None: + """Prepare the entity for simulation after the spec has been compiled. + + Computes global index mappings, initializes actuators, and allocates all nworld + state and target tensors in ``EntityData``. Called once by the scene during + environment construction. + """ + indexing = self._compute_indexing(mj_model, device) + self.indexing = indexing + nworld = data.nworld + + for act in self._actuators: + act.initialize(mj_model, model, data, device) + + # Vectorize built-in actuators, then fuse ideal PD actuators; we'll loop + # through whatever custom actuators remain. + builtin_group, custom_actuators = BuiltinActuatorGroup.process(self._actuators) + builtin_group.initialize(nworld, device) + self._builtin_group = builtin_group + fused_actuator_group, custom_actuators = FusedActuatorGroup.process( + custom_actuators + ) + fused_actuator_group.initialize(nworld, device) + self._fused_actuator_group = fused_actuator_group + self._custom_actuators = custom_actuators + + # Root state. + default_root_state = self._build_default_root_state(nworld, device) + + # Joint state. + if self.is_articulated: + if self.cfg.init_state.joint_pos is None: + # Use keyframe joint positions. The scene keyframe holds every entity's qpos. + key_qpos = torch.tensor( + mj_model.key("init_state").qpos, dtype=torch.float, device=device + ) + default_joint_pos = key_qpos[indexing.joint_q_adr][None].repeat(nworld, 1) + else: + default_joint_pos = torch.tensor( + resolve_expr(self.cfg.init_state.joint_pos, self.joint_names, 0.0), + device=device, + )[None].repeat(nworld, 1) + default_joint_vel = torch.tensor( + resolve_expr(self.cfg.init_state.joint_vel, self.joint_names, 0.0), + device=device, + )[None].repeat(nworld, 1) + + # Joint limits. + joint_ids_list = [j.id for j in self._non_free_joints] + dof_limits = model.jnt_range[:, joint_ids_list] + default_joint_pos_limits = dof_limits.clone() + joint_pos_limits = default_joint_pos_limits.clone() + + joint_pos_mean = (joint_pos_limits[..., 0] + joint_pos_limits[..., 1]) / 2 + joint_pos_range = joint_pos_limits[..., 1] - joint_pos_limits[..., 0] + + # Soft limits. + soft_limit_factor = ( + self.cfg.articulation.soft_joint_pos_limit_factor + if self.cfg.articulation + else 1.0 + ) + soft_joint_pos_limits = torch.stack( + [ + joint_pos_mean - 0.5 * joint_pos_range * soft_limit_factor, + joint_pos_mean + 0.5 * joint_pos_range * soft_limit_factor, + ], + dim=-1, + ) + + # Unlimited joints have jnt_range=[0,0] in MuJoCo, which makes all + # the computed limits [0,0]. Override to [-inf, inf] so downstream + # clamping becomes a no-op. (Can't do this before soft-limit math + # because inf - inf = NaN.) + unlimited = ~torch.tensor( + mj_model.jnt_limited[joint_ids_list], device=device, dtype=torch.bool + ) + for limits in (joint_pos_limits, default_joint_pos_limits, soft_joint_pos_limits): + limits[:, unlimited, 0] = float("-inf") + limits[:, unlimited, 1] = float("inf") + else: + empty_shape = (nworld, 0) + default_joint_pos = torch.empty(*empty_shape, dtype=torch.float, device=device) + default_joint_vel = torch.empty(*empty_shape, dtype=torch.float, device=device) + default_joint_pos_limits = torch.empty( + *empty_shape, 2, dtype=torch.float, device=device + ) + joint_pos_limits = torch.empty(*empty_shape, 2, dtype=torch.float, device=device) + soft_joint_pos_limits = torch.empty( + *empty_shape, 2, dtype=torch.float, device=device + ) + + if self.is_actuated: + joint_pos_target = torch.zeros( + (nworld, self.num_joints), dtype=torch.float, device=device + ) + joint_vel_target = torch.zeros( + (nworld, self.num_joints), dtype=torch.float, device=device + ) + joint_effort_target = torch.zeros( + (nworld, self.num_joints), dtype=torch.float, device=device + ) + else: + joint_pos_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + joint_vel_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + joint_effort_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + + # Only allocate tendon targets if there are actuators using tendon transmission. + if self.has_tendon_actuators: + num_tendons = len(self.tendon_names) + tendon_len_target = torch.zeros( + (nworld, num_tendons), dtype=torch.float, device=device + ) + tendon_vel_target = torch.zeros( + (nworld, num_tendons), dtype=torch.float, device=device + ) + tendon_effort_target = torch.zeros( + (nworld, num_tendons), dtype=torch.float, device=device + ) + else: + tendon_len_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + tendon_vel_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + tendon_effort_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + + # Only allocate site targets if there are actuators using site transmission. + if self.has_site_actuators: + num_sites = len(self.site_names) + site_effort_target = torch.zeros( + (nworld, num_sites), dtype=torch.float, device=device + ) + else: + site_effort_target = torch.empty(nworld, 0, dtype=torch.float, device=device) + + # Encoder bias for simulating encoder calibration errors. + # Shape: (num_envs, num_joints). Defaults to zero (no bias). + if self.is_articulated: + encoder_bias = torch.zeros( + (nworld, self.num_joints), dtype=torch.float, device=device + ) + else: + encoder_bias = torch.empty(nworld, 0, dtype=torch.float, device=device) + + self._data = EntityData( + indexing=indexing, + data=data, + model=model, + device=device, + default_root_state=default_root_state, + default_joint_pos=default_joint_pos, + default_joint_vel=default_joint_vel, + default_joint_pos_limits=default_joint_pos_limits, + joint_pos_limits=joint_pos_limits, + soft_joint_pos_limits=soft_joint_pos_limits, + gravity_vec_w=torch.tensor([0.0, 0.0, -1.0], device=device).repeat(nworld, 1), + forward_vec_b=torch.tensor([1.0, 0.0, 0.0], device=device).repeat(nworld, 1), + is_fixed_base=self.is_fixed_base, + is_articulated=self.is_articulated, + is_actuated=self.is_actuated, + joint_pos_target=joint_pos_target, + joint_vel_target=joint_vel_target, + joint_effort_target=joint_effort_target, + tendon_len_target=tendon_len_target, + tendon_vel_target=tendon_vel_target, + tendon_effort_target=tendon_effort_target, + site_effort_target=site_effort_target, + encoder_bias=encoder_bias, + ) + + def update(self, dt: float) -> None: + """Advance actuator internal state by one physics substep. + + Called after each ``sim.step()`` within the decimation loop. + """ + for act in self._actuators: + act.update(dt) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + """Zero actuator targets and reset actuator internal state. + + Called by the scene when environments are reset at episode boundaries, + and by commands that teleport the robot to a new pose mid-episode. + """ + self._data.clear_state(env_ids) + + for act in self._actuators: + act.reset(env_ids) + + def clear_state(self, env_ids: torch.Tensor | slice | None = None) -> None: + """Deprecated. Use ``reset`` instead.""" + warnings.warn( + "Entity.clear_state() is deprecated. Use Entity.reset().", + DeprecationWarning, + stacklevel=2, + ) + self.reset(env_ids) + + def write_data_to_sim(self) -> None: + """Convert actuator targets into low-level controls and write them to the sim. + + Called before each ``sim.step()`` within the decimation loop. Builtin actuators are + applied in a single batched operation; custom actuators are applied individually. + """ + self._apply_actuator_controls() + + def write_ctrl_to_sim( + self, + ctrl: torch.Tensor, + ctrl_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Write control inputs to the simulation. + + Args: + ctrl: A tensor of control inputs. + ctrl_ids: A tensor of control indices. + env_ids: Optional tensor or slice specifying which environments to set. + If None, all environments are set. + """ + self._data.write_ctrl(ctrl, ctrl_ids, env_ids) + + def write_root_state_to_sim( + self, root_state: torch.Tensor, env_ids: torch.Tensor | slice | None = None + ) -> None: + """Set the root state into the simulation. + + The root state consists of position (3), orientation as a (w, x, y, z) + quaternion (4), linear velocity (3), and angular velocity (3), for a total + of 13 values. All of the quantities are in the world frame. + + Args: + root_state: Tensor of shape (N, 13) where N is the number of environments. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_root_state(root_state, env_ids) + + def write_root_link_pose_to_sim( + self, + root_pose: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the root pose into the simulation. Like `write_root_state_to_sim()` + but only sets position and orientation. + + Args: + root_pose: Tensor of shape (N, 7) where N is the number of environments. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_root_pose(root_pose, env_ids) + + def write_root_link_velocity_to_sim( + self, + root_velocity: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the root link (body origin) velocity into the simulation. Like + `write_root_state_to_sim()` but only sets linear and angular velocity. + + Args: + root_velocity: Tensor of shape (N, 6) where N is the number of environments. + Contains linear velocity (3) at body origin and angular velocity (3), + both in world frame. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_root_velocity(root_velocity, env_ids) + + def write_root_link_velocity_b_to_sim( + self, + root_velocity_b: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ): + """Like `write_root_link_velocity_to_sim()` but the velocity is expressed + in the root link's body frame. Reads the orientation from qpos, so it is + safe to call during a reset before forward() runs. + + Args: + root_velocity_b: Tensor of shape (N, 6) where N is the number of + environments. Contains linear velocity (3) at body origin and angular + velocity (3), both in the root link's body frame. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_root_velocity_b(root_velocity_b, env_ids) + + def write_root_com_velocity_to_sim( + self, + root_velocity: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the root COM velocity into the simulation. + + Args: + root_velocity: Tensor of shape (N, 6) where N is the number of environments. + Contains linear velocity (3) at COM and angular velocity (3), + both in world frame. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_root_com_velocity(root_velocity, env_ids) + + def write_joint_state_to_sim( + self, + position: torch.Tensor, + velocity: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the joint state into the simulation. + + The joint state consists of joint positions and velocities. It does not include + the root state. + + Args: + position: Tensor of shape (N, num_joints) where N is the number of environments. + velocity: Tensor of shape (N, num_joints) where N is the number of environments. + joint_ids: Optional tensor or slice specifying which joints to set. If None, + all joints are set. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_joint_state(position, velocity, joint_ids, env_ids) + + def write_joint_position_to_sim( + self, + position: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the joint positions into the simulation. Like `write_joint_state_to_sim()` + but only sets joint positions. + + Args: + position: Tensor of shape (N, num_joints) where N is the number of environments. + joint_ids: Optional tensor or slice specifying which joints to set. If None, + all joints are set. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_joint_position(position, joint_ids, env_ids) + + def write_joint_velocity_to_sim( + self, + velocity: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ): + """Set the joint velocities into the simulation. Like `write_joint_state_to_sim()` + but only sets joint velocities. + + Args: + velocity: Tensor of shape (N, num_joints) where N is the number of environments. + joint_ids: Optional tensor or slice specifying which joints to set. If None, + all joints are set. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_joint_velocity(velocity, joint_ids, env_ids) + + def set_joint_position_target( + self, + position: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set joint position targets. + + Args: + position: Target joint poisitions with shape (N, num_joints). + joint_ids: Optional joint indices to set. If None, set all joints. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if joint_ids is None: + joint_ids = slice(None) + env_ids, joint_ids = _outer_index(env_ids, joint_ids) + self._data.joint_pos_target[env_ids, joint_ids] = position + + def set_joint_velocity_target( + self, + velocity: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set joint velocity targets. + + Args: + velocity: Target joint velocities with shape (N, num_joints). + joint_ids: Optional joint indices to set. If None, set all joints. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if joint_ids is None: + joint_ids = slice(None) + env_ids, joint_ids = _outer_index(env_ids, joint_ids) + self._data.joint_vel_target[env_ids, joint_ids] = velocity + + def set_joint_effort_target( + self, + effort: torch.Tensor, + joint_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set joint effort targets. + + Args: + effort: Target joint efforts with shape (N, num_joints). + joint_ids: Optional joint indices to set. If None, set all joints. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if joint_ids is None: + joint_ids = slice(None) + env_ids, joint_ids = _outer_index(env_ids, joint_ids) + self._data.joint_effort_target[env_ids, joint_ids] = effort + + def set_tendon_len_target( + self, + length: torch.Tensor, + tendon_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set tendon length targets. + + Args: + length: Target tendon lengths with shape (N, num_tendons). + tendon_ids: Optional tendon indices to set. If None, set all tendons. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if tendon_ids is None: + tendon_ids = slice(None) + env_ids, tendon_ids = _outer_index(env_ids, tendon_ids) + self._data.tendon_len_target[env_ids, tendon_ids] = length + + def set_tendon_vel_target( + self, + velocity: torch.Tensor, + tendon_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set tendon velocity targets. + + Args: + velocity: Target tendon velocities with shape (N, num_tendons). + tendon_ids: Optional tendon indices to set. If None, set all tendons. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if tendon_ids is None: + tendon_ids = slice(None) + env_ids, tendon_ids = _outer_index(env_ids, tendon_ids) + self._data.tendon_vel_target[env_ids, tendon_ids] = velocity + + def set_tendon_effort_target( + self, + effort: torch.Tensor, + tendon_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set tendon effort targets. + + Args: + effort: Target tendon efforts with shape (N, num_tendons). + tendon_ids: Optional tendon indices to set. If None, set all tendons. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if tendon_ids is None: + tendon_ids = slice(None) + env_ids, tendon_ids = _outer_index(env_ids, tendon_ids) + self._data.tendon_effort_target[env_ids, tendon_ids] = effort + + def set_site_effort_target( + self, + effort: torch.Tensor, + site_ids: torch.Tensor | slice | None = None, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set site effort targets. + + Args: + effort: Target site efforts with shape (N, num_sites). + site_ids: Optional site indices to set. If None, set all sites. + env_ids: Optional environment indices. If None, set all environments. + """ + if env_ids is None: + env_ids = slice(None) + if site_ids is None: + site_ids = slice(None) + env_ids, site_ids = _outer_index(env_ids, site_ids) + self._data.site_effort_target[env_ids, site_ids] = effort + + def write_external_wrench_to_sim( + self, + forces: torch.Tensor, + torques: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + body_ids: Sequence[int] | slice | None = None, + ) -> None: + """Apply external wrenches to bodies in the simulation. + + Underneath the hood, this sets the `xfrc_applied` field in the MuJoCo data + structure. The wrenches are specified in the world frame and persist until + the next call to this function or until the simulation is reset. + + Args: + forces: Tensor of shape (N, num_bodies, 3) where N is the number of + environments. + torques: Tensor of shape (N, num_bodies, 3) where N is the number of + environments. + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + body_ids: Optional list of body indices or slice specifying which bodies to + apply the wrenches to. If None, wrenches are applied to all bodies. + """ + self._data.write_external_wrench(forces, torques, body_ids, env_ids) + + def write_mocap_pose_to_sim( + self, + mocap_pose: torch.Tensor, + env_ids: torch.Tensor | slice | None = None, + ) -> None: + """Set the pose of a mocap body into the simulation. + + Args: + mocap_pose: Tensor of shape (N, 7) where N is the number of environments. + Format: [pos_x, pos_y, pos_z, quat_w, quat_x, quat_y, quat_z] + env_ids: Optional tensor or slice specifying which environments to set. If + None, all environments are set. + """ + self._data.write_mocap_pose(mocap_pose, env_ids) + + ## + # Private methods. + ## + + def _build_default_root_state(self, nworld: int, device: str) -> torch.Tensor: + """Build default root state tensor, uniform across all worlds.""" + base = self.cfg.init_state + components: list[tuple[float, ...]] = [base.pos, base.rot] + if not self.is_fixed_base: + components.extend([base.lin_vel, base.ang_vel]) + return torch.tensor( + sum((tuple(c) for c in components), ()), + dtype=torch.float, + device=device, + ).repeat(nworld, 1) + + def _compute_indexing(self, model: mujoco.MjModel, device: str) -> EntityIndexing: + bodies = tuple([b for b in self.spec.bodies[1:]]) + joints = self._non_free_joints + geoms = tuple(self.spec.geoms) + sites = tuple(self.spec.sites) + tendons = tuple(self.spec.tendons) + cameras = tuple(self.spec.cameras) + lights = tuple(self.spec.lights) + materials = tuple(self.spec.materials) + textures = tuple(self.spec.textures) + pairs = tuple(self.spec.pairs) + + body_ids = torch.tensor([b.id for b in bodies], dtype=torch.int, device=device) + geom_ids = torch.tensor([g.id for g in geoms], dtype=torch.int, device=device) + site_ids = torch.tensor([s.id for s in sites], dtype=torch.int, device=device) + tendon_ids = torch.tensor([t.id for t in tendons], dtype=torch.int, device=device) + cam_ids = torch.tensor([c.id for c in cameras], dtype=torch.int, device=device) + light_ids = torch.tensor([lt.id for lt in lights], dtype=torch.int, device=device) + mat_ids = torch.tensor([m.id for m in materials], dtype=torch.int, device=device) + tex_ids = torch.tensor([t.id for t in textures], dtype=torch.int, device=device) + pair_ids = torch.tensor([p.id for p in pairs], dtype=torch.int, device=device) + joint_ids = torch.tensor([j.id for j in joints], dtype=torch.int, device=device) + + if self.is_actuated: + actuators = tuple(self.spec.actuators) + ctrl_ids = torch.tensor([a.id for a in actuators], dtype=torch.int, device=device) + else: + actuators = None + ctrl_ids = torch.empty(0, dtype=torch.int, device=device) + + joint_q_adr = [] + joint_v_adr = [] + free_joint_q_adr = [] + free_joint_v_adr = [] + for joint in self.spec.joints: + jnt = model.joint(joint.name) + jnt_type = jnt.type[0] + vadr = jnt.dofadr[0] + qadr = jnt.qposadr[0] + if jnt_type == mujoco.mjtJoint.mjJNT_FREE: + free_joint_v_adr.extend(range(vadr, vadr + 6)) + free_joint_q_adr.extend(range(qadr, qadr + 7)) + else: + joint_v_adr.extend(range(vadr, vadr + dof_width(jnt_type))) + joint_q_adr.extend(range(qadr, qadr + qpos_width(jnt_type))) + joint_q_adr = torch.tensor(joint_q_adr, dtype=torch.int, device=device) + joint_v_adr = torch.tensor(joint_v_adr, dtype=torch.int, device=device) + free_joint_v_adr = torch.tensor(free_joint_v_adr, dtype=torch.int, device=device) + free_joint_q_adr = torch.tensor(free_joint_q_adr, dtype=torch.int, device=device) + + if self.is_fixed_base and self.is_mocap: + mocap_id = int(model.body_mocapid[self.root_body.id]) + else: + mocap_id = None + + return EntityIndexing( + bodies=bodies, + joints=joints, + geoms=geoms, + sites=sites, + tendons=tendons, + cameras=cameras, + lights=lights, + materials=materials, + textures=textures, + pairs=pairs, + actuators=actuators, + body_ids=body_ids, + geom_ids=geom_ids, + site_ids=site_ids, + tendon_ids=tendon_ids, + cam_ids=cam_ids, + light_ids=light_ids, + mat_ids=mat_ids, + tex_ids=tex_ids, + pair_ids=pair_ids, + ctrl_ids=ctrl_ids, + joint_ids=joint_ids, + mocap_id=mocap_id, + joint_q_adr=joint_q_adr, + joint_v_adr=joint_v_adr, + free_joint_q_adr=free_joint_q_adr, + free_joint_v_adr=free_joint_v_adr, + ) + + def _apply_actuator_controls(self) -> None: + self._builtin_group.apply_controls(self._data) + self._fused_actuator_group.apply_controls(self._data) + for act in self._custom_actuators: + command = act.get_command(self._data) + command = act.apply_delay(command) + self._data.write_ctrl(act.compute(command), act.ctrl_ids) diff --git a/rl/mjlab/entity/variants.py b/rl/mjlab/entity/variants.py new file mode 100644 index 0000000..f867f2b --- /dev/null +++ b/rl/mjlab/entity/variants.py @@ -0,0 +1,1442 @@ +"""Per-world mesh variant support. + +A single batched simulation can run with different mesh assets in different +parallel worlds. World 0 may simulate a cube, world 1 a sphere, world 2 a +bowl, all sharing the same compiled scene and the same kinematic structure. +The result is a heterogeneous batch in which the mesh and its derived +constants vary across worlds while everything else (body tree, joint +structure, contact and solver setup) is fixed. + +The feature spans two phases, both of which live in this file: + +1. **Authoring** (entity-build time, MjSpec only). The user supplies a + ``VariantEntityCfg`` whose ``variants`` map names to spec callables + instances, each with its own ``spec_fn``. ``build_merged_variant_spec`` + validates that all source specs share kinematic topology, computes + ``(body_path, role, ordinal)`` slots so each variable mesh geom has a + structural identity, and merges the source specs into one padded + template spec. It returns ``(template_spec, VariantMetadata)``. +2. **Realization** (sim-init time, mujoco_warp). ``build_variant_model`` + takes the scene-attached padded spec plus the per-entity + ``VariantMetadata`` and produces a heterogeneous Warp model: a per-world + ``geom_dataid`` table plus per-world arrays for the geometry-dependent + fields listed in ``VARIANT_DEPENDENT_FIELDS``. Each unique variant is + compiled once on the host with its source-spec geom semantics restored, + and the resulting reference fields are scattered into per-world arrays + keyed by world-to-variant assignment. + +Slot identity is the throughline. ``SlotKey(body_path, role, ordinal)`` is +fixed across variants because mujoco_warp's ``geom_contype``/ +``geom_conaffinity`` are 1D shared, so each slot's role (visual vs +collision) cannot change per world. ``VariantGeomSpec`` records each +variant's source-geom attributes per slot so the per-variant reference +compile restores them verbatim instead of inheriting whatever the template +variant set. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Callable, Literal, Sequence, cast + +import mujoco +import mujoco_warp as mjwarp +import numpy as np +import warp as wp + +from mjlab.entity.entity import EntityCfg +from mjlab.utils.mujoco import dof_width, qpos_width +from mjlab.utils.spec import copy_material_data, copy_mesh_data, copy_texture_data + +# Reserved name prefixes for synthesized template entities. Source variant +# specs must not create geoms, bodies, meshes, or other named items under +# these prefixes; validation rejects them so the merge pass can introduce +# padding geoms without colliding with user names. +RESERVED_NAME_PREFIXES: tuple[str, ...] = ("mjlab/pad/",) + +# Fields that depend on mesh geometry and must be compiled per-variant. +VARIANT_DEPENDENT_FIELDS = ( + "geom_size", + "geom_rbound", + "geom_aabb", + "geom_pos", + "geom_quat", + "body_mass", + "body_subtreemass", + "body_inertia", + "body_invweight0", + "body_ipos", + "body_iquat", +) + +GeomRole = Literal["visual", "collision"] +InertialMode = Literal["mesh-derived", "diagonal", "fullinertia"] + + +## +# Input: what the user writes. +## + + +def _variant_spec_fn_unset() -> mujoco.MjSpec: + """Sentinel default for ``VariantEntityCfg.spec_fn``. + + ``VariantEntityCfg`` builds its spec from ``variants`` via + ``build_merged_variant_spec``; the inherited ``spec_fn`` field is unused. + Identity comparison against this sentinel detects accidental user overrides. + """ + raise AssertionError( + "VariantEntityCfg.spec_fn should never be called; the merged spec is " + "built from `variants`." + ) + + +@dataclass +class VariantEntityCfg(EntityCfg): + """Entity config for per-world mesh variants. + + Provide a dict of named variants (each value is a callable returning + an ``MjSpec``) and optionally an ``assignment`` describing how worlds + map to variants. The merged spec (with all variant meshes and padded + geoms) is built automatically. + + All variants must share the same kinematic structure (same bodies, + joints, joint types). Only mesh geoms can differ. + + Variant assignment is fixed at ``Simulation`` initialization; it does + not resample on episode reset. + """ + + variants: dict[str, Callable[[], mujoco.MjSpec]] = field(default_factory=dict) + """Named mesh variants. Each value is a callable returning an ``MjSpec``.""" + + assignment: dict[str, float] | Callable[[int], Sequence[int]] | None = None + """How worlds get mapped to variants. Three shapes: + + * ``None`` (default): uniform allocation across variants via + largest-remainder. + * ``dict[str, float]``: per-variant weights for largest-remainder + allocation. Variants not listed default to weight 1.0. + * ``Callable[[int], Sequence[int]]``: an explicit assignment + function, called with ``num_envs`` at simulation init and required + to return a length-``num_envs`` sequence of variant indices in + ``[0, len(variants))``.""" + + spec_fn: Callable[[], mujoco.MjSpec] = field(default=_variant_spec_fn_unset) + """Unused on ``VariantEntityCfg``; the merged spec is built from ``variants``.""" + + def __post_init__(self) -> None: + if self.spec_fn is not _variant_spec_fn_unset: + raise ValueError( + "VariantEntityCfg.spec_fn cannot be set; pass per-variant spec " + "callables in `variants` instead." + ) + if isinstance(self.assignment, dict): + extras = set(self.assignment) - set(self.variants) + if extras: + raise ValueError( + f"VariantEntityCfg.assignment has keys not present in variants: " + f"{sorted(extras)}." + ) + + +## +# Output: data records that flow to scene/sim. +## + + +@dataclass(frozen=True) +class SlotKey: + """Structural identity of a mesh slot in the merged template.""" + + body_path: str + role: GeomRole + ordinal: int + + +@dataclass(frozen=True) +class VariantGeomSpec: + """Per-variant per-slot geom attributes captured from the source spec. + + The slot-aware merge uses this to restore exact source semantics + (contact bits, friction, material, etc.) for each variant's reference + compilation, instead of inheriting whatever the template variant set. + """ + + mesh_name: str + geom_name: str | None + contype: int + conaffinity: int + condim: int + group: int + priority: int + material: str | None + rgba: tuple[float, float, float, float] + friction: tuple[float, float, float] + margin: float + gap: float + solref: tuple[float, float] + solimp: tuple[float, float, float, float, float] + # ``mass``/``density`` default to 0.0 in the source spec, which MuJoCo + # interprets as "infer from mesh volume". The slot-aware compile path + # passes the source value through verbatim. + mass: float + density: float + + +@dataclass(frozen=True) +class VariantSlot: + """One slot in the merged template body. + + ``template_geom_name`` is the synthesized name on the template body's + geom that backs this slot. It always uses the reserved + ``mjlab/pad///`` prefix so it cannot + collide with user-named source geoms (validation rejects user geoms + starting with ``mjlab/pad/``). + + ``source_geom_names`` records, per variant, the original source geom + name for diagnostics. ``None`` here can mean either "variant doesn't + fill the slot" or "variant fills the slot with an unnamed geom"; the + authoritative source for fill status is + ``VariantMetadata.variant_slot_specs[v][s] is None``. + """ + + key: SlotKey + template_geom_name: str + source_geom_names: tuple[str | None, ...] + + +@dataclass +class VariantMetadata: + """Bookkeeping produced when merging variant specs. + + Slots are ordered by ``(body_path, role, ordinal)`` and + ``variant_slot_specs[v]`` aligns positionally with ``slots`` so + ``variant_slot_specs[v][s]`` describes how variant ``v`` fills slot + ``s`` (``None`` if variant ``v`` leaves the slot unfilled). + + ``variant_source_specs[v]`` is the original ``MjSpec`` produced by + variant ``v``'s ``spec_fn`` (pre-merge). The sim-time build path + compiles each one in isolation rather than recompiling a copy of the + merged scene, which keeps construction cost linear in the number of + variants instead of quadratic. + + ``variant_mesh_names`` and ``num_mesh_geoms`` are kept as derived + ``@property`` views over ``variant_slot_specs`` for back-compat with + consumers that index by slot. + """ + + variant_names: tuple[str, ...] + assignment: tuple[float, ...] | Callable[[int], Sequence[int]] = () + slots: tuple[VariantSlot, ...] = () + variant_slot_specs: tuple[tuple[VariantGeomSpec | None, ...], ...] = () + variant_source_specs: tuple[mujoco.MjSpec, ...] = () + + @property + def variant_mesh_names(self) -> tuple[tuple[str | None, ...], ...]: + """Per-variant slot-aligned mesh names with variant prefix. + + ``None`` at position ``s`` means variant ``v`` does not fill slot + ``s``. Derived from ``variant_slot_specs``. + """ + return tuple( + tuple( + None if ss is None else f"{self.variant_names[v_idx]}/{ss.mesh_name}" + for ss in slot_specs + ) + for v_idx, slot_specs in enumerate(self.variant_slot_specs) + ) + + @property + def num_mesh_geoms(self) -> int: + """Max number of filled mesh slots across variants (i.e. the longest + variant's source mesh count). Derived from ``variant_slot_specs``.""" + if not self.variant_slot_specs: + return 0 + return max( + sum(1 for ss in slot_specs if ss is not None) + for slot_specs in self.variant_slot_specs + ) + + +@dataclass +class MeshVariantResult: + """Output of :func:`build_variant_model`.""" + + wp_model: mjwarp.Model + mj_model: mujoco.MjModel + # Maps entity prefix -> array of variant indices per world. + world_to_variant: dict[str, np.ndarray] + + +## +# Tree helpers (shared by validation and slot computation). +## + + +def _iter_body_paths( + body: mujoco.MjsBody, parent_path: str = "" +) -> list[tuple[str, mujoco.MjsBody]]: + """Return ``[(path, body), ...]`` for the recursive body tree. + + Path is the body's path relative to the variant root (e.g. ``/prop``, + ``/prop/lid``), built by joining body names with ``/``. + """ + body_name = body.name or "" + path = f"{parent_path}/{body_name}" + out: list[tuple[str, mujoco.MjsBody]] = [(path, body)] + for child in body.bodies: + out.extend(_iter_body_paths(child, path)) + return out + + +def _classify_geom_role(geom: mujoco.MjsGeom) -> GeomRole: + """Derive a geom's slot role from its contact bits. + + Visual: ``contype == 0`` and ``conaffinity == 0``. Collision: anything + else. Shared by validation and slot computation. + """ + if int(geom.contype) == 0 and int(geom.conaffinity) == 0: + return "visual" + return "collision" + + +def _iter_body_tree(body: mujoco.MjsBody): + yield body + for child in body.bodies: + yield from _iter_body_tree(child) + + +## +# Validation pass: cross-variant topology comparison. +## + + +@dataclass(frozen=True) +class JointSignature: + """Joint identity for cross-variant matching. + + Captures the structural attributes that must match across variants: + name, type, and qpos/qvel widths. Parameter attributes (axis, range, + stiffness, damping, etc.) are not part of the structural signature; + they may legitimately differ across variants once registered as + per-world fields. + """ + + name: str + type: int + qpos_width: int + qvel_width: int + + +@dataclass(frozen=True) +class GeomSignature: + """Geom identity for cross-variant matching. + + ``role`` is derived from ``contype``/``conaffinity``: a geom that + participates in any contact pair is "collision"; otherwise "visual". + A slot's role is fixed across variants by construction because + mujoco_warp's ``geom_contype``/``geom_conaffinity`` are 1D shared. + """ + + name: str | None + type: int + role: GeomRole + + +@dataclass(frozen=True) +class BodySignature: + """Recursive body identity for cross-variant matching. + + ``path`` is the body path relative to its variant's worldbody, before + any scene-attachment prefix. ``children`` are nested signatures in + source order. + """ + + path: str + name: str + joints: tuple[JointSignature, ...] + geoms: tuple[GeomSignature, ...] + children: tuple["BodySignature", ...] + + +def _format_variant_error( + variant_name: str, message: str, hint: str | None = None +) -> str: + prefix = f"mjlab.entity: VariantEntityCfg '{variant_name}': " + body = message + if hint: + body += f" Hint: {hint}" + return prefix + body + + +def _extract_body_signature( + body: mujoco.MjsBody, parent_path: str = "" +) -> BodySignature: + body_name = body.name or "" + path = f"{parent_path}/{body_name}" + joints: list[JointSignature] = [] + for j in body.joints: + jt = int(j.type) + joints.append( + JointSignature( + name=j.name or "", + type=jt, + qpos_width=qpos_width(jt), + qvel_width=dof_width(jt), + ) + ) + geoms = tuple( + GeomSignature( + name=g.name if g.name else None, + type=int(g.type), + role=_classify_geom_role(g), + ) + for g in body.geoms + ) + children = tuple(_extract_body_signature(child, path) for child in body.bodies) + return BodySignature( + path=path, + name=body_name, + joints=tuple(joints), + geoms=geoms, + children=children, + ) + + +def _detect_inertial_mode(body: mujoco.MjsBody) -> InertialMode: + # `body.fullinertia` defaults to [nan, 0, 0, 0, 0, 0]; a non-NaN first + # element means the user assigned fullinertia (any assignment, even + # zeros, flips the user-specified flag at compile time). + if not math.isnan(float(body.fullinertia[0])): + return "fullinertia" + if body.explicitinertial: + return "diagonal" + return "mesh-derived" + + +def _collect_inertial_modes( + body: mujoco.MjsBody, parent_path: str = "" +) -> dict[str, InertialMode]: + body_name = body.name or "" + path = f"{parent_path}/{body_name}" + modes: dict[str, InertialMode] = {path: _detect_inertial_mode(body)} + for child in body.bodies: + modes.update(_collect_inertial_modes(child, path)) + return modes + + +def _check_reserved_names(spec: mujoco.MjSpec, variant_name: str) -> None: + collections: tuple[tuple[str, str], ...] = ( + ("bodies", "body"), + ("geoms", "geom"), + ("meshes", "mesh"), + ("materials", "material"), + ("textures", "texture"), + ("joints", "joint"), + ("actuators", "actuator"), + ("tendons", "tendon"), + ("sensors", "sensor"), + ("cameras", "camera"), + ("lights", "light"), + ("sites", "site"), + ("equalities", "equality"), + ) + for attr, item_kind in collections: + items = getattr(spec, attr, None) + if items is None: + continue + for item in items: + name = getattr(item, "name", None) or "" + for prefix in RESERVED_NAME_PREFIXES: + if name.startswith(prefix): + raise ValueError( + _format_variant_error( + variant_name, + f"reserved name prefix '{prefix}' is used by source " + f"{item_kind} '{name}'.", + "rename user geoms/bodies/assets away from this prefix.", + ) + ) + + +def _compare_geoms_for_body( + ref_variant: str, + ref_geoms: tuple[GeomSignature, ...], + other_variant: str, + other_geoms: tuple[GeomSignature, ...], + body_path: str, +) -> None: + """Compare geoms within a matching body across two variants. + + Mesh geoms may differ in count per ``(body, role)``; they form padded + slots and are not part of the structural signature. Non-mesh + primitive geoms must match exactly in count, name, type, role, and + ordering relative to other primitives. + """ + mesh_type = int(mujoco.mjtGeom.mjGEOM_MESH) + ref_primitives = tuple(g for g in ref_geoms if g.type != mesh_type) + other_primitives = tuple(g for g in other_geoms if g.type != mesh_type) + if len(ref_primitives) != len(other_primitives): + raise ValueError( + _format_variant_error( + other_variant, + f"body '{body_path}' has {len(other_primitives)} non-mesh " + f"geoms, but '{ref_variant}' has {len(ref_primitives)}.", + "primitive geoms must match across all variants; only mesh " + "geom counts may differ.", + ) + ) + for i, (rp, op) in enumerate(zip(ref_primitives, other_primitives, strict=False)): + if rp != op: + raise ValueError( + _format_variant_error( + other_variant, + f"body '{body_path}' primitive geom #{i} differs from " + f"'{ref_variant}': got {op}, expected {rp}.", + "primitive geom name, type, and role must match across variants.", + ) + ) + + +def _compare_body_signatures( + ref_variant: str, + ref: BodySignature, + other_variant: str, + other: BodySignature, +) -> None: + if ref.path != other.path: + raise ValueError( + _format_variant_error( + other_variant, + f"body path '{other.path}' has no counterpart in " + f"'{ref_variant}' (expected '{ref.path}').", + "variants must share the same recursive body tree with matching body names.", + ) + ) + if len(ref.joints) != len(other.joints): + raise ValueError( + _format_variant_error( + other_variant, + f"body '{other.path}' has {len(other.joints)} joints, but " + f"'{ref_variant}' has {len(ref.joints)}.", + "variants may change mesh assets, not add/remove joints.", + ) + ) + for i, (rj, oj) in enumerate(zip(ref.joints, other.joints, strict=False)): + if rj != oj: + raise ValueError( + _format_variant_error( + other_variant, + f"body '{other.path}' joint #{i} differs from " + f"'{ref_variant}': got {oj}, expected {rj}.", + "joint name, type, and qpos/qvel width must match across variants.", + ) + ) + _compare_geoms_for_body( + ref_variant, ref.geoms, other_variant, other.geoms, other.path + ) + if len(ref.children) != len(other.children): + raise ValueError( + _format_variant_error( + other_variant, + f"body '{other.path}' has {len(other.children)} child bodies, " + f"but '{ref_variant}' has {len(ref.children)}.", + "variants must share the same recursive body tree.", + ) + ) + for ref_child, other_child in zip(ref.children, other.children, strict=False): + _compare_body_signatures(ref_variant, ref_child, other_variant, other_child) + + +def _compare_inertial_modes( + ref_variant: str, + ref_modes: dict[str, InertialMode], + other_variant: str, + other_modes: dict[str, InertialMode], +) -> None: + for path in sorted(set(ref_modes) | set(other_modes)): + rm = ref_modes.get(path) + om = other_modes.get(path) + if rm is None or om is None: + # Path mismatch is reported by the body-tree comparison. + continue + if rm != om: + raise ValueError( + _format_variant_error( + other_variant, + f"body '{path}' uses inertial representation '{om}', but " + f"'{ref_variant}' uses '{rm}'.", + "use the same inertial representation in every variant: " + "all diagonal, all fullinertia, or all mesh-derived.", + ) + ) + + +def _compare_collection_topology( + *, + item_kind: str, + ref_variant: str, + ref_items: list[Any], + other_variant: str, + other_items: list[Any], + key_fn: Callable[[Any], tuple], + count_hint: str, + item_hint: str, +) -> None: + """Compare a flat spec-level collection (actuators, sensors, ...). + + Length must match; per-element ``key_fn(item)`` tuples must match. + Per-collection error specificity is preserved through ``item_kind``, + ``count_hint``, and ``item_hint`` strings supplied by the caller. + """ + if len(ref_items) != len(other_items): + raise ValueError( + _format_variant_error( + other_variant, + f"{item_kind} count {len(other_items)} differs from " + f"'{ref_variant}' ({len(ref_items)}).", + count_hint, + ) + ) + for i, (r, o) in enumerate(zip(ref_items, other_items, strict=False)): + rk, ok = key_fn(r), key_fn(o) + if rk != ok: + raise ValueError( + _format_variant_error( + other_variant, + f"{item_kind} #{i} differs from '{ref_variant}': got {ok}, expected {rk}.", + item_hint, + ) + ) + + +def _actuator_key(a: mujoco.MjsActuator) -> tuple[str, int, str]: + return (a.name or "", int(a.trntype), a.target or "") + + +def _sensor_key(s: mujoco.MjsSensor) -> tuple[str, int, int, str]: + return (s.name or "", int(s.type), int(s.objtype), s.objname or "") + + +def _tendon_key(t: mujoco.MjsTendon) -> tuple[str]: + return (t.name or "",) + + +def _equality_key(e: mujoco.MjsEquality) -> tuple[str, int, str, str]: + return (e.name or "", int(e.type), e.name1 or "", e.name2 or "") + + +def validate_variant_specs( + names: list[str], + specs: list[mujoco.MjSpec], +) -> None: + """Validate that all variant specs share the same kinematic topology. + + Performs recursive body-tree, joint, and primitive-geom comparison; + spec-level actuator/sensor/tendon/equality topology comparison; + per-body inertial-mode consistency; and reserved-name-prefix + rejection. Raises ``ValueError`` with a message of the form: + + mjlab.entity: VariantEntityCfg '': . Hint: . + + Mesh geom counts may differ per ``(body, role)``; this is the + intended source of variant-to-variant variation. Primitive (non-mesh) + geom counts and structure must match. + + Args: + names: Variant names. Must be non-empty and align with ``specs``. + specs: Variant specs (pre-attachment). Each must have exactly one + root body under worldbody. + """ + if len(names) != len(specs): + raise ValueError("names and specs must have the same length.") + if not names: + raise ValueError("at least one variant is required.") + + for name, spec in zip(names, specs, strict=False): + _check_reserved_names(spec, name) + + root_bodies: list[mujoco.MjsBody] = [] + for name, spec in zip(names, specs, strict=False): + children = list(spec.worldbody.bodies) + if len(children) != 1: + raise ValueError( + _format_variant_error( + name, + f"variant must have exactly one root body under worldbody, " + f"got {len(children)}.", + "place the variant's root body directly under worldbody.", + ) + ) + root_bodies.append(children[0]) + + ref_name = names[0] + ref_signature = _extract_body_signature(root_bodies[0]) + ref_modes = _collect_inertial_modes(root_bodies[0]) + ref_spec = specs[0] + + for i in range(1, len(names)): + other_name = names[i] + other_signature = _extract_body_signature(root_bodies[i]) + other_modes = _collect_inertial_modes(root_bodies[i]) + other_spec = specs[i] + + _compare_body_signatures(ref_name, ref_signature, other_name, other_signature) + _compare_inertial_modes(ref_name, ref_modes, other_name, other_modes) + _compare_collection_topology( + item_kind="actuator", + ref_variant=ref_name, + ref_items=list(ref_spec.actuators), + other_variant=other_name, + other_items=list(other_spec.actuators), + key_fn=_actuator_key, + count_hint="keep control and observation dimensions fixed across variants.", + item_hint=( + "actuator topology (name, transmission type, target) must match " + "across variants." + ), + ) + _compare_collection_topology( + item_kind="sensor", + ref_variant=ref_name, + ref_items=list(ref_spec.sensors), + other_variant=other_name, + other_items=list(other_spec.sensors), + key_fn=_sensor_key, + count_hint="keep sensor topology identical across variants.", + item_hint="sensor topology (name, type, target object) must match.", + ) + _compare_collection_topology( + item_kind="tendon", + ref_variant=ref_name, + ref_items=list(ref_spec.tendons), + other_variant=other_name, + other_items=list(other_spec.tendons), + key_fn=_tendon_key, + count_hint="keep tendon topology identical across variants.", + item_hint="tendon names must match across variants.", + ) + _compare_collection_topology( + item_kind="equality", + ref_variant=ref_name, + ref_items=list(ref_spec.equalities), + other_variant=other_name, + other_items=list(other_spec.equalities), + key_fn=_equality_key, + count_hint="keep equality constraints identical across variants.", + item_hint="equality constraint topology must match across variants.", + ) + + +## +# Slot computation: assign (body_path, role, ordinal) keys to mesh geoms. +## + + +def _extract_variant_geom_spec(g: mujoco.MjsGeom) -> VariantGeomSpec: + """Snapshot a source mesh geom's attributes for slot-aware compile.""" + return VariantGeomSpec( + mesh_name=g.meshname, + geom_name=g.name if g.name else None, + contype=int(g.contype), + conaffinity=int(g.conaffinity), + condim=int(g.condim), + group=int(g.group), + priority=int(g.priority), + material=g.material if g.material else None, + rgba=( + float(g.rgba[0]), + float(g.rgba[1]), + float(g.rgba[2]), + float(g.rgba[3]), + ), + friction=( + float(g.friction[0]), + float(g.friction[1]), + float(g.friction[2]), + ), + margin=float(g.margin), + gap=float(g.gap), + solref=(float(g.solref[0]), float(g.solref[1])), + solimp=( + float(g.solimp[0]), + float(g.solimp[1]), + float(g.solimp[2]), + float(g.solimp[3]), + float(g.solimp[4]), + ), + mass=float(g.mass), + density=float(g.density), + ) + + +def _compute_slot_metadata( + variant_names: tuple[str, ...], + variant_specs: list[mujoco.MjSpec], +) -> tuple[ + tuple[VariantSlot, ...], + tuple[tuple[VariantGeomSpec | None, ...], ...], +]: + """Compute slot list and per-variant slot specs. + + Walks each variant's body tree, buckets mesh geoms by ``(body_path, + role)``, and assigns ordinals deterministically from source order. + Slot count per ``(body_path, role)`` equals the max across variants; + variants with fewer geoms in that bucket leave trailing slots + unfilled (``None`` in ``variant_slot_specs``). + """ + mesh_type = int(mujoco.mjtGeom.mjGEOM_MESH) + + # Per-variant {body_path -> {role -> [VariantGeomSpec]}}. + per_variant_buckets: list[dict[str, dict[GeomRole, list[VariantGeomSpec]]]] = [] + for spec in variant_specs: + root = list(spec.worldbody.bodies)[0] + buckets: dict[str, dict[GeomRole, list[VariantGeomSpec]]] = {} + for path, body in _iter_body_paths(root): + role_buckets = buckets.setdefault(path, {"visual": [], "collision": []}) + for g in body.geoms: + if int(g.type) != mesh_type: + continue + role_buckets[_classify_geom_role(g)].append(_extract_variant_geom_spec(g)) + per_variant_buckets.append(buckets) + + # All body paths present in any variant. (Validation ensures the path + # sets agree across variants, but use the union here for safety.) + all_paths: set[str] = set() + for buckets in per_variant_buckets: + all_paths.update(buckets.keys()) + + slots: list[VariantSlot] = [] + per_variant_slot_specs: list[list[VariantGeomSpec | None]] = [ + [] for _ in variant_names + ] + + # Deterministic order: sort body paths lexicographically, then visual + # before collision per body, then by ordinal. + for body_path in sorted(all_paths): + for role in ("visual", "collision"): + max_count = 0 + for buckets in per_variant_buckets: + count = len(buckets.get(body_path, {}).get(role, [])) + if count > max_count: + max_count = count + for ordinal in range(max_count): + source_names: list[str | None] = [] + slot_specs: list[VariantGeomSpec | None] = [] + for variant_buckets in per_variant_buckets: + variant_geoms = variant_buckets.get(body_path, {}).get(role, []) + if ordinal < len(variant_geoms): + geom_spec = variant_geoms[ordinal] + source_names.append(geom_spec.geom_name) + slot_specs.append(geom_spec) + else: + source_names.append(None) + slot_specs.append(None) + template_name = f"mjlab/pad{body_path}/{role}/{ordinal}" + slots.append( + VariantSlot( + key=SlotKey( + body_path=body_path, + role=role, + ordinal=ordinal, + ), + template_geom_name=template_name, + source_geom_names=tuple(source_names), + ) + ) + for variant_idx, slot_spec in enumerate(slot_specs): + per_variant_slot_specs[variant_idx].append(slot_spec) + + return tuple(slots), tuple(tuple(s) for s in per_variant_slot_specs) + + +## +# Build pipeline (entity-time): merge variant specs into one padded template. +## + + +def build_merged_variant_spec( + cfg: VariantEntityCfg, +) -> tuple[mujoco.MjSpec, VariantMetadata]: + """Merge a ``VariantEntityCfg``'s variants into a single padded template. + + Validates that all variants share the same kinematic structure, merges + every variant's mesh assets into the template namespace, slot-renames + variant 0's mesh geoms to ``mjlab/pad///``, and + synthesizes padding geoms (per-(body, role)) for slots that variant 0 + does not fill, so the template's geom count and bodyid layout cover + every variant's needs. + + Returns the merged template spec and a ``VariantMetadata`` describing + the slot layout, per-variant slot specs, and explicit body inertials. + The reference compile in :func:`build_variant_model` overrides every + per-variant attribute via ``VariantGeomSpec``, so variant 0 being "the + template" does not privilege its attributes. + """ + variants = cfg.variants + if not variants: + raise ValueError("VariantEntityCfg.variants must contain at least one entry.") + + variant_names: list[str] = [] + variant_specs: list[mujoco.MjSpec] = [] + for name, spec_fn in variants.items(): + variant_names.append(name) + variant_specs.append(spec_fn()) + + # Resolve cfg.assignment (None | dict | callable) into the metadata's + # unified ``assignment`` field. None and dict both produce a weights + # tuple keyed by variant declaration order; a callable passes through. + resolved_assignment: tuple[float, ...] | Callable[[int], Sequence[int]] + cfg_assignment = cfg.assignment + if cfg_assignment is None: + resolved_assignment = (1.0,) * len(variant_names) + elif isinstance(cfg_assignment, dict): + weights_dict = cast(dict[str, float], cfg_assignment) + resolved_assignment = tuple(weights_dict.get(n, 1.0) for n in variant_names) + else: + resolved_assignment = cfg_assignment + + # Validate cross-variant topology (recursive body tree, joints, + # primitive geoms, actuators, sensors, tendons, equalities, inertial + # mode consistency, and reserved-prefix collisions). The validator + # enforces "exactly one root body under worldbody" itself. + validate_variant_specs(variant_names, variant_specs) + + variant_bodies: list[mujoco.MjsBody] = [ + list(spec.worldbody.bodies)[0] for spec in variant_specs + ] + + # Variant entities must be floating-base. Mocap auto-wrap is not applied + # for variant entities, so fixed-base variants would silently stack at + # the world origin. Variants share joint structure (validated above), so + # checking the first is sufficient. + ref_joints = list(variant_bodies[0].joints) + if not ref_joints or ref_joints[0].type != mujoco.mjtJoint.mjJNT_FREE: + raise ValueError( + "VariantEntityCfg requires floating-base variants. Each variant's " + "root body must declare a free joint via body.add_freejoint(); " + "fixed-base variants are not supported." + ) + + # Compute slot-based metadata from source specs BEFORE any mesh + # renaming or merging. This captures (body_path, role, ordinal) keyed + # slots and per-variant geom attributes for the slot-aware build path. + slots, variant_slot_specs = _compute_slot_metadata( + tuple(variant_names), variant_specs + ) + + # Snapshot a clean copy of every source spec BEFORE the merge mutates + # variant 0's spec (mesh renames, slot-name geom renames, padding geom + # synthesis). The sim-time build compiles each of these in isolation + # to populate per-world fields, avoiding the O(N) cost of recompiling + # the merged scene per unique variant assignment. + variant_source_specs = tuple(s.copy() for s in variant_specs) + + # Use variant 0's spec as the template. The merge: + # 1. prefixes variant 0's mesh names with its variant name and + # copies every other variant's mesh assets into the template; + # 2. renames variant 0's mesh geoms to slot template names so + # every mesh geom in the merged template is addressable by + # ``slot.template_geom_name``; + # 3. synthesizes padding geoms (per-(body, role)) for slots that + # variant 0 does not fill, so the template's geom count and + # bodyid layout cover every variant's needs. + template_spec = variant_specs[0] + template_body = variant_bodies[0] + + # (1) Rename template meshes and update mesh references on the + # template body's geoms (recursive: child bodies may also reference + # the template variant's meshes). + template_prefix = f"{variant_names[0]}/" + old_to_new: dict[str, str] = {} + for mesh in template_spec.meshes: + new_name = f"{template_prefix}{mesh.name}" + old_to_new[mesh.name] = new_name + mesh.name = new_name + for _, body in _iter_body_paths(template_body): + for g in body.geoms: + if g.meshname in old_to_new: + g.meshname = old_to_new[g.meshname] + + # Copy mesh assets from other variants into the template namespace. + for i in range(1, len(variant_specs)): + prefix = f"{variant_names[i]}/" + for mesh in variant_specs[i].meshes: + new_mesh = template_spec.add_mesh() + new_mesh.name = f"{prefix}{mesh.name}" + copy_mesh_data(mesh, new_mesh) + + # Mirror of the mesh treatment above for textures and materials. + texture_old_to_new: dict[str, str] = {} + for tex in template_spec.textures: + new_name = f"{template_prefix}{tex.name}" + texture_old_to_new[tex.name] = new_name + tex.name = new_name + material_old_to_new: dict[str, str] = {} + for mat in template_spec.materials: + new_name = f"{template_prefix}{mat.name}" + material_old_to_new[mat.name] = new_name + mat.name = new_name + mat.textures = [texture_old_to_new.get(t, t) for t in mat.textures] + for _, body in _iter_body_paths(template_body): + for g in body.geoms: + if g.material in material_old_to_new: + g.material = material_old_to_new[g.material] + + # Copy texture/material assets from other variants into the template. + for i in range(1, len(variant_specs)): + prefix = f"{variant_names[i]}/" + src_to_dst_tex: dict[str, str] = {} + for tex in variant_specs[i].textures: + new_tex = template_spec.add_texture() + new_tex.name = f"{prefix}{tex.name}" + src_to_dst_tex[tex.name] = new_tex.name + copy_texture_data(tex, new_tex) + for mat in variant_specs[i].materials: + new_mat = template_spec.add_material() + new_mat.name = f"{prefix}{mat.name}" + copy_material_data(mat, new_mat) + new_mat.textures = [src_to_dst_tex.get(t, t) for t in new_mat.textures] + + # (2) Slot-driven rename of variant 0's existing mesh geoms. Walk + # the template body tree; within each (body, role) bucket, the + # n-th mesh geom (in source order) maps to slot ordinal n. + mesh_type = mujoco.mjtGeom.mjGEOM_MESH + slot_by_key: dict[SlotKey, VariantSlot] = {s.key: s for s in slots} + template_bodies_by_path: dict[str, mujoco.MjsBody] = {} + filled_template_slot_keys: set[SlotKey] = set() + for body_path, body in _iter_body_paths(template_body): + template_bodies_by_path[body_path] = body + role_ordinals: dict[GeomRole, int] = {"visual": 0, "collision": 0} + for g in body.geoms: + if g.type != mesh_type: + continue + role = _classify_geom_role(g) + ordinal = role_ordinals[role] + role_ordinals[role] += 1 + key = SlotKey(body_path=body_path, role=role, ordinal=ordinal) + slot = slot_by_key[key] + g.name = slot.template_geom_name + filled_template_slot_keys.add(key) + + # (3) Synthesize per-(body, role) padding geoms for slots variant 0 + # doesn't fill. Each padding geom uses the slot's reserved + # ``mjlab/pad/...`` template name, the role's union contact bits, + # and a placeholder mesh from the lowest-index variant that fills + # the slot. ``mass=0`` and ``density=0`` keep the placeholder from + # contributing to the template's body inertial inference; the + # per-variant reference compile handles further masking. + for s_idx, slot in enumerate(slots): + if slot.key in filled_template_slot_keys: + continue + target_body = template_bodies_by_path[slot.key.body_path] + pad_geom = target_body.add_geom() + pad_geom.type = mesh_type + pad_geom.name = slot.template_geom_name + if slot.key.role == "visual": + pad_geom.contype = 0 + pad_geom.conaffinity = 0 + else: + pad_geom.contype = 1 + pad_geom.conaffinity = 1 + placeholder_set = False + for v_idx, slot_specs in enumerate(variant_slot_specs): + gs = slot_specs[s_idx] + if gs is None: + continue + pad_geom.meshname = f"{variant_names[v_idx]}/{gs.mesh_name}" + placeholder_set = True + break + assert placeholder_set, ( + f"slot {slot.key} has no filling variant; " + "_compute_slot_metadata must not emit unfilled slots." + ) + pad_geom.mass = 0.0 + pad_geom.density = 0.0 + + metadata = VariantMetadata( + variant_names=tuple(variant_names), + assignment=resolved_assignment, + slots=slots, + variant_slot_specs=variant_slot_specs, + variant_source_specs=variant_source_specs, + ) + return template_spec, metadata + + +## +# Build pipeline (sim-time): produce a heterogeneous Warp model. +# +# The module-level entry point is ``build_variant_model`` at the bottom +# of this section. ``allocate_worlds`` is its world-assignment step and +# is also exposed for unit testing the largest-remainder math in +# isolation. Everything between is private to ``build_variant_model`` +# (or to ``_populate_dependent_fields``, which is itself private). +## + + +def _qualified_mesh_name(entity_prefix: str, variant_name: str, mesh_name: str) -> str: + """Mesh asset name in the merged template's namespace. + + Source mesh names are prefixed with the variant name during merge + (``/``); after scene attach the entity prefix is + added (``//``). + """ + return f"{entity_prefix}{variant_name}/{mesh_name}" + + +def _qualified_material_name( + entity_prefix: str, variant_name: str, material_name: str +) -> str: + """Material asset name in the merged template's namespace. + + Source materials are prefixed with the variant name during merge + (``/``); after scene attach the entity prefix is + added (``//``). + """ + return f"{entity_prefix}{variant_name}/{material_name}" + + +def _qualified_slot_geom_name(entity_prefix: str, template_geom_name: str) -> str: + """Slot geom name in the merged template's namespace after attach.""" + return f"{entity_prefix}{template_geom_name}" + + +def _populate_dependent_fields( + m: mjwarp.Model, + padded_model: mujoco.MjModel, + nworld: int, + variant_info: list[tuple[str, VariantMetadata]], + world_to_variant: dict[str, np.ndarray], +) -> None: + """Populate per-world Warp arrays for the geometry-dependent fields. + + Compiles each variant's *source* spec in isolation (small, single-entity + spec) and scatters per-body and per-geom fields into per-world arrays. + Total cost is the sum of per-source compiles across all variant + entities, i.e. linear in the number of variants. This avoids the + O(N^2) cost of recompiling the merged scene per unique variant + assignment. + + For non-variant bodies and non-slot geoms, values come from a single + base scene compile (``padded_model``) broadcast across worlds. For + variant entity bodies and slot geoms, the per-world value comes from + the corresponding variant's source compile. ``body_subtreemass`` for + ancestors of each variant entity's root body is updated per-world by + applying the per-variant subtreemass delta. + """ + ngeom = padded_model.ngeom + nbody = padded_model.nbody + + # Initialize per-world arrays from the base scene compile. Bodies and + # geoms not touched by any variant entity inherit these values. We + # broadcast then copy so writes to per-world rows don't alias. + def _broadcast(src: np.ndarray, shape: tuple[int, ...]) -> np.ndarray: + return np.broadcast_to(src.astype(np.float32), shape).copy() + + geom_size = _broadcast(padded_model.geom_size, (nworld, ngeom, 3)) + geom_rbound = _broadcast(padded_model.geom_rbound, (nworld, ngeom)) + geom_aabb = _broadcast( + padded_model.geom_aabb.reshape(ngeom, 2, 3), (nworld, ngeom, 2, 3) + ) + geom_pos = _broadcast(padded_model.geom_pos, (nworld, ngeom, 3)) + geom_quat = _broadcast(padded_model.geom_quat, (nworld, ngeom, 4)) + body_mass = _broadcast(padded_model.body_mass, (nworld, nbody)) + body_subtreemass = _broadcast(padded_model.body_subtreemass, (nworld, nbody)) + body_inertia = _broadcast(padded_model.body_inertia, (nworld, nbody, 3)) + body_invweight0 = _broadcast(padded_model.body_invweight0, (nworld, nbody, 2)) + body_ipos = _broadcast(padded_model.body_ipos, (nworld, nbody, 3)) + body_iquat = _broadcast(padded_model.body_iquat, (nworld, nbody, 4)) + + mesh_type = mujoco.mjtGeom.mjGEOM_MESH + + for entity_prefix, metadata in variant_info: + w2v = world_to_variant[entity_prefix] + + # Use any variant's source spec to enumerate body paths and the + # entity's slot geom names. Validation guarantees variants share + # the same body tree and slot keys, so the choice is arbitrary. + sample_spec = metadata.variant_source_specs[0] + sample_root = list(sample_spec.worldbody.bodies)[0] + + # Map each variant-entity body path to its scene body id. Body + # names in the scene are the bare body name prefixed by the + # entity's attach prefix. + scene_body_id_by_path: dict[str, int] = {} + body_name_by_path: dict[str, str] = {} + for body_path, body in _iter_body_paths(sample_root): + body_name_by_path[body_path] = body.name or "" + scene_name = f"{entity_prefix}{body.name or ''}" + bid = mujoco.mj_name2id(padded_model, mujoco.mjtObj.mjOBJ_BODY, scene_name) + if bid < 0: + raise ValueError( + f"variant entity body '{scene_name}' not found in compiled scene." + ) + scene_body_id_by_path[body_path] = bid + + # Map each slot key to its scene geom id. + scene_geom_id_by_slot: dict[SlotKey, int] = {} + for slot in metadata.slots: + full_geom_name = _qualified_slot_geom_name(entity_prefix, slot.template_geom_name) + gid = mujoco.mj_name2id(padded_model, mujoco.mjtObj.mjOBJ_GEOM, full_geom_name) + if gid < 0: + raise ValueError( + f"slot geom '{full_geom_name}' (entity '{entity_prefix}') not " + f"found in compiled scene." + ) + scene_geom_id_by_slot[slot.key] = gid + + # Variant entity's root body (shortest body path), used to compute + # the subtreemass delta to propagate to ancestors. The root path is + # always the lex-smallest among the entity's body paths (parents + # sort before children). + sorted_paths = sorted(scene_body_id_by_path.keys()) + root_body_path = sorted_paths[0] + root_scene_bid = scene_body_id_by_path[root_body_path] + + # Walk up the scene's parent chain from the variant root to + # worldbody (inclusive). The variant root's own subtreemass is + # written from the per-variant compile; each ancestor receives an + # additive delta equal to (variant_subtreemass - base_subtreemass). + ancestor_ids: list[int] = [] + parent_id = int(padded_model.body_parentid[root_scene_bid]) + while True: + ancestor_ids.append(parent_id) + if parent_id == 0: + break + parent_id = int(padded_model.body_parentid[parent_id]) + base_root_subtreemass = float(padded_model.body_subtreemass[root_scene_bid]) + + # Compile each variant's source spec in isolation and scatter values + # into the per-world arrays for the worlds assigned to that variant. + for v_idx in range(len(metadata.variant_names)): + worlds = np.where(w2v == v_idx)[0] + if worlds.size == 0: + continue + + source_spec = metadata.variant_source_specs[v_idx].copy() + v_model = source_spec.compile() + + source_root = list(source_spec.worldbody.bodies)[0] + source_body_id_by_path: dict[str, int] = {} + for body_path, body in _iter_body_paths(source_root): + source_body_id_by_path[body_path] = body.id + + # Scatter per-body fields for every body in the variant entity's + # subtree. Bodies without explicit inertials inherit the + # mesh-derived inertia from the variant's mesh assignment in the + # source compile, which is exactly what we want. + for body_path, scene_bid in scene_body_id_by_path.items(): + source_bid = source_body_id_by_path[body_path] + body_mass[worlds, scene_bid] = v_model.body_mass[source_bid] + body_subtreemass[worlds, scene_bid] = v_model.body_subtreemass[source_bid] + body_inertia[worlds, scene_bid] = v_model.body_inertia[source_bid] + body_invweight0[worlds, scene_bid] = v_model.body_invweight0[source_bid] + body_ipos[worlds, scene_bid] = v_model.body_ipos[source_bid] + body_iquat[worlds, scene_bid] = v_model.body_iquat[source_bid] + + # Scatter per-geom fields for each slot this variant fills. Map + # each (body_path, role, ordinal) slot to the corresponding geom + # in the source compile by re-walking the source body tree (same + # ordering rules as ``_compute_slot_metadata``). + for body_path, body in _iter_body_paths(source_root): + role_ordinals: dict[GeomRole, int] = {"visual": 0, "collision": 0} + for g in body.geoms: + if g.type != mesh_type: + continue + role = _classify_geom_role(g) + ordinal = role_ordinals[role] + role_ordinals[role] += 1 + slot_key = SlotKey(body_path=body_path, role=role, ordinal=ordinal) + scene_gid = scene_geom_id_by_slot.get(slot_key) + if scene_gid is None: + continue + source_gid = g.id + geom_size[worlds, scene_gid] = v_model.geom_size[source_gid] + geom_rbound[worlds, scene_gid] = v_model.geom_rbound[source_gid] + geom_aabb[worlds, scene_gid] = v_model.geom_aabb[source_gid].reshape(2, 3) + geom_pos[worlds, scene_gid] = v_model.geom_pos[source_gid] + geom_quat[worlds, scene_gid] = v_model.geom_quat[source_gid] + + # Propagate the variant root's subtreemass delta up the ancestor + # chain. Base ancestor subtreemass already includes the placeholder + # contribution from the merged scene compile; adding the delta + # swaps that placeholder contribution for the variant's actual + # contribution. + delta = ( + float(v_model.body_subtreemass[source_body_id_by_path[root_body_path]]) + - base_root_subtreemass + ) + for anc_bid in ancestor_ids: + body_subtreemass[worlds, anc_bid] += delta + + m.geom_size = wp.array(geom_size, dtype=wp.vec3) + m.geom_rbound = wp.array(geom_rbound, dtype=float) + m.geom_aabb = wp.array(geom_aabb, dtype=wp.vec3) + m.geom_pos = wp.array(geom_pos, dtype=wp.vec3) + m.geom_quat = wp.array(geom_quat, dtype=wp.quat) + m.body_mass = wp.array(body_mass, dtype=float) + m.body_subtreemass = wp.array(body_subtreemass, dtype=float) + m.body_inertia = wp.array(body_inertia, dtype=wp.vec3) + m.body_invweight0 = wp.array(body_invweight0, dtype=wp.vec2) + m.body_ipos = wp.array(body_ipos, dtype=wp.vec3) + m.body_iquat = wp.array(body_iquat, dtype=wp.quat) + + +def allocate_worlds( + weights: tuple[float, ...], + nworld: int, +) -> list[int]: + """Assign worlds proportionally by weight (largest-remainder method). + + Returns a list of length *nworld* containing variant indices. Weights + must be non-negative with at least one positive entry. + """ + if any(w < 0 for w in weights): + raise ValueError(f"weights must be non-negative, got {weights}.") + total = sum(weights) + if total <= 0: + raise ValueError(f"weights must have a positive sum, got {weights}.") + quotas = [(w / total) * nworld for w in weights] + floors = [int(q) for q in quotas] + remainders = sorted( + ((quotas[i] - floors[i], i) for i in range(len(weights))), + key=lambda x: -x[0], + ) + allocated = sum(floors) + for j in range(nworld - allocated): + floors[remainders[j][1]] += 1 + assignment: list[int] = [] + for idx, count in enumerate(floors): + assignment.extend([idx] * count) + return assignment + + +def build_variant_model( + spec: mujoco.MjSpec, + nworld: int, + variant_info: list[tuple[str, VariantMetadata]], + configure_model: Callable[[mujoco.MjModel], None] | None = None, +) -> MeshVariantResult: + """Build a Warp Model with per-world mesh assignments. + + Args: + spec: Scene spec (already merged with padded variant geoms). + nworld: Number of simulation worlds. + variant_info: List of ``(entity_prefix, metadata)`` pairs for + entities that have mesh variants. + configure_model: Optional callback to configure the compiled + MjModel before ``put_model`` (e.g., setting solver options). + + Returns: + A :class:`MeshVariantResult` containing the warp model, host + model, and per-entity world-to-variant mappings. + """ + spec = spec.copy() + model = spec.compile() + if configure_model is not None: + configure_model(model) + + # Start from base dataid tiled for all worlds. + base_dataid = model.geom_dataid.copy() + dataid_table = np.tile(base_dataid, (nworld, 1)) + + # Same treatment for matid so each variant can use its own material. + base_matid = model.geom_matid.copy() + matid_table = np.tile(base_matid, (nworld, 1)) + + world_to_variant: dict[str, np.ndarray] = {} + + for entity_prefix, metadata in variant_info: + slots = metadata.slots + nslots = len(slots) + nvariants = len(metadata.variant_names) + + # Resolve world-to-variant assignment. The metadata's ``assignment`` + # is either a tuple of weights (from None or dict on the cfg) or a + # callable. Use ``callable()`` for the dispatch since tuples are + # not callable. + assignment_spec = metadata.assignment + if callable(assignment_spec): + assignment = list(assignment_spec(nworld)) + if len(assignment) != nworld: + raise ValueError( + f"VariantEntityCfg.assignment (entity '{entity_prefix}') " + f"returned {len(assignment)} indices but nworld={nworld}." + ) + for w, v in enumerate(assignment): + if not (0 <= int(v) < nvariants): + raise ValueError( + f"VariantEntityCfg.assignment (entity '{entity_prefix}') " + f"returned variant index {v} at world {w}, but only " + f"{nvariants} variants are declared." + ) + else: + assignment = allocate_worlds(assignment_spec, nworld) + w2v = np.array(assignment, dtype=np.int32) + world_to_variant[entity_prefix] = w2v + + # Resolve slot geom IDs by template name (deterministic, no + # positional or unnamed-padding heuristics). + slot_geom_ids = np.zeros(nslots, dtype=np.int64) + for s_idx, slot in enumerate(slots): + full_geom_name = _qualified_slot_geom_name(entity_prefix, slot.template_geom_name) + gid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_GEOM, full_geom_name) + if gid < 0: + raise ValueError( + f"Slot geom '{full_geom_name}' (entity '{entity_prefix}', " + f"slot {slot.key}) not found in compiled model." + ) + slot_geom_ids[s_idx] = gid + + # Resolve every (variant, slot) -> (mesh_id, mat_id). ``-1`` denotes + # an unfilled slot (mesh) or a slot rendered without a material + # (matid). + variant_slot_mesh_ids = np.full((nvariants, nslots), -1, dtype=np.int64) + variant_slot_matids = np.full((nvariants, nslots), -1, dtype=np.int64) + for v_idx, slot_specs in enumerate(metadata.variant_slot_specs): + variant_name = metadata.variant_names[v_idx] + for s_idx, gspec in enumerate(slot_specs): + if gspec is None: + continue + full_mesh_name = _qualified_mesh_name( + entity_prefix, variant_name, gspec.mesh_name + ) + mid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_MESH, full_mesh_name) + if mid < 0: + raise ValueError( + f"Mesh '{full_mesh_name}' (variant '{variant_name}', " + f"slot {slots[s_idx].key}) not found in compiled model." + ) + variant_slot_mesh_ids[v_idx, s_idx] = mid + if gspec.material: + full_mat_name = _qualified_material_name( + entity_prefix, variant_name, gspec.material + ) + matid = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_MATERIAL, full_mat_name) + if matid < 0: + raise ValueError( + f"Material '{full_mat_name}' (variant '{variant_name}', " + f"slot {slots[s_idx].key}) not found in compiled model." + ) + variant_slot_matids[v_idx, s_idx] = matid + + # Vectorized scatter: per-world row from variant assignment. + dataid_table[:, slot_geom_ids] = variant_slot_mesh_ids[w2v] + matid_table[:, slot_geom_ids] = variant_slot_matids[w2v] + + # Build warp model. + m = mjwarp.put_model(model) + m.geom_dataid = wp.array(dataid_table, dtype=int) + m.geom_matid = wp.array(matid_table, dtype=int) + + # Populate dependent per-world fields. + _populate_dependent_fields(m, model, nworld, variant_info, world_to_variant) + + return MeshVariantResult( + wp_model=m, + mj_model=model, + world_to_variant=world_to_variant, + ) diff --git a/rl/mjlab/envs/__init__.py b/rl/mjlab/envs/__init__.py new file mode 100644 index 0000000..f56adab --- /dev/null +++ b/rl/mjlab/envs/__init__.py @@ -0,0 +1,4 @@ +from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv as ManagerBasedRlEnv +from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnvCfg as ManagerBasedRlEnvCfg +from mjlab.envs.types import VecEnvObs as VecEnvObs +from mjlab.envs.types import VecEnvStepReturn as VecEnvStepReturn diff --git a/rl/mjlab/envs/manager_based_rl_env.py b/rl/mjlab/envs/manager_based_rl_env.py new file mode 100644 index 0000000..6665930 --- /dev/null +++ b/rl/mjlab/envs/manager_based_rl_env.py @@ -0,0 +1,618 @@ +import math +from dataclasses import dataclass, field +from typing import Any + +import mujoco +import numpy as np +import torch +import warp as wp +from prettytable import PrettyTable + +from mjlab.envs import types +from mjlab.envs.mdp.events import reset_scene_to_default +from mjlab.managers.action_manager import ActionManager, ActionTermCfg +from mjlab.managers.command_manager import ( + CommandManager, + CommandTermCfg, + NullCommandManager, +) +from mjlab.managers.curriculum_manager import ( + CurriculumManager, + CurriculumTermCfg, + NullCurriculumManager, +) +from mjlab.managers.event_manager import EventManager, EventTermCfg +from mjlab.managers.metrics_manager import ( + MetricsManager, + MetricsTermCfg, + NullMetricsManager, +) +from mjlab.managers.observation_manager import ObservationGroupCfg, ObservationManager +from mjlab.managers.recorder_manager import ( + NullRecorderManager, + RecorderManager, + RecorderTermCfg, +) +from mjlab.managers.reward_manager import RewardManager, RewardTermCfg +from mjlab.managers.termination_manager import TerminationManager, TerminationTermCfg +from mjlab.scene import Scene +from mjlab.scene.scene import SceneCfg +from mjlab.sim import SimulationCfg +from mjlab.sim.sim import Simulation +from mjlab.utils import random as random_utils +from mjlab.utils.logging import print_info +from mjlab.utils.spaces import Box +from mjlab.utils.spaces import Dict as DictSpace +from mjlab.viewer.debug_visualizer import DebugVisualizer +from mjlab.viewer.offscreen_renderer import OffscreenRenderer +from mjlab.viewer.viewer_config import ViewerConfig + + +@dataclass(kw_only=True) +class ManagerBasedRlEnvCfg: + """Configuration for a manager-based RL environment. + + This config defines all aspects of an RL environment: the physical scene, + observations, actions, rewards, terminations, and optional features like + commands and curriculum learning. + + The environment step size is ``sim.mujoco.timestep * decimation``. For example, + with a 2ms physics timestep and decimation=10, the environment runs at 50Hz. + """ + + # Base environment configuration. + + decimation: int + """Number of physics simulation steps per environment step. Higher values mean + coarser control frequency. Environment step duration = physics_dt * decimation.""" + + scene: SceneCfg + """Scene configuration defining terrain, entities, and sensors. The scene + specifies ``num_envs``, the number of parallel environments.""" + + observations: dict[str, ObservationGroupCfg] = field(default_factory=dict) + """Observation groups configuration. Each group (e.g., "actor", "critic") contains + observation terms that are concatenated. Groups can have different settings for + noise, history, and delay.""" + + actions: dict[str, ActionTermCfg] = field(default_factory=dict) + """Action terms configuration. Each term controls a specific entity/aspect + (e.g., joint positions). Action dimensions are concatenated across terms.""" + + events: dict[str, EventTermCfg] = field( + default_factory=lambda: { + "reset_scene_to_default": EventTermCfg( + func=reset_scene_to_default, + mode="reset", + ) + } + ) + """Event terms for domain randomization and state resets. Default includes + ``reset_scene_to_default`` which resets entities to their initial state. + Can be set to empty to disable all events including default reset.""" + + seed: int | None = None + """Random seed for reproducibility. If None, a random seed is used. The actual + seed used is stored back into this field after initialization.""" + + sim: SimulationCfg = field(default_factory=SimulationCfg) + """Simulation configuration including physics timestep, solver iterations, + contact parameters, and NaN guarding.""" + + viewer: ViewerConfig = field(default_factory=ViewerConfig) + """Viewer configuration for rendering (camera position, resolution, etc.).""" + + # RL-specific configuration. + + episode_length_s: float = 0.0 + """Duration of an episode (in seconds). + + Episode length in steps is computed as: + ceil(episode_length_s / (sim.mujoco.timestep * decimation)) + """ + + rewards: dict[str, RewardTermCfg] = field(default_factory=dict) + """Reward terms configuration.""" + + terminations: dict[str, TerminationTermCfg] = field(default_factory=dict) + """Termination terms configuration. If empty, episodes never reset. Use + ``mdp.time_out`` with ``time_out=True`` for episode time limits.""" + + commands: dict[str, CommandTermCfg] = field(default_factory=dict) + """Command generator terms (e.g., velocity targets).""" + + curriculum: dict[str, CurriculumTermCfg] = field(default_factory=dict) + """Curriculum terms for adaptive difficulty.""" + + metrics: dict[str, MetricsTermCfg] = field(default_factory=dict) + """Custom metric terms for logging per-step values as episode averages.""" + + recorders: dict[str, RecorderTermCfg] = field(default_factory=dict) + """Recorder terms for logging observations, actions, or other data during rollouts. + If empty, a no-op manager is used with zero overhead.""" + + is_finite_horizon: bool = False + """Whether the task has a finite or infinite horizon. Defaults to False (infinite). + + - **Finite horizon (True)**: The time limit defines the task boundary. When reached, + no future value exists beyond it, so the agent receives a terminal done signal. + - **Infinite horizon (False)**: The time limit is an artificial cutoff. The agent + receives a truncated done signal to bootstrap the value of continuing beyond the + limit. + """ + + auto_reset: bool = True + """Whether to automatically reset environments that terminate or time out. + + When True (default), ``step()`` resets done environments and returns post-reset + observations. When False, ``step()`` returns the true terminal observation and the + caller must explicitly call ``reset(env_ids=...)`` for done environments before the + next ``step()``. + + Note: mjlab's bundled ``train.py`` goes through rsl_rl's ``OnPolicyRunner``, which + does not drive manual resets. ``auto_reset=False`` is intended for users running + their own training loop (or a wrapper that handles the reset between steps). + """ + + scale_rewards_by_dt: bool = True + """Whether to multiply rewards by the environment step duration (dt). + + When True (default), reward values are scaled by step_dt to normalize cumulative + episodic rewards across different simulation frequencies. Set to False for + algorithms that expect unscaled reward signals (e.g., HER, static reward scaling). + """ + + +class ManagerBasedRlEnv: + """Manager-based RL environment.""" + + is_vector_env = True + metadata = { + "render_modes": [None, "rgb_array"], + "mujoco_version": mujoco.__version__, + "warp_version": wp.config.version, + } + cfg: ManagerBasedRlEnvCfg + + def __init__( + self, + cfg: ManagerBasedRlEnvCfg, + device: str, + render_mode: str | None = None, + **kwargs, + ) -> None: + # Initialize base environment state. + self.cfg = cfg + if self.cfg.seed is not None: + self.cfg.seed = self.seed(self.cfg.seed) + self._sim_step_counter = 0 + self.extras = {} + self.obs_buf = {} + self._manual_reset_pending = torch.zeros( + self.cfg.scene.num_envs, dtype=torch.bool, device=device + ) + # Scratch buffer for per-env command dt; see step(). + self._command_dt = torch.zeros(self.cfg.scene.num_envs, device=device) + + # Initialize scene and simulation. + self.scene = Scene(self.cfg.scene, device=device) + self.sim = Simulation( + num_envs=self.scene.num_envs, + cfg=self.cfg.sim, + spec=self.scene.spec, + variant_info=self.scene.collect_variant_info(), + device=device, + ) + + self.scene.initialize( + mj_model=self.sim.mj_model, + model=self.sim.model, + data=self.sim.data, + ) + + # Wire sensor context to simulation for sense_graph. + if self.scene.sensor_context is not None: + self.sim.set_sensor_context(self.scene.sensor_context) + + # Print environment info. + print_info("") + table = PrettyTable() + table.title = "Base Environment" + table.field_names = ["Property", "Value"] + table.align["Property"] = "l" + table.align["Value"] = "l" + table.add_row(["Number of environments", self.num_envs]) + table.add_row(["Environment device", self.device]) + table.add_row(["Environment seed", self.cfg.seed]) + table.add_row(["Physics step-size", self.physics_dt]) + table.add_row(["Environment step-size", self.step_dt]) + print_info(table.get_string()) + print_info("") + + # Initialize RL-specific state. + self.common_step_counter = 0 + self.episode_length_buf = torch.zeros( + cfg.scene.num_envs, device=device, dtype=torch.long + ) + self.render_mode = render_mode + self._offline_renderer: OffscreenRenderer | None = None + if self.render_mode == "rgb_array": + renderer = OffscreenRenderer( + model=self.sim.mj_model, + cfg=self.cfg.viewer, + scene=self.scene, + sim_model=self.sim.model, + expanded_fields=self.sim.expanded_fields, + ) + renderer.initialize() + self._offline_renderer = renderer + self.metadata["render_fps"] = 1.0 / self.step_dt + + # Load all managers. + self.load_managers() + self.setup_manager_visualizers() + + # Properties. + + @property + def num_envs(self) -> int: + """Number of parallel environments.""" + return self.scene.num_envs + + @property + def physics_dt(self) -> float: + """Physics simulation step size.""" + return self.cfg.sim.mujoco.timestep + + @property + def step_dt(self) -> float: + """Environment step size (physics_dt * decimation).""" + return self.cfg.sim.mujoco.timestep * self.cfg.decimation + + @property + def device(self) -> str: + """Device for computation.""" + return self.sim.device + + @property + def max_episode_length_s(self) -> float: + """Maximum episode length in seconds.""" + return self.cfg.episode_length_s + + @property + def max_episode_length(self) -> int: + """Maximum episode length in steps.""" + return math.ceil(self.max_episode_length_s / self.step_dt) + + @property + def unwrapped(self) -> "ManagerBasedRlEnv": + """Get the unwrapped environment (base case for wrapper chains).""" + return self + + # Methods. + + def setup_manager_visualizers(self) -> None: + self.manager_visualizers = {} + if getattr(self.command_manager, "active_terms", None): + self.manager_visualizers["command_manager"] = self.command_manager + self.manager_visualizers["event_manager"] = self.event_manager + self.manager_visualizers["reward_manager"] = self.reward_manager + + def load_managers(self) -> None: + """Load and initialize all managers. + + Order is important! Event and command managers must be loaded first, + then action and observation managers, then other RL managers. + """ + # Event manager (required before everything else for domain randomization). + self.event_manager = EventManager(self.cfg.events, self) + print_info(f"[INFO] {self.event_manager}") + + self.sim.expand_model_fields(self.event_manager.domain_randomization_fields) + + # Command manager (must be before observation manager since observations + # may reference commands). + if len(self.cfg.commands) > 0: + self.command_manager = CommandManager(self.cfg.commands, self) + else: + self.command_manager = NullCommandManager() + print_info(f"[INFO] {self.command_manager}") + + # Action and observation managers. + self.action_manager = ActionManager(self.cfg.actions, self) + print_info(f"[INFO] {self.action_manager}") + self.observation_manager = ObservationManager(self.cfg.observations, self) + print_info(f"[INFO] {self.observation_manager}") + + # Other RL-specific managers. + + self.termination_manager = TerminationManager(self.cfg.terminations, self) + print_info(f"[INFO] {self.termination_manager}") + self.reward_manager = RewardManager( + self.cfg.rewards, self, scale_by_dt=self.cfg.scale_rewards_by_dt + ) + print_info(f"[INFO] {self.reward_manager}") + if len(self.cfg.curriculum) > 0: + self.curriculum_manager = CurriculumManager(self.cfg.curriculum, self) + else: + self.curriculum_manager = NullCurriculumManager() + print_info(f"[INFO] {self.curriculum_manager}") + if len(self.cfg.metrics) > 0: + self.metrics_manager = MetricsManager(self.cfg.metrics, self) + else: + self.metrics_manager = NullMetricsManager() + print_info(f"[INFO] {self.metrics_manager}") + if len(self.cfg.recorders) > 0: + self.recorder_manager = RecorderManager(self.cfg.recorders, self) + else: + self.recorder_manager = NullRecorderManager() + print_info(f"[INFO] {self.recorder_manager}") + + # Configure spaces for the environment. + self._configure_gym_env_spaces() + + # Initialize startup events if defined. + if "startup" in self.event_manager.available_modes: + self.event_manager.apply(mode="startup") + + def reset( + self, + *, + seed: int | None = None, + env_ids: torch.Tensor | None = None, + options: dict[str, Any] | None = None, + ) -> tuple[types.VecEnvObs, dict]: + del options # Unused. + if env_ids is None: + env_ids = torch.arange(self.num_envs, dtype=torch.int64, device=self.device) + if seed is not None: + self.seed(seed) + self.extras["log"] = dict() + self._reset_idx(env_ids) + self.scene.write_data_to_sim() + self.sim.forward() + # Scoped to env_ids so a partial reset does not advance stateful commands in the + # other envs. + self.command_manager.compute(dt=0.0, env_ids=env_ids) + self.sim.sense() + # Scoped to env_ids: only the reset envs' history/delay buffers receive + # the post-reset frame; other envs' observation timelines are untouched. + self.obs_buf = self.observation_manager.compute( + update_history=True, env_ids=env_ids + ) + self.recorder_manager.record_post_reset(env_ids) + return self.obs_buf, self.extras + + def step(self, action: torch.Tensor) -> types.VecEnvStepReturn: + """Run one environment step: apply actions, simulate, compute RL signals. + + When ``auto_reset=True`` (default), environments that terminate or time out are + reset in place and the returned observation is the post-reset state. When + ``auto_reset=False``, the reset is skipped and the returned observation is the + terminal state; the caller must call ``reset(env_ids=...)`` for done envs before + the next ``step()``. + + **Forward-call placement.** MuJoCo's ``mj_step`` runs forward kinematics *before* + integration, so after stepping, derived quantities (``xpos``, ``xquat``, + ``site_xpos``, ``cvel``, ``sensordata``) lag ``qpos``/``qvel`` by one physics + substep. Rather than calling ``sim.forward()`` twice (once after the decimation + loop and once after the reset block), this method calls it **once**, right + before observation computation. This single call refreshes derived quantities + for *all* envs: non-reset envs pick up post-decimation kinematics, reset envs + pick up post-reset kinematics. + + The tradeoff is that termination, reward, and step/interval event managers see + derived quantities that are stale by one physics substep (the last ``mj_step`` + ran ``mj_forward`` from *pre*-integration ``qpos``). In practice, the staleness + is negligible for reward shaping, termination checks, and event perturbations. + Critically, the staleness is *consistent*: every env, every step, always sees + the same lag, so the MDP is well-defined and the value function can learn the + correct mapping. + + **Auto-reset parity.** Auto-reset reproduces the explicit ``reset()`` flow: + events fire on the terminal state before the reset, and freshly reset envs + advance commands with ``dt=0`` so their timers start full. + + .. note:: + + Events and reset-path resamples all run *before* ``forward()``, so state + they write (e.g. a velocity push) is refreshed automatically and visible + in this step's observations. Command updates run *after* + the call, so a command that writes sim state must refresh derived + quantities itself (see ``MotionCommand._update_command``). In all cases, + do not read derived quantities (``root_link_pose_w``, ``body_link_vel_w``, + etc.) in the same function that writes state (``write_root_state_to_sim``, + ``write_joint_state_to_sim``, etc.). See :ref:`faq` for details. + """ + if not self.cfg.auto_reset and torch.any(self._manual_reset_pending): + pending_ids = self._manual_reset_pending.nonzero(as_tuple=False).squeeze(-1) + raise RuntimeError( + f"Environments {pending_ids.cpu().tolist()} must be reset via " + "reset(env_ids=...) before calling step() again when auto_reset=False." + ) + + self.extras["log"] = dict() + self.action_manager.process_action(action.to(self.device)) + + for _ in range(self.cfg.decimation): + self._sim_step_counter += 1 + self.action_manager.apply_action() + self.scene.write_data_to_sim() + self.sim.step() + self.scene.update(dt=self.physics_dt) + self.metrics_manager.compute_substep() + + # Update env counters. + self.episode_length_buf += 1 + self.common_step_counter += 1 + + # Check terminations and compute rewards. + # NOTE: Derived quantities (xpos, xquat, ...) are stale by one physics + # substep here. See the docstring above for why this is acceptable. + self.reset_buf = self.termination_manager.compute() + self.reset_terminated = self.termination_manager.terminated + self.reset_time_outs = self.termination_manager.time_outs + + self.reward_buf = self.reward_manager.compute(dt=self.step_dt) + self.metrics_manager.compute() + + # Events fire before auto-reset, on the terminal state, matching the + # explicit flow. Staleness and forward() rules are the same as for + # termination/reward (see docstring above). + if "step" in self.event_manager.available_modes: + self.event_manager.apply(mode="step", dt=self.step_dt) + if "interval" in self.event_manager.available_modes: + self.event_manager.apply(mode="interval", dt=self.step_dt) + + # Reset envs that terminated/timed-out and log the episode info. + reset_env_ids = self.reset_buf.nonzero(as_tuple=False).squeeze(-1) + if self.cfg.auto_reset and len(reset_env_ids) > 0: + self.recorder_manager.record_pre_reset(reset_env_ids) + self._reset_idx(reset_env_ids) + self.scene.write_data_to_sim() + + # Single forward() call: recompute derived quantities from current + # qpos/qvel for every env. For non-reset envs this resolves the + # one-substep staleness left by mj_step; for reset envs it picks up + # the freshly written reset state. + self.sim.forward() + + # Pass dt=0 for freshly reset envs so their command timers start full, + # matching reset(). + if self.cfg.auto_reset and len(reset_env_ids) > 0: + command_dt = self._command_dt + command_dt.fill_(self.step_dt) + command_dt[reset_env_ids] = 0.0 + self.command_manager.compute(dt=command_dt) + else: + self.command_manager.compute(dt=self.step_dt) + + self.sim.sense() + self.obs_buf = self.observation_manager.compute(update_history=True) + + if self.cfg.auto_reset and len(reset_env_ids) > 0: + self.recorder_manager.record_post_reset(reset_env_ids) + elif len(reset_env_ids) > 0: + self._manual_reset_pending[reset_env_ids] = True + + self.recorder_manager.record_post_step() + + return ( + self.obs_buf, + self.reward_buf, + self.reset_terminated, + self.reset_time_outs, + self.extras, + ) + + def get_observations(self) -> dict: + return self.observation_manager.compute() + + def render(self) -> np.ndarray | None: + if self.render_mode == "human" or self.render_mode is None: + return None + elif self.render_mode == "rgb_array": + if self._offline_renderer is None: + raise ValueError("Offline renderer not initialized") + debug_callback = ( + self.update_visualizers if hasattr(self, "update_visualizers") else None + ) + self._offline_renderer.update(self.sim.data, debug_vis_callback=debug_callback) + return self._offline_renderer.render() + else: + raise NotImplementedError( + f"Render mode {self.render_mode} is not supported. " + f"Please use: {self.metadata['render_modes']}." + ) + + def close(self) -> None: + if self._offline_renderer is not None: + self._offline_renderer.close() + self.recorder_manager.close() + + @staticmethod + def seed(seed: int = -1) -> int: + if seed == -1: + seed = np.random.randint(0, 10_000) + print_info(f"Setting seed: {seed}") + random_utils.seed_rng(seed) + return seed + + def update_visualizers(self, visualizer: DebugVisualizer) -> None: + for mod in self.manager_visualizers.values(): + mod.debug_vis(visualizer) + for sensor in self.scene.sensors.values(): + sensor.debug_vis(visualizer) + + # Private methods. + + def _configure_gym_env_spaces(self) -> None: + from mjlab.utils.spaces import batch_space + + self.single_observation_space = DictSpace() + for group_name, group_term_names in self.observation_manager.active_terms.items(): + has_concatenated_obs = self.observation_manager.group_obs_concatenate[group_name] + group_dim = self.observation_manager.group_obs_dim[group_name] + if has_concatenated_obs: + assert isinstance(group_dim, tuple) + self.single_observation_space.spaces[group_name] = Box( + shape=group_dim, low=-math.inf, high=math.inf + ) + else: + assert not isinstance(group_dim, tuple) + group_term_cfgs = self.observation_manager._group_obs_term_cfgs[group_name] + # Create a nested dict for this group. + group_space = DictSpace() + for term_name, term_dim, _term_cfg in zip( + group_term_names, group_dim, group_term_cfgs, strict=False + ): + group_space.spaces[term_name] = Box( + shape=term_dim, low=-math.inf, high=math.inf + ) + self.single_observation_space.spaces[group_name] = group_space + + action_dim = sum(self.action_manager.action_term_dim) + self.single_action_space = Box(shape=(action_dim,), low=-math.inf, high=math.inf) + + self.observation_space = batch_space(self.single_observation_space, self.num_envs) + self.action_space = batch_space(self.single_action_space, self.num_envs) + + def _reset_idx(self, env_ids: torch.Tensor | None = None) -> None: + self.curriculum_manager.compute(env_ids=env_ids) + self.sim.reset(env_ids) + self.scene.reset(env_ids) + + if "reset" in self.event_manager.available_modes: + env_step_count = self._sim_step_counter // self.cfg.decimation + self.event_manager.apply( + mode="reset", env_ids=env_ids, global_env_step_count=env_step_count + ) + + # NOTE: This is order sensitive. + # observation manager. + info = self.observation_manager.reset(env_ids) + self.extras["log"].update(info) + # action manager. + info = self.action_manager.reset(env_ids) + self.extras["log"].update(info) + # rewards manager. + info = self.reward_manager.reset(env_ids) + self.extras["log"].update(info) + # metrics manager. + info = self.metrics_manager.reset(env_ids) + self.extras["log"].update(info) + # curriculum manager. + info = self.curriculum_manager.reset(env_ids) + self.extras["log"].update(info) + # command manager. + info = self.command_manager.reset(env_ids) + self.extras["log"].update(info) + # event manager. + info = self.event_manager.reset(env_ids) + self.extras["log"].update(info) + # termination manager. + info = self.termination_manager.reset(env_ids) + self.extras["log"].update(info) + # reset the episode length buffer. + self.episode_length_buf[env_ids] = 0 + self._manual_reset_pending[env_ids] = False diff --git a/rl/mjlab/envs/mdp/__init__.py b/rl/mjlab/envs/mdp/__init__.py new file mode 100644 index 0000000..a299ae0 --- /dev/null +++ b/rl/mjlab/envs/mdp/__init__.py @@ -0,0 +1,8 @@ +from . import dr as dr +from .actions import * # noqa: F403 +from .curriculums import * # noqa: F403 +from .events import * # noqa: F403 +from .metrics import * # noqa: F403 +from .observations import * # noqa: F403 +from .rewards import * # noqa: F403 +from .terminations import * # noqa: F403 diff --git a/rl/mjlab/envs/mdp/actions/__init__.py b/rl/mjlab/envs/mdp/actions/__init__.py new file mode 100644 index 0000000..a3eea95 --- /dev/null +++ b/rl/mjlab/envs/mdp/actions/__init__.py @@ -0,0 +1,36 @@ +from mjlab.envs.mdp.actions.actions import JointEffortAction as JointEffortAction +from mjlab.envs.mdp.actions.actions import JointEffortActionCfg as JointEffortActionCfg +from mjlab.envs.mdp.actions.actions import JointPositionAction as JointPositionAction +from mjlab.envs.mdp.actions.actions import ( + JointPositionActionCfg as JointPositionActionCfg, +) +from mjlab.envs.mdp.actions.actions import JointVelocityAction as JointVelocityAction +from mjlab.envs.mdp.actions.actions import ( + JointVelocityActionCfg as JointVelocityActionCfg, +) +from mjlab.envs.mdp.actions.actions import ( + RelativeJointPositionAction as RelativeJointPositionAction, +) +from mjlab.envs.mdp.actions.actions import ( + RelativeJointPositionActionCfg as RelativeJointPositionActionCfg, +) +from mjlab.envs.mdp.actions.actions import SiteEffortAction as SiteEffortAction +from mjlab.envs.mdp.actions.actions import SiteEffortActionCfg as SiteEffortActionCfg +from mjlab.envs.mdp.actions.actions import TendonEffortAction as TendonEffortAction +from mjlab.envs.mdp.actions.actions import ( + TendonEffortActionCfg as TendonEffortActionCfg, +) +from mjlab.envs.mdp.actions.actions import TendonLengthAction as TendonLengthAction +from mjlab.envs.mdp.actions.actions import ( + TendonLengthActionCfg as TendonLengthActionCfg, +) +from mjlab.envs.mdp.actions.actions import TendonVelocityAction as TendonVelocityAction +from mjlab.envs.mdp.actions.actions import ( + TendonVelocityActionCfg as TendonVelocityActionCfg, +) +from mjlab.envs.mdp.actions.differential_ik import ( + DifferentialIKAction as DifferentialIKAction, +) +from mjlab.envs.mdp.actions.differential_ik import ( + DifferentialIKActionCfg as DifferentialIKActionCfg, +) diff --git a/rl/mjlab/envs/mdp/actions/actions.py b/rl/mjlab/envs/mdp/actions/actions.py new file mode 100644 index 0000000..b9ba67d --- /dev/null +++ b/rl/mjlab/envs/mdp/actions/actions.py @@ -0,0 +1,386 @@ +"""Actions that control actuator transmissions (e.g., joints, tendons, sites).""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import torch + +from mjlab.actuator.actuator import TransmissionType +from mjlab.managers.action_manager import ActionTerm, ActionTermCfg +from mjlab.utils.lab_api.string import resolve_matching_names_values + +if TYPE_CHECKING: + from mjlab.entity import Entity + from mjlab.envs import ManagerBasedRlEnv + + +@dataclass(kw_only=True) +class BaseActionCfg(ActionTermCfg): + """Configuration for actions that control actuator transmissions.""" + + transmission_type: TransmissionType = TransmissionType.JOINT + """Type of transmission to control.""" + + actuator_names: tuple[str, ...] | list[str] + """Actuator names to control.""" + + scale: float | dict[str, float] = 1.0 + """Action scale. Float or dict mapping actuator names to scales.""" + + offset: float | dict[str, float] = 0.0 + """Action offset. Float or dict mapping actuator names to offsets.""" + + preserve_order: bool = False + """Whether to preserve the order of actuator names.""" + + +class BaseAction(ActionTerm): + """Apply actions to actuator transmissions with scale/offset processing. + + Supports controlling different transmission types (e.g., joints, tendons, + sites) with configurable affine transformations applied to raw actions. + """ + + cfg: BaseActionCfg + _entity: Entity + + def __init__(self, cfg: BaseActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + # Find targets based on transmission type. + target_ids, target_names = self._find_targets(cfg) + self._target_ids = torch.tensor(target_ids, device=self.device, dtype=torch.long) + self._target_names = target_names + + self._num_targets = len(target_ids) + self._action_dim = len(target_ids) + + self._raw_actions = torch.zeros(self.num_envs, self.action_dim, device=self.device) + self._processed_actions = torch.zeros_like(self._raw_actions) + + if isinstance(cfg.scale, (float, int)): + self._scale = float(cfg.scale) + elif isinstance(cfg.scale, dict): + self._scale = torch.ones(self.num_envs, self.action_dim, device=self.device) + index_list, _, value_list = resolve_matching_names_values( + cfg.scale, self._target_names + ) + self._scale[:, index_list] = torch.tensor(value_list, device=self.device) + else: + raise ValueError( + f"Unsupported scale type: {type(cfg.scale)}. " + f"Supported types are float and dict." + ) + + if isinstance(cfg.offset, (float, int)): + self._offset = float(cfg.offset) + elif isinstance(cfg.offset, dict): + self._offset = torch.zeros_like(self._raw_actions) + index_list, _, value_list = resolve_matching_names_values( + cfg.offset, self._target_names + ) + self._offset[:, index_list] = torch.tensor(value_list, device=self.device) + else: + raise ValueError( + f"Unsupported offset type: {type(cfg.offset)}. " + f"Supported types are float and dict." + ) + + if cfg.clip is not None: + self._clip = torch.tensor( + [[-float("inf"), float("inf")]], device=self.device + ).repeat(self.num_envs, self.action_dim, 1) + index_list, _, value_list = resolve_matching_names_values( + cfg.clip, self._target_names + ) + self._clip[:, index_list] = torch.tensor(value_list, device=self.device) + + def _find_targets(self, cfg: BaseActionCfg) -> tuple[list[int], list[str]]: + """Find target IDs and names based on transmission type. + + Args: + cfg: Action configuration. + + Returns: + Tuple of (target_ids, target_names). + """ + if cfg.transmission_type == TransmissionType.JOINT: + return self._entity.find_joints_by_actuator_names(cfg.actuator_names) + elif cfg.transmission_type == TransmissionType.TENDON: + return self._entity.find_tendons( + cfg.actuator_names, preserve_order=cfg.preserve_order + ) + elif cfg.transmission_type == TransmissionType.SITE: + return self._entity.find_sites( + cfg.actuator_names, preserve_order=cfg.preserve_order + ) + else: + raise ValueError(f"Unknown transmission type: {cfg.transmission_type}") + + # Properties. + + @property + def scale(self) -> torch.Tensor | float: + """Action scale.""" + return self._scale + + @property + def offset(self) -> torch.Tensor | float: + """Action offset.""" + return self._offset + + @property + def raw_action(self) -> torch.Tensor: + """Raw actions (before scale/offset).""" + return self._raw_actions + + @property + def action_dim(self) -> int: + """Dimension of the action space.""" + return self._action_dim + + @property + def target_ids(self) -> torch.Tensor: + """Target IDs for the controlled transmission.""" + return self._target_ids + + @property + def target_names(self) -> list[str]: + """Target names for the controlled transmission.""" + return self._target_names + + def process_actions(self, actions: torch.Tensor): + """Process raw actions by applying scale, offset, and optional clip.""" + self._raw_actions[:] = actions + self._processed_actions = self._raw_actions * self._scale + self._offset + if self.cfg.clip is not None: + self._processed_actions = torch.clamp( + self._processed_actions, + min=self._clip[:, :, 0], + max=self._clip[:, :, 1], + ) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + """Reset raw actions to zero for specified environments.""" + self._raw_actions[env_ids] = 0.0 + + +## +# Joint actions. +## + + +@dataclass(kw_only=True) +class JointPositionActionCfg(BaseActionCfg): + """Configuration for joint position control.""" + + use_default_offset: bool = True + + def __post_init__(self): + self.transmission_type = TransmissionType.JOINT + + def build(self, env: ManagerBasedRlEnv) -> JointPositionAction: + return JointPositionAction(self, env) + + +@dataclass(kw_only=True) +class JointVelocityActionCfg(BaseActionCfg): + """Configuration for joint velocity control.""" + + use_default_offset: bool = True + + def __post_init__(self): + self.transmission_type = TransmissionType.JOINT + + def build(self, env: ManagerBasedRlEnv) -> JointVelocityAction: + return JointVelocityAction(self, env) + + +@dataclass(kw_only=True) +class JointEffortActionCfg(BaseActionCfg): + """Configuration for joint effort (torque) control.""" + + def __post_init__(self): + self.transmission_type = TransmissionType.JOINT + + def build(self, env: ManagerBasedRlEnv) -> JointEffortAction: + return JointEffortAction(self, env) + + +class JointPositionAction(BaseAction): + """Control joints via position targets.""" + + def __init__(self, cfg: JointPositionActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + if cfg.use_default_offset: + self._offset = self._entity.data.default_joint_pos[:, self._target_ids].clone() + + def apply_actions(self) -> None: + encoder_bias = self._entity.data.encoder_bias[:, self._target_ids] + target = self._processed_actions - encoder_bias + self._entity.set_joint_position_target(target, joint_ids=self._target_ids) + + +@dataclass(kw_only=True) +class RelativeJointPositionActionCfg(BaseActionCfg): + """Configuration for joint position control relative to current positions. + + target = current_joint_pos + action * scale + + The ``offset`` field inherited from ``BaseActionCfg`` is not supported and + must remain at its default of ``0.0``. The ``clip`` field is supported and + clamps the delta (``action * scale``) before it is added to the current + position. + """ + + def __post_init__(self): + self.transmission_type = TransmissionType.JOINT + if self.offset != 0.0: + raise ValueError( + "RelativeJointPositionActionCfg does not support 'offset'. " + "The target is current_pos + action * scale; a fixed offset has no meaning." + ) + + def build(self, env: ManagerBasedRlEnv) -> RelativeJointPositionAction: + return RelativeJointPositionAction(self, env) + + +class RelativeJointPositionAction(BaseAction): + """Control joints via position targets relative to current positions.""" + + def apply_actions(self) -> None: + current_pos = self._entity.data.joint_pos[:, self._target_ids] + target = current_pos + self._processed_actions + self._entity.set_joint_position_target(target, joint_ids=self._target_ids) + + +class JointVelocityAction(BaseAction): + """Control joints via velocity targets.""" + + def __init__(self, cfg: JointVelocityActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + if cfg.use_default_offset: + self._offset = self._entity.data.default_joint_vel[:, self._target_ids].clone() + + def apply_actions(self) -> None: + self._entity.set_joint_velocity_target( + self._processed_actions, joint_ids=self._target_ids + ) + + +class JointEffortAction(BaseAction): + """Control joints via effort (torque) targets.""" + + def __init__(self, cfg: JointEffortActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + def apply_actions(self) -> None: + self._entity.set_joint_effort_target( + self._processed_actions, joint_ids=self._target_ids + ) + + +## +# Tendon actions. +## + + +@dataclass(kw_only=True) +class TendonLengthActionCfg(BaseActionCfg): + """Configuration for tendon length control.""" + + def __post_init__(self): + self.transmission_type = TransmissionType.TENDON + + def build(self, env: ManagerBasedRlEnv) -> TendonLengthAction: + return TendonLengthAction(self, env) + + +@dataclass(kw_only=True) +class TendonVelocityActionCfg(BaseActionCfg): + """Configuration for tendon velocity control.""" + + def __post_init__(self): + self.transmission_type = TransmissionType.TENDON + + def build(self, env: ManagerBasedRlEnv) -> TendonVelocityAction: + return TendonVelocityAction(self, env) + + +@dataclass(kw_only=True) +class TendonEffortActionCfg(BaseActionCfg): + """Configuration for tendon effort control.""" + + def __post_init__(self): + self.transmission_type = TransmissionType.TENDON + + def build(self, env: ManagerBasedRlEnv) -> TendonEffortAction: + return TendonEffortAction(self, env) + + +class TendonLengthAction(BaseAction): + """Control tendons via length targets.""" + + def __init__(self, cfg: TendonLengthActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + def apply_actions(self) -> None: + self._entity.set_tendon_len_target( + self._processed_actions, tendon_ids=self._target_ids + ) + + +class TendonVelocityAction(BaseAction): + """Control tendons via velocity targets.""" + + def __init__(self, cfg: TendonVelocityActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + def apply_actions(self) -> None: + self._entity.set_tendon_vel_target( + self._processed_actions, tendon_ids=self._target_ids + ) + + +class TendonEffortAction(BaseAction): + """Control tendons via effort targets.""" + + def __init__(self, cfg: TendonEffortActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + def apply_actions(self) -> None: + self._entity.set_tendon_effort_target( + self._processed_actions, tendon_ids=self._target_ids + ) + + +## +# Site actions. +## + + +@dataclass(kw_only=True) +class SiteEffortActionCfg(BaseActionCfg): + """Configuration for site effort control.""" + + def __post_init__(self): + self.transmission_type = TransmissionType.SITE + + def build(self, env: ManagerBasedRlEnv) -> SiteEffortAction: + return SiteEffortAction(self, env) + + +class SiteEffortAction(BaseAction): + """Control sites via effort targets.""" + + def __init__(self, cfg: SiteEffortActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + def apply_actions(self) -> None: + self._entity.set_site_effort_target( + self._processed_actions, site_ids=self._target_ids + ) diff --git a/rl/mjlab/envs/mdp/actions/differential_ik.py b/rl/mjlab/envs/mdp/actions/differential_ik.py new file mode 100644 index 0000000..c228c5e --- /dev/null +++ b/rl/mjlab/envs/mdp/actions/differential_ik.py @@ -0,0 +1,314 @@ +"""Differential IK action space for task-space control.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import mujoco_warp as mjwarp +import torch +import warp as wp + +from mjlab.managers.action_manager import ActionTerm, ActionTermCfg +from mjlab.utils.lab_api.math import ( + apply_delta_pose, + compute_pose_error, + quat_from_matrix, +) +from mjlab.utils.string import resolve_expr + +if TYPE_CHECKING: + from mjlab.entity import Entity + from mjlab.envs import ManagerBasedRlEnv + + +@dataclass(kw_only=True) +class DifferentialIKActionCfg(ActionTermCfg): + """Configuration for differential IK action space. + + Converts task-space commands (position and/or orientation) into + joint-space targets via damped least-squares (DLS) IK, applied + every decimation substep. + + The action dimension is determined by the active objectives: + + - ``position_weight > 0, orientation_weight == 0`` -> 3D + - ``orientation_weight > 0, use_relative_mode=True`` -> 6D + - ``orientation_weight > 0, use_relative_mode=False`` -> 7D + """ + + actuator_names: tuple[str, ...] | list[str] + """Actuator name expressions to resolve the controlled joints.""" + + frame_type: Literal["body", "site", "geom"] = "body" + """Element type of the target frame.""" + + frame_name: str + """Name of the target frame element on the entity.""" + + use_relative_mode: bool = True + """If True, actions are deltas applied to the current frame pose. + If False, actions are absolute targets.""" + + delta_pos_scale: float = 1.0 + """Scaling factor for position components in relative mode. + + Maps raw policy outputs to position deltas (meters). + Ignored in absolute mode. + """ + + delta_ori_scale: float = 1.0 + """Scaling factor for orientation components in relative mode. + + Maps raw policy outputs to orientation deltas (radians). + Ignored in absolute mode. + """ + + damping: float = 0.05 + """Damping coefficient (lambda) for the DLS pseudoinverse.""" + + max_dq: float = 0.5 + """Maximum joint displacement per IK solve (rad/step).""" + + position_weight: float = 1.0 + """Weight for the position residual in the DLS system.""" + + orientation_weight: float = 1.0 + """Weight for the orientation residual (0 = position-only mode).""" + + joint_limit_weight: float = 0.0 + """Weight for soft joint-limit residuals (0 = disabled). + + When positive, joint-limit violations are added as extra rows + in the DLS system so the solver steers away from limits. + """ + + posture_weight: float = 0.0 + """Weight for posture regularization (0 = disabled). + + When positive, a residual ``w * (q_target - q_current)`` is added + to bias the solver toward the posture target. Useful with + redundant manipulators or position-only tracking to control + null-space behaviour. + """ + + posture_target: dict[str, float] | None = None + """Target joint positions for posture regularization. + + Maps joint name patterns to target values (same format as + ``EntityCfg.InitialStateCfg.joint_pos``). Joints not listed + default to their initial (qpos0) value. Only used when + ``posture_weight > 0``. + """ + + def build(self, env: ManagerBasedRlEnv) -> DifferentialIKAction: + return DifferentialIKAction(self, env) + + +class DifferentialIKAction(ActionTerm): + """Converts task-space commands into joint position targets via damped least-squares + IK each decimation substep. + """ + + cfg: DifferentialIKActionCfg + _entity: Entity + + def __init__(self, cfg: DifferentialIKActionCfg, env: ManagerBasedRlEnv): + super().__init__(cfg=cfg, env=env) + + joint_ids, _ = self._entity.find_joints_by_actuator_names(cfg.actuator_names) + self._joint_ids = torch.tensor(joint_ids, device=self.device, dtype=torch.long) + self._num_joints = len(joint_ids) + self._joint_dof_ids = self._entity.indexing.joint_v_adr[self._joint_ids] + + self._frame_type = cfg.frame_type + self._resolve_frame(cfg.frame_name) + + if cfg.orientation_weight > 0: + self._action_dim = 6 if cfg.use_relative_mode else 7 + else: + self._action_dim = 3 + self._raw_actions = torch.zeros(self.num_envs, self._action_dim, device=self.device) + + self._desired_pos = torch.zeros(self.num_envs, 3, device=self.device) + self._desired_quat = torch.zeros(self.num_envs, 4, device=self.device) + self._desired_quat[:, 0] = 1.0 + + # Joint-limit buffers. + limits = self._entity.data.soft_joint_pos_limits + self._joint_lower = limits[:, self._joint_ids, 0] + self._joint_upper = limits[:, self._joint_ids, 1] + + # Posture regularization target. + q_target = self._entity.data.default_joint_pos[:, self._joint_ids].clone() + if cfg.posture_target is not None: + joint_names = tuple(self._entity.joint_names[i] for i in joint_ids) + overrides = resolve_expr(cfg.posture_target, joint_names) + for j, val in enumerate(overrides): + if val is not None: + q_target[:, j] = val + self._posture_target = q_target + + nworld = self.num_envs + nv = self._env.sim.mj_model.nv + + with wp.ScopedDevice(self._env.sim.wp_device): + self._jacp_wp = wp.zeros((nworld, 3, nv), dtype=float) + self._jacr_wp = wp.zeros((nworld, 3, nv), dtype=float) + self._point_wp = wp.zeros(nworld, dtype=wp.vec3) + self._body_wp = wp.zeros(nworld, dtype=wp.int32) + self._body_wp.fill_(self._body_id) + + self._jacp_torch = wp.to_torch(self._jacp_wp) + self._jacr_torch = wp.to_torch(self._jacr_wp) + self._point_torch = wp.to_torch(self._point_wp).view(nworld, 3) + + @property + def action_dim(self) -> int: + return self._action_dim + + @property + def raw_action(self) -> torch.Tensor: + return self._raw_actions + + def process_actions(self, actions: torch.Tensor) -> None: + self._raw_actions[:] = actions + + frame_pos, frame_quat = self._get_frame_pose() + if self._action_dim == 3: + if self.cfg.use_relative_mode: + self._desired_pos[:] = frame_pos + actions * self.cfg.delta_pos_scale + else: + self._desired_pos[:] = actions + self._desired_quat[:] = frame_quat + elif self._action_dim == 6: + delta = actions.clone() + delta[:, :3] *= self.cfg.delta_pos_scale + delta[:, 3:] *= self.cfg.delta_ori_scale + target_pos, target_quat = apply_delta_pose(frame_pos, frame_quat, delta) + self._desired_pos[:] = target_pos + self._desired_quat[:] = target_quat + else: + assert self._action_dim == 7 + self._desired_pos[:] = actions[:, :3] + self._desired_quat[:] = actions[:, 3:7] + + def compute_dq(self) -> torch.Tensor: + """Run one DLS IK step and return joint displacement. + + Returns: + Joint displacement tensor of shape ``(num_envs, num_joints)``. + """ + frame_pos, frame_quat = self._get_frame_pose() + pos_error, rot_error = compute_pose_error( + frame_pos, frame_quat, self._desired_pos, self._desired_quat + ) + + self._point_torch[:] = frame_pos + self._compute_jacobian() + jacp = self._jacp_torch[:, :, self._joint_dof_ids] + jacr = self._jacr_torch[:, :, self._joint_dof_ids] + + w_pos = self.cfg.position_weight + w_ori = self.cfg.orientation_weight + w_lim = self.cfg.joint_limit_weight + w_post = self.cfg.posture_weight + lam = max(self.cfg.damping, 1e-6) + + # Joint-space normal equations: (J^T W J + λ²I) dq = J^T W dx. + # Equivalent to task-space DLS but solves a smaller n×n system instead of the + # (6 + 2n)-square task-space system. + wp2, wo2 = w_pos * w_pos, w_ori * w_ori + JTJ = wp2 * torch.einsum("bti,btj->bij", jacp, jacp) + wo2 * torch.einsum( + "bti,btj->bij", jacr, jacr + ) + JTdx = wp2 * torch.einsum("bti,bt->bi", jacp, pos_error) + wo2 * torch.einsum( + "bti,bt->bi", jacr, rot_error + ) + + # Joint-limit penalty (diagonal contribution). + q = self._entity.data.joint_pos[:, self._joint_ids] + r_limit = (self._joint_upper - q).clamp(max=0) + (self._joint_lower - q).clamp( + min=0 + ) + violated = (r_limit != 0).float() + wl2 = w_lim * w_lim + JTJ.diagonal(dim1=-2, dim2=-1).add_(wl2 * violated) + JTdx.add_(wl2 * violated * r_limit) + + # Posture regularization (identity Jacobian contribution). + r_posture = self._posture_target - q + wpost2 = w_post * w_post + JTJ.diagonal(dim1=-2, dim2=-1).add_(wpost2) + JTdx.add_(wpost2 * r_posture) + + # Damping. + JTJ.diagonal(dim1=-2, dim2=-1).add_(lam * lam) + + dq = torch.linalg.solve(JTJ, JTdx) + return dq.clamp(-self.cfg.max_dq, self.cfg.max_dq) + + def apply_actions(self) -> None: + dq = self.compute_dq() + q_current = self._entity.data.joint_pos[:, self._joint_ids] + q_target = q_current + dq + self._entity.set_joint_position_target(q_target, joint_ids=self._joint_ids) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + if env_ids is None: + env_ids = slice(None) + self._raw_actions[env_ids] = 0.0 + self._desired_pos[env_ids] = 0.0 + self._desired_quat[env_ids] = 0.0 + self._desired_quat[env_ids, 0] = 1.0 + + # Private + + def _resolve_frame(self, frame_name: str) -> None: + """Resolve the frame element to its global ID and parent body ID.""" + if self._frame_type == "body": + ids, _ = self._entity.find_bodies(frame_name) + local_id = ids[0] + self._frame_id = int(self._entity.indexing.body_ids[local_id].item()) + self._body_id = self._frame_id + elif self._frame_type == "site": + ids, _ = self._entity.find_sites(frame_name) + local_id = ids[0] + self._frame_id = int(self._entity.indexing.site_ids[local_id].item()) + self._body_id = int(self._env.sim.mj_model.site_bodyid[self._frame_id]) + elif self._frame_type == "geom": + ids, _ = self._entity.find_geoms(frame_name) + local_id = ids[0] + self._frame_id = int(self._entity.indexing.geom_ids[local_id].item()) + self._body_id = int(self._env.sim.mj_model.geom_bodyid[self._frame_id]) + else: + raise ValueError(f"Unknown frame_type: {self._frame_type}") + + def _get_frame_pose(self) -> tuple[torch.Tensor, torch.Tensor]: + """Return current frame (position, quaternion) from sim data.""" + data = self._env.sim.data + if self._frame_type == "body": + pos = data.xpos[:, self._frame_id] + quat = data.xquat[:, self._frame_id] + elif self._frame_type == "site": + pos = data.site_xpos[:, self._frame_id] + xmat = data.site_xmat[:, self._frame_id] + quat = quat_from_matrix(xmat) + else: + assert self._frame_type == "geom" + pos = data.geom_xpos[:, self._frame_id] + xmat = data.geom_xmat[:, self._frame_id] + quat = quat_from_matrix(xmat) + return pos, quat + + def _compute_jacobian(self) -> None: + """Compute the frame Jacobian.""" + with wp.ScopedDevice(self._env.sim.wp_device): + mjwarp.jac( + self._env.sim.wp_model, + self._env.sim.wp_data, + self._jacp_wp, + self._jacr_wp, + self._point_wp, + self._body_wp, + ) diff --git a/rl/mjlab/envs/mdp/curriculums.py b/rl/mjlab/envs/mdp/curriculums.py new file mode 100644 index 0000000..f19cce8 --- /dev/null +++ b/rl/mjlab/envs/mdp/curriculums.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, TypedDict + +import torch + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + from mjlab.managers.curriculum_manager import CurriculumTermCfg + + +# Stage schemas. + + +class _RewardCurriculumStageOptional(TypedDict, total=False): + weight: float + params: dict[str, Any] + + +class RewardCurriculumStage(_RewardCurriculumStageOptional): + step: int + + +class _TerminationCurriculumStageOptional(TypedDict, total=False): + params: dict[str, Any] + time_out: bool + + +class TerminationCurriculumStage(_TerminationCurriculumStageOptional): + step: int + + +# Shared engine. Stage dicts are passed directly from the public TypedDict +# schemas. Any key that isn't "step" or "params" is treated as a top-level +# field on the target term config (e.g. "weight" on RewardTermCfg). + +_RESERVED_KEYS = {"step", "params"} + + +def _validate_stages( + term_cfg: Any, + term_name: str, + stages: Sequence[Any], +) -> None: + """Validate stage ordering, field existence, and param keys.""" + for i in range(1, len(stages)): + if stages[i]["step"] < stages[i - 1]["step"]: + raise ValueError( + f"Curriculum stages must be in nondecreasing step order," + f" but stage {i} has step" + f" {stages[i]['step']} < {stages[i - 1]['step']}." + ) + for stage in stages: + for key in stage: + if key not in _RESERVED_KEYS and not hasattr(term_cfg, key): + raise AttributeError( + f"Field '{key}' does not exist on the resolved term config for '{term_name}'." + ) + for stage in stages: + unknown = stage.get("params", {}).keys() - term_cfg.params.keys() + if unknown: + raise KeyError( + f"Stage at step {stage['step']} sets unknown param(s)" + f" {unknown} on term '{term_name}'. Check for typos." + ) + + +def _apply_stages( + term_cfg: Any, + step_counter: int, + stages: Sequence[Any], +) -> dict[str, torch.Tensor]: + """Apply staged updates and return a logging snapshot.""" + for stage in stages: + if step_counter >= stage["step"]: + for key, value in stage.items(): + if key not in _RESERVED_KEYS: + setattr(term_cfg, key, value) + if "params" in stage: + term_cfg.params.update(stage["params"]) + # Only log values that stages actually reference. + logged_fields: set[str] = set() + logged_params: set[str] = set() + for stage in stages: + for key in stage: + if key not in _RESERVED_KEYS: + logged_fields.add(key) + for key in stage.get("params", {}): + logged_params.add(key) + result: dict[str, torch.Tensor] = {} + for key in logged_fields: + value = getattr(term_cfg, key) + if isinstance(value, (int, float, bool)): + result[key] = torch.tensor(value) + elif isinstance(value, torch.Tensor): + result[key] = value + for key in logged_params: + v = term_cfg.params[key] + if isinstance(v, (int, float, bool)): + result[key] = torch.tensor(v) + elif isinstance(v, torch.Tensor): + result[key] = v + return result + + +# Public wrappers. + + +class reward_curriculum: + """Update a reward term's weight and/or params based on training steps. + + Each stage specifies a ``step`` threshold and optionally a ``weight`` + and/or ``params`` dict. When ``env.common_step_counter`` reaches a + stage's ``step``, the corresponding values are applied. Later stages + take precedence when multiple thresholds are reached. + + Example:: + + CurriculumTermCfg( + func=mdp.reward_curriculum, + params={ + "reward_name": "joint_vel_hinge", + "stages": [ + {"step": 0, "weight": -0.01}, + {"step": 12000, "weight": -0.1}, + {"step": 24000, "weight": -1.0, "params": {"max_vel": 1.0}}, + ], + }, + ) + """ + + def __init__(self, cfg: CurriculumTermCfg, env: ManagerBasedRlEnv): + reward_name: str = cfg.params["reward_name"] + stages: list[RewardCurriculumStage] = cfg.params["stages"] + self._term_cfg = env.reward_manager.get_term_cfg(reward_name) + self._stages = stages + _validate_stages(self._term_cfg, reward_name, self._stages) + + def __call__( + self, + env: ManagerBasedRlEnv, + env_ids: torch.Tensor, + reward_name: str, + stages: list[RewardCurriculumStage], + ) -> dict[str, torch.Tensor]: + del env_ids, reward_name, stages + return _apply_stages(self._term_cfg, env.common_step_counter, self._stages) + + +class termination_curriculum: + """Update a termination term's params based on training steps. + + Each stage specifies a ``step`` threshold and a ``params`` dict. When + ``env.common_step_counter`` reaches a stage's ``step``, the params are + applied. Later stages take precedence. + + Example:: + + CurriculumTermCfg( + func=mdp.termination_curriculum, + params={ + "termination_name": "energy", + "stages": [ + {"step": 12000, "params": {"threshold": 1000.0}}, + {"step": 24000, "params": {"threshold": 700.0}}, + ], + }, + ) + """ + + def __init__(self, cfg: CurriculumTermCfg, env: ManagerBasedRlEnv): + termination_name: str = cfg.params["termination_name"] + stages: list[TerminationCurriculumStage] = cfg.params["stages"] + self._term_cfg = env.termination_manager.get_term_cfg(termination_name) + self._stages = stages + _validate_stages(self._term_cfg, termination_name, self._stages) + + def __call__( + self, + env: ManagerBasedRlEnv, + env_ids: torch.Tensor, + termination_name: str, + stages: list[TerminationCurriculumStage], + ) -> dict[str, torch.Tensor]: + del env_ids, termination_name, stages + return _apply_stages(self._term_cfg, env.common_step_counter, self._stages) diff --git a/rl/mjlab/envs/mdp/dr/__init__.py b/rl/mjlab/envs/mdp/dr/__init__.py new file mode 100644 index 0000000..db4a2a2 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/__init__.py @@ -0,0 +1,95 @@ +"""Domain randomization functions.""" + +# Types and built-in instances. +# isort: split +from ._types import Distribution as Distribution +from ._types import Operation as Operation +from ._types import abs as abs +from ._types import add as add +from ._types import gaussian as gaussian +from ._types import log_uniform as log_uniform +from ._types import scale as scale +from ._types import uniform as uniform + +# Geom. +# isort: split +from .geom import geom_friction as geom_friction +from .geom import geom_matid as geom_matid +from .geom import geom_pos as geom_pos +from .geom import geom_quat as geom_quat +from .geom import geom_rgba as geom_rgba +from .geom import geom_size as geom_size + +# Body. +# isort: split +from .body import body_com_offset as body_com_offset +from .body import body_ipos as body_ipos +from .body import body_mass as body_mass +from .body import body_pos as body_pos +from .body import body_quat as body_quat +from .body import pseudo_inertia as pseudo_inertia + +# Joint / DOF. +# isort: split +from .joint import dof_armature as dof_armature +from .joint import dof_damping as dof_damping +from .joint import dof_frictionloss as dof_frictionloss +from .joint import encoder_bias as encoder_bias +from .joint import jnt_range as jnt_range +from .joint import jnt_stiffness as jnt_stiffness +from .joint import joint_armature as joint_armature +from .joint import joint_damping as joint_damping +from .joint import joint_default_pos as joint_default_pos +from .joint import joint_friction as joint_friction +from .joint import joint_limits as joint_limits +from .joint import joint_stiffness as joint_stiffness +from .joint import qpos0 as qpos0 + +# Site. +# isort: split +from .site import site_pos as site_pos +from .site import site_quat as site_quat + +# Tendon. +# isort: split +from .tendon import tendon_armature as tendon_armature +from .tendon import tendon_damping as tendon_damping +from .tendon import tendon_friction as tendon_friction +from .tendon import tendon_frictionloss as tendon_frictionloss +from .tendon import tendon_stiffness as tendon_stiffness + +# Camera. +# isort: split +from .camera import cam_fovy as cam_fovy +from .camera import cam_intrinsic as cam_intrinsic +from .camera import cam_pos as cam_pos +from .camera import cam_quat as cam_quat + +# Light. +# isort: split +from .light import light_ambient as light_ambient +from .light import light_attenuation as light_attenuation +from .light import light_cutoff as light_cutoff +from .light import light_diffuse as light_diffuse +from .light import light_dir as light_dir +from .light import light_exponent as light_exponent +from .light import light_pos as light_pos +from .light import light_specular as light_specular + +# Material. +# isort: split +from .material import mat_emission as mat_emission +from .material import mat_rgba as mat_rgba +from .material import mat_shininess as mat_shininess +from .material import mat_specular as mat_specular +from .material import mat_texid as mat_texid +from .material import mat_texrepeat as mat_texrepeat + +# Pair. +# isort: split +from .pair import pair_friction as pair_friction + +# Actuator. +# isort: split +from .actuator import effort_limits as effort_limits +from .actuator import pd_gains as pd_gains diff --git a/rl/mjlab/envs/mdp/dr/_core.py b/rl/mjlab/envs/mdp/dr/_core.py new file mode 100644 index 0000000..f2beff9 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/_core.py @@ -0,0 +1,446 @@ +"""Shared private engine for domain randomization.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +import torch + +from mjlab.entity import Entity, EntityIndexing +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.lab_api.math import quat_from_euler_xyz, quat_mul + +from ._types import ( + Distribution, + Operation, + resolve_distribution, + resolve_operation, +) + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + +Ranges = ( + tuple[float, float] | dict[int, tuple[float, float]] | dict[str, tuple[float, float]] +) + +_DEFAULT_ASSET_CFG = SceneEntityCfg("robot") + +# Mapping from entity_type to the names attribute on Entity / SceneEntityCfg. +_ENTITY_NAMES_ATTR: dict[str, str] = { + "dof": "joint_names", + "joint": "joint_names", + "body": "body_names", + "geom": "geom_names", + "site": "site_names", + "actuator": "actuator_names", + "tendon": "tendon_names", + "camera": "camera_names", + "light": "light_names", + "material": "material_names", + "texture": "texture_names", + "pair": "pair_names", +} + +# Private engine. + + +def _randomize_model_field( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + field: str, + *, + entity_type: str, + ranges: Ranges, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + axes: list[int] | None = None, + shared_random: bool = False, + default_axes: list[int] | None = None, + valid_axes: list[int] | None = None, + use_address: bool = False, +) -> torch.Tensor: + """Core randomization engine for model fields. Returns the written entity indices.""" + operation = resolve_operation(operation) + distribution = resolve_distribution(distribution) + + # Handle string-keyed ranges: resolve each pattern and recurse. + if isinstance(ranges, dict) and ranges and isinstance(next(iter(ranges)), str): + return _randomize_with_string_ranges( + env, + env_ids, + field, + entity_type=entity_type, + ranges=ranges, # type: ignore[arg-type] + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=default_axes, + valid_axes=valid_axes, + use_address=use_address, + ) + + # At this point, string-keyed ranges have been handled above. + narrowed_ranges: tuple[float, float] | dict[int, tuple[float, float]] = ranges # type: ignore[assignment] + + asset = env.scene[asset_cfg.name] + + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + entity_indices = _get_entity_indices( + asset.indexing, asset_cfg, entity_type, use_address + ) + + model_field = getattr(env.sim.model, field) + target_axes = _determine_target_axes( + model_field, axes, narrowed_ranges, default_axes, valid_axes + ) + axis_ranges = _prepare_axis_ranges(narrowed_ranges, target_axes, field) + + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + indexed_data = model_field[env_grid, entity_grid] + + if operation.uses_defaults: + base_values = _select_default_values(env, field, env_ids, entity_indices) + else: + base_values = indexed_data + + if shared_random: + single_entity_values = base_values[:, :1] + random_values = _generate_random_values( + distribution, + axis_ranges, + single_entity_values, + target_axes, + env.device, + operation, + ) + random_values = random_values.expand_as(base_values) + else: + random_values = _generate_random_values( + distribution, + axis_ranges, + base_values, + target_axes, + env.device, + operation, + ) + + combined = operation.combine(base_values, random_values) + + # Only write back the axes we targeted. Non-target axes in ``combined`` hold the + # operation identity applied to ``base_values`` (e.g. ``default * 1`` for scale), + # so writing the full slice would clobber modifications made by earlier events + # that targeted different axes of the same field. We loop with an integer axis + # index (basic indexing) rather than a device index tensor to avoid a + # per-call host-to-device copy; ``target_axes`` is tiny (at most 4). + if combined.ndim > 2: + for axis in target_axes: + model_field[env_grid, entity_grid, axis] = combined[..., axis] + else: + model_field[env_grid, entity_grid] = combined + return entity_indices + + +def _randomize_with_string_ranges( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + field: str, + *, + entity_type: str, + ranges: dict[str, tuple[float, float]], + distribution: Distribution | str, + operation: Operation | str, + asset_cfg: SceneEntityCfg, + axes: list[int] | None, + shared_random: bool, + default_axes: list[int] | None, + valid_axes: list[int] | None, + use_address: bool, +) -> torch.Tensor: + """Resolve string-keyed ranges to entity names and recurse.""" + asset: Entity = env.scene[asset_cfg.name] + names_attr = _ENTITY_NAMES_ATTR[entity_type] + all_names = list(getattr(asset, names_attr)) + + written: list[torch.Tensor] = [] + for pattern, range_val in ranges.items(): + matched = [n for n in all_names if re.fullmatch(pattern, n)] + if not matched: + raise ValueError( + f"Pattern '{pattern}' matched no {entity_type} names in " + f"entity '{asset_cfg.name}'. Available: {all_names}" + ) + sub_cfg = SceneEntityCfg(asset_cfg.name) + setattr(sub_cfg, names_attr, tuple(matched)) + sub_cfg.resolve(env.scene) + indices = _randomize_model_field( + env, + env_ids, + field, + entity_type=entity_type, + ranges=range_val, + distribution=distribution, + operation=operation, + asset_cfg=sub_cfg, + axes=axes, + shared_random=shared_random, + default_axes=default_axes, + valid_axes=valid_axes, + use_address=use_address, + ) + written.append(indices) + return torch.cat(written) + + +def _get_entity_indices( + indexing: EntityIndexing, + asset_cfg: SceneEntityCfg, + entity_type: str, + use_address: bool, +) -> torch.Tensor: + match entity_type: + case "dof": + return indexing.joint_v_adr[asset_cfg.joint_ids] + case "joint" if use_address: + return indexing.joint_q_adr[asset_cfg.joint_ids] + case "joint": + return indexing.joint_ids[asset_cfg.joint_ids] + case "body": + return indexing.body_ids[asset_cfg.body_ids] + case "geom": + return indexing.geom_ids[asset_cfg.geom_ids] + case "site": + return indexing.site_ids[asset_cfg.site_ids] + case "actuator": + assert indexing.ctrl_ids is not None + return indexing.ctrl_ids[asset_cfg.actuator_ids] + case "tendon": + return indexing.tendon_ids[asset_cfg.tendon_ids] + case "camera": + return indexing.cam_ids[asset_cfg.camera_ids] + case "light": + return indexing.light_ids[asset_cfg.light_ids] + case "material": + return indexing.mat_ids[asset_cfg.material_ids] + case "texture": + return indexing.tex_ids[asset_cfg.texture_ids] + case "pair": + return indexing.pair_ids[asset_cfg.pair_ids] + case _: + raise ValueError(f"Unknown entity type: {entity_type}") + + +def _determine_target_axes( + model_field: torch.Tensor, + axes: list[int] | None, + ranges: Ranges, + default_axes: list[int] | None, + valid_axes: list[int] | None, +) -> list[int]: + """Determine which axes to randomize.""" + field_ndim = len(model_field.shape) - 1 + + if axes is not None: + target_axes = axes + elif isinstance(ranges, dict) and ranges and isinstance(next(iter(ranges)), int): + target_axes = [k for k in ranges if isinstance(k, int)] + elif default_axes is not None: + target_axes = default_axes + else: + if field_ndim > 1: + target_axes = list(range(model_field.shape[-1])) + else: + target_axes = [0] + + if valid_axes is not None: + invalid_axes = set(target_axes) - set(valid_axes) + if invalid_axes: + raise ValueError( + f"Invalid axes {invalid_axes} for field. Valid axes: {valid_axes}" + ) + + return target_axes + + +def _prepare_axis_ranges( + ranges: tuple[float, float] | dict[int, tuple[float, float]], + target_axes: list[int], + field: str, +) -> dict[int, tuple[float, float]]: + """Convert ranges to a consistent dictionary format.""" + if isinstance(ranges, tuple): + return {axis: ranges for axis in target_axes} + missing_axes = set(target_axes) - set(ranges.keys()) + if missing_axes: + raise ValueError( + f"Missing ranges for axes {missing_axes} in field '{field}'. " + f"Required axes: {target_axes}" + ) + return {axis: ranges[axis] for axis in target_axes} + + +def _generate_random_values( + distribution: Distribution, + axis_ranges: dict[int, tuple[float, float]], + indexed_data: torch.Tensor, + target_axes: list[int], + device: str, + operation: Operation, +) -> torch.Tensor: + """Generate random values for the specified axes.""" + result = operation.initialize(indexed_data) + + for axis in target_axes: + lower, upper = axis_ranges[axis] + lower_bound = torch.tensor([lower], device=device) + upper_bound = torch.tensor([upper], device=device) + + if len(indexed_data.shape) > 2: + shape = (*indexed_data.shape[:-1], 1) + else: + shape = indexed_data.shape + + random_vals = distribution.sample(lower_bound, upper_bound, shape, device) + + if len(indexed_data.shape) > 2: + result[..., axis] = random_vals.squeeze(-1) + else: + result = random_vals + + return result + + +# Categorical engine. + + +def _randomize_categorical_field( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + field: str, + *, + entity_type: str, + pool: torch.Tensor, + asset_cfg: SceneEntityCfg, + shared_random: bool = False, + axis: int | None = None, +) -> None: + """Core engine for categorical fields: assign each entry a value from ``pool``.""" + asset = env.scene[asset_cfg.name] + + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + entity_indices = _get_entity_indices(asset.indexing, asset_cfg, entity_type, False) + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + + sample_shape = (env_ids.numel(), 1) if shared_random else env_grid.shape + pick = torch.randint(0, int(pool.numel()), sample_shape, device=env.device) + pick = pick.expand(env_grid.shape) + + model_field = getattr(env.sim.model, field) + if axis is None: + model_field[env_grid, entity_grid] = pool[pick] + else: + model_field[env_grid, entity_grid, axis] = pool[pick] + + +# Quaternion helpers. + + +def _sample_angle( + distribution: Distribution | str, + range_: tuple[float, float], + shape: tuple[int, ...], + device: str, +) -> torch.Tensor: + dist = resolve_distribution(distribution) + lower = torch.tensor(range_[0], device=device) + upper = torch.tensor(range_[1], device=device) + return dist.sample(lower, upper, shape, device) + + +def _randomize_quat_field( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + field: str, + *, + entity_type: str, + roll_range: tuple[float, float], + pitch_range: tuple[float, float], + yaw_range: tuple[float, float], + distribution: Distribution | str, + asset_cfg: SceneEntityCfg, +) -> None: + """Core implementation for quaternion randomization via RPY composition. + + Composes a sampled RPY perturbation with the default quaternion to + produce a valid unit quaternion. Axes with range ``(0.0, 0.0)`` sample + zero and leave that rotation component unchanged. + """ + asset = env.scene[asset_cfg.name] + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + entity_indices = _get_entity_indices(asset.indexing, asset_cfg, entity_type, False) + n_envs = len(env_ids) + n_entities = len(entity_indices) + shape = (n_envs, n_entities) + + roll = _sample_angle(distribution, roll_range, shape, env.device) + pitch = _sample_angle(distribution, pitch_range, shape, env.device) + yaw = _sample_angle(distribution, yaw_range, shape, env.device) + + # quat_from_euler_xyz requires 1-D inputs; reshape back after. + q_perturb = quat_from_euler_xyz( + roll.flatten(), pitch.flatten(), yaw.flatten() + ).reshape(n_envs, n_entities, 4) + + # Expand default to (n_envs, n_entities, 4) so quat_mul shapes match. + q_default_exp = _select_default_values( + env, field, env_ids, entity_indices + ).contiguous() + + q_new = quat_mul(q_perturb, q_default_exp) # (n_envs, n_entities, 4) + + model_field = getattr(env.sim.model, field) + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + model_field[env_grid, entity_grid] = q_new + + +def _select_default_values( + env, + field: str, + env_ids: torch.Tensor, + entity_indices: torch.Tensor, +) -> torch.Tensor: + """Return default model values indexed as ``(env, entity, ...)``. + + Standard model defaults are stored as ``(nentity, ...)`` and are shared by + every env. Per-world mesh compilation snapshots variant-dependent defaults as + ``(nworld, nentity, ...)`` so downstream DR must preserve each world's variant + baseline instead of indexing the first dimension as entities. + """ + default_field = env.sim.get_default_field(field) + if field in env.sim.per_world_default_fields: + assert default_field.shape[0] == env.num_envs, ( + f"Field '{field}' is registered as per-world but its default has " + f"shape {tuple(default_field.shape)}; expected leading dim " + f"{env.num_envs}." + ) + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + return default_field[env_grid, entity_grid] + + values = default_field[entity_indices].unsqueeze(0) + return values.expand((env_ids.numel(),) + values.shape[1:]) diff --git a/rl/mjlab/envs/mdp/dr/_types.py b/rl/mjlab/envs/mdp/dr/_types.py new file mode 100644 index 0000000..137d114 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/_types.py @@ -0,0 +1,165 @@ +"""Extensible Operation and Distribution types for domain randomization.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +import torch + +from mjlab.utils.lab_api.math import ( + sample_gaussian, + sample_log_uniform, + sample_uniform, +) + + +@dataclass(frozen=True) +class Operation: + """A domain randomization operation. + + Built-in operations (``abs``, ``scale``, ``add``) are provided as module-level + instances. Users can create custom operations by instantiating this class directly + and passing them wherever an operation string is accepted. + + The DR engine samples random values for each target axis, then computes:: + + combined = operation.combine(base, random) + + and writes back only the targeted axes of ``combined`` (so events targeting + different axes of the same field compose instead of clobbering each other). + ``base`` is either the compile-time default or the current model value, + depending on ``uses_defaults``. + + Args: + name: Human-readable identifier (used in error messages). + initialize: Called once per randomization call to create the result tensor that + gets filled axis by axis with sampled values. Receives the base values and + returns a tensor of the same shape. For example, ``scale`` starts from ones so + that unsampled axes multiply by 1 (no change), while ``add`` starts from zeros. + combine: Called after all axes have been sampled. Receives the base values and the + random values and returns the final tensor to write into the model field. For + example, ``scale`` returns ``base * random`` and ``add`` returns + ``base + random``. + uses_defaults: When ``True``, the engine reads the original default field values + captured at model compile time (preventing accumulation across repeated calls). + When ``False``, the engine reads the current model values instead. + """ + + name: str + initialize: Callable[[torch.Tensor], torch.Tensor] + combine: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] + uses_defaults: bool + + def __repr__(self) -> str: + return f"Operation({self.name!r})" + + +@dataclass(frozen=True) +class Distribution: + """A domain randomization sampling distribution. + + Built-in distributions (``uniform``, ``log_uniform``, ``gaussian``) are provided as + module-level instances. Users can create custom distributions by instantiating this + class directly. + + Args: + name: Human-readable identifier (used in error messages). + sample: Called for each target axis. Receives ``(lower, upper, shape, device)`` and + returns a tensor of random samples with the given shape on the given device. + ``lower`` and ``upper`` come from the ``ranges`` parameter passed to the DR + function. + """ + + name: str + sample: Callable[[torch.Tensor, torch.Tensor, tuple[int, ...], str], torch.Tensor] + + def __repr__(self) -> str: + return f"Distribution({self.name!r})" + + +# Built-in operations. + +abs = Operation( + name="abs", + initialize=torch.Tensor.clone, + combine=lambda base, random: random, + uses_defaults=False, +) +scale = Operation( + name="scale", + initialize=torch.ones_like, + combine=torch.mul, + uses_defaults=True, +) +add = Operation( + name="add", + initialize=torch.zeros_like, + combine=torch.add, + uses_defaults=True, +) + +# Built-in distributions. + +uniform = Distribution( + name="uniform", + sample=lambda lo, hi, shape, device: sample_uniform(lo, hi, shape, device=device), +) +log_uniform = Distribution( + name="log_uniform", + sample=lambda lo, hi, shape, device: sample_log_uniform(lo, hi, shape, device=device), +) +gaussian = Distribution( + name="gaussian", + sample=lambda lo, hi, shape, device: sample_gaussian(lo, hi, shape, device=device), +) + +# Resolution helpers. + +_OPERATION_REGISTRY: dict[str, Operation] = { + "abs": abs, + "scale": scale, + "add": add, +} + +_DISTRIBUTION_REGISTRY: dict[str, Distribution] = { + "uniform": uniform, + "log_uniform": log_uniform, + "gaussian": gaussian, +} + + +def resolve_operation(op: Operation | str) -> Operation: + """Resolve a string or Operation instance to an Operation. + + Raises: + ValueError: If the string does not match a built-in operation name. + """ + if isinstance(op, Operation): + return op + try: + return _OPERATION_REGISTRY[op] + except KeyError: + raise ValueError( + f"Unknown operation: {op!r}. " + f"Built-in operations: {list(_OPERATION_REGISTRY)}. " + f"Pass an Operation instance for custom operations." + ) from None + + +def resolve_distribution(dist: Distribution | str) -> Distribution: + """Resolve a string or Distribution instance to a Distribution. + + Raises: + ValueError: If the string does not match a built-in distribution name. + """ + if isinstance(dist, Distribution): + return dist + try: + return _DISTRIBUTION_REGISTRY[dist] + except KeyError: + raise ValueError( + f"Unknown distribution: {dist!r}. " + f"Built-in distributions: {list(_DISTRIBUTION_REGISTRY)}. " + f"Pass a Distribution instance for custom distributions." + ) from None diff --git a/rl/mjlab/envs/mdp/dr/actuator.py b/rl/mjlab/envs/mdp/dr/actuator.py new file mode 100644 index 0000000..09c0546 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/actuator.py @@ -0,0 +1,307 @@ +"""Domain randomization functions for actuators.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal + +import torch + +from mjlab.actuator import ( + BuiltinDcMotorActuator, + BuiltinPdActuator, + BuiltinPositionActuator, + IdealPdActuator, +) +from mjlab.actuator.actuator import TransmissionType +from mjlab.actuator.builtin_actuator import DcMotorInputMode +from mjlab.actuator.xml_actuator import XmlActuator +from mjlab.entity import Entity +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import _DEFAULT_ASSET_CFG +from ._types import Operation, resolve_distribution, resolve_operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("actuator_gainprm", "actuator_biasprm") +def pd_gains( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + kp_range: tuple[float, float], + kd_range: tuple[float, float], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Literal["uniform", "log_uniform"] = "uniform", + operation: Operation | str = "scale", +) -> None: + """Randomize PD stiffness and damping gains. + + Args: + env: The environment. + env_ids: Environment IDs to randomize. If None, randomizes all. + kp_range: (min, max) for proportional gain randomization. + kd_range: (min, max) for derivative gain randomization. + asset_cfg: Asset configuration specifying which entity and actuators. + distribution: Distribution type ("uniform" or "log_uniform"). + operation: "scale" multiplies default gains by sampled values, "abs" sets + absolute values. + """ + op = resolve_operation(operation) + if op.name not in ("scale", "abs"): + raise ValueError( + f"pd_gains only supports 'scale' and 'abs' operations, got {op.name!r}" + ) + asset: Entity = env.scene[asset_cfg.name] + + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + if isinstance(asset_cfg.actuator_ids, list): + actuators = [asset.actuators[i] for i in asset_cfg.actuator_ids] + elif isinstance(asset_cfg.actuator_ids, slice): + actuators = asset.actuators[asset_cfg.actuator_ids] + else: + actuators = [asset.actuators[asset_cfg.actuator_ids]] + + for actuator in actuators: + ctrl_ids = actuator.global_ctrl_ids + # Each target needs one kp draw and one kd draw. For single-element + # actuators that's len(ctrl_ids) of each; for BuiltinPd the ctrl tensor + # has 2*N entries but only N independent kp/kd values, so we sample + # num_targets to avoid throwing the other half away. + n_gains = ( + actuator.num_targets if isinstance(actuator, BuiltinPdActuator) else len(ctrl_ids) + ) + + dist = resolve_distribution(distribution) + kp_samples = dist.sample( + torch.tensor(kp_range[0], device=env.device), + torch.tensor(kp_range[1], device=env.device), + (len(env_ids), n_gains), + env.device, + ) + kd_samples = dist.sample( + torch.tensor(kd_range[0], device=env.device), + torch.tensor(kd_range[1], device=env.device), + (len(env_ids), n_gains), + env.device, + ) + + if isinstance(actuator, BuiltinPositionActuator) or ( + isinstance(actuator, XmlActuator) and actuator.command_field == "position" + ): + if op.name == "scale": + default_gainprm = env.sim.get_default_field("actuator_gainprm") + default_biasprm = env.sim.get_default_field("actuator_biasprm") + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 0] = ( + default_gainprm[ctrl_ids, 0] * kp_samples + ) + env.sim.model.actuator_biasprm[env_ids[:, None], ctrl_ids, 1] = ( + default_biasprm[ctrl_ids, 1] * kp_samples + ) + env.sim.model.actuator_biasprm[env_ids[:, None], ctrl_ids, 2] = ( + default_biasprm[ctrl_ids, 2] * kd_samples + ) + else: + assert op.name == "abs" + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 0] = kp_samples + env.sim.model.actuator_biasprm[env_ids[:, None], ctrl_ids, 1] = -kp_samples + env.sim.model.actuator_biasprm[env_ids[:, None], ctrl_ids, 2] = -kd_samples + + elif isinstance(actuator, BuiltinDcMotorActuator): + if actuator.cfg.mode == DcMotorInputMode.VOLTAGE: + raise ValueError( + "dr.pd_gains does not apply to BuiltinDcMotorActuator in VOLTAGE " + "mode (no internal PID gains to scale)." + ) + # DC motor stores kp at gainprm[4] and kd at gainprm[6] (set via + # set_to_dcmotor). The bias slots carry back-EMF / cogging, not the PD, + # so we only touch gainprm. + if op.name == "scale": + default_gainprm = env.sim.get_default_field("actuator_gainprm") + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 4] = ( + default_gainprm[ctrl_ids, 4] * kp_samples + ) + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 6] = ( + default_gainprm[ctrl_ids, 6] * kd_samples + ) + else: + assert op.name == "abs" + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 4] = kp_samples + env.sim.model.actuator_gainprm[env_ids[:, None], ctrl_ids, 6] = kd_samples + + elif isinstance(actuator, BuiltinPdActuator): + # ctrl_ids is laid out as [pos_0..pos_{N-1}, vel_0..vel_{N-1}], so the + # first N rows carry kp and the next N carry kd. + n = actuator.num_targets + pos_ids = ctrl_ids[:n] + vel_ids = ctrl_ids[n:] + if op.name == "scale": + default_gainprm = env.sim.get_default_field("actuator_gainprm") + default_biasprm = env.sim.get_default_field("actuator_biasprm") + env.sim.model.actuator_gainprm[env_ids[:, None], pos_ids, 0] = ( + default_gainprm[pos_ids, 0] * kp_samples + ) + env.sim.model.actuator_biasprm[env_ids[:, None], pos_ids, 1] = ( + default_biasprm[pos_ids, 1] * kp_samples + ) + env.sim.model.actuator_gainprm[env_ids[:, None], vel_ids, 0] = ( + default_gainprm[vel_ids, 0] * kd_samples + ) + env.sim.model.actuator_biasprm[env_ids[:, None], vel_ids, 2] = ( + default_biasprm[vel_ids, 2] * kd_samples + ) + else: + assert op.name == "abs" + env.sim.model.actuator_gainprm[env_ids[:, None], pos_ids, 0] = kp_samples + env.sim.model.actuator_biasprm[env_ids[:, None], pos_ids, 1] = -kp_samples + env.sim.model.actuator_gainprm[env_ids[:, None], vel_ids, 0] = kd_samples + env.sim.model.actuator_biasprm[env_ids[:, None], vel_ids, 2] = -kd_samples + # biasprm[2] on the position half stays zero by construction. Writing + # anything else here would inject damping into the position element on + # top of the velocity element, silently double-counting kd. + + elif isinstance(actuator, IdealPdActuator): + assert actuator.stiffness is not None + assert actuator.damping is not None + if op.name == "scale": + assert actuator.default_stiffness is not None + assert actuator.default_damping is not None + actuator.set_gains( + env_ids, + kp=actuator.default_stiffness[env_ids] * kp_samples, + kd=actuator.default_damping[env_ids] * kd_samples, + ) + else: + assert op.name == "abs" + actuator.set_gains(env_ids, kp=kp_samples, kd=kd_samples) + + else: + raise TypeError( + f"pd_gains only supports BuiltinPositionActuator, BuiltinPdActuator, " + f"BuiltinDcMotorActuator (position/velocity mode), XmlActuator (position), " + f"and IdealPdActuator, got {type(actuator).__name__}" + ) + + +@requires_model_fields("actuator_forcerange", "jnt_actfrcrange", "tendon_actfrcrange") +def effort_limits( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + effort_limit_range: tuple[float, float], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Literal["uniform", "log_uniform"] = "uniform", + operation: Operation | str = "scale", +) -> None: + """Randomize actuator effort limits. + + Args: + env: The environment. + env_ids: Environment IDs to randomize. If None, randomizes all. + effort_limit_range: (min, max) for effort limit randomization. + asset_cfg: Asset configuration specifying which entity and actuators. + distribution: Distribution type ("uniform" or "log_uniform"). + operation: "scale" multiplies default limits, "abs" sets absolute values. + """ + op = resolve_operation(operation) + if op.name not in ("scale", "abs"): + raise ValueError( + f"effort_limits only supports 'scale' and 'abs' operations, got {op.name!r}" + ) + asset: Entity = env.scene[asset_cfg.name] + + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + if isinstance(asset_cfg.actuator_ids, list): + actuators = [asset.actuators[i] for i in asset_cfg.actuator_ids] + else: + actuators = asset.actuators[asset_cfg.actuator_ids] + + if not isinstance(actuators, list): + actuators = [actuators] + + for actuator in actuators: + ctrl_ids = actuator.global_ctrl_ids + # One effort sample per target. For single-element actuators this matches + # ctrl_ids; for BuiltinPd the limit lives on the joint/tendon, so one + # sample per target is sufficient regardless of the two-element ctrl. + n_samples = ( + actuator.num_targets if isinstance(actuator, BuiltinPdActuator) else len(ctrl_ids) + ) + + dist = resolve_distribution(distribution) + effort_samples = dist.sample( + torch.tensor(effort_limit_range[0], device=env.device), + torch.tensor(effort_limit_range[1], device=env.device), + (len(env_ids), n_samples), + env.device, + ) + + if isinstance(actuator, (BuiltinPositionActuator, BuiltinDcMotorActuator)) or ( + isinstance(actuator, XmlActuator) and actuator.command_field == "position" + ): + if op.name == "scale": + default_forcerange = env.sim.get_default_field("actuator_forcerange") + env.sim.model.actuator_forcerange[env_ids[:, None], ctrl_ids, 0] = ( + default_forcerange[ctrl_ids, 0] * effort_samples + ) + env.sim.model.actuator_forcerange[env_ids[:, None], ctrl_ids, 1] = ( + default_forcerange[ctrl_ids, 1] * effort_samples + ) + else: + assert op.name == "abs" + env.sim.model.actuator_forcerange[ + env_ids[:, None], ctrl_ids, 0 + ] = -effort_samples + env.sim.model.actuator_forcerange[env_ids[:, None], ctrl_ids, 1] = ( + effort_samples + ) + + elif isinstance(actuator, IdealPdActuator): + assert actuator.force_limit is not None + if op.name == "scale": + assert actuator.default_force_limit is not None + actuator.set_effort_limit( + env_ids, + effort_limit=actuator.default_force_limit[env_ids] * effort_samples, + ) + else: + assert op.name == "abs" + actuator.set_effort_limit(env_ids, effort_limit=effort_samples) + + elif isinstance(actuator, BuiltinPdActuator): + # BuiltinPd's effort_limit lives on the joint/tendon as a sum-clamp + # (jnt_actfrcrange / tendon_actfrcrange), not on per-element forcerange. + if actuator.transmission_type == TransmissionType.JOINT: + field = "jnt_actfrcrange" + target_global_ids = asset.indexing.joint_ids[actuator.target_ids] + else: + field = "tendon_actfrcrange" + target_global_ids = asset.indexing.tendon_ids[actuator.target_ids] + arr = getattr(env.sim.model, field) + if op.name == "scale": + default = env.sim.get_default_field(field) + arr[env_ids[:, None], target_global_ids, 0] = ( + default[target_global_ids, 0] * effort_samples + ) + arr[env_ids[:, None], target_global_ids, 1] = ( + default[target_global_ids, 1] * effort_samples + ) + else: + assert op.name == "abs" + arr[env_ids[:, None], target_global_ids, 0] = -effort_samples + arr[env_ids[:, None], target_global_ids, 1] = effort_samples + + else: + raise TypeError( + f"effort_limits only supports BuiltinPositionActuator, BuiltinPdActuator, " + f"BuiltinDcMotorActuator, XmlActuator (position), and IdealPdActuator, " + f"got {type(actuator).__name__}" + ) diff --git a/rl/mjlab/envs/mdp/dr/body.py b/rl/mjlab/envs/mdp/dr/body.py new file mode 100644 index 0000000..ab22666 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/body.py @@ -0,0 +1,558 @@ +"""Domain randomization functions for body fields.""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import RecomputeLevel, requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.lab_api.math import ( + matrix_from_quat, + quat_from_matrix, +) + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _get_entity_indices, + _randomize_model_field, + _randomize_quat_field, + _sample_angle, + _select_default_values, +) +from ._types import Distribution, Operation, log_uniform, resolve_distribution + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + +# Number of Jacobi sweeps for 3x3 eigendecomposition. Each sweep applies 3 Givens +# rotations (one per off-diagonal pair). 5 sweeps are more than enough for 3x3 matrices +# to converge to machine precision. +_JACOBI_SWEEPS = 5 + +# Pseudo-inertia helpers. + + +def _cholesky_4x4(A: torch.Tensor) -> torch.Tensor: + """Analytical Cholesky for batched 4x4 SPD matrices. + + Avoids ``torch.linalg.cholesky`` (and the cuSOLVER library it loads), which allocates + several GB of persistent GPU memory on first use. + + Args: + A: ``(*batch, 4, 4)`` symmetric positive-definite matrix. + + Returns: + L: ``(*batch, 4, 4)`` lower-triangular Cholesky factor. + """ + L = torch.zeros_like(A) + L[..., 0, 0] = torch.sqrt(A[..., 0, 0]) + L[..., 1, 0] = A[..., 1, 0] / L[..., 0, 0] + L[..., 2, 0] = A[..., 2, 0] / L[..., 0, 0] + L[..., 3, 0] = A[..., 3, 0] / L[..., 0, 0] + L[..., 1, 1] = torch.sqrt(A[..., 1, 1] - L[..., 1, 0] ** 2) + L[..., 2, 1] = (A[..., 2, 1] - L[..., 2, 0] * L[..., 1, 0]) / L[..., 1, 1] + L[..., 3, 1] = (A[..., 3, 1] - L[..., 3, 0] * L[..., 1, 0]) / L[..., 1, 1] + L[..., 2, 2] = torch.sqrt(A[..., 2, 2] - L[..., 2, 0] ** 2 - L[..., 2, 1] ** 2) + L[..., 3, 2] = ( + A[..., 3, 2] - L[..., 3, 0] * L[..., 2, 0] - L[..., 3, 1] * L[..., 2, 1] + ) / L[..., 2, 2] + L[..., 3, 3] = torch.sqrt( + A[..., 3, 3] - L[..., 3, 0] ** 2 - L[..., 3, 1] ** 2 - L[..., 3, 2] ** 2 + ) + return L + + +def _eigh_3x3_jacobi( + A: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Batched eigendecomposition for 3x3 symmetric matrices via Jacobi. + + Avoids ``torch.linalg.eigh`` (and the cuSOLVER library it loads), which allocates + several GB of persistent GPU memory on first use. + + Uses cyclic Jacobi rotations over the three off-diagonal pairs. For 3x3 matrices, + ``_JACOBI_SWEEPS`` sweeps converge to machine precision. + + Args: + A: ``(*batch, 3, 3)`` symmetric matrix. + + Returns: + eigenvalues: ``(*batch, 3)`` in ascending order. + V: ``(*batch, 3, 3)`` orthogonal eigenvectors (columns). + """ + D = A.clone() + V = torch.eye(3, device=A.device, dtype=A.dtype).expand_as(D).clone() + + for _ in range(_JACOBI_SWEEPS): + for p, q in ((0, 1), (0, 2), (1, 2)): + r = 3 - p - q + apq = D[..., p, q] + app = D[..., p, p] + aqq = D[..., q, q] + + # Jacobi rotation: tau = (aqq - app) / (2*apq), + # t = sign(tau) / (|tau| + sqrt(1 + tau^2)), with sign(0) = 1 so that + # equal diagonal entries get a 45 degree rotation. + diff = aqq - app + denom = 2 * apq + tau = diff / denom + sign = torch.where(tau >= 0, 1.0, -1.0) + t = sign / (torch.abs(tau) + torch.sqrt(1 + tau * tau)) + # When apq ≈ 0, skip rotation (t = 0). + t = torch.where(torch.abs(denom) > 1e-30, t, torch.zeros_like(t)) + + c = 1 / torch.sqrt(1 + t * t) + s = t * c + + # Update diagonal and off-diagonal elements. + D[..., p, p] = app - t * apq + D[..., q, q] = aqq + t * apq + D[..., p, q] = D[..., q, p] = 0 + + arp = D[..., r, p].clone() + arq = D[..., r, q].clone() + D[..., r, p] = D[..., p, r] = c * arp - s * arq + D[..., r, q] = D[..., q, r] = s * arp + c * arq + + # Accumulate eigenvectors: V[:, p/q] = c*Vp ∓ s*Vq. + vp = V[..., :, p].clone() + vq = V[..., :, q].clone() + c_exp = c.unsqueeze(-1) + s_exp = s.unsqueeze(-1) + V[..., :, p] = c_exp * vp - s_exp * vq + V[..., :, q] = s_exp * vp + c_exp * vq + + eigenvalues = torch.stack([D[..., 0, 0], D[..., 1, 1], D[..., 2, 2]], dim=-1) + + # Sort in ascending order to match torch.linalg.eigh convention. + idx = eigenvalues.argsort(dim=-1) + eigenvalues = eigenvalues.gather(-1, idx) + V = V.gather(-1, idx.unsqueeze(-2).expand_as(V)) + + return eigenvalues, V + + +def _reconstruct_pseudo_inertia_J( + mass: torch.Tensor, + ipos: torch.Tensor, + inertia: torch.Tensor, + iquat: torch.Tensor, +) -> torch.Tensor: + """Build the 4x4 pseudo-inertia matrix J from MuJoCo body fields. + + 1. Rotate principal moments into body frame via ``body_iquat``. + 2. Apply the parallel-axis theorem to shift inertia from COM to body origin. + + Args: + mass: ``(*batch,)``. + ipos: COM in body frame, ``(*batch, 3)``. + inertia: Principal moments, ``(*batch, 3)``. + iquat: Principal-to-body quaternion (wxyz), ``(*batch, 4)``. + + Returns: + J: ``(*batch, 4, 4)``. + """ + I3 = torch.eye(3, device=mass.device, dtype=mass.dtype) + + # Rotate principal moments into body frame (body_iquat maps principal->body). + R = matrix_from_quat(iquat) # (*batch, 3, 3) + I_com = R @ torch.diag_embed(inertia) @ R.mT # (*batch, 3, 3) + + # Parallel-axis theorem: shift inertia from COM to body origin. + c = ipos + c_sq = (c * c).sum(dim=-1) # (*batch,) + c_outer = c.unsqueeze(-1) * c.unsqueeze(-2) # (*batch, 3, 3) + m = mass.unsqueeze(-1).unsqueeze(-1) # (*batch, 1, 1) + I_origin = I_com + m * ( + c_sq.unsqueeze(-1).unsqueeze(-1) * I3 - c_outer + ) # (*batch, 3, 3) + + # sigma = 0.5 * Tr(I_origin) * I3 - I_origin + trace = I_origin.diagonal(dim1=-2, dim2=-1).sum(dim=-1) # (*batch,) + sigma = 0.5 * trace.unsqueeze(-1).unsqueeze(-1) * I3 - I_origin + + h = mass.unsqueeze(-1) * ipos # first mass moment + + batch_shape = mass.shape + J = torch.zeros(*batch_shape, 4, 4, device=mass.device, dtype=mass.dtype) + J[..., :3, :3] = sigma + J[..., :3, 3] = h + J[..., 3, :3] = h + J[..., 3, 3] = mass + return J + + +def _decompose_pseudo_inertia_J( + J: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Decompose pseudo-inertia matrix to MuJoCo body fields (exact). + + Extracts ``body_mass``, ``body_ipos``, ``body_inertia`` (principal moments), and + ``body_iquat`` (principal-frame rotation) by diagonalizing the full inertia tensor + via eigendecomposition. This is exact for any perturbation magnitude, including large + shear. + + Args: + J: 4x4 pseudo-inertia matrix, shape ``(*batch, 4, 4)``. + + Returns: + Tuple of (mass, ipos, inertia, iquat) with shapes ``(*batch,)``, + ``(*batch, 3)``, ``(*batch, 3)``, ``(*batch, 4)``. + """ + mass = J[..., 3, 3] + h = J[..., :3, 3] + ipos = h / mass.unsqueeze(-1) + + sigma = J[..., :3, :3] + trace_sigma = sigma.diagonal(dim1=-2, dim2=-1).sum(dim=-1) # (*batch,) + I3 = torch.eye(3, device=J.device, dtype=J.dtype) + # Invert sigma = 0.5*Tr(I)*I3 - I => I_origin = Tr(sigma)*I3 - sigma + I_origin = trace_sigma.unsqueeze(-1).unsqueeze(-1) * I3 - sigma # (*batch, 3, 3) + + # Inverse parallel-axis: shift inertia from body origin back to COM. + c = ipos + c_sq = (c * c).sum(dim=-1) # (*batch,) + c_outer = c.unsqueeze(-1) * c.unsqueeze(-2) # (*batch, 3, 3) + m = mass.unsqueeze(-1).unsqueeze(-1) # (*batch, 1, 1) + I_com = I_origin - m * ( + c_sq.unsqueeze(-1).unsqueeze(-1) * I3 - c_outer + ) # (*batch, 3, 3) + + # Columns of V are principal axes in body frame; eigenvalues are principal moments. + principal_moments, V = _eigh_3x3_jacobi(I_com) + + # Ensure V is a proper rotation (det = +1). Eigenvector signs are arbitrary, so + # rebuild the third axis from the first two. + V[..., :, 2] = torch.linalg.cross(V[..., :, 0], V[..., :, 1]) + + # MuJoCo body_iquat is principal->body, i.e. it represents R = V. + iquat = quat_from_matrix(V) # (*batch, 4), wxyz + + return mass, ipos, principal_moments, iquat + + +def _build_perturbation_U( + alpha: torch.Tensor, + d1: torch.Tensor, + d2: torch.Tensor, + d3: torch.Tensor, + s12: torch.Tensor, + s13: torch.Tensor, + s23: torch.Tensor, + t1: torch.Tensor, + t2: torch.Tensor, + t3: torch.Tensor, +) -> torch.Tensor: + """Build the upper-triangular perturbation matrix U from 10 parameters. + + .. code-block:: + + U = e^alpha * [[e^d1, s12, s13, t1], + [0, e^d2, s23, t2], + [0, 0, e^d3, t3], + [0, 0, 0, 1 ]] + + All arguments have shape ``(*batch,)``. + + Returns: + U: shape ``(*batch, 4, 4)``. + """ + scale = torch.exp(alpha) # (*batch,) + batch_shape = alpha.shape + U = torch.zeros(*batch_shape, 4, 4, device=alpha.device, dtype=alpha.dtype) + U[..., 0, 0] = scale * torch.exp(d1) + U[..., 0, 1] = scale * s12 + U[..., 0, 2] = scale * s13 + U[..., 0, 3] = scale * t1 + U[..., 1, 1] = scale * torch.exp(d2) + U[..., 1, 2] = scale * s23 + U[..., 1, 3] = scale * t2 + U[..., 2, 2] = scale * torch.exp(d3) + U[..., 2, 3] = scale * t3 + U[..., 3, 3] = scale # e^alpha × 1 + return U + + +# Per-field functions. + + +@requires_model_fields("body_mass", recompute=RecomputeLevel.set_const) +def body_mass( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "scale", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize body mass. Triggers ``set_const`` recomputation. + + .. warning:: + + This function only changes ``body_mass`` and leaves ``body_inertia`` + unchanged. For a uniform density change (the typical DR use case), + inertia should scale proportionally with mass. Use + :func:`pseudo_inertia` with ``alpha_range`` instead, which scales both + correctly. ``body_mass`` alone is only appropriate when modelling a + point mass added at the COM (which contributes zero inertia). + """ + warnings.warn( + "dr.body_mass only randomizes mass and leaves the inertia tensor " + "unchanged. For a physically consistent density change, use " + "dr.pseudo_inertia(alpha_range=...) instead, which scales both mass " + "and inertia together. dr.body_mass is only appropriate when modelling " + "a point mass added at the COM.", + UserWarning, + stacklevel=2, + ) + _randomize_model_field( + env, + env_ids, + "body_mass", + entity_type="body", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("body_ipos", recompute=RecomputeLevel.set_const) +def body_com_offset( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "add", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize body COM offset (body_ipos). Triggers ``set_const``.""" + _randomize_model_field( + env, + env_ids, + "body_ipos", + entity_type="body", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +# Raw alias. +body_ipos = body_com_offset + + +@requires_model_fields("body_pos", recompute=RecomputeLevel.set_const_0) +def body_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "add", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize body position. Triggers ``set_const_0``.""" + _randomize_model_field( + env, + env_ids, + "body_pos", + entity_type="body", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("body_quat", recompute=RecomputeLevel.set_const_0) +def body_quat( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + roll_range: tuple[float, float] = (0.0, 0.0), + pitch_range: tuple[float, float] = (0.0, 0.0), + yaw_range: tuple[float, float] = (0.0, 0.0), + distribution: Distribution | str = "uniform", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Randomize body orientation by composing an RPY perturbation. + + Ranges are in radians. The sampled perturbation is composed with the default + quaternion (not the current one), so repeated calls do not accumulate. The result is + always a valid unit quaternion. Triggers ``set_const_0`` recomputation. + """ + _randomize_quat_field( + env, + env_ids, + "body_quat", + entity_type="body", + roll_range=roll_range, + pitch_range=pitch_range, + yaw_range=yaw_range, + distribution=distribution, + asset_cfg=asset_cfg, + ) + + +@requires_model_fields( + "body_mass", + "body_ipos", + "body_inertia", + "body_iquat", + recompute=RecomputeLevel.set_const, +) +def pseudo_inertia( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + alpha_range: tuple[float, float] = (0.0, 0.0), + d_range: tuple[float, float] | None = None, + d1_range: tuple[float, float] = (0.0, 0.0), + d2_range: tuple[float, float] = (0.0, 0.0), + d3_range: tuple[float, float] = (0.0, 0.0), + s12_range: tuple[float, float] = (0.0, 0.0), + s13_range: tuple[float, float] = (0.0, 0.0), + s23_range: tuple[float, float] = (0.0, 0.0), + t_range: tuple[float, float] | None = None, + t1_range: tuple[float, float] = (0.0, 0.0), + t2_range: tuple[float, float] = (0.0, 0.0), + t3_range: tuple[float, float] = (0.0, 0.0), + distribution: Distribution | str = "uniform", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + r"""Physics-consistent inertial randomization via the pseudo-inertia matrix. + + Jointly randomizes ``body_mass``, ``body_ipos``, ``body_inertia``, and ``body_iquat`` + while guaranteeing exact physical consistency for any perturbation magnitude. + Triggers ``set_const`` recomputation. + + The parameterization follows `Rucker & Wensing, 2022 + `_: the pseudo-inertia matrix + :math:`J \succ 0` is factored via + Cholesky as :math:`J = LL^\top`, then perturbed by an upper-triangular matrix + U: :math:`J' = (UL)(UL)^\top`. The result is diagonalized via eigendecomposition to + extract principal moments (``body_inertia``) and principal frame rotation + (``body_iquat``), so it is exact for any perturbation magnitude. + + Bodies without a positive-definite pseudo-inertia, such as massless bodies, have + nothing to randomize and keep their default values. + + The 10 parameters and their physical effects: + + - ``alpha``: global mass-density scale — mass and inertia scale by + :math:`e^{2\alpha}`, COM unchanged. + - ``d1, d2, d3``: axis-aligned stretch/compress. Use ``d_range`` as a convenience to + set all three to the same range. + - ``s12, s13, s23``: shear in the xy, xz, and yz planes. + - ``t1, t2, t3``: COM shift along x, y, z axes (in body frame). Use ``t_range`` as a + convenience to set all three to the same range. + + Args: + env: The RL environment. + env_ids: Environment indices to randomize. If ``None``, all envs. + alpha_range: Range for global mass-density log-scale. + d_range: Convenience shorthand — sets ``d1_range=d2_range=d3_range``. + d1_range: Stretch/compress along the x axis. + d2_range: Stretch/compress along the y axis. + d3_range: Stretch/compress along the z axis. + s12_range: Shear in the xy plane. + s13_range: Shear in the xz plane. + s23_range: Shear in the yz plane. + t_range: Convenience shorthand — sets ``t1_range=t2_range=t3_range``. + t1_range: COM shift along the x axis (body frame). + t2_range: COM shift along the y axis (body frame). + t3_range: COM shift along the z axis (body frame). + distribution: Sampling distribution for all parameters. ``log_uniform`` is not + supported because the parameters are log-scale offsets centered on zero. + asset_cfg: Asset and body selection. + """ + if resolve_distribution(distribution) is log_uniform: + raise ValueError( + "dr.pseudo_inertia does not support the log_uniform distribution: its " + "parameters are already log-scale offsets centered on zero. Use uniform " + "or gaussian instead." + ) + if d_range is not None: + d1_range = d2_range = d3_range = d_range + if t_range is not None: + t1_range = t2_range = t3_range = t_range + + asset = env.scene[asset_cfg.name] + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + entity_indices = _get_entity_indices(asset.indexing, asset_cfg, "body", False) + n_envs = len(env_ids) + n_bodies = len(entity_indices) + shape = (n_envs, n_bodies) + + def_mass = _select_default_values(env, "body_mass", env_ids, entity_indices) + def_ipos = _select_default_values(env, "body_ipos", env_ids, entity_indices) + def_inertia = _select_default_values(env, "body_inertia", env_ids, entity_indices) + def_iquat = _select_default_values(env, "body_iquat", env_ids, entity_indices) + + # Reconstruct J_default for each env/body: (n_envs, n_bodies, 4, 4). + J_default = _reconstruct_pseudo_inertia_J(def_mass, def_ipos, def_inertia, def_iquat) + + # Cholesky factor L: (n_envs, n_bodies, 4, 4), lower triangular. + L = _cholesky_4x4(J_default) + # A non-positive (or NaN) pivot means J is not positive definite. + valid = (L.diagonal(dim1=-2, dim2=-1) > 0).all(dim=-1) # (n_envs, n_bodies) + + # Sample perturbation parameters, each (n_envs, n_bodies). + def sample(r: tuple[float, float]) -> torch.Tensor: + return _sample_angle(distribution, r, shape, env.device) + + alpha = sample(alpha_range) + d1 = sample(d1_range) + d2 = sample(d2_range) + d3 = sample(d3_range) + s12 = sample(s12_range) + s13 = sample(s13_range) + s23 = sample(s23_range) + t1 = sample(t1_range) + t2 = sample(t2_range) + t3 = sample(t3_range) + + # Build U: (n_envs, n_bodies, 4, 4), upper triangular. + U = _build_perturbation_U(alpha, d1, d2, d3, s12, s13, s23, t1, t2, t3) + + # L_new = U @ L. + L_new = torch.matmul(U, L) # (n_envs, n_bodies, 4, 4) + + # New pseudo-inertia: J' = L_new @ L_newᵀ. + J_new = torch.matmul(L_new, L_new.mT) # (n_envs, n_bodies, 4, 4) + + # Decompose back to MuJoCo fields via eigendecomposition (exact). + mass_new, ipos_new, inertia_new, iquat_new = _decompose_pseudo_inertia_J(J_new) + + mass_new = torch.where(valid, mass_new, def_mass) + valid = valid.unsqueeze(-1) + ipos_new = torch.where(valid, ipos_new, def_ipos) + inertia_new = torch.where(valid, inertia_new, def_inertia) + iquat_new = torch.where(valid, iquat_new, def_iquat) + + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + env.sim.model.body_mass[env_grid, entity_grid] = mass_new + env.sim.model.body_ipos[env_grid, entity_grid] = ipos_new + env.sim.model.body_inertia[env_grid, entity_grid] = inertia_new + env.sim.model.body_iquat[env_grid, entity_grid] = iquat_new diff --git a/rl/mjlab/envs/mdp/dr/camera.py b/rl/mjlab/envs/mdp/dr/camera.py new file mode 100644 index 0000000..69bcb32 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/camera.py @@ -0,0 +1,130 @@ +"""Domain randomization functions for camera fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _randomize_model_field, + _randomize_quat_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("cam_fovy") +def cam_fovy( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize camera field-of-view (vertical, degrees).""" + _randomize_model_field( + env, + env_ids, + "cam_fovy", + entity_type="camera", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("cam_pos") +def cam_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize camera positions.""" + _randomize_model_field( + env, + env_ids, + "cam_pos", + entity_type="camera", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("cam_quat") +def cam_quat( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + roll_range: tuple[float, float] = (0.0, 0.0), + pitch_range: tuple[float, float] = (0.0, 0.0), + yaw_range: tuple[float, float] = (0.0, 0.0), + distribution: Distribution | str = "uniform", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Randomize camera orientation by composing an RPY perturbation. + + Ranges are in radians. The sampled perturbation is composed with the default + quaternion (not the current one), so repeated calls do not accumulate. The result is + always a valid unit quaternion. + """ + _randomize_quat_field( + env, + env_ids, + "cam_quat", + entity_type="camera", + roll_range=roll_range, + pitch_range=pitch_range, + yaw_range=yaw_range, + distribution=distribution, + asset_cfg=asset_cfg, + ) + + +@requires_model_fields("cam_intrinsic") +def cam_intrinsic( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize camera intrinsic parameters.""" + _randomize_model_field( + env, + env_ids, + "cam_intrinsic", + entity_type="camera", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2, 3], + ) diff --git a/rl/mjlab/envs/mdp/dr/geom.py b/rl/mjlab/envs/mdp/dr/geom.py new file mode 100644 index 0000000..a4f4fde --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/geom.py @@ -0,0 +1,323 @@ +"""Domain randomization functions for geom fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import mujoco +import torch + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _get_entity_indices, + _randomize_categorical_field, + _randomize_model_field, + _randomize_quat_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + +# MuJoCo geom type integer constants. +_GEOM_SPHERE = mujoco.mjtGeom.mjGEOM_SPHERE.value +_GEOM_CAPSULE = mujoco.mjtGeom.mjGEOM_CAPSULE.value +_GEOM_ELLIPSOID = mujoco.mjtGeom.mjGEOM_ELLIPSOID.value +_GEOM_CYLINDER = mujoco.mjtGeom.mjGEOM_CYLINDER.value +_GEOM_BOX = mujoco.mjtGeom.mjGEOM_BOX.value + + +def _recompute_geom_bounds( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor, + entity_indices: torch.Tensor, +) -> None: + """Recompute ``geom_rbound`` and ``geom_aabb`` from current ``geom_size``. + + ``entity_indices`` are global geom ids. Only primitive types (sphere, capsule, + ellipsoid, cylinder, box) are handled. Plane, hfield, mesh, and SDF geoms are left + unchanged. + """ + env_grid, entity_grid = torch.meshgrid(env_ids, entity_indices, indexing="ij") + size = env.sim.model.geom_size[env_grid, entity_grid] # (n_envs, n_geoms, 3) + s0, s1, s2 = size[..., 0], size[..., 1], size[..., 2] + + # geom_type is (ngeom,) int, not per-world. + geom_type_all = torch.as_tensor(env.sim.model.geom_type, device=env.device) + gtype = geom_type_all[entity_indices] # (G,) int + + # Compute rbound per type via torch.where cascades. + rbound = torch.zeros_like(s0) + + is_sphere = gtype[None, :] == _GEOM_SPHERE + is_capsule = gtype[None, :] == _GEOM_CAPSULE + is_ellipsoid = gtype[None, :] == _GEOM_ELLIPSOID + is_cylinder = gtype[None, :] == _GEOM_CYLINDER + is_box = gtype[None, :] == _GEOM_BOX + + rbound = torch.where(is_sphere, s0, rbound) + rbound = torch.where(is_capsule, s0 + s1, rbound) + rbound = torch.where(is_ellipsoid, torch.maximum(s0, torch.maximum(s1, s2)), rbound) + rbound = torch.where(is_cylinder, torch.sqrt(s0 * s0 + s1 * s1), rbound) + rbound = torch.where(is_box, torch.sqrt(s0 * s0 + s1 * s1 + s2 * s2), rbound) + + # Compute aabb half-sizes per type. + aabb_half_x = torch.zeros_like(s0) + aabb_half_y = torch.zeros_like(s0) + aabb_half_z = torch.zeros_like(s0) + + # Sphere: (s0, s0, s0) + aabb_half_x = torch.where(is_sphere, s0, aabb_half_x) + aabb_half_y = torch.where(is_sphere, s0, aabb_half_y) + aabb_half_z = torch.where(is_sphere, s0, aabb_half_z) + # Capsule: (s0, s0, s0+s1) + aabb_half_x = torch.where(is_capsule, s0, aabb_half_x) + aabb_half_y = torch.where(is_capsule, s0, aabb_half_y) + aabb_half_z = torch.where(is_capsule, s0 + s1, aabb_half_z) + # Cylinder: (s0, s0, s1) + aabb_half_x = torch.where(is_cylinder, s0, aabb_half_x) + aabb_half_y = torch.where(is_cylinder, s0, aabb_half_y) + aabb_half_z = torch.where(is_cylinder, s1, aabb_half_z) + # Ellipsoid: (s0, s1, s2) + aabb_half_x = torch.where(is_ellipsoid, s0, aabb_half_x) + aabb_half_y = torch.where(is_ellipsoid, s1, aabb_half_y) + aabb_half_z = torch.where(is_ellipsoid, s2, aabb_half_z) + # Box: (s0, s1, s2) + aabb_half_x = torch.where(is_box, s0, aabb_half_x) + aabb_half_y = torch.where(is_box, s1, aabb_half_y) + aabb_half_z = torch.where(is_box, s2, aabb_half_z) + + is_supported = is_sphere | is_capsule | is_ellipsoid | is_cylinder | is_box + + if not is_supported.all(): + unsupported = gtype[~is_supported[0]] + names = [mujoco.mjtGeom(t.item()).name for t in unsupported] + raise ValueError( + f"dr.geom_size only supports primitive geom types (sphere, capsule, " + f"ellipsoid, cylinder, box). Got unsupported types: {names}" + ) + + env.sim.model.geom_rbound[env_grid, entity_grid] = rbound + + # aabb shape is (*, ngeom, 2, 3) — [0] is center, [1] is half-size. + # Center stays (0,0,0) for all primitives; only update half-size. + aabb_half = torch.stack([aabb_half_x, aabb_half_y, aabb_half_z], dim=-1) + env.sim.model.geom_aabb[env_grid, entity_grid, 1] = aabb_half + + +# Per-field functions. + + +@requires_model_fields("geom_friction") +def geom_friction( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize geom friction. + + Default axis is 0 (tangential friction only). Axes 1 (torsional) and 2 (rolling) only + affect contacts with ``condim >= 4``; pass ``axes=[0, 1, 2]`` explicitly if your + model uses high-dimensional contacts. + """ + _randomize_model_field( + env, + env_ids, + "geom_friction", + entity_type="geom", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0], + valid_axes=[0, 1, 2], + ) + + +@requires_model_fields("geom_pos") +def geom_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize geom positions.""" + _randomize_model_field( + env, + env_ids, + "geom_pos", + entity_type="geom", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("geom_quat") +def geom_quat( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + roll_range: tuple[float, float] = (0.0, 0.0), + pitch_range: tuple[float, float] = (0.0, 0.0), + yaw_range: tuple[float, float] = (0.0, 0.0), + distribution: Distribution | str = "uniform", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Randomize geom orientation by composing an RPY perturbation. + + Ranges are in radians. The sampled perturbation is composed with the default + quaternion (not the current one), so repeated calls do not accumulate. The result is + always a valid unit quaternion. + """ + _randomize_quat_field( + env, + env_ids, + "geom_quat", + entity_type="geom", + roll_range=roll_range, + pitch_range=pitch_range, + yaw_range=yaw_range, + distribution=distribution, + asset_cfg=asset_cfg, + ) + + +@requires_model_fields("geom_rgba") +def geom_rgba( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize geom RGBA colors. + + Only affects rendering when the geom has no material assigned. If a material is + assigned, its color (or texture) takes precedence over ``geom_rgba``. + Use :func:`~mjlab.envs.mdp.dr.mat_rgba` instead, or clear the material assignment + first. + + Args: + env: The environment instance. + env_ids: Environment indices to randomize. ``None`` means all. + ranges: Value range(s) for sampling. + asset_cfg: Entity and geom selection. + distribution: Sampling distribution. + operation: How to combine sampled values with the base. + axes: Which RGBA channels to randomize. Defaults to ``[0, 1, 2, 3]``. + shared_random: If ``True``, all selected geoms receive the same sampled value per + environment. + """ + _randomize_model_field( + env, + env_ids, + "geom_rgba", + entity_type="geom", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2, 3], + ) + + +@requires_model_fields("geom_size", "geom_rbound", "geom_aabb") +def geom_size( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "scale", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize geom size and recompute broadphase bounds. + + After writing new values to ``geom_size``, this function automatically recomputes + ``geom_rbound`` (bounding sphere) and ``geom_aabb`` (local bounding box) so that the + broadphase remains consistent. Only primitive geom types (sphere, capsule, ellipsoid, + cylinder, box) are supported. + """ + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + geom_ids = _randomize_model_field( + env, + env_ids, + "geom_size", + entity_type="geom", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + _recompute_geom_bounds(env, env_ids, geom_ids) + + +@requires_model_fields("geom_matid") +def geom_matid( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + shared_random: bool = False, +) -> None: + """Randomize which baked material each selected geom uses. + + Each geom is assigned a material drawn uniformly from ``asset_cfg.material_names``. + Assigning a material to a geom that had none (``matid < 0``) switches its color + source from ``geom_rgba`` to the material's ``mat_rgba``/texture. + + Args: + env: The environment instance. + env_ids: Environment indices to randomize. ``None`` means all. + asset_cfg: Entity, geom, and material selection. + shared_random: If ``True``, all selected geoms receive the same sampled value per + environment. + """ + asset = env.scene[asset_cfg.name] + mat_ids = _get_entity_indices(asset.indexing, asset_cfg, "material", False) + if mat_ids.numel() == 0: + raise ValueError( + f"No materials selected for entity '{asset_cfg.name}'. Set " + "SceneEntityCfg.material_names to a non-empty selection." + ) + + _randomize_categorical_field( + env, + env_ids, + "geom_matid", + entity_type="geom", + pool=mat_ids, + asset_cfg=asset_cfg, + shared_random=shared_random, + ) diff --git a/rl/mjlab/envs/mdp/dr/joint.py b/rl/mjlab/envs/mdp/dr/joint.py new file mode 100644 index 0000000..1bfe8a2 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/joint.py @@ -0,0 +1,241 @@ +"""Domain randomization functions for joint and DOF fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.entity import Entity +from mjlab.managers.event_manager import RecomputeLevel, requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.lab_api.math import sample_uniform + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _randomize_model_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("dof_damping") +def joint_damping( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize joint damping (dof_damping).""" + _randomize_model_field( + env, + env_ids, + "dof_damping", + entity_type="dof", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + use_address=True, + ) + + +# Raw alias. +dof_damping = joint_damping + + +@requires_model_fields("dof_armature", recompute=RecomputeLevel.set_const_0) +def joint_armature( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize joint armature. Triggers ``set_const_0``.""" + _randomize_model_field( + env, + env_ids, + "dof_armature", + entity_type="dof", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + use_address=True, + ) + + +# Raw alias. +dof_armature = joint_armature + + +@requires_model_fields("dof_frictionloss") +def joint_friction( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize joint friction loss (dof_frictionloss).""" + _randomize_model_field( + env, + env_ids, + "dof_frictionloss", + entity_type="dof", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + use_address=True, + ) + + +# Raw alias. +dof_frictionloss = joint_friction + + +@requires_model_fields("jnt_stiffness") +def joint_stiffness( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize joint stiffness (jnt_stiffness).""" + _randomize_model_field( + env, + env_ids, + "jnt_stiffness", + entity_type="joint", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +# Raw alias. +jnt_stiffness = joint_stiffness + + +@requires_model_fields("jnt_range") +def joint_limits( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize joint limits (jnt_range).""" + _randomize_model_field( + env, + env_ids, + "jnt_range", + entity_type="joint", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +# Raw alias. +jnt_range = joint_limits + + +@requires_model_fields("qpos0", recompute=RecomputeLevel.set_const_0) +def joint_default_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "add", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize default joint positions (qpos0). Triggers ``set_const_0``.""" + _randomize_model_field( + env, + env_ids, + "qpos0", + entity_type="joint", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + use_address=True, + ) + + +# Raw alias. +qpos0 = joint_default_pos + + +def encoder_bias( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + bias_range: tuple[float, float], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Randomize encoder bias to simulate joint encoder calibration errors.""" + asset: Entity = env.scene[asset_cfg.name] + + if env_ids is None: + env_ids = torch.arange(env.num_envs, device=env.device, dtype=torch.int) + else: + env_ids = env_ids.to(env.device, dtype=torch.int) + + joint_ids = asset_cfg.joint_ids + if isinstance(joint_ids, slice): + num_joints = asset.num_joints + joint_ids_tensor = torch.arange(num_joints, device=env.device) + else: + joint_ids_tensor = torch.tensor(joint_ids, device=env.device) + + num_joints = len(joint_ids_tensor) + bias_samples = sample_uniform( + torch.tensor(bias_range[0], device=env.device), + torch.tensor(bias_range[1], device=env.device), + (len(env_ids), num_joints), + env.device, + ) + + if isinstance(joint_ids, slice): + asset.data.encoder_bias[env_ids] = bias_samples + else: + asset.data.encoder_bias[env_ids[:, None], joint_ids_tensor] = bias_samples diff --git a/rl/mjlab/envs/mdp/dr/light.py b/rl/mjlab/envs/mdp/dr/light.py new file mode 100644 index 0000000..f25045b --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/light.py @@ -0,0 +1,234 @@ +"""Domain randomization functions for light fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _randomize_model_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("light_pos") +def light_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light positions.""" + _randomize_model_field( + env, + env_ids, + "light_pos", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_dir") +def light_dir( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light direction vectors.""" + _randomize_model_field( + env, + env_ids, + "light_dir", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_diffuse") +def light_diffuse( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light diffuse RGB color.""" + _randomize_model_field( + env, + env_ids, + "light_diffuse", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_specular") +def light_specular( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light specular RGB color.""" + _randomize_model_field( + env, + env_ids, + "light_specular", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_ambient") +def light_ambient( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light ambient RGB color.""" + _randomize_model_field( + env, + env_ids, + "light_ambient", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_attenuation") +def light_attenuation( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize light attenuation coefficients.""" + _randomize_model_field( + env, + env_ids, + "light_attenuation", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("light_cutoff") +def light_cutoff( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize spot light cutoff angles in degrees.""" + _randomize_model_field( + env, + env_ids, + "light_cutoff", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("light_exponent") +def light_exponent( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize spot light angular falloff exponents.""" + _randomize_model_field( + env, + env_ids, + "light_exponent", + entity_type="light", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) diff --git a/rl/mjlab/envs/mdp/dr/material.py b/rl/mjlab/envs/mdp/dr/material.py new file mode 100644 index 0000000..ce5c831 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/material.py @@ -0,0 +1,230 @@ +"""Domain randomization functions for material fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import mujoco + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _get_entity_indices, + _randomize_categorical_field, + _randomize_model_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + import torch + + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("mat_rgba") +def mat_rgba( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize material RGBA color. + + In MuJoCo, a material's RGBA plays two roles depending on whether a texture is + assigned: + + - **With texture**: ``mat_rgba`` is a multiplicative modulator. The rendered color + equals ``texture_color * mat_rgba`` per channel. ``(1, 1, 1, 1)`` leaves the + texture unchanged; values below 1 darken it. + - **Without texture**: ``mat_rgba`` directly sets the material's surface color, + similar to how :func:`~mjlab.envs.mdp.dr.geom_rgba` sets geom colors. + + Note: If a geom has no material assigned (``matid < 0``), its color is controlled by + ``geom_rgba``, not ``mat_rgba``. + + Args: + env: The environment instance. + env_ids: Environment indices to randomize. ``None`` means all. + ranges: Value range(s) for sampling. + asset_cfg: Entity and material selection. Use + ``SceneEntityCfg("entity", material_names=(...))`` to target specific materials. + distribution: Sampling distribution. + operation: How to combine sampled values with the base. + axes: Which RGBA channels to randomize. Defaults to ``[0, 1, 2, 3]``. + shared_random: If ``True``, all selected materials receive the same sampled + value per environment. + """ + _randomize_model_field( + env, + env_ids, + "mat_rgba", + entity_type="material", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2, 3], + ) + + +@requires_model_fields("mat_emission") +def mat_emission( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize material emission. + + MuJoCo Warp's RGB renderer adds ``mat_emission * base_color`` per shaded + pixel, so higher values make the material appear self-illuminated. + """ + _randomize_model_field( + env, + env_ids, + "mat_emission", + entity_type="material", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("mat_specular") +def mat_specular( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize material specular reflection strength. + + MuJoCo stores specular in ``[0, 1]``. MuJoCo Warp uses it to scale the + specular component of RGB rendering. + """ + _randomize_model_field( + env, + env_ids, + "mat_specular", + entity_type="material", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("mat_shininess") +def mat_shininess( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize material shininess. + + MuJoCo stores shininess in ``[0, 1]``. MuJoCo Warp maps it to a Phong + exponent in ``[0, 128]`` during RGB rendering. + """ + _randomize_model_field( + env, + env_ids, + "mat_shininess", + entity_type="material", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("mat_texrepeat") +def mat_texrepeat( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize material texture repeat in the S/T directions. + + Only affects textured materials. Values should remain positive; zero or + negative texture repeats are not meaningful for rendering. + """ + _randomize_model_field( + env, + env_ids, + "mat_texrepeat", + entity_type="material", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1], + ) + + +@requires_model_fields("mat_texid") +def mat_texid( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + role: int = mujoco.mjtTextureRole.mjTEXROLE_RGB.value, + shared_random: bool = False, +) -> None: + """Randomize which texture each selected material uses. + + A material stores one texture per ``mjtTextureRole`` slot; only the ``role`` + slot is modified, drawn uniformly from ``asset_cfg.texture_names``. + """ + asset = env.scene[asset_cfg.name] + tex_ids = _get_entity_indices(asset.indexing, asset_cfg, "texture", False) + if tex_ids.numel() == 0: + raise ValueError( + f"No textures selected for entity '{asset_cfg.name}'. Set " + "SceneEntityCfg.texture_names to a non-empty selection." + ) + + _randomize_categorical_field( + env, + env_ids, + "mat_texid", + entity_type="material", + pool=tex_ids, + asset_cfg=asset_cfg, + shared_random=shared_random, + axis=role, + ) diff --git a/rl/mjlab/envs/mdp/dr/pair.py b/rl/mjlab/envs/mdp/dr/pair.py new file mode 100644 index 0000000..4cecf74 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/pair.py @@ -0,0 +1,89 @@ +"""Domain randomization functions for geom pair fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import _DEFAULT_ASSET_CFG, Ranges, _randomize_model_field +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("pair_friction") +def pair_friction( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, + isotropic: bool = False, +) -> None: + """Randomize geom-pair friction overrides. + + Pair friction has 5 components: ``[tangent1, tangent2, spin, roll1, roll2]``. + Default axis is 0 (tangent1 only). Axis 0 requires ``condim >= 3`` + (the default); axes 1 and 2 require ``condim >= 4``; axes 3 and 4 + require ``condim = 6``. + + When ``isotropic=True``, tangent2 is set equal to tangent1 after sampling + (axis 1 is overwritten with axis 0). If axes 3 or 4 are targeted, roll2 is + also set equal to roll1 (axis 4 is overwritten with axis 3). This matches + MuJoCo's standard geom friction convention where both tangent directions and + both roll directions share one coefficient. + + Args: + env: The environment instance. + env_ids: Environment indices to randomize. ``None`` means all. + ranges: Value range(s) for sampling. + asset_cfg: Entity and pair selection. + distribution: Sampling distribution. + operation: How to combine sampled values with the base. + axes: Which friction components to randomize. Defaults to ``[0]`` (tangent1). + shared_random: If ``True``, all selected pairs receive the same sampled value per + environment. + isotropic: If ``True``, mirror tangent2 = tangent1 after sampling (and + roll2 = roll1 if roll axes are targeted). Use this to maintain isotropic + friction, matching MuJoCo's geom friction convention. + """ + pair_indices = _randomize_model_field( + env, + env_ids, + "pair_friction", + entity_type="pair", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0], + valid_axes=[0, 1, 2, 3, 4], + ) + + if isotropic: + _env_ids = ( + torch.arange(env.num_envs, device=env.device, dtype=torch.int) + if env_ids is None + else env_ids.to(env.device, dtype=torch.int) + ) + env_grid, pair_grid = torch.meshgrid(_env_ids, pair_indices, indexing="ij") + pf = env.sim.model.pair_friction + pf[env_grid, pair_grid, 1] = pf[env_grid, pair_grid, 0] + if axes is not None: + effective_axes = axes + elif isinstance(ranges, dict) and ranges and isinstance(next(iter(ranges)), int): + effective_axes = list(ranges.keys()) + else: + effective_axes = [0] + if 3 in effective_axes or 4 in effective_axes: + pf[env_grid, pair_grid, 4] = pf[env_grid, pair_grid, 3] diff --git a/rl/mjlab/envs/mdp/dr/site.py b/rl/mjlab/envs/mdp/dr/site.py new file mode 100644 index 0000000..54bcbe4 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/site.py @@ -0,0 +1,77 @@ +"""Domain randomization functions for site fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _randomize_model_field, + _randomize_quat_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("site_pos") +def site_pos( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize site positions.""" + _randomize_model_field( + env, + env_ids, + "site_pos", + entity_type="site", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + default_axes=[0, 1, 2], + ) + + +@requires_model_fields("site_quat") +def site_quat( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + roll_range: tuple[float, float] = (0.0, 0.0), + pitch_range: tuple[float, float] = (0.0, 0.0), + yaw_range: tuple[float, float] = (0.0, 0.0), + distribution: Distribution | str = "uniform", + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Randomize site orientation by composing an RPY perturbation. + + Ranges are in radians. The sampled perturbation is composed with the default + quaternion (not the current one), so repeated calls do not accumulate. The result is + always a valid unit quaternion. + """ + _randomize_quat_field( + env, + env_ids, + "site_quat", + entity_type="site", + roll_range=roll_range, + pitch_range=pitch_range, + yaw_range=yaw_range, + distribution=distribution, + asset_cfg=asset_cfg, + ) diff --git a/rl/mjlab/envs/mdp/dr/tendon.py b/rl/mjlab/envs/mdp/dr/tendon.py new file mode 100644 index 0000000..971c6a5 --- /dev/null +++ b/rl/mjlab/envs/mdp/dr/tendon.py @@ -0,0 +1,128 @@ +"""Domain randomization functions for tendon fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.event_manager import RecomputeLevel, requires_model_fields +from mjlab.managers.scene_entity_config import SceneEntityCfg + +from ._core import ( + _DEFAULT_ASSET_CFG, + Ranges, + _randomize_model_field, +) +from ._types import Distribution, Operation + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@requires_model_fields("tendon_damping") +def tendon_damping( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize tendon damping.""" + _randomize_model_field( + env, + env_ids, + "tendon_damping", + entity_type="tendon", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("tendon_stiffness") +def tendon_stiffness( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize tendon stiffness.""" + _randomize_model_field( + env, + env_ids, + "tendon_stiffness", + entity_type="tendon", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +@requires_model_fields("tendon_frictionloss") +def tendon_friction( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize tendon friction loss (tendon_frictionloss).""" + _randomize_model_field( + env, + env_ids, + "tendon_frictionloss", + entity_type="tendon", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) + + +# Raw alias. +tendon_frictionloss = tendon_friction + + +@requires_model_fields("tendon_armature", recompute=RecomputeLevel.set_const_0) +def tendon_armature( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + ranges: Ranges, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, + distribution: Distribution | str = "uniform", + operation: Operation | str = "abs", + axes: list[int] | None = None, + shared_random: bool = False, +) -> None: + """Randomize tendon armature. Triggers ``set_const_0``.""" + _randomize_model_field( + env, + env_ids, + "tendon_armature", + entity_type="tendon", + ranges=ranges, + distribution=distribution, + operation=operation, + asset_cfg=asset_cfg, + axes=axes, + shared_random=shared_random, + ) diff --git a/rl/mjlab/envs/mdp/events.py b/rl/mjlab/envs/mdp/events.py new file mode 100644 index 0000000..542f9ab --- /dev/null +++ b/rl/mjlab/envs/mdp/events.py @@ -0,0 +1,605 @@ +"""Useful methods for MDP events.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import torch + +from mjlab.entity import Entity +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.lab_api.math import ( + quat_apply, + quat_from_euler_xyz, + quat_mul, + sample_uniform, +) + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + from mjlab.viewer.debug_visualizer import DebugVisualizer + +_DEFAULT_ASSET_CFG = SceneEntityCfg("robot") +_SE3_KEYS = ("x", "y", "z", "roll", "pitch", "yaw") + + +def _sample_se3_range( + range_dict: dict[str, tuple[float, float]] | None, + shape: tuple[int, ...], + device: str, +) -> torch.Tensor: + """Sample uniform ``[x, y, z, roll, pitch, yaw]`` offsets. + + ``range_dict`` maps any subset of those keys to ``(min, max)`` ranges; missing + keys default to ``(0.0, 0.0)`` (no offset). ``None`` is treated as empty. The + returned tensor has the requested ``shape`` whose last dimension must be 6. + """ + range_dict = range_dict or {} + range_list = [range_dict.get(key, (0.0, 0.0)) for key in _SE3_KEYS] + ranges = torch.tensor(range_list, device=device) + return sample_uniform(ranges[:, 0], ranges[:, 1], shape, device=device) + + +def resolve_env_ids( + env: ManagerBasedRlEnv, env_ids: torch.Tensor | None +) -> torch.Tensor: + """Return ``env_ids`` unchanged, or all environment indices if ``None``. + + Event functions receive ``env_ids=None`` to mean "all environments" (a full + reset, or a global-time interval term). This normalizes that sentinel to a + concrete index tensor so the function body can assume a real ``torch.Tensor``. + """ + if env_ids is None: + return torch.arange(env.num_envs, device=env.device, dtype=torch.int) + return env_ids + + +def randomize_terrain(env: ManagerBasedRlEnv, env_ids: torch.Tensor | None) -> None: + """Randomize the sub-terrain for each environment on reset. + + This picks a random terrain type (column) and difficulty level (row) for each + environment. Useful for play/evaluation mode to test on varied terrains. + """ + env_ids = resolve_env_ids(env, env_ids) + + terrain = env.scene.terrain + if terrain is not None: + terrain.randomize_env_origins(env_ids) + + +def reset_scene_to_default( + env: ManagerBasedRlEnv, env_ids: torch.Tensor | None +) -> None: + """Reset all entities in the scene to their default states. + + For floating-base entities: Resets root state (position, orientation, velocities). + For fixed-base mocap entities: Resets mocap pose. + For all articulated entities: Resets joint positions and velocities. + + Automatically applies env_origins offset to position all entities correctly. + """ + env_ids = resolve_env_ids(env, env_ids) + + for entity in env.scene.entities.values(): + if not isinstance(entity, Entity): + continue + + # Reset root/mocap pose. + if entity.is_fixed_base and entity.is_mocap: + # Fixed-base mocap entity - reset mocap pose with env_origins. + default_root_state = entity.data.default_root_state[env_ids].clone() + mocap_pose = torch.zeros((len(env_ids), 7), device=env.device) + mocap_pose[:, 0:3] = default_root_state[:, 0:3] + env.scene.env_origins[env_ids] + mocap_pose[:, 3:7] = default_root_state[:, 3:7] + entity.write_mocap_pose_to_sim(mocap_pose, env_ids=env_ids) + elif not entity.is_fixed_base: + # Floating-base entity - reset root state with env_origins. + default_root_state = entity.data.default_root_state[env_ids].clone() + default_root_state[:, 0:3] += env.scene.env_origins[env_ids] + entity.write_root_state_to_sim(default_root_state, env_ids=env_ids) + + # Reset joint state for articulated entities. + if entity.is_articulated: + default_joint_pos = entity.data.default_joint_pos[env_ids].clone() + default_joint_vel = entity.data.default_joint_vel[env_ids].clone() + entity.write_joint_state_to_sim( + default_joint_pos, default_joint_vel, env_ids=env_ids + ) + + +def reset_root_state_uniform( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + pose_range: dict[str, tuple[float, float]], + velocity_range: dict[str, tuple[float, float]] | None = None, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Reset root state for floating-base or mocap fixed-base entities. + + For floating-base entities: Resets pose and velocity via write_root_state_to_sim(). + For fixed-base mocap entities: Resets pose only via write_mocap_pose_to_sim(). + + .. note:: + This function applies the env_origins offset to position entities in a grid. + For fixed-base robots, this is the ONLY way to position them per-environment. + Without calling this function in a reset event, fixed-base robots will stack + at (0,0,0). + + See FAQ: "Why are my fixed-base robots all stacked at the origin?" + + Args: + env: The environment. + env_ids: Environment IDs to reset. If None, resets all environments. + pose_range: Dictionary with keys {"x", "y", "z", "roll", "pitch", "yaw"}. + velocity_range: Velocity range (only used for floating-base entities). + asset_cfg: Asset configuration. + """ + env_ids = resolve_env_ids(env, env_ids) + + asset: Entity = env.scene[asset_cfg.name] + + # Pose. + pose_samples = _sample_se3_range(pose_range, (len(env_ids), 6), env.device) + + # Fixed-based entities with mocap=True. + if asset.is_fixed_base: + if not asset.is_mocap: + raise ValueError( + f"Cannot reset root state for fixed-base non-mocap entity '{asset_cfg.name}'." + ) + + default_root_state = asset.data.default_root_state + assert default_root_state is not None + root_states = default_root_state[env_ids].clone() + + positions = ( + root_states[:, 0:3] + pose_samples[:, 0:3] + env.scene.env_origins[env_ids] + ) + orientations_delta = quat_from_euler_xyz( + pose_samples[:, 3], pose_samples[:, 4], pose_samples[:, 5] + ) + orientations = quat_mul(root_states[:, 3:7], orientations_delta) + + asset.write_mocap_pose_to_sim( + torch.cat([positions, orientations], dim=-1), env_ids=env_ids + ) + return + + # Floating-base entities. + default_root_state = asset.data.default_root_state + assert default_root_state is not None + root_states = default_root_state[env_ids].clone() + + positions = ( + root_states[:, 0:3] + pose_samples[:, 0:3] + env.scene.env_origins[env_ids] + ) + orientations_delta = quat_from_euler_xyz( + pose_samples[:, 3], pose_samples[:, 4], pose_samples[:, 5] + ) + orientations = quat_mul(root_states[:, 3:7], orientations_delta) + + # Velocities. + vel_samples = _sample_se3_range(velocity_range, (len(env_ids), 6), env.device) + velocities = root_states[:, 7:13] + vel_samples + + asset.write_root_link_pose_to_sim( + torch.cat([positions, orientations], dim=-1), env_ids=env_ids + ) + + asset.write_root_link_velocity_to_sim(velocities, env_ids=env_ids) + + +def reset_root_state_from_flat_patches( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + patch_name: str = "spawn", + pose_range: dict[str, tuple[float, float]] | None = None, + velocity_range: dict[str, tuple[float, float]] | None = None, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Reset root state by placing the asset on a randomly chosen flat patch. + + Selects a random flat patch from the terrain for each environment and positions + the asset there. Falls back to ``reset_root_state_uniform`` if the terrain has + no flat patches. + + Args: + env: The environment. + env_ids: Environment IDs to reset. If None, resets all environments. + patch_name: Key into ``terrain.flat_patches`` to use. + pose_range: Optional random offset applied on top of the patch position. + Keys: ``{"x", "y", "z", "roll", "pitch", "yaw"}``. + velocity_range: Optional velocity range (floating-base only). + asset_cfg: Asset configuration. + """ + env_ids = resolve_env_ids(env, env_ids) + + terrain = env.scene.terrain + if terrain is None or patch_name not in terrain.flat_patches: + reset_root_state_uniform( + env, + env_ids, + pose_range=pose_range or {}, + velocity_range=velocity_range, + asset_cfg=asset_cfg, + ) + return + + patches = terrain.flat_patches[patch_name] # (num_rows, num_cols, num_patches, 3) + num_patches = patches.shape[2] + + # Look up terrain level (row) and type (col) for each env. + levels = terrain.terrain_levels[env_ids] + types = terrain.terrain_types[env_ids] + + # Randomly select a patch index for each env. + patch_ids = torch.randint(0, num_patches, (len(env_ids),), device=env.device) + positions = patches[levels, types, patch_ids] + + asset: Entity = env.scene[asset_cfg.name] + default_root_state = asset.data.default_root_state + assert default_root_state is not None + root_states = default_root_state[env_ids].clone() + + # Apply optional pose range offset. + pose_samples = _sample_se3_range(pose_range, (len(env_ids), 6), env.device) + + # Position: flat patch position + optional offset. Use patch z instead of default. + final_positions = positions.clone() + final_positions[:, 0] += pose_samples[:, 0] + final_positions[:, 1] += pose_samples[:, 1] + final_positions[:, 2] += root_states[:, 2] + pose_samples[:, 2] + + orientations_delta = quat_from_euler_xyz( + pose_samples[:, 3], pose_samples[:, 4], pose_samples[:, 5] + ) + orientations = quat_mul(root_states[:, 3:7], orientations_delta) + + if asset.is_fixed_base: + if not asset.is_mocap: + raise ValueError( + f"Cannot reset root state for fixed-base non-mocap entity '{asset_cfg.name}'." + ) + asset.write_mocap_pose_to_sim( + torch.cat([final_positions, orientations], dim=-1), env_ids=env_ids + ) + return + + # Velocities. + vel_samples = _sample_se3_range(velocity_range, (len(env_ids), 6), env.device) + velocities = root_states[:, 7:13] + vel_samples + + asset.write_root_link_pose_to_sim( + torch.cat([final_positions, orientations], dim=-1), env_ids=env_ids + ) + asset.write_root_link_velocity_to_sim(velocities, env_ids=env_ids) + + +def reset_joints_by_offset( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + position_range: tuple[float, float], + velocity_range: tuple[float, float], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + env_ids = resolve_env_ids(env, env_ids) + + asset: Entity = env.scene[asset_cfg.name] + default_joint_pos = asset.data.default_joint_pos + assert default_joint_pos is not None + default_joint_vel = asset.data.default_joint_vel + assert default_joint_vel is not None + soft_joint_pos_limits = asset.data.soft_joint_pos_limits + assert soft_joint_pos_limits is not None + + joint_pos = default_joint_pos[env_ids][:, asset_cfg.joint_ids].clone() + joint_pos += sample_uniform(*position_range, joint_pos.shape, env.device) + joint_pos_limits = soft_joint_pos_limits[env_ids][:, asset_cfg.joint_ids] + joint_pos = joint_pos.clamp_(joint_pos_limits[..., 0], joint_pos_limits[..., 1]) + + joint_vel = default_joint_vel[env_ids][:, asset_cfg.joint_ids].clone() + joint_vel += sample_uniform(*velocity_range, joint_vel.shape, env.device) + + joint_ids = asset_cfg.joint_ids + if isinstance(joint_ids, list): + joint_ids = torch.tensor(joint_ids, device=env.device) + + asset.write_joint_state_to_sim( + joint_pos.view(len(env_ids), -1), + joint_vel.view(len(env_ids), -1), + env_ids=env_ids, + joint_ids=joint_ids, + ) + + +def push_by_setting_velocity( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + velocity_range: dict[str, tuple[float, float]], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Push an entity by overwriting its root velocity with a sampled offset. + + This is an *instantaneous, mass-independent* kick: it adds a uniformly sampled + delta directly to the root velocity, ignoring inertia and contact dynamics. It + is the cheapest disturbance and the standard locomotion "push the robot" term. + Use with ``mode="interval"``. + + For force-based disturbances that respect the entity's dynamics, see + :func:`apply_external_force_torque` (a constant wrench you manage yourself) or + :class:`apply_body_impulse` (transient, self-managing impulses). + """ + env_ids = resolve_env_ids(env, env_ids) + asset: Entity = env.scene[asset_cfg.name] + vel_w = asset.data.root_link_vel_w[env_ids] + vel_w += _sample_se3_range(velocity_range, vel_w.shape, env.device) + asset.write_root_link_velocity_to_sim(vel_w, env_ids=env_ids) + + +def apply_external_force_torque( + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + force_range: tuple[float, float], + torque_range: tuple[float, float], + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> None: + """Apply a single *constant* external wrench to bodies. + + Samples a force and torque once and writes them to ``xfrc_applied``. The wrench + is **stateless and never expires**: MuJoCo holds it constant on every physics + step until something overwrites or zeroes it. There is no duration, cooldown, + or auto-clear. + + **When to use this vs.** :class:`apply_body_impulse`: + + - Use ``apply_external_force_torque`` for a *steady, episode-long* disturbance + such as a fixed payload, a constant wind, or a sustained load. The intended + pattern is ``mode="reset"``: re-randomize the wrench each episode so it holds + for that episode's duration. Because it never turns itself off, **you are + responsible for clearing or overwriting it** (e.g. via the next reset). It is + *not* suited to transient bumps on its own. + + - Use :class:`apply_body_impulse` for *transient, repeated, randomized* + disturbances during an episode (bumps, gusts, collisions). It runs a full + cooldown -> trigger -> sustain -> expire lifecycle per environment, zeroing + the wrench automatically when each impulse ends, and ticks on ``mode="step"``. + + For an instantaneous, mass-independent kick instead of a force, see + :func:`push_by_setting_velocity`. + """ + env_ids = resolve_env_ids(env, env_ids) + asset: Entity = env.scene[asset_cfg.name] + num_bodies = ( + len(asset_cfg.body_ids) + if isinstance(asset_cfg.body_ids, list) + else asset.num_bodies + ) + size = (len(env_ids), num_bodies, 3) + forces = sample_uniform(*force_range, size, env.device) + torques = sample_uniform(*torque_range, size, env.device) + asset.write_external_wrench_to_sim( + forces, torques, env_ids=env_ids, body_ids=asset_cfg.body_ids + ) + + +class apply_body_impulse: + """Apply random impulses to bodies for a sampled duration. + + Simulates transient external disturbances such as bumps, wind gusts, or + collisions with unseen objects. A constant force/torque wrench is applied + to one or more bodies for a randomly sampled duration, followed by a + cooldown period of silence before the next impulse. + + **Lifecycle of a single impulse:** + + 1. **Cooldown.** The event is idle for a random duration sampled from ``cooldown_s``. + No force is applied. + 2. **Trigger.** A force vector is sampled uniformly per component from ``force_range`` + and written to ``xfrc_applied`` on the selected bodies. + 3. **Sustain.** The force is held constant for a random duration sampled from + ``duration_s``. + 4. **Expire.** The force is zeroed and the cooldown restarts at step 1. + + Each environment runs its own independent timer so impulses are decorrelated across + the batch. + + **Application point.** By default, forces act at each body's center of mass. + ``body_point_offset`` shifts the application point in the body's local frame, for + example ``(0, 0, 0.1)`` for 10 cm above the CoM. The offset produces additional + torque via the cross product ``offset x force``, causing the body to tip rather than + just translate. This is analogous to choosing where on the body an external push is + applied. + + Use with ``mode="step"``. + + For a *constant* episode-long wrench instead of transient impulses, see + :func:`apply_external_force_torque`. For an instantaneous, mass-independent + velocity kick, see :func:`push_by_setting_velocity`. + """ + + @dataclass + class VizCfg: + """Arrow visualization settings for active impulse forces.""" + + rgba: tuple[float, float, float, float] = (0.9, 0.2, 0.8, 0.9) + """Arrow color (RGBA).""" + scale: float = 0.005 + """Arrow length in meters per Newton of force.""" + width: float = 0.015 + """Arrow shaft width in meters.""" + min_force: float = 1.0 + """Minimum force magnitude (N) below which arrows are hidden.""" + + def __init__(self, cfg, env: ManagerBasedRlEnv): + self._asset: Entity = env.scene[cfg.params["asset_cfg"].name] + self._body_ids = cfg.params["asset_cfg"].body_ids + self._num_envs = env.num_envs + self._device = env.device + self._step_dt = env.step_dt + self._viz_cfg: apply_body_impulse.VizCfg = cfg.params.get( + "viz_cfg", apply_body_impulse.VizCfg() + ) + offset = cfg.params.get("body_point_offset", None) + self._body_point_offset: torch.Tensor | None = ( + torch.tensor(offset, device=self._device, dtype=torch.float32) + if offset is not None + else None + ) + + self._num_bodies = ( + len(self._body_ids) + if isinstance(self._body_ids, list) + else self._asset.num_bodies + ) + + self._cooldown_s: tuple[float, float] = cfg.params["cooldown_s"] + self._time_remaining = torch.zeros(self._num_envs, device=self._device) + self._active = torch.zeros(self._num_envs, device=self._device, dtype=torch.bool) + # Pre-sample the initial cooldown so the first impulse is preceded by a cooldown + # rather than firing immediately at t=0. + self._interval_time_left = self._sample_cooldown(self._num_envs) + + def _sample_cooldown(self, n: int) -> torch.Tensor: + low, high = self._cooldown_s + return sample_uniform(low, high, n, self._device) + + def __call__( + self, + env: ManagerBasedRlEnv, + env_ids: torch.Tensor | None, + force_range: tuple[float, float], + torque_range: tuple[float, float], + duration_s: tuple[float, float], + cooldown_s: tuple[float, float], + asset_cfg: SceneEntityCfg, + body_point_offset: tuple[float, float, float] | None = None, + ) -> None: + """Tick impulse state: expire old impulses, trigger new ones. + + Args: + env: The environment instance. + env_ids: Unused (step events always operate on all envs). + force_range: ``(min, max)`` uniform range for each force component (N). + torque_range: ``(min, max)`` uniform range for each torque component (Nm). + duration_s: ``(min, max)`` uniform range for impulse duration in seconds. + cooldown_s: ``(min, max)`` uniform range for the cooldown between consecutive + impulses in seconds. Captured at init so the first impulse can be + preceded by a sampled cooldown; the kwarg passed here is unused. + asset_cfg: Entity and body selection. ``body_ids`` on the config selects which + bodies receive forces. + body_point_offset: Optional ``(x, y, z)`` offset in the body frame where the + force is applied. Generates additional torque via ``cross(offset, force)``. + """ + del env, env_ids, asset_cfg, cooldown_s # Unused at call time. + dt = self._step_dt + + # Decrement timers for active envs. + self._time_remaining[self._active] -= dt + + # Clear expired impulses and resample their interval timers. + expired = self._active & (self._time_remaining <= 0) + if expired.any(): + expired_ids = expired.nonzero(as_tuple=False).squeeze(-1) + zeros = torch.zeros((len(expired_ids), self._num_bodies, 3), device=self._device) + self._asset.write_external_wrench_to_sim( + zeros, zeros, env_ids=expired_ids, body_ids=self._body_ids + ) + self._active[expired_ids] = False + self._time_remaining[expired_ids] = 0.0 + self._interval_time_left[expired_ids] = self._sample_cooldown(len(expired_ids)) + + # Decrement interval timers. + self._interval_time_left -= dt + + # Trigger new impulses for eligible envs. + eligible = (~self._active) & (self._interval_time_left <= 0) + if not eligible.any(): + return + + trigger_ids = eligible.nonzero(as_tuple=False).squeeze(-1) + n = len(trigger_ids) + + # Sample forces and torques. + size = (n, self._num_bodies, 3) + forces = sample_uniform(*force_range, size, self._device) + torques = sample_uniform(*torque_range, size, self._device) + + # Adjust torque for off-CoM application point. + if body_point_offset is not None: + offset_local = torch.tensor( + body_point_offset, device=self._device, dtype=torch.float32 + ) + body_quat = self._asset.data.body_link_quat_w[trigger_ids][:, self._body_ids] + # Rotate offset into world frame: (n, num_bodies, 3). + offset_w = quat_apply( + body_quat.reshape(-1, 4), offset_local.expand(n * self._num_bodies, 3) + ).reshape(n, self._num_bodies, 3) + torques = torques + torch.cross(offset_w, forces, dim=-1) + + self._asset.write_external_wrench_to_sim( + forces, torques, env_ids=trigger_ids, body_ids=self._body_ids + ) + + # Sample duration and set timers. + dur_low, dur_high = duration_s + self._time_remaining[trigger_ids] = ( + torch.rand(n, device=self._device) * (dur_high - dur_low) + dur_low + ) + self._active[trigger_ids] = True + + # Resample interval timers. + self._interval_time_left[trigger_ids] = self._sample_cooldown(n) + + def debug_vis(self, visualizer: DebugVisualizer) -> None: + """Draw arrows for active impulse forces.""" + if not self._active.any(): + return + viz = self._viz_cfg + min_sq = viz.min_force * viz.min_force + wrench = self._asset.data.body_external_wrench # (nworld, nbody, 6) + com_pos = self._asset.data.body_com_pos_w # (nworld, nbody, 3) + offset = self._body_point_offset + body_quat = self._asset.data.body_link_quat_w if offset is not None else None + for env_idx in visualizer.get_env_indices(self._num_envs): + if not self._active[env_idx]: + continue + for i in range(wrench.shape[1]): + force = wrench[env_idx, i, :3] + if (force * force).sum().item() < min_sq: + continue + force_np = force.cpu().numpy() + start_np = com_pos[env_idx, i].cpu().numpy() + if offset is not None and body_quat is not None: + offset_w = quat_apply(body_quat[env_idx, i], offset) + start_np = start_np + offset_w.cpu().numpy() + end_np = start_np + force_np * viz.scale + visualizer.add_arrow( + start=start_np, + end=end_np, + color=viz.rgba, + width=viz.width, + ) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + if env_ids is None: + env_ids = slice(None) + + if self._active[env_ids].any(): + if isinstance(env_ids, slice): + active_ids = self._active.nonzero(as_tuple=False).squeeze(-1) + else: + active_ids = env_ids[self._active[env_ids]] + if len(active_ids) > 0: + zeros = torch.zeros( + (len(active_ids), self._num_bodies, 3), + device=self._device, + ) + self._asset.write_external_wrench_to_sim( + zeros, zeros, env_ids=active_ids, body_ids=self._body_ids + ) + + n = self._num_envs if isinstance(env_ids, slice) else len(env_ids) + self._time_remaining[env_ids] = 0.0 + self._interval_time_left[env_ids] = self._sample_cooldown(n) + self._active[env_ids] = False diff --git a/rl/mjlab/envs/mdp/metrics.py b/rl/mjlab/envs/mdp/metrics.py new file mode 100644 index 0000000..09f956f --- /dev/null +++ b/rl/mjlab/envs/mdp/metrics.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +def mean_action_acc(env: ManagerBasedRlEnv) -> torch.Tensor: + """Mean absolute action acceleration. + + Lower values indicate smoother actions. + + Returns: + Per-environment scalar. Shape: ``(B,)``. + """ + # Discrete second derivative: a_t - 2 * a_{t-1} + a_{t-2}. (B, N) + action_acc = ( + env.action_manager.action + - 2 * env.action_manager.prev_action + + env.action_manager.prev_prev_action + ) + return torch.mean(torch.abs(action_acc), dim=-1) # (B,) diff --git a/rl/mjlab/envs/mdp/observations.py b/rl/mjlab/envs/mdp/observations.py new file mode 100644 index 0000000..92cbf40 --- /dev/null +++ b/rl/mjlab/envs/mdp/observations.py @@ -0,0 +1,163 @@ +"""Useful methods for MDP observations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.entity import Entity +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.sensor import BuiltinSensor, RayCastSensor + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + +_DEFAULT_ASSET_CFG = SceneEntityCfg("robot") + + +## +# Root state. +## + + +def base_lin_vel( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + return asset.data.root_link_lin_vel_b + + +def base_ang_vel( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + return asset.data.root_link_ang_vel_b + + +def projected_gravity( + env: ManagerBasedRlEnv, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + return asset.data.projected_gravity_b + + +## +# Joint state. +## + + +def joint_pos_rel( + env: ManagerBasedRlEnv, + biased: bool = False, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + default_joint_pos = asset.data.default_joint_pos + assert default_joint_pos is not None + jnt_ids = asset_cfg.joint_ids + joint_pos = asset.data.joint_pos_biased if biased else asset.data.joint_pos + return joint_pos[:, jnt_ids] - default_joint_pos[:, jnt_ids] + + +def joint_vel_rel( + env: ManagerBasedRlEnv, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + default_joint_vel = asset.data.default_joint_vel + assert default_joint_vel is not None + jnt_ids = asset_cfg.joint_ids + return asset.data.joint_vel[:, jnt_ids] - default_joint_vel[:, jnt_ids] + + +## +# Actions. +## + + +def last_action(env: ManagerBasedRlEnv, action_name: str | None = None) -> torch.Tensor: + if action_name is None: + return env.action_manager.action + return env.action_manager.get_term(action_name).raw_action + + +## +# Commands. +## + + +def generated_commands(env: ManagerBasedRlEnv, command_name: str) -> torch.Tensor: + command = env.command_manager.get_command(command_name) + assert command is not None + return command + + +## +# Sensors. +## + + +def builtin_sensor(env: ManagerBasedRlEnv, sensor_name: str) -> torch.Tensor: + """Get observation from a built-in sensor by name.""" + sensor = env.scene[sensor_name] + assert isinstance(sensor, BuiltinSensor) + return sensor.data + + +def projected_gravity_from_sensor( + env: ManagerBasedRlEnv, sensor_name: str +) -> torch.Tensor: + """Projected gravity from a ``framezaxis`` up-vector sensor. + + The sensor is expected to output the world Z-axis expressed in the sensor's frame + (e.g. ``framezaxis`` with ``objtype=body objname=world`` and ``reftype=site``). That + is the body-frame "up" vector, so it is negated to point along gravity. + + Unlike :func:`projected_gravity`, which uses the root body orientation, this reads + the sensor's site frame and therefore reflects IMU site pose randomization. + """ + sensor = env.scene[sensor_name] + assert isinstance(sensor, BuiltinSensor) + return -sensor.data + + +def height_scan( + env: ManagerBasedRlEnv, + sensor_name: str, + offset: float = 0.0, + miss_value: float | None = None, +) -> torch.Tensor: + """Height scan from a raycast sensor. + + Returns the height of the sensor frame above each hit point. + Supports multi-frame sensors: each ray uses its own frame's Z. + + Args: + env: The environment. + sensor_name: Name of a RayCastSensor in the scene. + offset: Constant offset subtracted from heights. + miss_value: Value to use for rays that miss (distance < 0). + Defaults to the sensor's ``max_distance``. + + Returns: + Tensor of shape [B, N] where N = num_frames * num_rays_per_frame. + Rays are ordered frame-major (all rays for frame 0, then frame 1, etc.). + """ + sensor: RayCastSensor = env.scene[sensor_name] + if miss_value is None: + miss_value = sensor.cfg.max_distance + + data = sensor.data + F, N = sensor.num_frames, sensor.num_rays_per_frame + B = data.distances.shape[0] + + # Each ray's height = its frame's Z - hit Z. For single-frame sensors (F=1) this + # reduces to the original pos_w[:, 2] - hit_z broadcast. + frame_z = data.frame_pos_w[:, :, 2:3] # [B, F, 1] + hit_z = data.hit_pos_w[..., 2].view(B, F, N) # [B, F, N] + heights = (frame_z - hit_z - offset).view(B, F * N) + + miss_mask = data.distances < 0 + return torch.where(miss_mask, torch.full_like(heights, miss_value), heights) diff --git a/rl/mjlab/envs/mdp/rewards.py b/rl/mjlab/envs/mdp/rewards.py new file mode 100644 index 0000000..6658334 --- /dev/null +++ b/rl/mjlab/envs/mdp/rewards.py @@ -0,0 +1,160 @@ +"""Useful methods for MDP rewards.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.entity import Entity +from mjlab.managers.reward_manager import RewardTermCfg +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.lab_api.string import ( + resolve_matching_names_values, +) + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + +_DEFAULT_ASSET_CFG = SceneEntityCfg("robot") + + +def is_alive(env: ManagerBasedRlEnv) -> torch.Tensor: + """Reward for being alive.""" + return (~env.termination_manager.terminated).float() + + +def is_terminated(env: ManagerBasedRlEnv) -> torch.Tensor: + """Penalize terminated episodes that don't correspond to episodic timeouts.""" + return env.termination_manager.terminated.float() + + +def joint_torques_l2( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + """Penalize joint torques applied on the articulation using L2 squared kernel.""" + asset: Entity = env.scene[asset_cfg.name] + return torch.sum( + torch.square(asset.data.actuator_force[:, asset_cfg.actuator_ids]), dim=1 + ) + + +def joint_vel_l2( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + """Penalize joint velocities on the articulation using L2 squared kernel.""" + asset: Entity = env.scene[asset_cfg.name] + return torch.sum(torch.square(asset.data.joint_vel[:, asset_cfg.joint_ids]), dim=1) + + +def joint_acc_l2( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + """Penalize joint accelerations on the articulation using L2 squared kernel.""" + asset: Entity = env.scene[asset_cfg.name] + return torch.sum(torch.square(asset.data.joint_acc[:, asset_cfg.joint_ids]), dim=1) + + +def action_rate_l2(env: ManagerBasedRlEnv) -> torch.Tensor: + """Penalize the rate of change of the actions using L2 squared kernel. + + Operates on raw policy output (before per-term scale/offset). + """ + return torch.sum( + torch.square(env.action_manager.action - env.action_manager.prev_action), dim=1 + ) + + +def action_acc_l2(env: ManagerBasedRlEnv) -> torch.Tensor: + """Penalize the acceleration of the actions using L2 squared kernel. + + Operates on raw policy output (before per-term scale/offset). + """ + action_acc = ( + env.action_manager.action + - 2 * env.action_manager.prev_action + + env.action_manager.prev_prev_action + ) + return torch.sum(torch.square(action_acc), dim=1) + + +def joint_pos_limits( + env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG +) -> torch.Tensor: + """Penalize joint positions if they cross the soft limits.""" + asset: Entity = env.scene[asset_cfg.name] + soft_joint_pos_limits = asset.data.soft_joint_pos_limits + assert soft_joint_pos_limits is not None + out_of_limits = -( + asset.data.joint_pos[:, asset_cfg.joint_ids] + - soft_joint_pos_limits[:, asset_cfg.joint_ids, 0] + ).clip(max=0.0) + out_of_limits += ( + asset.data.joint_pos[:, asset_cfg.joint_ids] + - soft_joint_pos_limits[:, asset_cfg.joint_ids, 1] + ).clip(min=0.0) + return torch.sum(out_of_limits, dim=1) + + +class posture: + """Penalize the deviation of the joint positions from the default positions. + + Note: This is implemented as a class so that we can resolve the standard deviation + dictionary into a tensor and thereafter use it in the __call__ method. + """ + + def __init__(self, cfg: RewardTermCfg, env: ManagerBasedRlEnv): + asset: Entity = env.scene[cfg.params["asset_cfg"].name] + default_joint_pos = asset.data.default_joint_pos + assert default_joint_pos is not None + self.default_joint_pos = default_joint_pos + + asset_cfg = cfg.params["asset_cfg"] + _, joint_names = asset.find_joints( + asset_cfg.joint_names, preserve_order=asset_cfg.preserve_order + ) + + _, _, std = resolve_matching_names_values( + data=cfg.params["std"], + list_of_strings=joint_names, + ) + self.std = torch.tensor(std, device=env.device, dtype=torch.float32) + + def __call__( + self, env: ManagerBasedRlEnv, std, asset_cfg: SceneEntityCfg + ) -> torch.Tensor: + del std # Unused. + asset: Entity = env.scene[asset_cfg.name] + current_joint_pos = asset.data.joint_pos[:, asset_cfg.joint_ids] + desired_joint_pos = self.default_joint_pos[:, asset_cfg.joint_ids] + error_squared = torch.square(current_joint_pos - desired_joint_pos) + return torch.exp(-torch.mean(error_squared / (self.std**2), dim=1)) + + +class electrical_power_cost: + """Penalize electrical power consumption of actuators.""" + + def __init__(self, cfg: RewardTermCfg, env: ManagerBasedRlEnv): + asset: Entity = env.scene[cfg.params["asset_cfg"].name] + + joint_ids, _ = asset.find_joints( + cfg.params["asset_cfg"].joint_names, + ) + self._joint_ids = torch.tensor(joint_ids, device=env.device, dtype=torch.long) + + def __call__(self, env: ManagerBasedRlEnv, asset_cfg: SceneEntityCfg) -> torch.Tensor: + asset: Entity = env.scene[asset_cfg.name] + tau = asset.data.qfrc_actuator[:, self._joint_ids] + qd = asset.data.joint_vel[:, self._joint_ids] + mech = tau * qd + mech_pos = torch.clamp(mech, min=0.0) # Don't penalize regen. + return torch.sum(mech_pos, dim=1) + + +def flat_orientation_l2( + env: ManagerBasedRlEnv, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> torch.Tensor: + """Penalize non-flat base orientation.""" + asset: Entity = env.scene[asset_cfg.name] + return torch.sum(torch.square(asset.data.projected_gravity_b[:, :2]), dim=1) diff --git a/rl/mjlab/envs/mdp/terminations.py b/rl/mjlab/envs/mdp/terminations.py new file mode 100644 index 0000000..8f32240 --- /dev/null +++ b/rl/mjlab/envs/mdp/terminations.py @@ -0,0 +1,49 @@ +"""Useful methods for MDP terminations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from mjlab.managers.scene_entity_config import SceneEntityCfg +from mjlab.utils.nan_guard import NanGuard + +if TYPE_CHECKING: + from mjlab.entity import Entity + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + +_DEFAULT_ASSET_CFG = SceneEntityCfg("robot") + + +def time_out(env: ManagerBasedRlEnv) -> torch.Tensor: + """Terminate when the episode length exceeds its maximum.""" + return env.episode_length_buf >= env.max_episode_length + + +def bad_orientation( + env: ManagerBasedRlEnv, + limit_angle: float, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +): + """Terminate when the asset's orientation exceeds the limit angle.""" + asset: Entity = env.scene[asset_cfg.name] + projected_gravity = asset.data.projected_gravity_b + return ( + torch.acos(torch.clamp(-projected_gravity[:, 2], -1.0, 1.0)).abs() > limit_angle + ) + + +def root_height_below_minimum( + env: ManagerBasedRlEnv, + minimum_height: float, + asset_cfg: SceneEntityCfg = _DEFAULT_ASSET_CFG, +) -> torch.Tensor: + """Terminate when the asset's root height is below the minimum height.""" + asset: Entity = env.scene[asset_cfg.name] + return asset.data.root_link_pos_w[:, 2] < minimum_height + + +def nan_detection(env: ManagerBasedRlEnv) -> torch.Tensor: + """Terminate environments that have NaN/Inf values in their physics state.""" + return NanGuard.detect_nans(env.sim.data) diff --git a/rl/mjlab/envs/types.py b/rl/mjlab/envs/types.py new file mode 100644 index 0000000..7d49acd --- /dev/null +++ b/rl/mjlab/envs/types.py @@ -0,0 +1,6 @@ +from typing import Dict + +import torch + +VecEnvObs = Dict[str, torch.Tensor | Dict[str, torch.Tensor]] +VecEnvStepReturn = tuple[VecEnvObs, torch.Tensor, torch.Tensor, torch.Tensor, dict] diff --git a/rl/mjlab/licenses/IsaacLab.txt b/rl/mjlab/licenses/IsaacLab.txt new file mode 100644 index 0000000..dee9ba5 --- /dev/null +++ b/rl/mjlab/licenses/IsaacLab.txt @@ -0,0 +1,30 @@ +Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). + +All rights reserved. + +SPDX-License-Identifier: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt b/rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt new file mode 100644 index 0000000..12db770 --- /dev/null +++ b/rl/mjlab/licenses/mujoco_menagerie-i2rt_yam.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 i2rt robotics + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt b/rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt new file mode 100644 index 0000000..62637c8 --- /dev/null +++ b/rl/mjlab/licenses/mujoco_menagerie-unitree_g1.txt @@ -0,0 +1,27 @@ +Copyright (c) 2016-2023 HangZhou YuShu TECHNOLOGY CO.,LTD. ("Unitree Robotics") +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rl/mjlab/licenses/mujoco_menagerie-unitree_go1.txt b/rl/mjlab/licenses/mujoco_menagerie-unitree_go1.txt new file mode 100644 index 0000000..e472bbd --- /dev/null +++ b/rl/mjlab/licenses/mujoco_menagerie-unitree_go1.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2016-2022 HangZhou YuShu TECHNOLOGY CO.,LTD. ("Unitree Robotics") +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rl/mjlab/managers/__init__.py b/rl/mjlab/managers/__init__.py new file mode 100644 index 0000000..316f8ed --- /dev/null +++ b/rl/mjlab/managers/__init__.py @@ -0,0 +1,37 @@ +"""Environment managers.""" + +from mjlab.managers.action_manager import ActionManager as ActionManager +from mjlab.managers.action_manager import ActionTerm as ActionTerm +from mjlab.managers.action_manager import ActionTermCfg as ActionTermCfg +from mjlab.managers.command_manager import CommandManager as CommandManager +from mjlab.managers.command_manager import CommandTerm as CommandTerm +from mjlab.managers.command_manager import CommandTermCfg as CommandTermCfg +from mjlab.managers.command_manager import NullCommandManager as NullCommandManager +from mjlab.managers.curriculum_manager import CurriculumManager as CurriculumManager +from mjlab.managers.curriculum_manager import CurriculumTermCfg as CurriculumTermCfg +from mjlab.managers.curriculum_manager import ( + NullCurriculumManager as NullCurriculumManager, +) +from mjlab.managers.event_manager import EventManager as EventManager +from mjlab.managers.event_manager import EventMode as EventMode +from mjlab.managers.event_manager import EventTermCfg as EventTermCfg +from mjlab.managers.manager_base import ManagerBase as ManagerBase +from mjlab.managers.manager_base import ManagerTermBase as ManagerTermBase +from mjlab.managers.manager_base import ManagerTermBaseCfg as ManagerTermBaseCfg +from mjlab.managers.metrics_manager import MetricsManager as MetricsManager +from mjlab.managers.metrics_manager import MetricsTermCfg as MetricsTermCfg +from mjlab.managers.metrics_manager import NullMetricsManager as NullMetricsManager +from mjlab.managers.observation_manager import ( + ObservationGroupCfg as ObservationGroupCfg, +) +from mjlab.managers.observation_manager import ObservationManager as ObservationManager +from mjlab.managers.observation_manager import ObservationTermCfg as ObservationTermCfg +from mjlab.managers.recorder_manager import NullRecorderManager as NullRecorderManager +from mjlab.managers.recorder_manager import RecorderManager as RecorderManager +from mjlab.managers.recorder_manager import RecorderTerm as RecorderTerm +from mjlab.managers.recorder_manager import RecorderTermCfg as RecorderTermCfg +from mjlab.managers.reward_manager import RewardManager as RewardManager +from mjlab.managers.reward_manager import RewardTermCfg as RewardTermCfg +from mjlab.managers.scene_entity_config import SceneEntityCfg as SceneEntityCfg +from mjlab.managers.termination_manager import TerminationManager as TerminationManager +from mjlab.managers.termination_manager import TerminationTermCfg as TerminationTermCfg diff --git a/rl/mjlab/managers/action_manager.py b/rl/mjlab/managers/action_manager.py new file mode 100644 index 0000000..ab275d3 --- /dev/null +++ b/rl/mjlab/managers/action_manager.py @@ -0,0 +1,208 @@ +"""Action manager for processing actions sent to the environment.""" + +from __future__ import annotations + +import abc +from dataclasses import dataclass +from typing import TYPE_CHECKING, Sequence + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBase + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@dataclass(kw_only=True) +class ActionTermCfg(abc.ABC): + """Configuration for an action term. + + Action terms process raw actions from the policy and apply them to entities + in the scene (e.g., setting joint positions, velocities, or efforts). + """ + + entity_name: str + """Name of the entity in the scene that this action term controls.""" + + clip: dict[str, tuple] | None = None + """Optional clipping bounds applied to processed actions (after scale + and offset). Dict maps actuator name regex patterns to (min, max) + tuples, resolved the same way as ``scale`` and ``offset``.""" + + @abc.abstractmethod + def build(self, env: ManagerBasedRlEnv) -> ActionTerm: + """Build the action term from this config.""" + raise NotImplementedError + + +class ActionTerm(ManagerTermBase): + """Base class for action terms. + + The action term is responsible for processing the raw actions sent to the environment + and applying them to the entity managed by the term. + """ + + def __init__(self, cfg: ActionTermCfg, env: ManagerBasedRlEnv): + self.cfg = cfg + super().__init__(env) + self._entity = self._env.scene[self.cfg.entity_name] + + @property + @abc.abstractmethod + def action_dim(self) -> int: + raise NotImplementedError + + @abc.abstractmethod + def process_actions(self, actions: torch.Tensor) -> None: + raise NotImplementedError + + @abc.abstractmethod + def apply_actions(self) -> None: + raise NotImplementedError + + @property + @abc.abstractmethod + def raw_action(self) -> torch.Tensor: + raise NotImplementedError + + +class ActionManager(ManagerBase): + """Manages action processing for the environment. + + The action manager aggregates multiple action terms, each controlling a different + entity or aspect of the simulation. It splits the policy's action tensor and + routes each slice to the appropriate action term. + """ + + def __init__(self, cfg: dict[str, ActionTermCfg], env: ManagerBasedRlEnv): + self.cfg = cfg + super().__init__(env=env) + + # Create buffers to store actions. + self._action = torch.zeros( + (self.num_envs, self.total_action_dim), device=self.device + ) + self._prev_action = torch.zeros_like(self._action) + self._prev_prev_action = torch.zeros_like(self._action) + + def __str__(self) -> str: + msg = f" contains {len(self._term_names)} active terms.\n" + table = PrettyTable() + table.title = f"Active Action Terms (shape: {self.total_action_dim})" + table.field_names = ["Index", "Name", "Dimension"] + table.align["Name"] = "l" + table.align["Dimension"] = "r" + for index, (name, term) in enumerate(self._terms.items()): + table.add_row([index, name, term.action_dim]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def total_action_dim(self) -> int: + return sum(self.action_term_dim) + + @property + def action_term_dim(self) -> list[int]: + return [term.action_dim for term in self._terms.values()] + + @property + def action(self) -> torch.Tensor: + """Raw policy output from the current step, before per-term + scale/offset. Shape: ``(num_envs, total_action_dim)``.""" + return self._action + + @property + def prev_action(self) -> torch.Tensor: + """Raw policy output from the previous step, before per-term + scale/offset. Shape: ``(num_envs, total_action_dim)``.""" + return self._prev_action + + @property + def prev_prev_action(self) -> torch.Tensor: + """Raw policy output from two steps ago, before per-term + scale/offset. Shape: ``(num_envs, total_action_dim)``.""" + return self._prev_prev_action + + @property + def active_terms(self) -> list[str]: + return self._term_names + + # Methods. + + def get_term(self, name: str) -> ActionTerm: + return self._terms[name] + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> dict[str, float]: + if env_ids is None: + env_ids = slice(None) + # Reset action history. + self._prev_action[env_ids] = 0.0 + self._prev_prev_action[env_ids] = 0.0 + self._action[env_ids] = 0.0 + # Reset action terms. + for term in self._terms.values(): + term.reset(env_ids=env_ids) + return {} + + def process_action(self, action: torch.Tensor) -> None: + """Store the raw policy output and route slices to each action term. + + Called once per policy step. The raw action tensor is saved into the + history buffers (``action``, ``prev_action``, ``prev_prev_action``) *before* + any per-term scale/offset is applied. Each term then receives its slice and + independently applies its own affine transformation via + :meth:`ActionTerm.process_actions`. + """ + if self.total_action_dim != action.shape[1]: + raise ValueError( + f"Invalid action shape, expected: {self.total_action_dim}," + f" received: {action.shape[1]}." + ) + # Shift history: prev_prev ← prev ← current ← new. + self._prev_prev_action[:] = self._prev_action + self._prev_action[:] = self._action + self._action[:] = action.to(self.device) + # Split the flat action vector and route each slice to its term. + idx = 0 + for term in self._terms.values(): + term_actions = action[:, idx : idx + term.action_dim] + term.process_actions(term_actions) + idx += term.action_dim + + def apply_action(self) -> None: + """Write processed actions to entity actuator targets. + + Called on every decimation substep (physics step), not just once per policy + step. Each term writes its most recently processed targets to the simulation. + """ + for term in self._terms.values(): + term.apply_actions() + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + idx = 0 + for name, term in self._terms.items(): + term_actions = self._action[env_idx, idx : idx + term.action_dim].cpu() + terms.append((name, term_actions.tolist())) + idx += term.action_dim + return terms + + def _prepare_terms(self): + self._term_names: list[str] = list() + self._terms: dict[str, ActionTerm] = dict() + + for term_name, term_cfg in self.cfg.items(): + term_cfg: ActionTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + term = term_cfg.build(self._env) + self._term_names.append(term_name) + self._terms[term_name] = term diff --git a/rl/mjlab/managers/command_manager.py b/rl/mjlab/managers/command_manager.py new file mode 100644 index 0000000..950a34b --- /dev/null +++ b/rl/mjlab/managers/command_manager.py @@ -0,0 +1,373 @@ +"""Command manager for generating and updating commands.""" + +from __future__ import annotations + +import abc +import inspect +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Sequence + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBase + +if TYPE_CHECKING: + import viser + + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + from mjlab.viewer.debug_visualizer import DebugVisualizer + + +@dataclass(kw_only=True) +class CommandTermCfg(abc.ABC): + """Configuration for a command generator term. + + Command terms generate goal commands for the agent (e.g., target velocity, + target position). Commands are automatically resampled at configurable + intervals and can track metrics for logging. + """ + + resampling_time_range: tuple[float, float] + """Time range in seconds for command resampling. When the timer expires, a new + command is sampled and the timer is reset to a value uniformly drawn from + ``[min, max]``. Set both values equal for fixed-interval resampling.""" + + debug_vis: bool = False + """Whether to enable debug visualization for this command term. When True, + the command term's ``_debug_vis_impl`` method is called each frame to render + visual aids (e.g., velocity arrows, target markers).""" + + @abc.abstractmethod + def build(self, env: ManagerBasedRlEnv) -> CommandTerm: + """Build the command term from this config.""" + raise NotImplementedError + + +class CommandTerm(ManagerTermBase): + """Base class for command terms.""" + + def __init__(self, cfg: CommandTermCfg, env: ManagerBasedRlEnv): + self.cfg = cfg + super().__init__(env) + self._check_update_command_signature() + self.metrics = dict() + self.time_left = torch.zeros(self.num_envs, device=self.device) + self.command_counter = torch.zeros( + self.num_envs, device=self.device, dtype=torch.long + ) + self._debug_vis_enabled: bool = True + + def debug_vis(self, visualizer: "DebugVisualizer") -> None: + if self.cfg.debug_vis and self._debug_vis_enabled: + self._debug_vis_impl(visualizer) + + def _debug_vis_impl(self, visualizer: "DebugVisualizer") -> None: + pass + + def create_gui( + self, + name: str, + server: viser.ViserServer, + get_env_idx: Callable[[], int], + on_change: Callable[[], None] | None = None, + request_action: Callable[[str, Any], None] | None = None, + ) -> None: + """Create interactive GUI controls for this command term. + + Override in subclasses to add task-specific controls (e.g., velocity sliders) to + the Viser viewer. Called once during viewer setup. + + The *name* argument is the term's key in the command manager config (e.g., + ``"twist"``). + """ + + def on_viewer_pause(self, paused: bool) -> None: + """Called when the viewer pause state changes.""" + + def apply_gui_reset(self, env_ids: torch.Tensor) -> bool: + """Apply GUI-selected state as an env reset override. + + Returns True if this term wrote state to sim. + """ + return False + + @property + @abc.abstractmethod + def command(self): + raise NotImplementedError + + def reset(self, env_ids: torch.Tensor | slice | None) -> dict[str, float]: + assert isinstance(env_ids, torch.Tensor) + extras = {} + for metric_name, metric_value in self.metrics.items(): + extras[metric_name] = torch.mean(metric_value[env_ids]).item() + metric_value[env_ids] = 0.0 + self.command_counter[env_ids] = 0 + self._resample(env_ids) + return extras + + def compute( + self, dt: float | torch.Tensor, env_ids: torch.Tensor | None = None + ) -> None: + """Advance the command state by dt. + + With env_ids=None (the per-step path) all envs are updated; with env_ids + (the reset path) timers and the command update are scoped to those envs. + Metrics are always refreshed. + + dt may be a scalar (all envs) or a per-env tensor (auto-reset path, + where freshly reset envs get zero to keep their timers full). A tensor + dt requires env_ids=None. + """ + self._update_metrics() + if env_ids is None: + self.time_left -= dt + resample_env_ids = (self.time_left <= 0.0).nonzero().flatten() + else: + assert not isinstance(dt, torch.Tensor) + self.time_left[env_ids] -= dt + resample_env_ids = env_ids[self.time_left[env_ids] <= 0.0] + if len(resample_env_ids) > 0: + self._resample(resample_env_ids) + self._update_command(env_ids) + + def _check_update_command_signature(self) -> None: + """Fail fast with a migration hint for terms with the old signature.""" + try: + sig = inspect.signature(self._update_command) + except (TypeError, ValueError): + return + if len(sig.parameters) == 0: + raise TypeError( + f"{type(self).__name__}._update_command must accept env_ids: " + "_update_command(self, env_ids: torch.Tensor | None). It receives " + "None on the per-step update and the reset env ids on reset(); " + "scope per-step state advances to env_ids." + ) + + def _resample(self, env_ids: torch.Tensor) -> None: + if len(env_ids) != 0: + self.time_left[env_ids] = self.time_left[env_ids].uniform_( + *self.cfg.resampling_time_range + ) + self._resample_command(env_ids) + self.command_counter[env_ids] += 1 + + @abc.abstractmethod + def _update_metrics(self) -> None: + """Update the metrics based on the current state.""" + raise NotImplementedError + + @abc.abstractmethod + def _resample_command(self, env_ids: torch.Tensor) -> None: + """Resample the command for the specified environments.""" + raise NotImplementedError + + @abc.abstractmethod + def _update_command(self, env_ids: torch.Tensor | None) -> None: + """Update the command based on the current state. + + env_ids is None on the per-step update (all envs) and the reset env ids on reset(). + Scope per-step state advances (e.g. a motion frame index) to env_ids; pure + functions of the current state may ignore it. + """ + raise NotImplementedError + + +class CommandManager(ManagerBase): + """Manages command generation for the environment. + + The command manager generates and updates goal commands for the agent (e.g., + target velocity, target position). Commands are resampled at configurable + intervals and can track metrics for logging. + """ + + _env: ManagerBasedRlEnv + + def __init__(self, cfg: dict[str, CommandTermCfg], env: ManagerBasedRlEnv): + self._terms: dict[str, CommandTerm] = dict() + + self.cfg = cfg + super().__init__(env) + self._commands = dict() + + def __str__(self) -> str: + msg = f" contains {len(self._terms.values())} active terms.\n" + table = PrettyTable() + table.title = "Active Command Terms" + table.field_names = ["Index", "Name", "Type"] + table.align["Name"] = "l" + for index, (name, term) in enumerate(self._terms.items()): + table.add_row([index, name, term.__class__.__name__]) + msg += table.get_string() + msg += "\n" + return msg + + def debug_vis(self, visualizer: "DebugVisualizer") -> None: + for term in self._terms.values(): + term.debug_vis(visualizer) + + def create_gui( + self, + server: viser.ViserServer, + get_env_idx: Callable[[], int], + on_change: Callable[[], None] | None = None, + request_action: Callable[[str, Any], None] | None = None, + ) -> None: + """Let each command term create its GUI controls.""" + for name, term in self._terms.items(): + term.create_gui( + name, + server, + get_env_idx, + on_change=on_change, + request_action=request_action, + ) + + def on_viewer_pause(self, paused: bool) -> None: + """Notify all command terms of viewer pause state change.""" + for term in self._terms.values(): + term.on_viewer_pause(paused) + + def apply_gui_reset(self, env_ids: torch.Tensor) -> bool: + """Apply GUI-selected state from all terms. Returns True if any applied.""" + applied = False + for term in self._terms.values(): + applied |= term.apply_gui_reset(env_ids) + return applied + + def create_debug_vis_gui( + self, + server: viser.ViserServer, + on_change: Callable[[], None] | None = None, + ) -> None: + """Add per-term debug visualization checkboxes.""" + vis_terms = {name: term for name, term in self._terms.items() if term.cfg.debug_vis} + if not vis_terms: + return + for name, term in vis_terms.items(): + cb = server.gui.add_checkbox( + name.capitalize(), + initial_value=term._debug_vis_enabled, + ) + + def _on_update(_ev, _term: CommandTerm = term, _cb=cb) -> None: + _term._debug_vis_enabled = _cb.value + if on_change is not None: + on_change() + + cb.on_update(_on_update) + + # Properties. + + @property + def active_terms(self) -> list[str]: + return list(self._terms.keys()) + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + idx = 0 + for name, term in self._terms.items(): + terms.append((name, term.command[env_idx].cpu().tolist())) + idx += term.command.shape[1] + return terms + + def reset(self, env_ids: torch.Tensor | None) -> dict[str, torch.Tensor]: + extras = {} + for name, term in self._terms.items(): + metrics = term.reset(env_ids=env_ids) + for metric_name, metric_value in metrics.items(): + extras[f"Metrics/{name}/{metric_name}"] = metric_value + return extras + + def compute(self, dt: float | torch.Tensor, env_ids: torch.Tensor | None = None): + for term in self._terms.values(): + term.compute(dt, env_ids) + + def get_command(self, name: str) -> torch.Tensor: + return self._terms[name].command + + def get_term(self, name: str) -> CommandTerm: + return self._terms[name] + + def get_term_cfg(self, name: str) -> CommandTermCfg: + return self.cfg[name] + + def _prepare_terms(self): + for term_name, term_cfg in self.cfg.items(): + term_cfg: CommandTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + term = term_cfg.build(self._env) + if not isinstance(term, CommandTerm): + raise TypeError( + f"Returned object for the term {term_name} is not of type CommandType." + ) + self._terms[term_name] = term + + +class NullCommandManager: + """Placeholder for absent command manager that safely no-ops all operations.""" + + def __init__(self): + self.active_terms: list[str] = [] + self._terms: dict[str, Any] = {} + self.cfg = None + + def __str__(self) -> str: + return " (inactive)" + + def __repr__(self) -> str: + return "NullCommandManager()" + + def debug_vis(self, visualizer: "DebugVisualizer") -> None: + pass + + def create_gui( + self, + server: viser.ViserServer, + get_env_idx: Callable[[], int], + on_change: Callable[[], None] | None = None, + request_action: Callable[[str, Any], None] | None = None, + ) -> None: + pass + + def on_viewer_pause(self, paused: bool) -> None: + pass + + def apply_gui_reset(self, env_ids: torch.Tensor) -> bool: + return False + + def create_debug_vis_gui( + self, + server: viser.ViserServer, + on_change: Callable[[], None] | None = None, + ) -> None: + pass + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + return [] + + def reset(self, env_ids: torch.Tensor | None = None) -> dict[str, torch.Tensor]: + return {} + + def compute( + self, dt: float | torch.Tensor, env_ids: torch.Tensor | None = None + ) -> None: + pass + + def get_command(self, name: str) -> None: + return None + + def get_term(self, name: str) -> None: + return None + + def get_term_cfg(self, name: str) -> None: + return None diff --git a/rl/mjlab/managers/curriculum_manager.py b/rl/mjlab/managers/curriculum_manager.py new file mode 100644 index 0000000..cafd2d2 --- /dev/null +++ b/rl/mjlab/managers/curriculum_manager.py @@ -0,0 +1,155 @@ +"""Curriculum manager for updating environment quantities subject to a training curriculum.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Sequence + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + + +@dataclass(kw_only=True) +class CurriculumTermCfg(ManagerTermBaseCfg): + """Configuration for a curriculum term. + + Curriculum terms modify environment parameters during training to implement + curriculum learning strategies (e.g., gradually increasing task difficulty). + """ + + pass + + +class CurriculumManager(ManagerBase): + """Manages curriculum learning for the environment. + + The curriculum manager updates environment parameters during training based + on agent performance. Each term can modify different aspects of the task + difficulty (e.g., terrain complexity, command ranges). + """ + + _env: ManagerBasedRlEnv + + def __init__(self, cfg: dict[str, CurriculumTermCfg], env: ManagerBasedRlEnv): + self._term_names: list[str] = list() + self._term_cfgs: list[CurriculumTermCfg] = list() + self._class_term_cfgs: list[CurriculumTermCfg] = list() + + self.cfg = deepcopy(cfg) + super().__init__(env) + + self._curriculum_state = dict() + for term_name in self._term_names: + self._curriculum_state[term_name] = None + + def __str__(self) -> str: + msg = f" contains {len(self._term_names)} active terms.\n" + table = PrettyTable() + table.title = "Active Curriculum Terms" + table.field_names = ["Index", "Name"] + table.align["Name"] = "l" + for index, name in enumerate(self._term_names): + table.add_row([index, name]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def active_terms(self) -> list[str]: + return self._term_names + + # Methods. + + def get_term_cfg(self, term_name: str) -> CurriculumTermCfg: + if term_name not in self._term_names: + raise ValueError(f"Term '{term_name}' not found in active terms.") + return self._term_cfgs[self._term_names.index(term_name)] + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + for term_name, term_state in self._curriculum_state.items(): + if term_state is not None: + data = [] + if isinstance(term_state, dict): + for _key, value in term_state.items(): + if isinstance(value, torch.Tensor): + value = value.item() + data.append(value) + else: + if isinstance(term_state, torch.Tensor): + term_state = term_state.item() + data.append(term_state) + terms.append((term_name, data)) + return terms + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> dict[str, float]: + extras = {} + for term_name, term_state in self._curriculum_state.items(): + if term_state is not None: + if isinstance(term_state, dict): + for key, value in term_state.items(): + if isinstance(value, torch.Tensor): + value = value.item() + extras[f"Curriculum/{term_name}/{key}"] = value + else: + if isinstance(term_state, torch.Tensor): + term_state = term_state.item() + extras[f"Curriculum/{term_name}"] = term_state + for term_cfg in self._class_term_cfgs: + term_cfg.func.reset(env_ids=env_ids) + return extras + + def compute(self, env_ids: torch.Tensor | slice | None = None): + if env_ids is None: + env_ids = slice(None) + for name, term_cfg in zip(self._term_names, self._term_cfgs, strict=False): + state = term_cfg.func(self._env, env_ids, **term_cfg.params) + self._curriculum_state[name] = state + + def _prepare_terms(self): + for term_name, term_cfg in self.cfg.items(): + term_cfg: CurriculumTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + self._resolve_common_term_cfg(term_name, term_cfg) + self._term_names.append(term_name) + self._term_cfgs.append(term_cfg) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._class_term_cfgs.append(term_cfg) + + +class NullCurriculumManager: + """Placeholder for absent curriculum manager that safely no-ops all operations.""" + + def __init__(self): + self.active_terms: list[str] = [] + self._curriculum_state: dict[str, Any] = {} + self.cfg = None + + def __str__(self) -> str: + return " (inactive)" + + def __repr__(self) -> str: + return "NullCurriculumManager()" + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + return [] + + def reset(self, env_ids: torch.Tensor | None = None) -> dict[str, float]: + return {} + + def compute(self, env_ids: torch.Tensor | None = None) -> None: + pass diff --git a/rl/mjlab/managers/event_manager.py b/rl/mjlab/managers/event_manager.py new file mode 100644 index 0000000..6215b09 --- /dev/null +++ b/rl/mjlab/managers/event_manager.py @@ -0,0 +1,390 @@ +"""Event manager for orchestrating operations based on different simulation events.""" + +from __future__ import annotations + +import enum +from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + from mjlab.viewer.debug_visualizer import DebugVisualizer + +F = Callable[..., None] + +EventMode = Literal["startup", "reset", "interval", "step"] + + +class RecomputeLevel(enum.IntEnum): + """Recomputation level for derived model constants after domain randomization. + + Higher values are more expensive and recompute a superset of the lower levels. + + .. note:: + + All levels above ``none`` are expensive (forward kinematics, mass matrix + factorization, etc.). Prefer ``startup`` or ``reset`` event modes for + DR terms that require recomputation. + """ + + none = 0 + """No recomputation needed (e.g. geom_friction, dof_damping).""" + + set_const_fixed = 1 + """Recompute ``body_subtreemass``. Use after modifying ``body_gravcomp``.""" + + set_const_0 = 2 + """Recompute ``dof_invweight0``, ``body_invweight0``, ``tendon_length0``, + ``tendon_invweight0``, and ``actuator_acc0``. Use after modifying + ``dof_armature``, ``body_inertia``, ``body_pos``, ``body_quat``, or ``qpos0``.""" + + set_const = 3 + """Full recomputation (superset of all lower levels). Use after modifying + ``body_mass`` or ``body_ipos``.""" + + +_DERIVED_FIELDS: dict[RecomputeLevel, tuple[str, ...]] = { + RecomputeLevel.none: (), + RecomputeLevel.set_const_fixed: ("body_subtreemass",), + RecomputeLevel.set_const_0: ( + "dof_invweight0", + "body_invweight0", + "tendon_length0", + "tendon_invweight0", + "actuator_acc0", + ), +} +_DERIVED_FIELDS[RecomputeLevel.set_const] = ( + _DERIVED_FIELDS[RecomputeLevel.set_const_fixed] + + _DERIVED_FIELDS[RecomputeLevel.set_const_0] +) + + +def requires_model_fields( + *fields: str, recompute: RecomputeLevel = RecomputeLevel.none +) -> Callable[[F], F]: + """Mark an event function as requiring specific model fields expanded per-world. + + Fields listed here are registered in ``EventManager.domain_randomization_fields`` + so that ``sim.expand_model_fields()`` allocates real per-world memory for them. + + Args: + *fields: Model field names to expand per-world. + recompute: Recomputation level after modifying these fields. + Derived fields are automatically appended to the model_fields list. + + Example:: + + @requires_model_fields("body_mass", recompute=RecomputeLevel.set_const) + def body_mass(env, env_ids, ...): + ... + """ + derived = _DERIVED_FIELDS[recompute] + all_fields = fields + tuple(f for f in derived if f not in fields) + + def decorator(func: F) -> F: + func.model_fields = all_fields # type: ignore[attr-defined] + func.recompute = recompute # type: ignore[attr-defined] + return func + + return decorator + + +@dataclass(kw_only=True) +class EventTermCfg(ManagerTermBaseCfg): + """Configuration for an event term. + + Event terms trigger operations at specific simulation events. They're commonly + used for domain randomization, state resets, and periodic perturbations. + + The four modes determine when the event fires: + + - ``"startup"``: Once when the environment initializes. Use for parameters that + should be randomized per-environment but stay constant within an episode (e.g., + domain randomization). + + - ``"reset"``: On every episode reset. Use for parameters that should vary between + episodes (e.g., initial robot pose, domain randomization). + + - ``"interval"``: Periodically during simulation, controlled by ``interval_range_s``. + Use for perturbations that should happen during episodes (e.g., pushing the robot, + external disturbances). + + - ``"step"``: Every environment step, unconditionally on all envs. Use for terms that + manage per-step state such as force lifetimes (e.g., ``apply_body_impulse``). + """ + + mode: EventMode + """When the event triggers: ``"startup"`` (once at init), ``"reset"`` (every + episode), ``"interval"`` (periodically during simulation), or ``"step"`` (every + environment step).""" + + interval_range_s: tuple[float, float] | None = None + """Time range in seconds for interval mode. The next trigger time is uniformly + sampled from ``[min, max]``. Required when ``mode="interval"``.""" + + is_global_time: bool = False + """Whether all environments share the same timer. If True, all envs trigger + simultaneously. If False (default), each env has an independent timer that + resets on episode reset. Only applies to ``mode="interval"``.""" + + min_step_count_between_reset: int = 0 + """Minimum environment steps between triggers. Prevents the event from firing + too frequently when episodes reset rapidly. Only applies to ``mode="reset"``. + Set to 0 (default) to trigger on every reset.""" + + +class EventManager(ManagerBase): + """Manages event-based operations for the environment. + + The event manager triggers operations at different simulation events: startup + (once at initialization), reset (on episode reset), or interval (periodically + during simulation). Common uses include domain randomization and state resets. + """ + + _env: ManagerBasedRlEnv + + def __init__(self, cfg: dict[str, EventTermCfg], env: ManagerBasedRlEnv): + self.cfg = deepcopy(cfg) + self._mode_term_names: dict[EventMode, list[str]] = dict() + self._mode_term_cfgs: dict[EventMode, list[EventTermCfg]] = dict() + self._mode_class_term_cfgs: dict[EventMode, list[EventTermCfg]] = dict() + self._domain_randomization_fields: list[str] = list() + + super().__init__(env=env) + + def __str__(self) -> str: + msg = f" contains {len(self._mode_term_names)} active terms.\n" + for mode in self._mode_term_names: + table = PrettyTable() + table.title = f"Active Event Terms in Mode: '{mode}'" + if mode == "interval": + table.field_names = ["Index", "Name", "Interval time range (s)"] + table.align["Name"] = "l" + for index, (name, cfg) in enumerate( + zip(self._mode_term_names[mode], self._mode_term_cfgs[mode], strict=False) + ): + table.add_row([index, name, cfg.interval_range_s]) + else: + table.field_names = ["Index", "Name"] + table.align["Name"] = "l" + for index, name in enumerate(self._mode_term_names[mode]): + table.add_row([index, name]) + msg += table.get_string() + msg += "\n" + if self._domain_randomization_fields: + table = PrettyTable() + table.title = "Domain Randomization Fields" + table.field_names = ["Index", "Field Name"] + table.align["Field Name"] = "l" + for index, field in enumerate(self._domain_randomization_fields): + table.add_row([index, field]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def active_terms(self) -> dict[EventMode, list[str]]: + return self._mode_term_names + + @property + def available_modes(self) -> list[EventMode]: + return list(self._mode_term_names.keys()) + + @property + def domain_randomization_fields(self) -> tuple[str, ...]: + return tuple(self._domain_randomization_fields) + + # Methods. + + def get_term_cfg(self, term_name: str) -> EventTermCfg: + """Get the configuration of a specific event term by name.""" + for mode in self._mode_term_names: + if term_name in self._mode_term_names[mode]: + index = self._mode_term_names[mode].index(term_name) + return self._mode_term_cfgs[mode][index] + raise ValueError(f"Event term '{term_name}' not found in active terms.") + + def reset(self, env_ids: torch.Tensor | None = None): + for mode_cfg in self._mode_class_term_cfgs.values(): + for term_cfg in mode_cfg: + term_cfg.func.reset(env_ids=env_ids) + if env_ids is None: + num_envs = self._env.num_envs + ids: torch.Tensor | slice = slice(None) + else: + num_envs = len(env_ids) + ids = env_ids + # Iterate the full interval term list: _interval_term_time_left is parallel + # to _mode_term_cfgs["interval"], not the class-only subset. + if "interval" in self._mode_term_cfgs: + for index, term_cfg in enumerate(self._mode_term_cfgs["interval"]): + if not term_cfg.is_global_time: + assert term_cfg.interval_range_s is not None + lower, upper = term_cfg.interval_range_s + sampled_interval = ( + torch.rand(num_envs, device=self.device) * (upper - lower) + lower + ) + self._interval_term_time_left[index][ids] = sampled_interval + return {} + + def apply( + self, + mode: EventMode, + env_ids: torch.Tensor | slice | None = None, + dt: float | None = None, + global_env_step_count: int | None = None, + ): + if mode == "interval" and dt is None: + raise ValueError( + f"Event mode '{mode}' requires the time-step of the environment." + ) + if mode == "interval" and env_ids is not None: + raise ValueError( + f"Event mode '{mode}' does not require environment indices. This is an undefined behavior" + " as the environment indices are computed based on the time left for each environment." + ) + if mode == "reset" and global_env_step_count is None: + raise ValueError( + f"Event mode '{mode}' requires the total number of environment steps to be provided." + ) + if mode == "step" and dt is None: + raise ValueError( + f"Event mode '{mode}' requires the time-step of the environment." + ) + + strongest_fired = RecomputeLevel.none + + for index, term_cfg in enumerate(self._mode_term_cfgs[mode]): + fired = False + if mode == "interval": + time_left = self._interval_term_time_left[index] + assert dt is not None + time_left -= dt + if term_cfg.is_global_time: + if time_left < 1e-6: + assert term_cfg.interval_range_s is not None + lower, upper = term_cfg.interval_range_s + sampled_interval = torch.rand(1) * (upper - lower) + lower + self._interval_term_time_left[index][:] = sampled_interval + term_cfg.func(self._env, None, **term_cfg.params) + fired = True + else: + valid_env_ids = (time_left < 1e-6).nonzero().flatten() + if len(valid_env_ids) > 0: + assert term_cfg.interval_range_s is not None + lower, upper = term_cfg.interval_range_s + sampled_time = ( + torch.rand(len(valid_env_ids), device=self.device) * (upper - lower) + + lower + ) + self._interval_term_time_left[index][valid_env_ids] = sampled_time + term_cfg.func(self._env, valid_env_ids, **term_cfg.params) + fired = True + elif mode == "step": + term_cfg.func(self._env, None, **term_cfg.params) + fired = True + elif mode == "reset": + assert global_env_step_count is not None + # Reset events require concrete indices: callers (e.g. ManagerBasedRlEnv) + # resolve None to all environments upstream. Enforce that here so a future + # caller passing None fails loudly instead of leaking a slice into event + # functions, which only understand None or a tensor. + assert env_ids is not None, "reset events require concrete env_ids, got None" + min_step_count = term_cfg.min_step_count_between_reset + if min_step_count == 0: + self._reset_term_last_triggered_step_id[index][env_ids] = ( + global_env_step_count + ) + self._reset_term_last_triggered_once[index][env_ids] = True + term_cfg.func(self._env, env_ids, **term_cfg.params) + fired = True + else: + last_triggered_step = self._reset_term_last_triggered_step_id[index][env_ids] + triggered_at_least_once = self._reset_term_last_triggered_once[index][env_ids] + steps_since_triggered = global_env_step_count - last_triggered_step + valid_trigger = steps_since_triggered >= min_step_count + valid_trigger |= (last_triggered_step == 0) & ~triggered_at_least_once + if isinstance(env_ids, torch.Tensor): + valid_env_ids = env_ids[valid_trigger] + else: + valid_env_ids = valid_trigger.nonzero().flatten() + if len(valid_env_ids) > 0: + self._reset_term_last_triggered_once[index][valid_env_ids] = True + self._reset_term_last_triggered_step_id[index][valid_env_ids] = ( + global_env_step_count + ) + term_cfg.func(self._env, valid_env_ids, **term_cfg.params) + fired = True + else: + term_cfg.func(self._env, env_ids, **term_cfg.params) + fired = True + + if fired: + level = getattr(term_cfg.func, "recompute", RecomputeLevel.none) + strongest_fired = max(strongest_fired, level) + + if strongest_fired != RecomputeLevel.none: + self._env.sim.recompute_constants(strongest_fired) + + def debug_vis(self, visualizer: "DebugVisualizer") -> None: + """Delegate debug visualization to class-based event terms.""" + for mode_cfgs in self._mode_class_term_cfgs.values(): + for term_cfg in mode_cfgs: + if hasattr(term_cfg.func, "debug_vis"): + term_cfg.func.debug_vis(visualizer) + + def _prepare_terms(self) -> None: + self._interval_term_time_left: list[torch.Tensor] = list() + self._reset_term_last_triggered_step_id: list[torch.Tensor] = list() + self._reset_term_last_triggered_once: list[torch.Tensor] = list() + + for term_name, term_cfg in self.cfg.items(): + term_cfg: EventTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + self._resolve_common_term_cfg(term_name, term_cfg) + if term_cfg.mode not in self._mode_term_names: + self._mode_term_names[term_cfg.mode] = list() + self._mode_term_cfgs[term_cfg.mode] = list() + self._mode_class_term_cfgs[term_cfg.mode] = list() + self._mode_term_names[term_cfg.mode].append(term_name) + self._mode_term_cfgs[term_cfg.mode].append(term_cfg) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._mode_class_term_cfgs[term_cfg.mode].append(term_cfg) + if term_cfg.mode == "interval": + if term_cfg.interval_range_s is None: + raise ValueError( + f"Event term '{term_name}' has mode 'interval' but 'interval_range_s' is not specified." + ) + if term_cfg.is_global_time: + lower, upper = term_cfg.interval_range_s + time_left = torch.rand(1) * (upper - lower) + lower + self._interval_term_time_left.append(time_left) + else: + lower, upper = term_cfg.interval_range_s + time_left = ( + torch.rand(self.num_envs, device=self.device) * (upper - lower) + lower + ) + self._interval_term_time_left.append(time_left) + elif term_cfg.mode == "reset": + step_count = torch.zeros(self.num_envs, device=self.device, dtype=torch.int32) + self._reset_term_last_triggered_step_id.append(step_count) + no_trigger = torch.zeros(self.num_envs, device=self.device, dtype=torch.bool) + self._reset_term_last_triggered_once.append(no_trigger) + + func = term_cfg.func + if hasattr(func, "model_fields"): + for field in func.model_fields: + if field not in self._domain_randomization_fields: + self._domain_randomization_fields.append(field) diff --git a/rl/mjlab/managers/manager_base.py b/rl/mjlab/managers/manager_base.py new file mode 100644 index 0000000..364bd45 --- /dev/null +++ b/rl/mjlab/managers/manager_base.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import abc +import inspect +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +import torch + +from mjlab.managers.scene_entity_config import SceneEntityCfg + +if TYPE_CHECKING: + from mjlab.envs import ManagerBasedRlEnv + + +@dataclass +class ManagerTermBaseCfg: + """Base configuration for manager terms. + + This is the base config for terms in observation, reward, termination, curriculum, + and event managers. It provides a common interface for specifying a callable + and its parameters. + + The ``func`` field accepts either a function or a class: + + **Function-based terms** are simpler and suitable for stateless computations: + + .. code-block:: python + + RewardTermCfg(func=mdp.joint_torques_l2, weight=-0.01) + + **Class-based terms** are instantiated with ``(cfg, env)`` and useful when you need + to: + + - Cache computed values at initialization (e.g., resolve regex patterns to indices) + - Maintain state across calls + - Perform expensive setup once rather than every call + + .. code-block:: python + + class posture: + def __init__(self, cfg: RewardTermCfg, env: ManagerBasedRlEnv): + # Resolve std dict to tensor once at init + self.std = resolve_std_to_tensor(cfg.params["std"], env) + + def __call__(self, env, **kwargs) -> torch.Tensor: + # Use cached self.std + return compute_posture_reward(env, self.std) + + RewardTermCfg(func=posture, params={"std": {".*knee.*": 0.3}}, weight=1.0) + + Class-based terms can optionally implement ``reset(env_ids)`` for per-episode state. + """ + + func: Any + """The callable that computes this term's value. Can be a function or a class. + Classes are auto-instantiated with ``(cfg=term_cfg, env=env)``.""" + + params: dict[str, Any] = field(default_factory=lambda: {}) + """Additional keyword arguments passed to func when called.""" + + +class ManagerTermBase: + def __init__(self, env: ManagerBasedRlEnv): + self._env = env + + # Properties. + + @property + def num_envs(self) -> int: + return self._env.num_envs + + @property + def device(self) -> str: + return self._env.device + + @property + def name(self) -> str: + return self.__class__.__name__ + + # Methods. + + def reset(self, env_ids: torch.Tensor | slice | None) -> Any: + """Resets the manager term.""" + del env_ids # Unused. + pass + + def __call__(self, *args, **kwargs) -> Any: + """Returns the value of the term required by the manager.""" + raise NotImplementedError + + +class ManagerBase(abc.ABC): + """Base class for all managers.""" + + def __init__(self, env: ManagerBasedRlEnv): + self._env = env + + self._prepare_terms() + + # Properties. + + @property + def num_envs(self) -> int: + return self._env.num_envs + + @property + def device(self) -> str: + return self._env.device + + @property + @abc.abstractmethod + def active_terms(self) -> list[str] | dict[Any, list[str]]: + raise NotImplementedError + + # Methods. + + def reset(self, env_ids: torch.Tensor) -> dict[str, Any]: + """Resets the manager and returns logging info for the current step.""" + del env_ids # Unused. + return {} + + def _check_term_shape(self, term_name: str, value: torch.Tensor) -> None: + if value.shape != (self.num_envs,): + manager_name = type(self).__name__ + raise ValueError( + f"{manager_name} term '{term_name}' returned shape {tuple(value.shape)}, " + f"expected ({self.num_envs},)." + ) + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + raise NotImplementedError + + @abc.abstractmethod + def _prepare_terms(self): + raise NotImplementedError + + def _resolve_common_term_cfg(self, term_name: str, term_cfg: ManagerTermBaseCfg): + del term_name # Unused. + for value in term_cfg.params.values(): + if isinstance(value, SceneEntityCfg): + value.resolve(self._env.scene) + if inspect.isclass(term_cfg.func): + term_cfg.func = term_cfg.func(cfg=term_cfg, env=self._env) diff --git a/rl/mjlab/managers/metrics_manager.py b/rl/mjlab/managers/metrics_manager.py new file mode 100644 index 0000000..020f8ab --- /dev/null +++ b/rl/mjlab/managers/metrics_manager.py @@ -0,0 +1,249 @@ +"""Metrics manager for logging custom per-step metrics during training.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Sequence + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + +REDUCE_OPTIONS = ("last", "max", "mean", "sum") + + +@dataclass(kw_only=True) +class MetricsTermCfg(ManagerTermBaseCfg): + """Configuration for a metrics term. + + Attributes: + per_substep: If True, evaluate this term once per physics substep inside + the decimation loop and report the per-step mean. Only the integrated + state (qpos, qvel, act) is current mid-loop; all derived quantities + (xpos, xquat, site_xpos, actuator_force, contacts, ...) are stale. + + reduce: How to aggregate per-step values into an episode metric. + - ``"mean"`` (default) reports ``sum / step_count``. + - ``"last"`` reports the value from the final step of the episode, + useful for binary success metrics that should not be averaged over + timesteps. + - ``"max"`` reports the highest value seen during the episode, useful + for peak metrics like maximum power or contact force. + - ``"sum"`` reports the accumulated total over the episode, useful for + cumulative quantities like episodic reward or total distance + traveled. + """ + + per_substep: bool = False + reduce: Literal["last", "max", "mean", "sum"] = "mean" + + +class MetricsManager(ManagerBase): + """Accumulates per-step metric values, reports episode averages. + + Unlike rewards, metrics have no weight, no dt scaling, and no + normalization by episode length. Episode values are true per-step + averages (sum / step_count), so a metric in [0,1] stays in [0,1] + in the logger. + """ + + _env: ManagerBasedRlEnv + + def __init__(self, cfg: dict[str, MetricsTermCfg], env: ManagerBasedRlEnv): + self._term_names: list[str] = list() + self._term_cfgs: list[MetricsTermCfg] = list() + self._class_term_cfgs: list[MetricsTermCfg] = list() + self._step_term_indices: list[int] = list() + self._substep_term_indices: list[int] = list() + + self.cfg = deepcopy(cfg) + super().__init__(env=env) + + self._episode_sums: dict[str, torch.Tensor] = {} + self._episode_max: dict[str, torch.Tensor] = {} + for idx, term_name in enumerate(self._term_names): + if self._term_cfgs[idx].reduce not in REDUCE_OPTIONS: + msg = ( + f"The reduce method '{self._term_cfgs[idx].reduce}' for metric '{term_name}' " + f"is unknown. Valid options are {REDUCE_OPTIONS}." + ) + raise ValueError(msg) + + self._episode_sums[term_name] = torch.zeros( + self.num_envs, dtype=torch.float, device=self.device + ) + + if self._term_cfgs[idx].reduce == "max": + self._episode_max[term_name] = torch.full( + (self.num_envs,), float("-inf"), dtype=torch.float, device=self.device + ) + # Pre-resolved tensor refs for substep terms to avoid dict lookups in + # the hot loop. + self._substep_accum: list[torch.Tensor] = [] + self._substep_episode_sums: list[torch.Tensor] = [] + self._substep_episode_max: list[torch.Tensor | None] = [] + for idx in self._substep_term_indices: + name = self._term_names[idx] + buf = torch.zeros(self.num_envs, dtype=torch.float, device=self.device) + self._substep_accum.append(buf) + self._substep_episode_sums.append(self._episode_sums[name]) + self._substep_episode_max.append(self._episode_max.get(name)) + self._substep_count: int = 0 + self._step_count = torch.zeros(self.num_envs, dtype=torch.long, device=self.device) + self._step_values = torch.zeros( + (self.num_envs, len(self._term_names)), dtype=torch.float, device=self.device + ) + + def __str__(self) -> str: + msg = f" contains {len(self._term_names)} active terms.\n" + table = PrettyTable() + table.title = "Active Metrics Terms" + table.field_names = ["Index", "Name"] + table.align["Name"] = "l" + for index, name in enumerate(self._term_names): + table.add_row([index, name]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def active_terms(self) -> list[str]: + return self._term_names + + # Methods. + + def reset( + self, env_ids: torch.Tensor | slice | None = None + ) -> dict[str, torch.Tensor]: + if env_ids is None: + env_ids = slice(None) + extras = {} + counts = self._step_count[env_ids].float() + # Avoid division by zero for envs that haven't stepped. + safe_counts = torch.clamp(counts, min=1.0) + for idx, key in enumerate(self._episode_sums): + reduce = self._term_cfgs[idx].reduce + if reduce == "max": + extras["Episode_Metrics/" + key] = torch.mean(self._episode_max[key][env_ids]) + self._episode_max[key][env_ids] = float("-inf") + + elif reduce == "last": + extras["Episode_Metrics/" + key] = torch.mean(self._step_values[env_ids, idx]) + + elif reduce == "sum": + extras["Episode_Metrics/" + key] = torch.mean(self._episode_sums[key][env_ids]) + + else: + extras["Episode_Metrics/" + key] = torch.mean( + self._episode_sums[key][env_ids] / safe_counts + ) + + self._episode_sums[key][env_ids] = 0.0 + self._step_count[env_ids] = 0 + + for buf in self._substep_accum: + buf[env_ids] = 0.0 + + for term_cfg in self._class_term_cfgs: + term_cfg.func.reset(env_ids=env_ids) + + return extras + + def compute_substep(self) -> None: + """Accumulate per-substep metric values inside the decimation loop. + + No-op when no ``per_substep`` terms are configured. + """ + if not self._substep_term_indices: + return + for i, idx in enumerate(self._substep_term_indices): + value = self._compute_term(idx) + self._substep_accum[i] += value + self._substep_count += 1 + + def compute(self) -> None: + self._step_count += 1 + if self._substep_term_indices and self._substep_count > 0: + for i, idx in enumerate(self._substep_term_indices): + avg = self._substep_accum[i] / self._substep_count + self._substep_episode_sums[i] += avg + self._step_values[:, idx] = avg + max_buf = self._substep_episode_max[i] + if max_buf is not None: + torch.maximum(max_buf, avg, out=max_buf) + self._substep_accum[i].zero_() + self._substep_count = 0 + for idx in self._step_term_indices: + name = self._term_names[idx] + value = self._compute_term(idx) + self._episode_sums[name] += value + self._step_values[:, idx] = value + if name in self._episode_max: + torch.maximum(self._episode_max[name], value, out=self._episode_max[name]) + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + for idx, name in enumerate(self._term_names): + terms.append((name, [self._step_values[env_idx, idx].cpu().item()])) + return terms + + def _prepare_terms(self): + for term_name, term_cfg in self.cfg.items(): + term_cfg: MetricsTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + self._resolve_common_term_cfg(term_name, term_cfg) + idx = len(self._term_names) + self._term_names.append(term_name) + self._term_cfgs.append(term_cfg) + if term_cfg.per_substep: + self._substep_term_indices.append(idx) + else: + self._step_term_indices.append(idx) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._class_term_cfgs.append(term_cfg) + + def _compute_term(self, idx: int) -> torch.Tensor: + name = self._term_names[idx] + term_cfg = self._term_cfgs[idx] + value = term_cfg.func(self._env, **term_cfg.params) + self._check_term_shape(name, value) + return value + + +class NullMetricsManager: + """Placeholder for absent metrics manager that safely no-ops all operations.""" + + def __init__(self): + self.active_terms: list[str] = [] + self.cfg = None + + def __str__(self) -> str: + return " (inactive)" + + def __repr__(self) -> str: + return "NullMetricsManager()" + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + return [] + + def reset(self, env_ids: torch.Tensor | None = None) -> dict[str, float]: + return {} + + def compute_substep(self) -> None: + pass + + def compute(self) -> None: + pass diff --git a/rl/mjlab/managers/observation_manager.py b/rl/mjlab/managers/observation_manager.py new file mode 100644 index 0000000..bca6352 --- /dev/null +++ b/rl/mjlab/managers/observation_manager.py @@ -0,0 +1,515 @@ +"""Observation manager for computing observations.""" + +from copy import deepcopy +from dataclasses import dataclass +from typing import Literal, Sequence + +import numpy as np +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg +from mjlab.utils.buffers import CircularBuffer, DelayBuffer +from mjlab.utils.noise import noise_cfg, noise_model +from mjlab.utils.noise.noise_cfg import NoiseCfg, NoiseModelCfg + + +@dataclass +class ObservationTermCfg(ManagerTermBaseCfg): + """Configuration for an observation term. + + Processing pipeline: compute → noise → clip → scale → delay → history. + Delay models sensor latency. History provides temporal context. Both are optional + and can be combined. + """ + + noise: NoiseCfg | NoiseModelCfg | None = None + """Noise model to apply to the observation.""" + + clip: tuple[float, float] | None = None + """Range (min, max) to clip the observation values.""" + + scale: tuple[float, ...] | float | torch.Tensor | None = None + """Scaling factor(s) to multiply the observation by.""" + + delay_min_lag: int = 0 + """Minimum lag (in steps) for delayed observations. Lag sampled uniformly from + [min_lag, max_lag]. Convert to ms: lag * (1000 / control_hz).""" + + delay_max_lag: int = 0 + """Maximum lag (in steps) for delayed observations. Use min=max for constant delay.""" + + delay_per_env: bool = True + """If True, each environment samples its own lag. If False, all environments share + the same lag at each step.""" + + delay_hold_prob: float = 0.0 + """Probability of reusing the previous lag instead of resampling. Useful for + temporally correlated latency patterns.""" + + delay_update_period: int = 0 + """Resample lag every N steps (models multi-rate sensors). If 0, update every step.""" + + delay_per_env_phase: bool = True + """If True and update_period > 0, stagger update timing across envs to avoid + synchronized resampling. Ignored when delay_per_env is False.""" + + history_length: int = 0 + """Number of past observations to keep in history. 0 = no history.""" + + flatten_history_dim: bool = True + """Whether to flatten the history dimension into observation. + + When True and concatenate_terms=True, uses term-major ordering: + [A_t0, A_t1, ..., A_tH-1, B_t0, B_t1, ..., B_tH-1, ...] + See docs/source/observation.rst for details on ordering.""" + + +@dataclass +class ObservationGroupCfg: + """Configuration for an observation group. + + An observation group bundles multiple observation terms together. Groups are + typically used to separate observations for different purposes (e.g., "actor" + for the actor, "critic" for the value function). + """ + + terms: dict[str, ObservationTermCfg] + """Dictionary mapping term names to their configurations.""" + + concatenate_terms: bool = True + """Whether to concatenate all terms into a single tensor. If False, returns + a dict mapping term names to their individual tensors.""" + + concatenate_dim: int = -1 + """Dimension along which to concatenate terms. Default -1 (last dimension).""" + + enable_corruption: bool = False + """Whether to apply noise corruption to observations. Set to True during + training for domain randomization, False during evaluation.""" + + history_length: int | None = None + """Group-level history length override. If set, applies to all terms in + this group. If None, each term uses its own ``history_length`` setting.""" + + flatten_history_dim: bool = True + """Whether to flatten history into the observation dimension. If True, + observations have shape ``(num_envs, obs_dim * history_length)``. If False, + shape is ``(num_envs, history_length, obs_dim)``.""" + + nan_policy: Literal["disabled", "warn", "sanitize", "error"] = "disabled" + """NaN/Inf handling policy for observations in this group. + + - 'disabled': No checks (default, fastest) + - 'warn': Log warning with term name and env IDs, then sanitize (debugging) + - 'sanitize': Silent sanitization to 0.0 like reward manager (safe for production) + - 'error': Raise ValueError on NaN/Inf (strict development mode) + """ + + nan_check_per_term: bool = True + """If True, check each observation term individually to identify NaN source. + If False, check only the final concatenated output (faster but less informative). + Only applies when nan_policy != 'disabled'.""" + + +class ObservationManager(ManagerBase): + """Manages observation computation for the environment. + + The observation manager computes observations from multiple terms organized + into groups. Each term can have noise, clipping, scaling, delay, and history + applied. Groups can optionally concatenate their terms into a single tensor. + """ + + def __init__(self, cfg: dict[str, ObservationGroupCfg], env): + self.cfg = deepcopy(cfg) + super().__init__(env=env) + + self._group_obs_dim: dict[str, tuple[int, ...] | list[tuple[int, ...]]] = dict() + + for group_name, group_term_dims in self._group_obs_term_dim.items(): + if self._group_obs_concatenate[group_name]: + term_dims = torch.stack( + [torch.tensor(dims, device="cpu") for dims in group_term_dims], dim=0 + ) + if len(term_dims.shape) > 1: + if self._group_obs_concatenate_dim[group_name] >= 0: + dim = self._group_obs_concatenate_dim[group_name] - 1 + else: + dim = self._group_obs_concatenate_dim[group_name] + dim_sum = torch.sum(term_dims[:, dim], dim=0) + term_dims[0, dim] = dim_sum + term_dims = term_dims[0] + else: + term_dims = torch.sum(term_dims, dim=0) + self._group_obs_dim[group_name] = tuple(term_dims.tolist()) + else: + self._group_obs_dim[group_name] = group_term_dims + + self._obs_buffer: dict[str, torch.Tensor | dict[str, torch.Tensor]] | None = None + + def __str__(self) -> str: + msg = f" contains {len(self._group_obs_term_names)} groups.\n" + for group_name, group_dim in self._group_obs_dim.items(): + table = PrettyTable() + table.title = f"Active Observation Terms in Group: '{group_name}'" + if self._group_obs_concatenate[group_name]: + table.title += f" (shape: {group_dim})" # type: ignore + table.field_names = ["Index", "Name", "Shape"] + table.align["Name"] = "l" + obs_terms = zip( + self._group_obs_term_names[group_name], + self._group_obs_term_dim[group_name], + self._group_obs_term_cfgs[group_name], + strict=False, + ) + for index, (name, dims, term_cfg) in enumerate(obs_terms): + if term_cfg.history_length > 0 and term_cfg.flatten_history_dim: + # Flattened history: show (9,) ← 3×(3,) + original_size = int(np.prod(dims)) // term_cfg.history_length + original_shape = (original_size,) if len(dims) == 1 else dims[1:] + shape_str = f"{dims} ← {term_cfg.history_length}×{original_shape}" + else: + shape_str = str(tuple(dims)) + table.add_row([index, name, shape_str]) + msg += table.get_string() + msg += "\n" + return msg + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + + if self._obs_buffer is None: + self.compute() + assert self._obs_buffer is not None + obs_buffer: dict[str, torch.Tensor | dict[str, torch.Tensor]] = self._obs_buffer + + for group_name, _ in self.group_obs_dim.items(): + if not self.group_obs_concatenate[group_name]: + buffers = obs_buffer[group_name] + assert isinstance(buffers, dict) + for name, term in buffers.items(): + terms.append((group_name + "-" + name, term[env_idx].cpu().tolist())) # type: ignore[unsupported-operator] + continue + + idx = 0 + data = obs_buffer[group_name] + assert isinstance(data, torch.Tensor) + for name, shape in zip( + self._group_obs_term_names[group_name], + self._group_obs_term_dim[group_name], + strict=False, + ): + data_length = np.prod(shape) + term = data[env_idx, idx : idx + data_length] + terms.append((group_name + "-" + name, term.cpu().tolist())) + idx += data_length + + return terms + + # Properties. + + @property + def active_terms(self) -> dict[str, list[str]]: + return self._group_obs_term_names + + @property + def group_obs_dim(self) -> dict[str, tuple[int, ...] | list[tuple[int, ...]]]: + return self._group_obs_dim + + @property + def group_obs_term_dim(self) -> dict[str, list[tuple[int, ...]]]: + return self._group_obs_term_dim + + @property + def group_obs_concatenate(self) -> dict[str, bool]: + return self._group_obs_concatenate + + # Methods. + + def get_term_cfg(self, group_name: str, term_name: str) -> ObservationTermCfg: + if group_name not in self._group_obs_term_names: + raise ValueError(f"Group '{group_name}' not found in active groups.") + if term_name not in self._group_obs_term_names[group_name]: + raise ValueError(f"Term '{term_name}' not found in group '{group_name}'.") + index = self._group_obs_term_names[group_name].index(term_name) + return self._group_obs_term_cfgs[group_name][index] + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> dict[str, float]: + # Invalidate cache since reset envs will have different observations. + self._obs_buffer = None + + for group_name, group_cfg in self._group_obs_class_term_cfgs.items(): + for term_cfg in group_cfg: + term_cfg.func.reset(env_ids=env_ids) + for term_name in self._group_obs_term_names[group_name]: + batch_ids = None if isinstance(env_ids, slice) else env_ids + if term_name in self._group_obs_term_delay_buffer[group_name]: + self._group_obs_term_delay_buffer[group_name][term_name].reset( + batch_ids=batch_ids + ) + if term_name in self._group_obs_term_history_buffer[group_name]: + self._group_obs_term_history_buffer[group_name][term_name].reset( + batch_ids=batch_ids + ) + for group_mods in self._group_obs_class_instances.values(): + for mod in group_mods.values(): + mod.reset(env_ids=env_ids) + return {} + + def _check_and_handle_nans( + self, tensor: torch.Tensor, context: str, policy: str + ) -> torch.Tensor: + """Check for NaN/Inf and handle according to policy. + + Args: + tensor: Observation tensor to check. + context: Context string for error/warning messages (e.g., "actor/base_lin_vel"). + policy: NaN handling policy ("disabled", "warn", "sanitize", "error"). + + Returns: + The tensor, potentially sanitized depending on policy. + + Raises: + ValueError: If policy is "error" and NaN/Inf detected. + """ + if policy == "disabled": + return tensor + + has_nan = torch.isnan(tensor).any() + has_inf = torch.isinf(tensor).any() + + if not (has_nan or has_inf): + return tensor + + if policy == "error": + nan_mask = torch.isnan(tensor).any(dim=-1) | torch.isinf(tensor).any(dim=-1) + nan_env_ids = torch.where(nan_mask)[0].cpu().tolist() + raise ValueError( + f"NaN/Inf detected in observation '{context}' " + f"for environments: {nan_env_ids[:10]}" + ) + + if policy == "warn": + nan_mask = torch.isnan(tensor).any(dim=-1) | torch.isinf(tensor).any(dim=-1) + nan_env_ids = torch.where(nan_mask)[0].cpu().tolist() + print( + f"[ObservationManager] NaN/Inf in '{context}' " + f"(envs: {nan_env_ids[:5]}). Sanitizing to 0." + ) + + # Sanitize (applies to both "warn" and "sanitize" policies). + return torch.nan_to_num(tensor, nan=0.0, posinf=0.0, neginf=0.0) + + def compute( + self, + update_history: bool = False, + env_ids: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor | dict[str, torch.Tensor]]: + """Compute observations for all groups. + + With env_ids=None (the per-step path), history and delay buffers advance + for all envs. With env_ids (the reset path), only the reset envs' buffers + receive their post-reset frame (a backfill); other envs' buffers, delay + schedules, and lag draws are untouched, so a partial reset does not + advance their observation timelines. + """ + assert env_ids is None or update_history, ( + "env_ids is only meaningful with update_history=True (the reset path)." + ) + # Return cached observations if not updating and cache exists. + # This prevents double-pushing to delay buffers when compute() is called + # multiple times per control step (e.g., in get_observations() after step()). + if not update_history and self._obs_buffer is not None: + return self._obs_buffer + + obs_buffer: dict[str, torch.Tensor | dict[str, torch.Tensor]] = dict() + for group_name in self._group_obs_term_names: + obs_buffer[group_name] = self.compute_group(group_name, update_history, env_ids) + self._obs_buffer = obs_buffer + return obs_buffer + + def compute_group( + self, + group_name: str, + update_history: bool = False, + env_ids: torch.Tensor | None = None, + ) -> torch.Tensor | dict[str, torch.Tensor]: + group_cfg = self.cfg[group_name] + group_term_names = self._group_obs_term_names[group_name] + group_obs: dict[str, torch.Tensor] = {} + obs_terms = zip( + group_term_names, self._group_obs_term_cfgs[group_name], strict=False + ) + for term_name, term_cfg in obs_terms: + obs: torch.Tensor = term_cfg.func(self._env, **term_cfg.params).clone() + if isinstance(term_cfg.noise, noise_cfg.NoiseCfg): + obs = term_cfg.noise.apply(obs) + elif isinstance(term_cfg.noise, noise_cfg.NoiseModelCfg): + obs = self._group_obs_class_instances[group_name][term_name](obs) + if term_cfg.clip: + obs = obs.clip_(min=term_cfg.clip[0], max=term_cfg.clip[1]) + if term_cfg.scale is not None: + scale = term_cfg.scale + assert isinstance(scale, torch.Tensor) + obs = obs.mul_(scale) + + # Check for NaN/Inf before delay/history buffers (per-term checking). + if group_cfg.nan_check_per_term and group_cfg.nan_policy != "disabled": + obs = self._check_and_handle_nans( + obs, context=f"{group_name}/{term_name}", policy=group_cfg.nan_policy + ) + + if term_cfg.delay_max_lag > 0: + delay_buffer = self._group_obs_term_delay_buffer[group_name][term_name] + if env_ids is None or not delay_buffer.is_initialized: + delay_buffer.append(obs) + obs = delay_buffer.compute() + else: + delay_buffer.backfill(obs, env_ids) + obs = delay_buffer.peek() + if term_cfg.history_length > 0: + circular_buffer = self._group_obs_term_history_buffer[group_name][term_name] + if env_ids is None or not circular_buffer.is_initialized: + if update_history or not circular_buffer.is_initialized: + circular_buffer.append(obs) + else: + circular_buffer.backfill(obs, env_ids) + + if term_cfg.flatten_history_dim: + group_obs[term_name] = circular_buffer.buffer.reshape(self._env.num_envs, -1) + else: + group_obs[term_name] = circular_buffer.buffer + else: + group_obs[term_name] = obs + + # Final NaN check for non-per-term checking. + if not group_cfg.nan_check_per_term and group_cfg.nan_policy != "disabled": + if self._group_obs_concatenate[group_name]: + # Will check after concatenation below. + pass + else: + for term_name in group_obs: + group_obs[term_name] = self._check_and_handle_nans( + group_obs[term_name], + context=f"{group_name}/{term_name}", + policy=group_cfg.nan_policy, + ) + + if self._group_obs_concatenate[group_name]: + result = torch.cat( + list(group_obs.values()), dim=self._group_obs_concatenate_dim[group_name] + ) + # Final check for concatenated result (non-per-term checking). + if not group_cfg.nan_check_per_term and group_cfg.nan_policy != "disabled": + result = self._check_and_handle_nans( + result, context=group_name, policy=group_cfg.nan_policy + ) + return result + return group_obs + + def _prepare_terms(self) -> None: + self._group_obs_term_names: dict[str, list[str]] = dict() + self._group_obs_term_dim: dict[str, list[tuple[int, ...]]] = dict() + self._group_obs_term_cfgs: dict[str, list[ObservationTermCfg]] = dict() + self._group_obs_class_term_cfgs: dict[str, list[ObservationTermCfg]] = dict() + self._group_obs_concatenate: dict[str, bool] = dict() + self._group_obs_concatenate_dim: dict[str, int] = dict() + self._group_obs_class_instances: dict[str, dict[str, noise_model.NoiseModel]] = {} + self._group_obs_term_delay_buffer: dict[str, dict[str, DelayBuffer]] = dict() + self._group_obs_term_history_buffer: dict[str, dict[str, CircularBuffer]] = dict() + + for group_name, group_cfg in self.cfg.items(): + group_cfg: ObservationGroupCfg | None + if group_cfg is None: + print(f"group: {group_name} set to None, skipping...") + continue + + if not any(t is not None for t in group_cfg.terms.values()): + print(f"group: {group_name} has no active terms, skipping...") + continue + + self._group_obs_term_names[group_name] = list() + self._group_obs_term_dim[group_name] = list() + self._group_obs_term_cfgs[group_name] = list() + self._group_obs_class_term_cfgs[group_name] = list() + self._group_obs_class_instances[group_name] = {} + group_entry_delay_buffer: dict[str, DelayBuffer] = dict() + group_entry_history_buffer: dict[str, CircularBuffer] = dict() + + self._group_obs_concatenate[group_name] = group_cfg.concatenate_terms + self._group_obs_concatenate_dim[group_name] = ( + group_cfg.concatenate_dim + 1 + if group_cfg.concatenate_dim >= 0 + else group_cfg.concatenate_dim + ) + + for term_name, term_cfg in group_cfg.terms.items(): + term_cfg: ObservationTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + + # NOTE: This deepcopy is important to avoid cross-group contamination of term + # configs. + term_cfg = deepcopy(term_cfg) + self._resolve_common_term_cfg(term_name, term_cfg) + + if not group_cfg.enable_corruption: + term_cfg.noise = None + if group_cfg.history_length is not None: + term_cfg.history_length = group_cfg.history_length + term_cfg.flatten_history_dim = group_cfg.flatten_history_dim + self._group_obs_term_names[group_name].append(term_name) + self._group_obs_term_cfgs[group_name].append(term_cfg) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._group_obs_class_term_cfgs[group_name].append(term_cfg) + + obs_dims = tuple(term_cfg.func(self._env, **term_cfg.params).shape) + + if term_cfg.scale is not None: + term_cfg.scale = torch.as_tensor( + term_cfg.scale, dtype=torch.float, device=self._env.device + ).clone() + + if term_cfg.noise is not None and isinstance( + term_cfg.noise, noise_cfg.NoiseModelCfg + ): + noise_model_cls = term_cfg.noise.class_type + assert issubclass(noise_model_cls, noise_model.NoiseModel), ( + f"Class type for observation term '{term_name}' NoiseModelCfg" + f" is not a subclass of 'NoiseModel'. Received: '{type(noise_model_cls)}'." + ) + self._group_obs_class_instances[group_name][term_name] = noise_model_cls( + term_cfg.noise, num_envs=self._env.num_envs, device=self._env.device + ) + + if term_cfg.delay_max_lag > 0: + group_entry_delay_buffer[term_name] = DelayBuffer( + min_lag=term_cfg.delay_min_lag, + max_lag=term_cfg.delay_max_lag, + batch_size=self._env.num_envs, + device=self._env.device, + per_env=term_cfg.delay_per_env, + hold_prob=term_cfg.delay_hold_prob, + update_period=term_cfg.delay_update_period, + per_env_phase=term_cfg.delay_per_env_phase, + ) + + if term_cfg.history_length > 0: + group_entry_history_buffer[term_name] = CircularBuffer( + max_len=term_cfg.history_length, + batch_size=self._env.num_envs, + device=self._env.device, + ) + old_dims = list(obs_dims) + old_dims.insert(1, term_cfg.history_length) + obs_dims = tuple(old_dims) + if term_cfg.flatten_history_dim: + obs_dims = (obs_dims[0], int(np.prod(obs_dims[1:]))) + + self._group_obs_term_dim[group_name].append(obs_dims[1:]) + + self._group_obs_term_delay_buffer[group_name] = group_entry_delay_buffer + self._group_obs_term_history_buffer[group_name] = group_entry_history_buffer diff --git a/rl/mjlab/managers/recorder_manager.py b/rl/mjlab/managers/recorder_manager.py new file mode 100644 index 0000000..cf1f011 --- /dev/null +++ b/rl/mjlab/managers/recorder_manager.py @@ -0,0 +1,265 @@ +"""Recorder manager for logging environment data during rollouts.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + + +@dataclass +class RecorderTermCfg(ManagerTermBaseCfg): + """Configuration for a recorder term. + + ``func`` must be a :class:`RecorderTerm` subclass. Function-based terms are not + supported because recorder terms are stateful (file handles, buffers, etc.). + """ + + +class RecorderTerm(ManagerTermBase): + """Base class for recorder terms. + + Override only the lifecycle methods you need. Each method is a no-op by default so + subclasses are not required to implement all of them. + + The environment is available as ``self._env``, giving access to ``self._env.obs_buf``, + ``self._env.action_manager.action``, and all other environment state. + + Example:: + + class CsvRecorder(RecorderTerm): + def __init__(self, cfg, env): + super().__init__(cfg, env) + self._file = open(cfg.params["path"], "w", newline="") + self._writer = csv.writer(self._file) + + def record_pre_reset(self, env_ids): + # Terminal transition: action is still intact here. + # It will be zeroed by _reset_idx immediately after this returns. + obs = self._env.obs_buf["actor"][env_ids].cpu().numpy() + act = self._env.action_manager.action[env_ids].cpu().numpy() + for o, a in zip(obs, act): + self._writer.writerow(o.tolist() + a.tolist()) + + def record_post_step(self): + # Skip envs that just reset: their terminal pair was written in record_pre_reset + # and their action is now zeroed. + mask = ~self._env.reset_buf + obs = self._env.obs_buf["actor"][mask].cpu().numpy() + act = self._env.action_manager.action[mask].cpu().numpy() + for o, a in zip(obs, act): + self._writer.writerow(o.tolist() + a.tolist()) + + def close(self): + self._file.close() + """ + + def __init__(self, cfg: RecorderTermCfg, env: ManagerBasedRlEnv): + super().__init__(env) # ManagerTermBase only accepts env + self.cfg = cfg + + def record_pre_reset(self, env_ids: torch.Tensor) -> None: + """Called in ``env.step()`` before terminated environments are reset. + + **What is available:** + + - ``obs_buf`` contains the observation from the *end of the previous step* (the + input the agent used to choose the terminal action). It does **not** contain the + post-action terminal observation (the state reached after applying the action), + which is never computed for resetting environments. + - ``action_manager.action`` contains the action applied during this step. This is + the correct terminal action. It will be zeroed for these environments by + ``_reset_idx`` immediately after this hook returns, so capture it here if you + need it later. + - ``reward_buf`` contains the reward for this terminal step. + - ``reset_terminated`` and ``reset_time_outs`` reflect why each environment is + resetting. + + This is the right hook to record the terminal transition + ``(obs_t, action_t, reward_t, done=True)`` for each resetting environment. + + Args: + env_ids: Indices of environments that are about to be reset. + """ + del env_ids # Unused in base implementation. + + def record_post_reset(self, env_ids: torch.Tensor) -> None: + """Called after a reset completes with fresh observations computed. + + Fires at the end of ``env.reset()`` (covering all environments on the initial call) + and within ``env.step()`` for each batch of environments that terminates, after + state has been overwritten and new observations computed. + + At this point ``obs_buf[env_ids]`` holds the initial observation of the new episode + and ``action_manager.action[env_ids]`` is zero (no action has been taken in the new + episode yet). + + Use this hook to initialize per-episode state or record the first observation of a + new episode. + + Args: + env_ids: Indices of environments that were reset. + """ + del env_ids # Unused in base implementation. + + def record_post_step(self) -> None: + """Called at the end of every ``env.step()`` with fresh observations. + + At this point ``obs_buf`` holds the new observation for every environment and + ``action_manager.action`` holds the action that was applied during this step. + **Exception:** for environments that reset during this step, + ``action_manager.action`` has been zeroed by ``_reset_idx`` and ``obs_buf`` holds + the initial observation of the new episode rather than the post-action terminal + observation. Use ``record_pre_reset`` to capture the terminal ``(obs, action)`` + pair for those environments. Resetting environments are identified by + ``self._env.reset_buf``. + """ + + def close(self) -> None: + """Called when the environment closes. + + Release file handles, flush write buffers, or finalize output here. + """ + + def __call__(self): + raise NotImplementedError( + "RecorderTerm is not invoked via __call__. " + "Override the lifecycle methods instead." + ) + + +class RecorderManager(ManagerBase): + """Orchestrates recorder terms during environment rollouts. + + Holds a collection of :class:`RecorderTerm` instances and calls their lifecycle + methods at the appropriate points in the environment loop. The manager has no opinion + on how data is stored; each term handles its own I/O entirely. + + Register terms by adding them to the ``recorders`` dict on + :class:`~mjlab.envs.ManagerBasedRlEnvCfg`. If the dict is empty, the environment + substitutes a :class:`NullRecorderManager` with zero overhead. + """ + + def __init__(self, cfg: dict[str, RecorderTermCfg], env: ManagerBasedRlEnv): + self._terms: dict[str, RecorderTerm] = {} + self.cfg = deepcopy(cfg) + super().__init__(env) # calls _prepare_terms() + + def __str__(self) -> str: + msg = f" contains {len(self._terms)} active terms.\n" + table = PrettyTable() + table.title = "Active Recorder Terms" + table.field_names = ["Index", "Name"] + for idx, name in enumerate(self._terms): + table.add_row([idx, name]) + return msg + table.get_string() + + def __contains__(self, name: str) -> bool: + """Return True if a term named ``name`` is registered.""" + return name in self._terms + + @property + def active_terms(self) -> list[str]: + """List of active term names.""" + return list(self._terms.keys()) + + def get_term(self, name: str) -> RecorderTerm: + """Return the recorder term registered under ``name``. + + Use this to reach a recorder's public methods (e.g. to start/stop logging) + from outside the env loop without touching private state. + + Args: + name: Term name as registered in ``ManagerBasedRlEnvCfg.recorders``. + + Raises: + KeyError: If no term is registered under ``name``. + """ + try: + return self._terms[name] + except KeyError: + msg = f"No recorder term named '{name}'. Active terms: {self.active_terms}" + raise KeyError(msg) from None + + def record_pre_reset(self, env_ids: torch.Tensor) -> None: + """Forward to each term's :meth:`RecorderTerm.record_pre_reset`.""" + for term in self._terms.values(): + term.record_pre_reset(env_ids) + + def record_post_reset(self, env_ids: torch.Tensor) -> None: + """Forward to each term's :meth:`RecorderTerm.record_post_reset`.""" + for term in self._terms.values(): + term.record_post_reset(env_ids) + + def record_post_step(self) -> None: + """Forward to each term's :meth:`RecorderTerm.record_post_step`.""" + for term in self._terms.values(): + term.record_post_step() + + def close(self) -> None: + """Forward to each term's :meth:`RecorderTerm.close`.""" + for term in self._terms.values(): + term.close() + + def _prepare_terms(self): + for name, cfg in self.cfg.items(): + if cfg is None: + continue + self._resolve_common_term_cfg(name, cfg) + # _resolve_common_term_cfg instantiates class-based terms in-place. + if not isinstance(cfg.func, RecorderTerm): + raise TypeError( + f"Recorder term '{name}': func must be a RecorderTerm subclass," + f" got {type(cfg.func).__name__}. Function-based terms are not" + " supported." + ) + self._terms[name] = cfg.func + + +class NullRecorderManager: + """No-op fallback used when no recorder terms are configured. + + All methods are no-ops. This class is not a :class:`ManagerBase` subclass + so it carries zero overhead. + """ + + def __init__(self): + self.active_terms: list[str] = [] + self.cfg = None + + def __str__(self) -> str: + return " (inactive)" + + def __repr__(self) -> str: + return "NullRecorderManager()" + + def __contains__(self, name: str) -> bool: + """Always returns False since there are no terms.""" + del name + return False + + def get_term(self, name: str) -> RecorderTerm: + """Always raises KeyError since there are no terms.""" + del name + msg = "NullRecorderManager has no terms." + raise KeyError(msg) + + def record_pre_reset(self, env_ids: torch.Tensor) -> None: + del env_ids + + def record_post_reset(self, env_ids: torch.Tensor) -> None: + del env_ids + + def record_post_step(self) -> None: + pass + + def close(self) -> None: + pass diff --git a/rl/mjlab/managers/reward_manager.py b/rl/mjlab/managers/reward_manager.py new file mode 100644 index 0000000..955e3ba --- /dev/null +++ b/rl/mjlab/managers/reward_manager.py @@ -0,0 +1,188 @@ +"""Reward manager for computing reward signals.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + from mjlab.viewer.debug_visualizer import DebugVisualizer + + +@dataclass(kw_only=True) +class RewardTermCfg(ManagerTermBaseCfg): + """Configuration for a reward term.""" + + func: Any + """The callable that computes this reward term's value.""" + + weight: float + """Weight multiplier for this reward term.""" + + +class RewardManager(ManagerBase): + """Manages reward computation by aggregating weighted reward terms. + + Reward Scaling Behavior: + By default, rewards are scaled by the environment step duration (dt). This + normalizes cumulative episodic rewards across different simulation frequencies. + The scaling can be disabled via the ``scale_by_dt`` parameter. + + When ``scale_by_dt=True`` (default): + - ``reward_buf`` (returned by ``compute()``) = raw_value * weight * dt + - ``_episode_sums`` (cumulative rewards) are scaled by dt + - ``Episode_Reward/*`` logged metrics are scaled by dt + + When ``scale_by_dt=False``: + - ``reward_buf`` = raw_value * weight (no dt scaling) + + Regardless of the scaling setting: + - ``_step_reward`` (via ``get_active_iterable_terms()``) always contains + the unscaled reward rate (raw_value * weight) + """ + + _env: ManagerBasedRlEnv + + def __init__( + self, + cfg: dict[str, RewardTermCfg], + env: ManagerBasedRlEnv, + *, + scale_by_dt: bool = True, + ): + self._term_names: list[str] = list() + self._term_cfgs: list[RewardTermCfg] = list() + self._class_term_cfgs: list[RewardTermCfg] = list() + self._scale_by_dt = scale_by_dt + + self.cfg = deepcopy(cfg) + super().__init__(env=env) + self._episode_sums = dict() + for term_name in self._term_names: + self._episode_sums[term_name] = torch.zeros( + self.num_envs, dtype=torch.float, device=self.device + ) + self._reward_buf = torch.zeros(self.num_envs, dtype=torch.float, device=self.device) + self._scale = 1.0 + self._step_reward = torch.zeros( + (self.num_envs, len(self._term_names)), dtype=torch.float, device=self.device + ) + + def __str__(self) -> str: + msg = f" contains {len(self._term_names)} active terms.\n" + table = PrettyTable() + table.title = "Active Reward Terms" + table.field_names = ["Index", "Name", "Weight"] + table.align["Name"] = "l" + table.align["Weight"] = "r" + for index, (name, term_cfg) in enumerate( + zip(self._term_names, self._term_cfgs, strict=False) + ): + table.add_row([index, name, term_cfg.weight]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def active_terms(self) -> list[str]: + return self._term_names + + @property + def term_rewards(self) -> torch.Tensor: + """Per-term rewards from the last step, scaled like ``reward_buf``.""" + return self._step_reward * self._scale + + # Methods. + + def reset( + self, env_ids: torch.Tensor | slice | None = None + ) -> dict[str, torch.Tensor]: + if env_ids is None: + env_ids = slice(None) + extras = {} + for key in self._episode_sums.keys(): + episodic_sum_avg = torch.mean(self._episode_sums[key][env_ids]) + extras["Episode_Reward/" + key] = ( + episodic_sum_avg / self._env.max_episode_length_s + ) + self._episode_sums[key][env_ids] = 0.0 + for term_cfg in self._class_term_cfgs: + term_cfg.func.reset(env_ids=env_ids) + return extras + + def compute(self, dt: float) -> torch.Tensor: + self._reward_buf[:] = 0.0 + scale = dt if self._scale_by_dt else 1.0 + self._scale = scale + for term_idx, (name, term_cfg) in enumerate( + zip(self._term_names, self._term_cfgs, strict=False) + ): + if term_cfg.weight == 0.0: + self._step_reward[:, term_idx] = 0.0 + continue + value = term_cfg.func(self._env, **term_cfg.params) + self._check_term_shape(name, value) + value = value * term_cfg.weight * scale + # NaN/Inf can occur from corrupted physics state; zero them to avoid policy crash. + value = torch.nan_to_num(value, nan=0.0, posinf=0.0, neginf=0.0) + self._reward_buf += value + self._episode_sums[name] += value + self._step_reward[:, term_idx] = value / scale + return self._reward_buf + + def debug_vis(self, visualizer: DebugVisualizer) -> None: + """Delegate debug visualization to class-based reward terms.""" + for _, func in self.get_visualizable_terms(): + func.debug_vis(visualizer) + + def get_visualizable_terms(self) -> list[tuple[str, Any]]: + """Return ``(name, func)`` pairs for class-based terms with debug_vis.""" + results: list[tuple[str, Any]] = [] + for term_cfg in self._class_term_cfgs: + if not hasattr(term_cfg.func, "debug_vis"): + continue + name = next( + n + for n, c in zip(self._term_names, self._term_cfgs, strict=False) + if c is term_cfg + ) + results.append((name, term_cfg.func)) + return results + + def get_active_iterable_terms(self, env_idx): + terms = [] + for idx, name in enumerate(self._term_names): + terms.append((name, [self._step_reward[env_idx, idx].cpu().item()])) + return terms + + def get_term(self, name: str) -> torch.Tensor: + """Return one term's column of ``term_rewards``, shape ``(num_envs,)``.""" + if name not in self._term_names: + raise ValueError(f"Term '{name}' not found in active terms.") + return self.term_rewards[:, self._term_names.index(name)] + + def get_term_cfg(self, term_name: str) -> RewardTermCfg: + if term_name not in self._term_names: + raise ValueError(f"Term '{term_name}' not found in active terms.") + return self._term_cfgs[self._term_names.index(term_name)] + + def _prepare_terms(self): + for term_name, term_cfg in self.cfg.items(): + term_cfg: RewardTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + self._resolve_common_term_cfg(term_name, term_cfg) + self._term_names.append(term_name) + self._term_cfgs.append(term_cfg) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._class_term_cfgs.append(term_cfg) diff --git a/rl/mjlab/managers/scene_entity_config.py b/rl/mjlab/managers/scene_entity_config.py new file mode 100644 index 0000000..65e470b --- /dev/null +++ b/rl/mjlab/managers/scene_entity_config.py @@ -0,0 +1,265 @@ +"""Configuration for scene entities used by manager terms.""" + +from dataclasses import dataclass, field +from typing import NamedTuple + +from mjlab.entity import Entity +from mjlab.scene import Scene + + +class _FieldConfig(NamedTuple): + """Configuration for a resolvable entity field.""" + + names_attr: str + ids_attr: str + find_method: str + num_attr: str + kind_label: str + + +_FIELD_CONFIGS = [ + _FieldConfig("joint_names", "joint_ids", "find_joints", "num_joints", "joint"), + _FieldConfig("body_names", "body_ids", "find_bodies", "num_bodies", "body"), + _FieldConfig("geom_names", "geom_ids", "find_geoms", "num_geoms", "geom"), + _FieldConfig("site_names", "site_ids", "find_sites", "num_sites", "site"), + _FieldConfig( + "actuator_names", "actuator_ids", "find_actuators", "num_actuators", "actuator" + ), + _FieldConfig("tendon_names", "tendon_ids", "find_tendons", "num_tendons", "tendon"), + _FieldConfig( + "camera_names", + "camera_ids", + "find_cameras", + "num_cameras", + "camera", + ), + _FieldConfig( + "light_names", + "light_ids", + "find_lights", + "num_lights", + "light", + ), + _FieldConfig( + "material_names", + "material_ids", + "find_materials", + "num_materials", + "material", + ), + _FieldConfig( + "texture_names", + "texture_ids", + "find_textures", + "num_textures", + "texture", + ), + _FieldConfig("pair_names", "pair_ids", "find_pairs", "num_pairs", "pair"), +] + + +@dataclass +class SceneEntityCfg: + """Configuration for a scene entity that is used by the manager's term. + + This configuration allows flexible specification of entity components either by name + or by ID. During resolution, it ensures consistency between names and IDs, and can + optimize to slice(None) when all components are selected. + """ + + name: str + """The name of the entity in the scene.""" + + joint_names: str | tuple[str, ...] | None = None + """Names of joints to include. Can be a single string or tuple.""" + + joint_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of joints to include. Can be a list or slice.""" + + body_names: str | tuple[str, ...] | None = None + """Names of bodies to include. Can be a single string or tuple.""" + + body_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of bodies to include. Can be a list or slice.""" + + geom_names: str | tuple[str, ...] | None = None + """Names of geometries to include. Can be a single string or tuple.""" + + geom_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of geometries to include. Can be a list or slice.""" + + site_names: str | tuple[str, ...] | None = None + """Names of sites to include. Can be a single string or tuple.""" + + site_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of sites to include. Can be a list or slice.""" + + actuator_names: str | list[str] | None = None + """Names of actuators to include. Can be a single string or list.""" + + actuator_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of actuators to include. Can be a list or slice.""" + + tendon_names: str | tuple[str, ...] | None = None + """Names of tendons to include. Can be a single string or tuple.""" + + tendon_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of tendons to include. Can be a list or slice.""" + + camera_names: str | tuple[str, ...] | None = None + """Names of cameras to include. Can be a single string or tuple.""" + + camera_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of cameras to include. Can be a list or slice.""" + + light_names: str | tuple[str, ...] | None = None + """Names of lights to include. Can be a single string or tuple.""" + + light_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of lights to include. Can be a list or slice.""" + + material_names: str | tuple[str, ...] | None = None + """Names of materials to include. Can be a single string or tuple.""" + + material_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of materials to include. Can be a list or slice.""" + + texture_names: str | tuple[str, ...] | None = None + """Names of textures to include. Can be a single string or tuple.""" + + texture_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of textures to include. Can be a list or slice.""" + + pair_names: str | tuple[str, ...] | None = None + """Names of contact pairs to include. Can be a single string or tuple.""" + + pair_ids: list[int] | slice = field(default_factory=lambda: slice(None)) + """IDs of contact pairs to include. Can be a list or slice.""" + + preserve_order: bool = False + """If True, maintains the order of components as specified.""" + + def resolve(self, scene: Scene) -> None: + """Resolve names and IDs for all configured fields. + + This method ensures consistency between names and IDs for each field type. + It handles three cases: + 1. Both names and IDs provided: Validates they match + 2. Only names provided: Computes IDs (optimizes to slice(None) if all selected) + 3. Only IDs provided: Computes names + + Args: + scene: The scene containing the entity to resolve against. + + Raises: + ValueError: If provided names and IDs are inconsistent. + KeyError: If the entity name is not found in the scene. + """ + entity = scene[self.name] + + for config in _FIELD_CONFIGS: + self._resolve_field(entity, config) + + def _resolve_field(self, entity: Entity, config: _FieldConfig) -> None: + """Resolve a single field's names and IDs. + + Args: + entity: The entity to resolve against. + config: Field configuration specifying attribute names and methods. + """ + names = getattr(self, config.names_attr) + ids = getattr(self, config.ids_attr) + + # Early return if nothing to resolve. + if names is None and not isinstance(ids, list): + return + + # Get entity metadata. + entity_all_names = getattr(entity, config.names_attr) + entity_count = getattr(entity, config.num_attr) + find_method = getattr(entity, config.find_method) + + # Normalize single values to lists for uniform processing. + names = self._normalize_to_list(names) + if isinstance(ids, (int, list)): + ids = self._normalize_to_list(ids) + setattr(self, config.ids_attr, ids) + + # Handle three resolution cases. + if names is not None and isinstance(ids, list): + setattr(self, config.names_attr, names) + self._validate_consistency( + names, ids, entity_all_names, find_method, config.kind_label + ) + elif names is not None: + self._resolve_names_to_ids( + names, + entity_all_names, + entity_count, + find_method, + config.names_attr, + config.ids_attr, + ) + elif isinstance(ids, list): + self._resolve_ids_to_names(ids, entity_all_names, config.names_attr) + + def _normalize_to_list(self, value: str | int | tuple | list | None) -> list | None: + """Convert single values to lists for uniform processing.""" + if value is None: + return None + if isinstance(value, (str, int)): + return [value] + if isinstance(value, list): + return value + return list(value) + + def _validate_consistency( + self, + names: list[str], + ids: list[int], + entity_all_names: list[str], + find_method, + kind_label: str, + ) -> None: + """Validate that provided names and IDs are consistent. + + Raises: + ValueError: If names and IDs don't match. + """ + found_ids, _ = find_method(names, preserve_order=self.preserve_order) + computed_names = [entity_all_names[i] for i in ids] + + if found_ids != ids or computed_names != names: + raise ValueError( + f"Inconsistent {kind_label} names and indices. " + f"Names {names} resolved to indices {found_ids}, " + f"but indices {ids} (mapping to names {computed_names}) were provided." + ) + + def _resolve_names_to_ids( + self, + names: list[str], + entity_all_names: list[str], + entity_count: int, + find_method, + names_attr: str, + ids_attr: str, + ) -> None: + """Resolve names to IDs, optimizing to slice(None) when all are selected.""" + found_ids, found_names = find_method(names, preserve_order=self.preserve_order) + + # Keep names and IDs in the same order. + setattr(self, names_attr, found_names) + + # Optimize to slice(None) if all components are selected in order. + if len(found_ids) == entity_count and found_names == list(entity_all_names): + setattr(self, ids_attr, slice(None)) + else: + setattr(self, ids_attr, found_ids) + + def _resolve_ids_to_names( + self, ids: list[int], entity_all_names: list[str], names_attr: str + ) -> None: + """Resolve IDs to their corresponding names.""" + resolved_names = [entity_all_names[i] for i in ids] + setattr(self, names_attr, resolved_names) diff --git a/rl/mjlab/managers/termination_manager.py b/rl/mjlab/managers/termination_manager.py new file mode 100644 index 0000000..d62ef35 --- /dev/null +++ b/rl/mjlab/managers/termination_manager.py @@ -0,0 +1,141 @@ +"""Termination manager for computing done signals.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Sequence + +import torch +from prettytable import PrettyTable + +from mjlab.managers.manager_base import ManagerBase, ManagerTermBaseCfg + +if TYPE_CHECKING: + from mjlab.envs.manager_based_rl_env import ManagerBasedRlEnv + + +@dataclass +class TerminationTermCfg(ManagerTermBaseCfg): + """Configuration for a termination term.""" + + time_out: bool = False + """Whether the term contributes towards episodic timeouts.""" + + +class TerminationManager(ManagerBase): + """Manages termination conditions for the environment. + + The termination manager aggregates multiple termination terms to compute + episode done signals. Terms can be either truncations (time-based) or + terminations (failure conditions). + """ + + _env: ManagerBasedRlEnv + + def __init__(self, cfg: dict[str, TerminationTermCfg], env: ManagerBasedRlEnv): + self._term_names: list[str] = list() + self._term_cfgs: list[TerminationTermCfg] = list() + self._class_term_cfgs: list[TerminationTermCfg] = list() + + self.cfg = deepcopy(cfg) + super().__init__(env) + + self._term_dones = dict() + for term_name in self._term_names: + self._term_dones[term_name] = torch.zeros( + self.num_envs, device=self.device, dtype=torch.bool + ) + self._truncated_buf = torch.zeros( + self.num_envs, device=self.device, dtype=torch.bool + ) + self._terminated_buf = torch.zeros_like(self._truncated_buf) + + def __str__(self) -> str: + msg = f" contains {len(self._term_names)} active terms.\n" + table = PrettyTable() + table.title = "Active Termination Terms" + table.field_names = ["Index", "Name", "Time Out"] + table.align["Name"] = "l" + for index, (name, term_cfg) in enumerate( + zip(self._term_names, self._term_cfgs, strict=False) + ): + table.add_row([index, name, term_cfg.time_out]) + msg += table.get_string() + msg += "\n" + return msg + + # Properties. + + @property + def active_terms(self) -> list[str]: + return self._term_names + + @property + def dones(self) -> torch.Tensor: + return self._truncated_buf | self._terminated_buf + + @property + def time_outs(self) -> torch.Tensor: + return self._truncated_buf + + @property + def terminated(self) -> torch.Tensor: + return self._terminated_buf + + # Methods. + + def reset( + self, env_ids: torch.Tensor | slice | None = None + ) -> dict[str, torch.Tensor]: + if env_ids is None: + env_ids = slice(None) + extras = {} + for key in self._term_dones.keys(): + extras["Episode_Termination/" + key] = torch.count_nonzero( + self._term_dones[key][env_ids] + ).item() + for term_cfg in self._class_term_cfgs: + term_cfg.func.reset(env_ids=env_ids) + return extras + + def compute(self) -> torch.Tensor: + self._truncated_buf[:] = False + self._terminated_buf[:] = False + for name, term_cfg in zip(self._term_names, self._term_cfgs, strict=False): + value = term_cfg.func(self._env, **term_cfg.params) + self._check_term_shape(name, value) + if term_cfg.time_out: + self._truncated_buf |= value + else: + self._terminated_buf |= value + self._term_dones[name][:] = value + return self._truncated_buf | self._terminated_buf + + def get_term(self, name: str) -> torch.Tensor: + return self._term_dones[name] + + def get_term_cfg(self, term_name: str) -> TerminationTermCfg: + if term_name not in self._term_names: + raise ValueError(f"Term '{term_name}' not found in active terms.") + return self._term_cfgs[self._term_names.index(term_name)] + + def get_active_iterable_terms( + self, env_idx: int + ) -> Sequence[tuple[str, Sequence[float]]]: + terms = [] + for key in self._term_dones.keys(): + terms.append((key, [self._term_dones[key][env_idx].float().cpu().item()])) + return terms + + def _prepare_terms(self): + for term_name, term_cfg in self.cfg.items(): + term_cfg: TerminationTermCfg | None + if term_cfg is None: + print(f"term: {term_name} set to None, skipping...") + continue + self._resolve_common_term_cfg(term_name, term_cfg) + self._term_names.append(term_name) + self._term_cfgs.append(term_cfg) + if hasattr(term_cfg.func, "reset") and callable(term_cfg.func.reset): + self._class_term_cfgs.append(term_cfg) diff --git a/rl/mjlab/py.typed b/rl/mjlab/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/rl/mjlab/rl/__init__.py b/rl/mjlab/rl/__init__.py new file mode 100644 index 0000000..e3d0de1 --- /dev/null +++ b/rl/mjlab/rl/__init__.py @@ -0,0 +1,6 @@ +from mjlab.rl.config import RslRlBaseRunnerCfg as RslRlBaseRunnerCfg +from mjlab.rl.config import RslRlModelCfg as RslRlModelCfg +from mjlab.rl.config import RslRlOnPolicyRunnerCfg as RslRlOnPolicyRunnerCfg +from mjlab.rl.config import RslRlPpoAlgorithmCfg as RslRlPpoAlgorithmCfg +from mjlab.rl.runner import MjlabOnPolicyRunner as MjlabOnPolicyRunner +from mjlab.rl.vecenv_wrapper import RslRlVecEnvWrapper as RslRlVecEnvWrapper diff --git a/rl/mjlab/rl/config.py b/rl/mjlab/rl/config.py new file mode 100644 index 0000000..8ce233d --- /dev/null +++ b/rl/mjlab/rl/config.py @@ -0,0 +1,145 @@ +"""RSL-RL configuration.""" + +from dataclasses import dataclass, field +from typing import Any, Literal, Tuple + + +@dataclass +class RslRlModelCfg: + """Config for a single neural network model (Actor or Critic).""" + + hidden_dims: Tuple[int, ...] = (128, 128, 128) + """The hidden dimensions of the network.""" + activation: str = "elu" + """The activation function.""" + obs_normalization: bool = False + """Whether to normalize the observations. Default is False.""" + cnn_cfg: dict[str, Any] | None = None + """CNN encoder config. When set, class_name should be "CNNModel". + + Passed to ``rsl_rl.modules.CNN``. Common keys: output_channels, + kernel_size, stride, padding, activation, global_pool, max_pool. + """ + distribution_cfg: dict[str, Any] | None = None + """Distribution config dict passed to rsl_rl. Example:: + + {"class_name": "GaussianDistribution", + "init_std": 1.0, "std_type": "scalar"} + + ``None`` means deterministic output (use for critic). + """ + rnn_type: str | None = None + """RNN type ("lstm" or "gru"). When set, class_name should be "RNNModel".""" + rnn_hidden_dim: int = 256 + """Hidden state dimension for the RNN.""" + rnn_num_layers: int = 1 + """Number of stacked RNN layers.""" + class_name: str = "MLPModel" + """Model class name resolved by RSL-RL (MLPModel, CNNModel, or RNNModel).""" + + +@dataclass +class RslRlPpoAlgorithmCfg: + """Config for the PPO algorithm.""" + + num_learning_epochs: int = 5 + """The number of learning epochs per update.""" + num_mini_batches: int = 4 + """The number of mini-batches per update. + mini batch size = num_envs * num_steps / num_mini_batches + """ + learning_rate: float = 1e-3 + """The learning rate.""" + schedule: Literal["adaptive", "fixed"] = "adaptive" + """The learning rate schedule.""" + gamma: float = 0.99 + """The discount factor.""" + lam: float = 0.95 + """The lambda parameter for Generalized Advantage Estimation (GAE).""" + entropy_coef: float = 0.005 + """The coefficient for the entropy loss.""" + desired_kl: float = 0.01 + """The desired KL divergence between the new and old policies.""" + max_grad_norm: float = 1.0 + """The maximum gradient norm for the policy.""" + value_loss_coef: float = 1.0 + """The coefficient for the value loss.""" + use_clipped_value_loss: bool = True + """Whether to use clipped value loss.""" + clip_param: float = 0.2 + """The clipping parameter for the policy.""" + normalize_advantage_per_mini_batch: bool = False + """Whether to normalize the advantage per mini-batch. Default is False. If True, the + advantage is normalized over the mini-batches only. Otherwise, the advantage is + normalized over the entire collected trajectories. + """ + optimizer: Literal["adam", "adamw", "sgd", "rmsprop"] = "adam" + """The optimizer to use.""" + share_cnn_encoders: bool = False + """Share CNN encoders between actor and critic.""" + class_name: str = "PPO" + """Algorithm class name resolved by RSL-RL.""" + + +@dataclass +class RslRlBaseRunnerCfg: + seed: int = 42 + """The seed for the experiment. Default is 42.""" + num_steps_per_env: int = 24 + """The number of steps per environment update.""" + max_iterations: int = 300 + """The maximum number of iterations.""" + obs_groups: dict[str, tuple[str, ...]] = field( + default_factory=lambda: {"actor": ("actor",), "critic": ("critic",)}, + ) + save_interval: int = 50 + """The number of iterations between saves.""" + experiment_name: str = "exp1" + """Directory name used to group runs under ``{log_root}/{experiment_name}/``. + The log root defaults to ``logs/rsl_rl`` and can be overridden with + ``--log-root`` on the CLI.""" + run_name: str = "" + """Optional label appended to the timestamped run directory + (e.g. ``2025-01-27_14-30-00_{run_name}``). Also becomes the + display name for the run in wandb.""" + logger: Literal["wandb", "tensorboard"] = "wandb" + """The logger to use. Default is wandb.""" + wandb_project: str = "mjlab" + """The wandb project name.""" + wandb_tags: Tuple[str, ...] = () + """Tags for the wandb run. Default is empty tuple.""" + resume: bool = False + """Whether to resume the experiment. Default is False.""" + load_run: str = ".*" + """The run directory to load. Default is ".*" which means all runs. If regex + expression, the latest (alphabetical order) matching run will be loaded. + """ + load_checkpoint: str = "model_.*.pt" + """The checkpoint file to load. Default is "model_.*.pt" (all). If regex expression, + the latest (alphabetical order) matching file will be loaded. + """ + clip_actions: float | None = None + """The clipping range for action values. If None (default), no clipping is applied.""" + upload_model: bool = True + """Whether to upload model files (.pt, .onnx) to W&B on save. Set to + False to keep metric logging but avoid storage usage. Default is True.""" + + +@dataclass +class RslRlOnPolicyRunnerCfg(RslRlBaseRunnerCfg): + class_name: str = "OnPolicyRunner" + """The runner class name. Default is OnPolicyRunner.""" + actor: RslRlModelCfg = field( + default_factory=lambda: RslRlModelCfg( + distribution_cfg={ + "class_name": "GaussianDistribution", + "init_std": 1.0, + "std_type": "scalar", + } + ) + ) + """The actor configuration.""" + critic: RslRlModelCfg = field(default_factory=RslRlModelCfg) + """The critic configuration.""" + algorithm: RslRlPpoAlgorithmCfg = field(default_factory=RslRlPpoAlgorithmCfg) + """The algorithm configuration.""" diff --git a/rl/mjlab/rl/exporter_utils.py b/rl/mjlab/rl/exporter_utils.py new file mode 100644 index 0000000..7ee4aad --- /dev/null +++ b/rl/mjlab/rl/exporter_utils.py @@ -0,0 +1,125 @@ +"""Shared utilities for ONNX policy export across RL tasks.""" + +import onnx +import torch + +from mjlab.entity import Entity +from mjlab.envs import ManagerBasedRlEnv +from mjlab.envs.mdp.actions import JointPositionAction + + +def list_to_csv_str( + arr, *, decimals: int = 3, delimiter: str = ",", sub_delimiter: str = ";" +) -> str: + """Convert list to CSV string with specified decimal precision. + + Elements that are themselves sequences (e.g. a per-dimension scale or a + [min, max] clip range) are joined with `sub_delimiter` instead of being + `str()`-formatted, which would otherwise embed a second, ambiguous set of + commas inside the top-level comma-delimited string. + """ + fmt = f"{{:.{decimals}f}}" + + def format_scalar(x) -> str: + return fmt.format(x) if isinstance(x, (int, float)) else str(x) + + def format_entry(x) -> str: + if isinstance(x, (list, tuple)): + return sub_delimiter.join(format_scalar(v) for v in x) + return format_scalar(x) + + return delimiter.join(format_entry(x) for x in arr) + + +def get_base_metadata( + env: ManagerBasedRlEnv, run_path: str +) -> dict[str, list | str | float]: + """Get base metadata common to all RL policy exports. + + Args: + env: The RL environment. + run_path: W&B run path or other identifier. + + Returns: + Dictionary of metadata fields that are common across all tasks. + """ + robot: Entity = env.scene["robot"] + joint_action = env.action_manager.get_term("joint_pos") + assert isinstance(joint_action, JointPositionAction) + # Build mapping from joint name to actuator ID for natural joint order. + # Each spec actuator controls exactly one joint (via its target field). + joint_name_to_ctrl_id = {} + for actuator in robot.spec.actuators: + joint_name = actuator.target.split("/")[-1] + joint_name_to_ctrl_id[joint_name] = actuator.id + # Get actuator IDs in natural joint order (same order as robot.joint_names). + ctrl_ids_natural = [ + joint_name_to_ctrl_id[jname] + for jname in robot.joint_names # global joint order + if jname in joint_name_to_ctrl_id # skip non-actuated joints + ] + joint_stiffness = env.sim.mj_model.actuator_gainprm[ctrl_ids_natural, 0] + joint_damping = -env.sim.mj_model.actuator_biasprm[ctrl_ids_natural, 2] + observation_term_scale: list = [] + observation_term_flatten_history_dim: list = [] + observation_term_history_length: list = [] + observation_term_clip: list = [] + observation_names = env.observation_manager.active_terms["actor"] + + for active_term in observation_names: + cfg = env.observation_manager.get_term_cfg("actor", active_term) + + if cfg.scale is None: + observation_term_scale.append(1.0) + else: + raw_scale = cfg.scale + scale = ( + raw_scale.cpu().tolist() if isinstance(raw_scale, torch.Tensor) else raw_scale + ) + observation_term_scale.append(scale) + + raw_clip = cfg.clip + if raw_clip is None: + observation_term_clip.append([float("-inf"), float("inf")]) + else: + observation_term_clip.append(list(raw_clip)) + + observation_term_flatten_history_dim.append(cfg.flatten_history_dim) + observation_term_history_length.append(cfg.history_length) + + return { + "run_path": run_path, + "joint_names": list(robot.joint_names), + "joint_stiffness": joint_stiffness.tolist(), + "joint_damping": joint_damping.tolist(), + "default_joint_pos": robot.data.default_joint_pos[0].cpu().tolist(), + "command_names": list(env.command_manager.active_terms), + "observation_names": observation_names, + "observation_terms_scale": observation_term_scale, + "observation_terms_flatten_history_dim": observation_term_flatten_history_dim, + "observation_terms_history_length": observation_term_history_length, + "observation_terms_clip": observation_term_clip, + "action_scale": joint_action._scale[0].cpu().tolist() + if isinstance(joint_action._scale, torch.Tensor) + else joint_action._scale, + } + + +def attach_metadata_to_onnx( + onnx_path: str, metadata: dict[str, list | str | float] +) -> None: + """Attach metadata to an ONNX model file. + + Args: + onnx_path: Path to the ONNX model file. + metadata: Dictionary of metadata key-value pairs to attach. + """ + model = onnx.load(onnx_path) + + for k, v in metadata.items(): + entry = onnx.StringStringEntryProto() + entry.key = k + entry.value = list_to_csv_str(v) if isinstance(v, list) else str(v) + model.metadata_props.append(entry) + + onnx.save(model, onnx_path) diff --git a/rl/mjlab/rl/runner.py b/rl/mjlab/rl/runner.py new file mode 100644 index 0000000..2934929 --- /dev/null +++ b/rl/mjlab/rl/runner.py @@ -0,0 +1,141 @@ +import os +from pathlib import Path + +import torch +from rsl_rl.env import VecEnv +from rsl_rl.runners import OnPolicyRunner + +from mjlab.rl.vecenv_wrapper import RslRlVecEnvWrapper + + +class MjlabOnPolicyRunner(OnPolicyRunner): + """Base runner that persists environment state across checkpoints.""" + + env: RslRlVecEnvWrapper + + def __init__( + self, + env: VecEnv, + train_cfg: dict, + log_dir: str | None = None, + device: str = "cpu", + ) -> None: + # Strip None-valued optional configs so MLPModel doesn't receive them. + for key in ("actor", "critic"): + if key in train_cfg: + for opt in ("cnn_cfg", "distribution_cfg"): + if train_cfg[key].get(opt) is None: + train_cfg[key].pop(opt, None) + if train_cfg[key].get("rnn_type") is None: + for opt in ("rnn_type", "rnn_hidden_dim", "rnn_num_layers"): + train_cfg[key].pop(opt, None) + super().__init__(env, train_cfg, log_dir, device) + + def export_policy_to_onnx( + self, path: str, filename: str = "policy.onnx", verbose: bool = False + ) -> None: + """Export policy to ONNX format using legacy export path. + + Overrides the base implementation to set dynamo=False, avoiding warnings about + dynamic_axes being deprecated with the new TorchDynamo export path + (torch>=2.9 default). + """ + onnx_model = self.alg.get_policy().as_onnx(verbose=verbose) + onnx_model.to("cpu") + onnx_model.eval() + os.makedirs(path, exist_ok=True) + torch.onnx.export( + onnx_model, + onnx_model.get_dummy_inputs(), # type: ignore[operator] + os.path.join(path, filename), + export_params=True, + opset_version=18, + verbose=verbose, + input_names=onnx_model.input_names, # type: ignore[arg-type] + output_names=onnx_model.output_names, # type: ignore[arg-type] + dynamic_axes={}, + dynamo=False, + ) + + @staticmethod + def _get_export_paths(checkpoint_path: str) -> tuple[Path, str, Path]: + """Resolve ONNX export paths from a checkpoint path.""" + export_dir = Path(checkpoint_path).parent + filename = f"{export_dir.name}.onnx" + return export_dir, filename, export_dir / filename + + def save(self, path: str, infos=None) -> None: + """Save checkpoint. + + Extends the base implementation to persist the environment's + common_step_counter and to respect the ``upload_model`` config flag. + """ + env_state = {"common_step_counter": self.env.unwrapped.common_step_counter} + infos = {**(infos or {}), "env_state": env_state} + # Inline base OnPolicyRunner.save() to conditionally gate W&B upload. + saved_dict = self.alg.save() + saved_dict["iter"] = self.current_learning_iteration + saved_dict["infos"] = infos + torch.save(saved_dict, path) + if self.cfg["upload_model"]: + self.logger.save_model(path, self.current_learning_iteration) + + def load( + self, + path: str, + load_cfg: dict | None = None, + strict: bool = True, + map_location: str | None = None, + ) -> dict: + """Load checkpoint. + + Extends the base implementation to: + 1. Restore common_step_counter to preserve curricula state. + 2. Migrate legacy checkpoints (actor.* -> mlp.*, actor_obs_normalizer.* + -> obs_normalizer.*) to the current format (rsl-rl>=4.0). + """ + loaded_dict = torch.load(path, map_location=map_location, weights_only=False) + + if "model_state_dict" in loaded_dict: + print(f"Detected legacy checkpoint at {path}. Migrating to new format...") + model_state_dict = loaded_dict.pop("model_state_dict") + actor_state_dict = {} + critic_state_dict = {} + + for key, value in model_state_dict.items(): + # Migrate actor keys. + if key.startswith("actor."): + new_key = key.replace("actor.", "mlp.") + actor_state_dict[new_key] = value + elif key.startswith("actor_obs_normalizer."): + new_key = key.replace("actor_obs_normalizer.", "obs_normalizer.") + actor_state_dict[new_key] = value + elif key in ["std", "log_std"]: + actor_state_dict[key] = value + + # Migrate critic keys. + if key.startswith("critic."): + new_key = key.replace("critic.", "mlp.") + critic_state_dict[new_key] = value + elif key.startswith("critic_obs_normalizer."): + new_key = key.replace("critic_obs_normalizer.", "obs_normalizer.") + critic_state_dict[new_key] = value + + loaded_dict["actor_state_dict"] = actor_state_dict + loaded_dict["critic_state_dict"] = critic_state_dict + + # Migrate rsl-rl 4.x actor keys to 5.x distribution keys. + actor_sd = loaded_dict.get("actor_state_dict", {}) + if "std" in actor_sd: + actor_sd["distribution.std_param"] = actor_sd.pop("std") + if "log_std" in actor_sd: + actor_sd["distribution.log_std_param"] = actor_sd.pop("log_std") + + load_iteration = self.alg.load(loaded_dict, load_cfg, strict) + if load_iteration: + self.current_learning_iteration = loaded_dict["iter"] + + infos = loaded_dict["infos"] + if infos and "env_state" in infos: + self.env.unwrapped.common_step_counter = infos["env_state"]["common_step_counter"] + return infos diff --git a/rl/mjlab/rl/spatial_softmax.py b/rl/mjlab/rl/spatial_softmax.py new file mode 100644 index 0000000..170bf45 --- /dev/null +++ b/rl/mjlab/rl/spatial_softmax.py @@ -0,0 +1,199 @@ +"""CNN with a spatial softmax layer.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn as nn +from rsl_rl.models.cnn_model import CNNModel +from rsl_rl.models.mlp_model import MLPModel +from rsl_rl.modules import CNN +from tensordict import TensorDict + + +class SpatialSoftmax(nn.Module): + """Spatial soft-argmax over feature maps. + + Given input of shape ``(B, C, H, W)``, computes a softmax over each channel's spatial + locations and returns the expected (x, y) coordinates, yielding output shape + ``(B, C * 2)``. + + Args: + height: Height of the input feature maps. + width: Width of the input feature maps. + temperature: Temperature for the spatial softmax. Lower values produce sharper + distributions. + """ + + def __init__(self, height: int, width: int, temperature: float = 1.0) -> None: + super().__init__() + # Create normalised coordinate grids in [-1, 1]. + pos_x, pos_y = torch.meshgrid( + torch.linspace(-1.0, 1.0, height), + torch.linspace(-1.0, 1.0, width), + indexing="ij", + ) + # Register as buffers so they move with the module's device. + self.register_buffer("pos_x", pos_x.reshape(1, 1, -1)) + self.register_buffer("pos_y", pos_y.reshape(1, 1, -1)) + self.temperature = temperature + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Compute spatial soft-argmax. + + Args: + x: Feature maps of shape ``(B, C, H, W)``. + + Returns: + Keypoint coordinates of shape ``(B, C * 2)``. + """ + B, C, H, W = x.shape + # Flatten spatial dims: (B, C, H*W). + features = x.reshape(B, C, -1) + # Spatial softmax. + weights = torch.softmax(features / self.temperature, dim=-1) + # Expected coordinates. + pos_x: torch.Tensor = self.pos_x # type: ignore[assignment] + pos_y: torch.Tensor = self.pos_y # type: ignore[assignment] + expected_x = (weights * pos_x).sum(dim=-1) + expected_y = (weights * pos_y).sum(dim=-1) + # Interleave: (B, C*2). + return torch.stack([expected_x, expected_y], dim=-1).reshape(B, C * 2) + + +class SpatialSoftmaxCNN(nn.Module): + """CNN encoder with spatial-softmax pooling. + + Wraps ``rsl_rl.modules.CNN`` (created with ``global_pool="none"`` + and ``flatten=False``) followed by :class:`SpatialSoftmax`. Exposes the same + interface as ``CNN``: :attr:`output_dim` (int) and :attr:`output_channels` + (``None``, signalling flattened output). + + Args: + input_dim: ``(H, W)`` of the input images. + input_channels: Number of input channels. + temperature: Temperature for the spatial softmax. + **cnn_kwargs: Remaining keyword arguments forwarded to ``rsl_rl.modules.CNN``. + """ + + def __init__( + self, + input_dim: tuple[int, int], + input_channels: int, + temperature: float = 1.0, + **cnn_kwargs: Any, + ) -> None: + super().__init__() + # Override pooling/flatten — spatial softmax replaces these. + cnn_kwargs.pop("global_pool", None) + cnn_kwargs.pop("flatten", None) + self.cnn = CNN( + input_dim=input_dim, + input_channels=input_channels, + global_pool="none", + flatten=False, + **cnn_kwargs, + ) + # cnn.output_dim is (H, W) when flatten=False. + out_h, out_w = self.cnn.output_dim # type: ignore[misc] + num_channels: int = self.cnn.output_channels # type: ignore[assignment] + self.spatial_softmax = SpatialSoftmax(out_h, out_w, temperature) + self._output_dim = num_channels * 2 + + @property + def output_dim(self) -> int: + """Total flattened output dimension (C * 2).""" + return self._output_dim + + @property + def output_channels(self) -> None: + """Always ``None`` (output is flattened keypoints).""" + return None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + features = self.cnn(x) + return self.spatial_softmax(features) + + +class SpatialSoftmaxCNNModel(CNNModel): + """CNN model that uses spatial-softmax pooling. + + Drop-in replacement for ``rsl_rl.models.CNNModel``. The only difference is that each + CNN encoder is wrapped with :class:`SpatialSoftmaxCNN` instead of a plain ``CNN``. + + The ``spatial_softmax_temperature`` parameter is extracted from ``cnn_cfg`` before + the remaining keys are forwarded to ``CNN``. + """ + + def __init__( + self, + obs: TensorDict, + obs_groups: dict[str, list[str]], + obs_set: str, + output_dim: int, + cnn_cfg: dict[str, dict] | dict[str, Any], + cnns: nn.ModuleDict | None = None, + hidden_dims: tuple[int] | list[int] = [256, 256, 256], # noqa: B006 + activation: str = "elu", + obs_normalization: bool = False, + distribution_cfg: dict[str, Any] | None = None, + ) -> None: + # Separate 1D / 2D observation groups (sets self.obs_groups_2d, + # obs_dims_2d, obs_channels_2d; returns 1D info for MLPModel). + self._get_obs_dim(obs, obs_groups, obs_set) + + if cnns is not None: + if set(cnns.keys()) != set(self.obs_groups_2d): + raise ValueError( + "The 2D observations must be identical for all models sharing CNN encoders." + ) + print( + "Sharing CNN encoders between models, the CNN " + "configurations of the receiving model are ignored." + ) + _cnns = cnns + else: + # Expand a single flat config to per-group configs. + if not all(isinstance(v, dict) for v in cnn_cfg.values()): + cnn_cfg = {group: cnn_cfg for group in self.obs_groups_2d} + assert len(cnn_cfg) == len(self.obs_groups_2d), ( + "The number of CNN configurations must match the " + "number of 2D observation groups." + ) + _cnns = {} + for idx, obs_group in enumerate(self.obs_groups_2d): + group_cfg = dict(cnn_cfg[obs_group]) + group_cfg.pop("spatial_softmax", None) + temperature = group_cfg.pop("spatial_softmax_temperature", 1.0) + _cnns[obs_group] = SpatialSoftmaxCNN( + input_dim=self.obs_dims_2d[idx], + input_channels=self.obs_channels_2d[idx], + temperature=temperature, + **group_cfg, + ) + + self.cnn_latent_dim = 0 + for cnn in _cnns.values(): + if cnn.output_channels is not None: + raise ValueError( + "The output of the CNN must be flattened before passing it to the MLP." + ) + self.cnn_latent_dim += int(cnn.output_dim) # type: ignore[arg-type] + + MLPModel.__init__( + self, + obs=obs, + obs_groups=obs_groups, + obs_set=obs_set, + output_dim=output_dim, + hidden_dims=hidden_dims, + activation=activation, + obs_normalization=obs_normalization, + distribution_cfg=distribution_cfg, + ) + + if isinstance(_cnns, nn.ModuleDict): + self.cnns = _cnns + else: + self.cnns = nn.ModuleDict(_cnns) diff --git a/rl/mjlab/rl/vecenv_wrapper.py b/rl/mjlab/rl/vecenv_wrapper.py new file mode 100644 index 0000000..28c207b --- /dev/null +++ b/rl/mjlab/rl/vecenv_wrapper.py @@ -0,0 +1,107 @@ +import torch +from rsl_rl.env import VecEnv +from tensordict import TensorDict + +from mjlab.envs import ManagerBasedRlEnv, ManagerBasedRlEnvCfg +from mjlab.utils.spaces import Space + + +class RslRlVecEnvWrapper(VecEnv): + def __init__( + self, + env: ManagerBasedRlEnv, + clip_actions: float | None = None, + ): + self.env = env + self.clip_actions = clip_actions + + self.num_envs = self.unwrapped.num_envs + self.device = torch.device(self.unwrapped.device) + self.max_episode_length = self.unwrapped.max_episode_length + self.num_actions = self.unwrapped.action_manager.total_action_dim + self._modify_action_space() + + # Reset at the start since rsl_rl does not call reset. + self.env.reset() + + @property + def cfg(self) -> ManagerBasedRlEnvCfg: + return self.unwrapped.cfg + + @property + def render_mode(self) -> str | None: + return self.env.render_mode + + @property + def observation_space(self) -> Space: + return self.env.observation_space + + @property + def action_space(self) -> Space: + return self.env.action_space + + @classmethod + def class_name(cls) -> str: + return cls.__name__ + + @property + def unwrapped(self) -> ManagerBasedRlEnv: + return self.env.unwrapped + + # Properties. + + @property + def episode_length_buf(self) -> torch.Tensor: + return self.unwrapped.episode_length_buf + + @episode_length_buf.setter + def episode_length_buf(self, value: torch.Tensor) -> None: # pyright: ignore[reportIncompatibleVariableOverride] + self.unwrapped.episode_length_buf = value + + def seed(self, seed: int = -1) -> int: + return self.unwrapped.seed(seed) + + def get_observations(self) -> TensorDict: + obs_dict = self.unwrapped.observation_manager.compute() + return TensorDict(obs_dict, batch_size=[self.num_envs]) + + def reset(self) -> tuple[TensorDict, dict]: + obs_dict, extras = self.env.reset() + return TensorDict(obs_dict, batch_size=[self.num_envs]), extras + + def step( + self, actions: torch.Tensor + ) -> tuple[TensorDict, torch.Tensor, torch.Tensor, dict]: + if self.clip_actions is not None: + actions = torch.clamp(actions, -self.clip_actions, self.clip_actions) + obs_dict, rew, terminated, truncated, extras = self.env.step(actions) + term_or_trunc = terminated | truncated + assert isinstance(rew, torch.Tensor) + assert isinstance(term_or_trunc, torch.Tensor) + dones = term_or_trunc.to(dtype=torch.long) + if not self.cfg.is_finite_horizon: + extras["time_outs"] = truncated + return ( + TensorDict(obs_dict, batch_size=[self.num_envs]), + rew, + dones, + extras, + ) + + def close(self) -> None: + return self.env.close() + + # Private methods. + + def _modify_action_space(self) -> None: + if self.clip_actions is None: + return + + from mjlab.utils.spaces import Box, batch_space + + self.unwrapped.single_action_space = Box( + shape=(self.num_actions,), low=-self.clip_actions, high=self.clip_actions + ) + self.unwrapped.action_space = batch_space( + self.unwrapped.single_action_space, self.num_envs + ) diff --git a/rl/mjlab/scene/__init__.py b/rl/mjlab/scene/__init__.py new file mode 100644 index 0000000..3cc68e6 --- /dev/null +++ b/rl/mjlab/scene/__init__.py @@ -0,0 +1,2 @@ +from mjlab.scene.scene import Scene as Scene +from mjlab.scene.scene import SceneCfg as SceneCfg diff --git a/rl/mjlab/scene/scene.py b/rl/mjlab/scene/scene.py new file mode 100644 index 0000000..608436b --- /dev/null +++ b/rl/mjlab/scene/scene.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +import mujoco +import mujoco_warp as mjwarp +import numpy as np +import torch + +from mjlab.entity import Entity, EntityCfg +from mjlab.entity.variants import VariantMetadata +from mjlab.sensor import BuiltinSensor, RayCastSensor, Sensor, SensorCfg +from mjlab.sensor.camera_sensor import CameraSensor +from mjlab.sensor.sensor_context import SensorContext +from mjlab.terrains.terrain_entity import TerrainEntity, TerrainEntityCfg +from mjlab.utils.spec import export_spec, non_default_option_fields + +_SCENE_XML = Path(__file__).parent / "scene.xml" + + +@dataclass(kw_only=True) +class SceneCfg: + """Configuration for a simulation scene.""" + + num_envs: int = 1 + """Number of parallel environments.""" + + env_spacing: float = 2.0 + """Spacing between environment origins in meters.""" + + terrain: TerrainEntityCfg | None = None + """Terrain configuration. If ``None``, no terrain is added.""" + + entities: dict[str, EntityCfg] = field(default_factory=dict) + """Mapping of entity names to their configurations.""" + + sensors: tuple[SensorCfg, ...] = field(default_factory=tuple) + """Sensor configurations to attach to the scene.""" + + extent: float | None = None + """Override for ``mjModel.stat.extent``. If ``None``, MuJoCo computes + it automatically.""" + + spec_fn: Callable[[mujoco.MjSpec], None] | None = None + """Optional callback to modify the ``MjSpec`` after entities and sensors + have been added but before compilation.""" + + +class Scene: + def __init__(self, scene_cfg: SceneCfg, device: str) -> None: + self._cfg = scene_cfg + self._device = device + self._entities: dict[str, Entity] = {} + self._sensors: dict[str, Sensor] = {} + self._terrain: TerrainEntity | None = None + self._default_env_origins: torch.Tensor | None = None + self._sensor_context: SensorContext | None = None + + self._spec = mujoco.MjSpec.from_file(str(_SCENE_XML)) + if self._cfg.extent is not None: + self._spec.stat.extent = self._cfg.extent + self._add_terrain() + self._add_entities() + self._add_sensors() + if self._cfg.spec_fn is not None: + self._cfg.spec_fn(self._spec) + + def compile(self) -> mujoco.MjModel: + return self._spec.compile() + + def write(self, output_dir: Path, *, zip: bool = False) -> None: + """Write the scene XML and mesh assets to a directory. + + Creates ``scene.xml`` and an ``assets/`` subdirectory containing mesh files + referenced by the scene. When *zip* is True the directory is compressed into a + ``.zip`` archive and the directory is removed. Operates on a copy of the spec to + avoid mutation. + + Args: + output_dir: Destination directory (created if it doesn't exist). + zip: If True, produce ``.zip`` instead of a directory. + """ + spec = self._spec.copy() + # Entity meshes compile straight from disk, so spec.assets holds no mesh bytes. + for ent in self._entities.values(): + for mesh in ent.spec.meshes: + path = Path(ent.spec.modelfiledir) / mesh.compiler.meshdir / mesh.file + if mesh.file and mesh.file not in spec.assets and path.is_file(): + spec.assets[mesh.file] = path.read_bytes() + export_spec(spec, output_dir, zip=zip) + + def to_zip(self, path: Path) -> None: + """Deprecated. Use ``write(output_dir, zip=True)`` instead.""" + warnings.warn( + "Scene.to_zip() is deprecated. Use Scene.write(path, zip=True).", + DeprecationWarning, + stacklevel=2, + ) + self.write(path, zip=True) + + # Attributes. + + @property + def spec(self) -> mujoco.MjSpec: + return self._spec + + @property + def env_origins(self) -> torch.Tensor: + if self._terrain is not None: + assert self._terrain.env_origins is not None + return self._terrain.env_origins + assert self._default_env_origins is not None + return self._default_env_origins + + @property + def env_spacing(self) -> float: + return self._cfg.env_spacing + + @property + def entities(self) -> dict[str, Entity]: + return self._entities + + @property + def sensors(self) -> dict[str, Sensor]: + return self._sensors + + @property + def terrain(self) -> TerrainEntity | None: + return self._terrain + + @property + def num_envs(self) -> int: + return self._cfg.num_envs + + @property + def device(self) -> str: + return self._device + + def collect_variant_info( + self, + ) -> list[tuple[str, VariantMetadata]]: + """Collect variant metadata for entities with mesh variants.""" + result: list[tuple[str, VariantMetadata]] = [] + for name, ent in self._entities.items(): + if ent.variant_metadata is not None: + result.append((f"{name}/", ent.variant_metadata)) + return result + + def __getitem__(self, key: str) -> Any: + if key in self._sensors: + return self._sensors[key] + if key in self._entities: + return self._entities[key] + + # Not found, raise helpful error. + available = list(self._entities.keys()) + list(self._sensors.keys()) + raise KeyError(f"Scene element '{key}' not found. Available: {available}") + + # Methods. + + @property + def sensor_context(self) -> SensorContext | None: + """Shared sensing resources, or None if no cameras/raycasts.""" + return self._sensor_context + + def initialize( + self, + mj_model: mujoco.MjModel, + model: mjwarp.Model, + data: mjwarp.Data, + ): + self._default_env_origins = torch.zeros( + (self._cfg.num_envs, 3), device=self._device, dtype=torch.float32 + ) + for ent in self._entities.values(): + ent.initialize(mj_model, model, data, self._device) + for sensor in self._sensors.values(): + sensor.initialize(mj_model, model, data, self._device) + + # Create SensorContext if any sensors require it. + ctx_sensors = [s for s in self._sensors.values() if s.requires_sensor_context] + if ctx_sensors: + camera_sensors = [s for s in ctx_sensors if isinstance(s, CameraSensor)] + raycast_sensors = [s for s in ctx_sensors if isinstance(s, RayCastSensor)] + self._sensor_context = SensorContext( + mj_model=mj_model, + model=model, + data=data, + camera_sensors=camera_sensors, + raycast_sensors=raycast_sensors, + device=self._device, + ) + + def reset(self, env_ids: torch.Tensor | slice | None = None) -> None: + for ent in self._entities.values(): + ent.reset(env_ids) + for sensor in self._sensors.values(): + sensor.reset(env_ids) + + def update(self, dt: float) -> None: + for ent in self._entities.values(): + ent.update(dt) + for sensor in self._sensors.values(): + sensor.update(dt) + + def write_data_to_sim(self) -> None: + for ent in self._entities.values(): + ent.write_data_to_sim() + + # Private methods. + + def _add_entities(self) -> None: + # Collect keyframes from each entity to merge into a single scene keyframe. + # Order matters: qpos/ctrl are concatenated in entity iteration order. + key_qpos: list[np.ndarray] = [] + key_ctrl: list[np.ndarray] = [] + for ent_name, ent_cfg in self._cfg.entities.items(): + ent = ent_cfg.build() + self._entities[ent_name] = ent + # Extract keyframe before attach (must delete before attach to avoid corruption). + if ent.spec.keys: + if len(ent.spec.keys) > 1: + warnings.warn( + f"Entity '{ent_name}' has {len(ent.spec.keys)} keyframes; only the " + "first one will be used.", + stacklevel=2, + ) + key_qpos.append(np.array(ent.spec.keys[0].qpos)) + key_ctrl.append(np.array(ent.spec.keys[0].ctrl)) + ent.spec.delete(ent.spec.keys[0]) + non_default = non_default_option_fields(ent.spec.option) + if non_default: + fields = ", ".join(non_default) + warnings.warn( + f"Entity '{ent_name}' has non-default