Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-15 14:13:49 +08:00
commit 7e4ef6f98b
7002 changed files with 1452249 additions and 0 deletions
+115
View File
@@ -0,0 +1,115 @@
# @package _global_
defaults:
- _self_
# === TASK CONFIGURATION ===
robot_type: xhand # "inspire", "allegro", "g1"
embodiment_type: bimanual # "left", "right", "bimanual", "CMU"
task: pick_spoon_bowl
seed: 0
# === DATASET CONFIGURATION ===
dataset_dir: example_datasets
dataset_name: oakink
data_id: 0
model_path: ""
data_path: ""
# === SIMULATOR CONFIGURATION ===
simulator: mjwp # "isaac" | "mujoco" | "mjwp" | "mjwp_cons" | "mjwp_eq" | "mjwp_cons_eq" | "kinematic"
device: cuda:0
# Simulation timing
sim_dt: 0.01 # simulation timestep
ctrl_dt: 0.4 # control timestep
ref_dt: 0.02 # reference data timestep (1/50.0)
render_dt: 0.02 # rendering timestep
horizon: 1.6 # planning horizon
knot_dt: 0.4 # knot point spacing
max_sim_steps: -1 # maximum simulation steps (-1 for unlimited)
# Simulation constraints
nconmax_per_env: 120 # max contacts per environment
njmax_per_env: 350 # max joints per environment
# Simulation annealing
num_dyn: 1 # number of environments for annealing, used for virtual contact constraint
# Domain randomization
num_dr: 1 # number of domain randomization groups, used for domain randomization
pair_margin_range: [-0.005, 0.005]
xy_offset_range: [-0.005, 0.005]
perturb_force: 0.0
perturb_torque: 0.0
contact_guidance: false
# === OPTIMIZER CONFIGURATION ===
# Sampling parameters
num_samples: 1024
temperature: 0.1
max_num_iterations: 32
improvement_threshold: 0.01
improvement_check_steps: 1
# Noise scheduling
first_ctrl_noise_scale: 0.5
last_ctrl_noise_scale: 1.0
final_noise_scale: 0.1
exploit_ratio: 0.01
exploit_noise_scale: 0.01
# Noise scaling by component
joint_noise_scale: 0.15
pos_noise_scale: 0.03
rot_noise_scale: 0.03
# Reward scaling
base_pos_rew_scale: 1.0
base_rot_rew_scale: 0.3
joint_rew_scale: 0.003
pos_rew_scale: 1.0
rot_rew_scale: 0.3
vel_rew_scale: 0.0001
terminal_rew_scale: 1.0
# Termination threshold
terminate_resample: false
object_pos_threshold: 0.1 # meters
object_rot_threshold: 0.3 # radians
# Gibbs sampling
gibbs_sampling: false
# === VISUALIZATION CONFIGURATION ===
show_viewer: true
viewer: mujoco-rerun # "mujoco" | "rerun" | "viser" | "isaac"
rerun_spawn: true
save_video: true
save_rerun: false
save_viser: false
save_metrics: true
save_info: true
save_config: true
# === TRACE RECORDING ===
trace_dt: 0.02 # 1/50.0
num_trace_uniform_samples: 4
num_trace_topk_samples: 2
trace_site_ids: []
# === AUTOMATICALLY SET PROPERTIES ===
# Computed timesteps
horizon_steps: -1
knot_steps: -1
ref_steps: -1
ctrl_steps: -1
# Model dimensions
nq_obj: -1 # object DOF
nq: -1 # total position DOF
nv: -1 # total velocity DOF
nu: -1 # total control DOF
npair: -1 # total pair DOF
# Computed tensors (these will be set programmatically)
noise_scale: null
beta_traj: -1.0
# Runtime state
env_params_list: []
viewer_body_entity_and_ids: []
output_dir: ""
# Hydra configuration
hydra:
run:
dir: .
output_subdir: null
+50
View File
@@ -0,0 +1,50 @@
# Configuration for DexMachina (Genesis) dynamics retargeting pipeline
defaults:
- _self_
# Dataset configuration (not used by DexMachina, but kept for compatibility)
dataset_dir: "example_datasets"
dataset_name: "arctic"
robot_type: "inspire_hand" # "inspire_hand", "xhand", "dex3_hand"
embodiment_type: "bimanual"
task: "box-30-230" # format: "object_name-start_frame-end_frame"
data_id: 0
viewer: "gs"
rerun_spawn: false
device: "cuda:0"
# Simulation and control parameters
simulator: dexmachina
sim_dt: 0.02 # Genesis simulation timestep
ctrl_dt: 0.20 # Control timestep (how often MPC updates)
knot_dt: 0.20 # Knot point spacing
ref_dt: 0.02 # Reference trajectory timestep
horizon: 0.40 # MPC horizon in seconds
show_viewer: true
save_video: true
save_info: true
max_sim_steps: -1 # -1 means use full trajectory length
nu: 36
# MPC optimization parameters
max_num_iterations: 32 # Number of iterations per MPC step
num_samples: 1024 # Number of parallel samples for CEM
last_ctrl_noise_scale: 1.0
first_ctrl_noise_scale: 0.5
final_noise_scale: 0.01
pos_noise_scale: 1.0
rot_noise_scale: 1.0
joint_noise_scale: 1.0
improvement_threshold: 0.01
improvement_check_steps: 4
terminal_rew_scale: 10.0
temperature: 1.0
# Reward mode
use_rl_reward: true # true to use dexmachina RL training reward formulation
# Domain randomization (curriculum learning)
num_dr: 1 # Number of domain randomization settings per iteration
# Output configuration
output_dir: "outputs/dexmachina"
+48
View File
@@ -0,0 +1,48 @@
# Configuration for HDMI (mjlab) humanoid retargeting pipeline
defaults:
- _self_
# Task configuration
dataset_name: "hdmi"
robot_type: "unitree_g1"
task: "move_suitcase" # HDMI task name (e.g., move_suitcase, open_drawer, etc.)
viewer: "mujoco-rerun" # "mujoco" for visualization, "none" for headless
rerun_spawn: false
embodiment_type: "humanoid_object" # don't use specific noise scaling for hands
device: "cuda:0"
joint_noise_scale: 0.2 # disable specific noise scaling for robot
# Simulation and control parameters
simulator: hdmi
sim_dt: 0.02 # mjlab simulation timestep
ctrl_dt: 0.04 # Control timestep (how often MPC updates)
knot_dt: 0.20 # Knot point spacing
ref_dt: 0.02 # Reference trajectory timestep
horizon: 0.80 # MPC horizon in seconds
show_viewer: true
save_video: true # Disable for now - need to configure camera names for HDMI
save_info: true
max_sim_steps: -1 # Maximum simulation steps (-1 for unlimited)
nu: -1 # Number of controls (will be set from environment)
# Optimization parameters
use_torch_compile: true
# MPC optimization parameters
max_num_iterations: 32 # Number of iterations per MPC step
num_samples: 1024 # Number of parallel samples for sampling
last_ctrl_noise_scale: 1.0
first_ctrl_noise_scale: 1.0
final_noise_scale: 0.1
improvement_threshold: 0.02
improvement_check_steps: 1
terminal_rew_scale: 1.0
temperature: 0.1
num_resamples: 0
resample_ratio: 0.2
# Domain randomization (not used for HDMI, kept for compatibility)
num_dr: 1
# Output configuration
output_dir: "outputs/hdmi"
+62
View File
@@ -0,0 +1,62 @@
# Configuration for ManipTrans (IsaacGym) dynamics retargeting pipeline
defaults:
- _self_
# Dataset configuration
dataset_dir: "example_datasets"
dataset_name: "maniptrans"
robot_type: "inspire" # "inspire", "shadow", "allegro", "xhand"
embodiment_type: "left" # "right", "left", "bimanual"
task: "0f900@0" # HO-Tracker data index used as task name (format: sequence_id@sub_id)
data_id: 0
seed: 0
viewer: "none" # "isaac" for isaac gym native viewer
rerun_spawn: false
device: "cuda:0"
# Simulation and control parameters
simulator: maniptrans
sim_dt: 0.0166667 # 1/60 second (IsaacGym default)
ctrl_dt: 0.2 # Control timestep (how often MPC updates)
knot_dt: 0.2 # Knot point spacing
ref_dt: 0.0166667 # Reference trajectory timestep
render_dt: 0.0166667 # Same as sim_dt
trace_dt: 0.0166667 # Must be divisible by sim_dt
horizon: 0.6 # MPC horizon in seconds
show_viewer: false
save_video: true
save_info: true
max_sim_steps: -1 # -1 means use full trajectory length
nu: 18 # 6 (wrist force/torque) + 12 (inspire DOFs), overridden by env
# MPC optimization parameters
max_num_iterations: 32 # Number of iterations per MPC step
num_samples: 1024 # Number of parallel samples for CEM
last_ctrl_noise_scale: 1.0
first_ctrl_noise_scale: 0.5
final_noise_scale: 0.1
# this part is handled by base controller in maniptrans
pos_noise_scale: 0.0
rot_noise_scale: 0.0
joint_noise_scale: 1.0
improvement_threshold: 0.001
improvement_check_steps: 4
terminal_rew_scale: 10.0
temperature: 1.0
# Domain randomization (not used for ManipTrans evaluation)
num_dr: 1
# ManipTrans-specific parameters (maniptrans_data_indices derived from task name)
maniptrans_episode_length: 1200
maniptrans_tighten_method: "None" # No curriculum tightening for MPC
maniptrans_tighten_factor: 1.0
maniptrans_tighten_steps: 3200
maniptrans_actions_moving_average: 0.4
maniptrans_use_pid: false
maniptrans_use_quat_rot: false
maniptrans_ctrl_freq_inv: 1
maniptrans_record_video: false # Record video after MPC (replay with 2-env camera)
# Output configuration (empty = use standard path under dataset_dir)
output_dir: ""
+43
View File
@@ -0,0 +1,43 @@
dataset_dir: ../../example_datasets
dataset_name: oakink
robot_type: allegro
embodiment_type: bimanual
task: pick_spoon_bowl
data_id: 0
sim_dt: 0.01
ctrl_dt: 0.2
ref_dt: 0.02
show_viewer: true
save_video: true
save_info: true
# Configuration for MJWP (MuJoCo + Warp) dynamics retargeting pipeline
defaults:
- _self_
- launcher: basic
# Configuration for MJWP (MuJoCo + Warp) dynamics retargeting pipeline
dataset_dir: "../example_datasets"
dataset_name: "oakink"
robot_type: "allegro"
embodiment_type: "bimanual"
all_tasks: null # if null, process all available tasks, otherwise specify list of tasks
overwrite: false # whether to overwrite existing mjwp trajectories
viewer: ""
device: "cuda:0"
method: "mjwp"
# MJWP-specific optimization parameters (optional overrides)
# Set to null to use defaults from Config class
horizon: null # default: 1.2
sim_dt: null # default: 0.01
ctrl_dt: null # default: 0.2
show_viewer: true # typically false for batch processing
save_video: true
save_info: true
max_sim_steps: -1 # -1 means use full trajectory length
# Hydra configuration
hydra:
run:
dir: .
output_subdir: null
@@ -0,0 +1,22 @@
# @package _global_
dataset_name: arcticv2
task: s01-box_use_01
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: 0.01
improvement_check_steps: 2
max_num_iterations: 16
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
@@ -0,0 +1,36 @@
# @package _global_
dataset_name: arcticv2
task: s01-box_use_01
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: -1.0
improvement_check_steps: 2
max_num_iterations: 4
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
object_rot_threshold: 2.5
max_local_retries: 3
# Arctic clips start ~2 s before contact (hand approaching, object resting on
# the table only via friction). The default sanity check holds ctrl_ref[0]
# for 3 s and requires the object not to drift — that's incompatible with a
# pre-grasp start frame, so disable it here.
sanity_check_seconds: 0.0
# Arctic objects (box / laptop / mixer / notebook / waffleiron) decompose
# into 25-30 convex parts each. With bimanual xhand the contact + joint-row
# counts overflow the default per-env limits ("nefc overflow - please
# increase njmax to ~420"). Bump both above the worst observed value with
# headroom.
nconmax_per_env: 256
njmax_per_env: 768
@@ -0,0 +1,25 @@
# @package _global_
dataset_name: fair_fre
task: pick_up
robot_type: xhand
embodiment_type: right
data_id: 4
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
ref_dt: 0.02
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: 0.01
improvement_check_steps: 3
terminal_rew_scale: 10.0
pos_rew_scale: 1.0
max_num_iterations: 16
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
@@ -0,0 +1,24 @@
# @package _global_
# Override configuration for fair_mon dataset
dataset_name: fair_mon
robot_type: allegro
viewer: rerun-mujoco
embodiment_type: right
task: cat
rerun_spawn: true
ref_dt: 0.05
max_sim_steps: -1
horizon: 0.8
ctrl_dt: 0.1
knot_dt: 0.2
num_dr: 1
perturb_torque: 0.0
pos_noise_scale: 0.05
rot_noise_scale: 0.05
joint_noise_scale: 0.1
improvement_check_steps: 2
max_num_iterations: 16
temperature: 1.0
# original reference is bad, so we put smaller scale on tracking wrist pose
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
@@ -0,0 +1,21 @@
# @package _global_
# Override configuration for gigahand dataset
dataset_name: gigahand
robot_type: xhand
embodiment_type: bimanual
task: p36-tea
temperature: 1.0
data_id: 0
ctrl_dt: 1.5
knot_dt: 0.5
horizon: 3.0
first_ctrl_noise_scale: 1.0
last_ctrl_noise_scale: 2.0
improvement_threshold: 0.01
improvement_check_steps: 2
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.03
pos_noise_scale: 0.03
rot_noise_scale: 0.01
final_noise_scale: 0.01
@@ -0,0 +1,33 @@
# @package _global_
task: p36-tea
dataset_name: gigahand
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
first_ctrl_noise_scale: 1.0
final_noise_scale: 0.3
sim_dt: 0.005
max_num_iterations: 32
gibbs_sampling: false
# guidance related
improvement_threshold: 0.0
contact_guidance: true
guidance_decay_ratio: 0.5
init_pos_actuator_gain: 2.0
init_pos_actuator_bias: 2.0
init_rot_actuator_gain: 0.3
init_rot_actuator_bias: 0.3
# with guidance, exploration noise can be higher
joint_noise_scale: 0.15
pos_noise_scale: 0.01
rot_noise_scale: 0.01
# with guidance, horizon can be shorter
horizon: 1.6
knot_dt: 0.4
ctrl_dt: 0.2
base_pos_rew_scale: 0.03
base_rot_rew_scale: 0.03
pos_rew_scale: 1.0
rot_rew_scale: 1.0
@@ -0,0 +1,23 @@
# @package _global_
dataset_name: gigahand
robot_type: xhand
embodiment_type: bimanual
task: p36-tea
temperature: 1.0
data_id: 0
ctrl_dt: 1.5
knot_dt: 0.5
horizon: 3.0
first_ctrl_noise_scale: 1.0
last_ctrl_noise_scale: 2.0
improvement_threshold: -1.0
improvement_check_steps: 2
max_num_iterations: 8
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.03
pos_noise_scale: 0.03
rot_noise_scale: 0.01
final_noise_scale: 0.01
object_rot_threshold: 0.5
max_local_retries: 3
@@ -0,0 +1,16 @@
# @package _global_
# Override configuration for gigahand dataset
dataset_name: gigahand
robot_type: xhand
embodiment_type: bimanual
task: p36-tea
data_id: 10
horizon: 1.6
temperature: 1.0
improvement_threshold: 0.01
improvement_check_steps: 1
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.03
pos_noise_scale: 0.03
rot_noise_scale: 0.01
@@ -0,0 +1,31 @@
# @package _global_
# Override configuration for fair_mon dataset
dataset_name: amass
robot_type: unitree_g1
embodiment_type: humanoid
task: sprint
rerun_spawn: true
sim_dt: 0.0166667
ref_dt: 0.0333333
trace_dt: 0.0333333
max_sim_steps: -1
horizon: 0.8
ctrl_dt: 0.1
knot_dt: 0.1
# annealing parameters
first_ctrl_noise_scale: 0.5
last_ctrl_noise_scale: 1.0
final_noise_scale: 0.1
joint_noise_scale: 0.1
# sampling
improvement_threshold: 0.01
max_num_iterations: 16
temperature: 0.6
num_samples: 2048
# reward
vel_rew_scale: 0.0
joint_rew_scale: 1.0
pos_rew_scale: 3.0
rot_rew_scale: 1.0
contact_rew_scale: 1.0
# note: above parameter is optimized for speed. you can have better performance with horizon=1.0
@@ -0,0 +1,33 @@
# @package _global_
# Override configuration for fair_mon dataset
dataset_name: omomo
robot_type: unitree_g1
viewer: mujoco-rerun
embodiment_type: humanoid_object
task: move_largebox
rerun_spawn: true
sim_dt: 0.0166667
ref_dt: 0.0333333
trace_dt: 0.0333333
max_sim_steps: -1
horizon: 0.8
ctrl_dt: 0.1
knot_dt: 0.1
# annealing parameters
first_ctrl_noise_scale: 0.5
last_ctrl_noise_scale: 1.0
final_noise_scale: 0.1
joint_noise_scale: 0.1
# sampling
improvement_threshold: 0.01
max_num_iterations: 16
temperature: 1.0
num_samples: 2048
# reward
vel_rew_scale: 0.0
joint_rew_scale: 1.0
base_pos_rew_scale: 1.0
base_rot_rew_scale: 1.0
pos_rew_scale: 1.0
rot_rew_scale: 0.3
# note: above parameter is optimized for speed. you can have better performance with horizon=1.0
+21
View File
@@ -0,0 +1,21 @@
# @package _global_
task: lift_board
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: 0.01
improvement_check_steps: 2
max_num_iterations: 16
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
@@ -0,0 +1,32 @@
# @package _global_
task: lift_board
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
first_ctrl_noise_scale: 1.0
final_noise_scale: 0.3
sim_dt: 0.005
max_num_iterations: 32
gibbs_sampling: false
# guidance related
improvement_threshold: 0.0
contact_guidance: true
guidance_decay_ratio: 0.8
init_pos_actuator_gain: 10.0
init_pos_actuator_bias: 10.0
init_rot_actuator_gain: 0.3
init_rot_actuator_bias: 0.3
# with guidance, exploration noise can be higher
joint_noise_scale: 0.02
pos_noise_scale: 0.005
rot_noise_scale: 0.01
# with guidance, horizon can be shorter
horizon: 0.4
knot_dt: 0.1
ctrl_dt: 0.05
base_pos_rew_scale: 0.3
base_rot_rew_scale: 0.3
pos_rew_scale: 1.0
rot_rew_scale: 1.0
@@ -0,0 +1,23 @@
# @package _global_
task: lift_board
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: -1.0
improvement_check_steps: 2
max_num_iterations: 8
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
object_rot_threshold: 0.5
max_local_retries: 3
@@ -0,0 +1,17 @@
# @package _global_
task: lift_board
robot_type: xhand
embodiment_type: bimanual
data_id: 0
temperature: 1.0
num_samples: 1024
horizon: 2.0
improvement_threshold: 0.01
improvement_check_steps: 2
max_num_iterations: 32
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
# rot_rew_scale: 0.3
joint_noise_scale: 0.03
pos_noise_scale: 0.01
rot_noise_scale: 0.01
@@ -0,0 +1,22 @@
# @package _global_
dataset_name: oakinkv2
task: pick_spoon_bowl
robot_type: xhand
embodiment_type: right
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: 0.01
improvement_check_steps: 2
max_num_iterations: 16
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
@@ -0,0 +1,24 @@
# @package _global_
dataset_name: oakinkv2
task: pick_spoon_bowl
robot_type: xhand
embodiment_type: right
data_id: 0
temperature: 1.0
num_samples: 1024
ctrl_dt: 1.5
knot_dt: 1.0
horizon: 3.0
first_ctrl_noise_scale: 2.0
last_ctrl_noise_scale: 4.0
improvement_threshold: -1.0
improvement_check_steps: 2
max_num_iterations: 4
base_pos_rew_scale: 0.1
base_rot_rew_scale: 0.03
joint_noise_scale: 0.1
pos_noise_scale: 0.01
rot_noise_scale: 0.01
final_noise_scale: 0.01
object_rot_threshold: 1.0
max_local_retries: 3