Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-15 14:13:49 +08:00
commit 7e4ef6f98b
7002 changed files with 1452249 additions and 0 deletions
@@ -0,0 +1,64 @@
+defaults:
+  - data: HOT3D
+  - optim
+  - _self_
+
+model:
+  use_init: True
+  opt_cams: False #True
+  opt_scale: True
+  async_tracks: True
+
+overwrite: False
+run_opt: True
+run_vis: True
+run_prior: False
+datatype: ${data.video_dir}
+vis:
+  phases:
+    # - motion_chunks
+    # - input
+    # - root_fit
+    - smooth_fit
+    - prior
+    # - init
+  render_views:
+    - above
+    - side
+    - front
+    - src_cam
+  make_grid: True
+  overwrite: True
+  render_keypoints: False  # Overlay GT 2D keypoints on source camera view
+
+MANO:
+  MODEL_PATH: ${paths.DATA_DIR}/mano
+  GENDER: neutral
+  NUM_HAND_JOINTS: 15
+  MEAN_PARAMS: ${paths.DATA_DIR}/mano_mean_params.npz
+  CREATE_BODY_POSE: FALSE
+
+paths:
+  base_dir: None
+  DATA_DIR: _DATA/data/
+  vposer: VPoser/pretrained/Vposer_right_mirrored # _DATA/body_models/vposer_v1_0
+
+HMP:
+  config: hmp_config.yaml
+  use_hposer: False
+  vid_path: ${data.root}/video/${data.seq}.mp4
+  exp_name: None
+
+fps: 30
+log_root: ../outputs/logs
+log_dir: ${log_root}/${data.type}-${data.split}
+exp_name: ${now:%Y-%m-%d}
+temporal_smooth: True
+gpu: 0
+is_static: False
+
+hydra:
+  job:
+    chdir: True
+  run:
+    dir: ${log_dir}/${exp_name}/${data.name}
@@ -0,0 +1,24 @@
+type: video
+split: custom
+root: /data/home/zy3023/code/hand/Dyn-HaMR/test  # put your videos in root/videos/vid.mp4
+video_dir: videos
+seq: demo1
+ext: mp4
+src_path: ${data.root}/${data.video_dir}/${data.seq}.${data.ext}
+frame_opts:
+  ext: jpg
+  fps: 25
+  start_sec: 0
+  end_sec: -1
+use_cams: True
+track_ids: "all"
+shot_idx: 0
+start_idx: 0
+end_idx: 500
+split_cameras: True
+name: ${data.seq}-${data.track_ids}-shot-${data.shot_idx}-${data.start_idx}-${data.end_idx}
+sources:
+  images: ${data.root}/images/${data.seq}
+  cameras: ${data.root}/dynhamr/cameras/${data.seq}/shot-${data.shot_idx}
+  tracks: ${data.root}/dynhamr/track_preds/${data.seq}
+  shots: ${data.root}/dynhamr/shot_idcs/${data.seq}.json
@@ -0,0 +1,29 @@
+type: video
+split: custom
+root: /data/home/zy3023/code/hand/Dyn-HaMR/test  # put your videos in root/videos/vid.mp4
+video_dir: videos
+seq: prod1
+ext: mp4
+src_path: ${data.root}/${data.video_dir}/${data.seq}.${data.ext}
+frame_opts:
+  ext: jpg
+  fps: 25
+  start_sec: 0
+  end_sec: -1
+use_cams: True
+track_ids: "all"
+shot_idx: 0
+start_idx: 0
+end_idx: -1
+split_cameras: True
+name: ${data.seq}-${data.track_ids}-shot-${data.shot_idx}-${data.start_idx}-${data.end_idx}
+sources:
+  images: ${data.root}/images/${data.seq}
+  cameras: ${data.root}/dynhamr/cameras/${data.seq}/shot-${data.shot_idx}
+  tracks: ${data.root}/dynhamr/track_preds/${data.seq}
+  shots: ${data.root}/dynhamr/shot_idcs/${data.seq}.json
+
+# VIPE-specific options
+use_vipe: True
+vipe_dir: /data/home/zy3023/code/hand/Dyn-HaMR/third-party/vipe/vipe_results
+
@@ -0,0 +1,13 @@
+defaults:
+  - data: posetrack
+  - _self_
+
+gap: 1
+log_root: outputs
+save_per_frame: False
+print_err: False
+stride: 48
+
+hydra:
+    run:
+        dir: ${log_root}/init/${data.type}-${data.seq}-${data.depth_dir}-${data.fov}fov-gap${gap}
@@ -0,0 +1,52 @@
+optim:
+  options:
+    robust_loss_type: "bisquare"
+    robust_tuning_const: 4.6851
+    joints2d_sigma: 100.0
+    lr: 1.0
+    lbfgs_max_iter: 20
+    save_every: 20
+    vis_every: -1
+    max_chunk_steps: 20
+    save_meshes: False
+
+  root:
+    num_iters: 50
+
+  smpl:
+    num_iters: 0
+
+  smooth:
+    opt_scale: False
+    num_iters: 300
+
+  motion_chunks:
+    chunk_size: 10
+    init_steps: 20
+    chunk_steps: 20
+    opt_cams: True
+
+  loss_weights:
+    penetration: [0,0,0] # [0, 1.0, 10.0]
+    # joints2d: [1, 1, 1] # Increased smooth_fit weight from 0.001 to 0.1 to better match 2D keypoints
+    joints2d: [10000, 10000, 10000] # Increased smooth_fit weight from 0.001 to 0.1 to better match 2D keypoints
+    bg2d: [0.0, 0.000, 0.000]
+    cam_R_smooth : [0.0, 0.0, 0.0]
+    cam_t_smooth : [0.0, 0.0, 0.0]
+    bio : [0.0, 0.0, 0.0] # [10, 10, 10]
+    depth_constraint: [100.0, 100.0, 0.0] # Prevent hand from being behind camera - MUST be stronger than 2D loss
+      #    bg2d: [0.0, 0.0001, 0.0001]
+      #    cam_R_smooth : [0.0, 1000.0, 1000.0]
+      #    cam_t_smooth : [0.0, 1000.0, 1000.0]
+    joints3d: [0.0, 0.0, 0.0]
+    joints3d_smooth: [1000, 10000.0, 0.0] # raw: [1.0, 10.0, 0.0]
+    # joints3d_smooth: [0, 0, 0.0] # raw: [1.0, 10.0, 0.0]
+    joints3d_rollout: [0.0, 0.0, 0.0]
+    verts3d: [0.0, 0.0, 0.0]
+    points3d: [0.0, 0.0, 0.0]
+    pose_prior: [1, 1, 1]
+    shape_prior: [0.05, 0.05, 0.05]
+    motion_prior: [0.0, 0.0, 0.075]
+    init_motion_prior: [0.0, 0.0, 0.075]
+    joint_consistency: [0.0, 0.0, 100.0]
+    bone_length: [0.0, 2000.0, 2000.0]