liyang/hand-motion-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
liyang
2026-09-15 14:13:49 +08:00
commit 7e4ef6f98b
7002 changed files with 1452249 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/Dyn-HaMR/dyn-hamr/body_model/__init__.py
Vendored
+4
View File
@@ -0,0 +1,4 @@
from .body_model import *
from .specs import *
from .utils import *
from .mano_wrapper import *
third_party/Dyn-HaMR/dyn-hamr/body_model/body_model.py
Vendored
+2415
View File
File diff suppressed because it is too large Load Diff
third_party/Dyn-HaMR/dyn-hamr/body_model/mano_wrapper.py
Vendored
+44
View File
@@ -0,0 +1,44 @@
import torch
import numpy as np
import pickle
from typing import Optional
from body_model import MANOLayer
from smplx.lbs import vertices2joints
from smplx.utils import MANOOutput, to_tensor
from smplx.vertex_ids import vertex_ids
class MANO(MANOLayer):
def __init__(self, *args, joint_regressor_extra: Optional[str] = None, **kwargs):
"""
Extension of the official MANO implementation to support more joints.
Args:
Same as MANOLayer.
joint_regressor_extra (str): Path to extra joint regressor.
"""
super(MANO, self).__init__(*args, **kwargs)
mano_to_openpose = [0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8, 9, 20]
#2, 3, 5, 4, 1
if joint_regressor_extra is not None:
self.register_buffer('joint_regressor_extra', torch.tensor(pickle.load(open(joint_regressor_extra, 'rb'), encoding='latin1'), dtype=torch.float32))
self.register_buffer('extra_joints_idxs', to_tensor(list(vertex_ids['mano'].values()), dtype=torch.long))
self.register_buffer('joint_map', torch.tensor(mano_to_openpose, dtype=torch.long))
self.pose2rot = kwargs.get('pose2rot')
# print([attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("__")])
# print(self.hand_mean, self.flat_hand_mean, self.num_pca_comps, self.use_pca)
# exit()
def forward(self, *args, **kwargs) -> MANOOutput:
"""
Run forward pass. Same as MANO and also append an extra set of joints if joint_regressor_extra is specified.
"""
mano_output = super(MANO, self).forward(*args, **kwargs)
extra_joints = torch.index_select(mano_output.vertices, 1, self.extra_joints_idxs)
joints = torch.cat([mano_output.joints, extra_joints], dim=1)
joints = joints[:, self.joint_map, :]
if hasattr(self, 'joint_regressor_extra'):
extra_joints = vertices2joints(self.joint_regressor_extra, mano_output.vertices)
joints = torch.cat([joints, extra_joints], dim=1)
mano_output.joints = joints
return mano_output
third_party/Dyn-HaMR/dyn-hamr/body_model/specs.py
Vendored
+575
View File
@@ -0,0 +1,575 @@
import numpy as np
# SMPL_JOINTS = {
# "hips": 0,
# "leftUpLeg": 1,
# "rightUpLeg": 2,
# "spine": 3,
# "leftLeg": 4,
# "rightLeg": 5,
# "spine1": 6,
# "leftFoot": 7,
# "rightFoot": 8,
# "spine2": 9,
# "leftToeBase": 10,
# "rightToeBase": 11,
# "neck": 12,
# "leftShoulder": 13,
# "rightShoulder": 14,
# "head": 15,
# "leftArm": 16,
# "rightArm": 17,
# "leftForeArm": 18,
# "rightForeArm": 19,
# "leftHand": 20,
# "rightHand": 21,
# }
MANO_JOINTS = {
'wrist': 0,
'index1': 1,
'index2': 2,
'index3': 3,
'middle1': 4,
'middle2': 5,
'middle3': 6,
'pinky1': 7,
'pinky2': 8,
'pinky3': 9,
'ring1': 10,
'ring2': 11,
'ring3': 12,
'thumb1': 13,
'thumb2': 14,
'thumb3': 15,
}
OP_IGNORE_JOINTS = None
OP_NUM_JOINTS = 21
# SMPL_PARENTS = [
# -1,
# 0,
# 0,
# 0,
# 1,
# 2,
# 3,
# 4,
# 5,
# 6,
# 7,
# 8,
# 9,
# 12,
# 12,
# 12,
# 13,
# 14,
# 16,
# 17,
# 18,
# 19,
# ]
SMPLH_PATH = "./body_models/smplh"
SMPLX_PATH = "./body_models/smplx"
SMPL_PATH = "./body_models/smpl"
VPOSER_PATH = "./body_models/vposer_v1_0"
# chosen virtual mocap markers that are "keypoints" to work with
# KEYPT_VERTS = [
# 4404,
# 920,
# 3076,
# 3169,
# 823,
# 4310,
# 1010,
# 1085,
# 4495,
# 4569,
# 6615,
# 3217,
# 3313,
# 6713,
# 6785,
# 3383,
# 6607,
# 3207,
# 1241,
# 1508,
# 4797,
# 4122,
# 1618,
# 1569,
# 5135,
# 5040,
# 5691,
# 5636,
# 5404,
# 2230,
# 2173,
# 2108,
# 134,
# 3645,
# 6543,
# 3123,
# 3024,
# 4194,
# 1306,
# 182,
# 3694,
# 4294,
# 744,
# ]
# """
# Openpose
# """
# OP_NUM_JOINTS = 25
# # OP_IGNORE_JOINTS = [1, 9, 12] # neck and left/right hip
# OP_IGNORE_JOINTS = [1] # neck
# OP_EDGE_LIST = [
# [1, 8],
# [1, 2],
# [1, 5],
# [2, 3],
# [3, 4],
# [5, 6],
# [6, 7],
# [8, 9],
# [9, 10],
# [10, 11],
# [8, 12],
# [12, 13],
# [13, 14],
# [1, 0],
# [0, 15],
# [15, 17],
# [0, 16],
# [16, 18],
# [14, 19],
# [19, 20],
# [14, 21],
# [11, 22],
# [22, 23],
# [11, 24],
# ]
# # indices to map an openpose detection to its flipped version
# OP_FLIP_MAP = [
# 0,
# 1,
# 5,
# 6,
# 7,
# 2,
# 3,
# 4,
# 8,
# 12,
# 13,
# 14,
# 9,
# 10,
# 11,
# 16,
# 15,
# 18,
# 17,
# 22,
# 23,
# 24,
# 19,
# 20,
# 21,
# ]
#
# From https://github.com/vchoutas/smplify-x/blob/master/smplifyx/utils.py
# Please see license for usage restrictions.
#
# def smpl_to_openpose(
# model_type="smplx",
# use_hands=True,
# use_face=True,
# use_face_contour=False,
# openpose_format="coco25",
# ):
# """Returns the indices of the permutation that maps SMPL to OpenPose
# Parameters
# ----------
# model_type: str, optional
# The type of SMPL-like model that is used. The default mapping
# returned is for the SMPLX model
# use_hands: bool, optional
# Flag for adding to the returned permutation the mapping for the
# hand keypoints. Defaults to True
# use_face: bool, optional
# Flag for adding to the returned permutation the mapping for the
# face keypoints. Defaults to True
# use_face_contour: bool, optional
# Flag for appending the facial contour keypoints. Defaults to False
# openpose_format: bool, optional
# The output format of OpenPose. For now only COCO-25 and COCO-19 is
# supported. Defaults to 'coco25'
# """
# if openpose_format.lower() == "coco25":
# if model_type == "smpl":
# return np.array(
# [
# 24,
# 12,
# 17,
# 19,
# 21,
# 16,
# 18,
# 20,
# 0,
# 2,
# 5,
# 8,
# 1,
# 4,
# 7,
# 25,
# 26,
# 27,
# 28,
# 29,
# 30,
# 31,
# 32,
# 33,
# 34,
# ],
# dtype=np.int32,
# )
# elif model_type == "smplh":
# body_mapping = np.array(
# [
# 52,
# 12,
# 17,
# 19,
# 21,
# 16,
# 18,
# 20,
# 0,
# 2,
# 5,
# 8,
# 1,
# 4,
# 7,
# 53,
# 54,
# 55,
# 56,
# 57,
# 58,
# 59,
# 60,
# 61,
# 62,
# ],
# dtype=np.int32,
# )
# mapping = [body_mapping]
# if use_hands:
# lhand_mapping = np.array(
# [
# 20,
# 34,
# 35,
# 36,
# 63,
# 22,
# 23,
# 24,
# 64,
# 25,
# 26,
# 27,
# 65,
# 31,
# 32,
# 33,
# 66,
# 28,
# 29,
# 30,
# 67,
# ],
# dtype=np.int32,
# )
# rhand_mapping = np.array(
# [
# 21,
# 49,
# 50,
# 51,
# 68,
# 37,
# 38,
# 39,
# 69,
# 40,
# 41,
# 42,
# 70,
# 46,
# 47,
# 48,
# 71,
# 43,
# 44,
# 45,
# 72,
# ],
# dtype=np.int32,
# )
# mapping += [lhand_mapping, rhand_mapping]
# return np.concatenate(mapping)
# # SMPLX
# elif model_type == "smplx":
# body_mapping = np.array(
# [
# 55,
# 12,
# 17,
# 19,
# 21,
# 16,
# 18,
# 20,
# 0,
# 2,
# 5,
# 8,
# 1,
# 4,
# 7,
# 56,
# 57,
# 58,
# 59,
# 60,
# 61,
# 62,
# 63,
# 64,
# 65,
# ],
# dtype=np.int32,
# )
# mapping = [body_mapping]
# if use_hands:
# lhand_mapping = np.array(
# [
# 20,
# 37,
# 38,
# 39,
# 66,
# 25,
# 26,
# 27,
# 67,
# 28,
# 29,
# 30,
# 68,
# 34,
# 35,
# 36,
# 69,
# 31,
# 32,
# 33,
# 70,
# ],
# dtype=np.int32,
# )
# rhand_mapping = np.array(
# [
# 21,
# 52,
# 53,
# 54,
# 71,
# 40,
# 41,
# 42,
# 72,
# 43,
# 44,
# 45,
# 73,
# 49,
# 50,
# 51,
# 74,
# 46,
# 47,
# 48,
# 75,
# ],
# dtype=np.int32,
# )
# mapping += [lhand_mapping, rhand_mapping]
# if use_face:
# # end_idx = 127 + 17 * use_face_contour
# face_mapping = np.arange(
# 76, 127 + 17 * use_face_contour, dtype=np.int32
# )
# mapping += [face_mapping]
# return np.concatenate(mapping)
# else:
# raise ValueError("Unknown model type: {}".format(model_type))
# elif openpose_format == "coco19":
# if model_type == "smpl":
# return np.array(
# [24, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4, 7, 25, 26, 27, 28],
# dtype=np.int32,
# )
# elif model_type == "smplh":
# body_mapping = np.array(
# [52, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4, 7, 53, 54, 55, 56],
# dtype=np.int32,
# )
# mapping = [body_mapping]
# if use_hands:
# lhand_mapping = np.array(
# [
# 20,
# 34,
# 35,
# 36,
# 57,
# 22,
# 23,
# 24,
# 58,
# 25,
# 26,
# 27,
# 59,
# 31,
# 32,
# 33,
# 60,
# 28,
# 29,
# 30,
# 61,
# ],
# dtype=np.int32,
# )
# rhand_mapping = np.array(
# [
# 21,
# 49,
# 50,
# 51,
# 62,
# 37,
# 38,
# 39,
# 63,
# 40,
# 41,
# 42,
# 64,
# 46,
# 47,
# 48,
# 65,
# 43,
# 44,
# 45,
# 66,
# ],
# dtype=np.int32,
# )
# mapping += [lhand_mapping, rhand_mapping]
# return np.concatenate(mapping)
# # SMPLX
# elif model_type == "smplx":
# body_mapping = np.array(
# [55, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4, 7, 56, 57, 58, 59],
# dtype=np.int32,
# )
# mapping = [body_mapping]
# if use_hands:
# lhand_mapping = np.array(
# [
# 20,
# 37,
# 38,
# 39,
# 60,
# 25,
# 26,
# 27,
# 61,
# 28,
# 29,
# 30,
# 62,
# 34,
# 35,
# 36,
# 63,
# 31,
# 32,
# 33,
# 64,
# ],
# dtype=np.int32,
# )
# rhand_mapping = np.array(
# [
# 21,
# 52,
# 53,
# 54,
# 65,
# 40,
# 41,
# 42,
# 66,
# 43,
# 44,
# 45,
# 67,
# 49,
# 50,
# 51,
# 68,
# 46,
# 47,
# 48,
# 69,
# ],
# dtype=np.int32,
# )
# mapping += [lhand_mapping, rhand_mapping]
# if use_face:
# face_mapping = np.arange(
# 70, 70 + 51 + 17 * use_face_contour, dtype=np.int32
# )
# mapping += [face_mapping]
# return np.concatenate(mapping)
# else:
# raise ValueError("Unknown model type: {}".format(model_type))
# else:
# raise ValueError("Unknown joint format: {}".format(openpose_format))
third_party/Dyn-HaMR/dyn-hamr/body_model/utils.py
Vendored
+76
View File
@@ -0,0 +1,76 @@
import torch
from .specs import MANO_JOINTS
def run_mano(body_model, trans, root_orient, body_pose, is_right, betas=None):
"""
Forward pass of the MANO model and populates pred_data accordingly with
joints3d, verts3d, points3d.
trans : B x T x 3
root_orient : B x T x 3
body_pose : B x T x J*3
betas : (optional) B x D
"""
B, T, _ = trans.shape
bm_batch_size = body_model.batch_size
assert bm_batch_size % B == 0
# assert (is_right[0]==is_right[0][0]).all(), f'{is_right}'
# assert (is_right[1]==is_right[1][0]).all(), f'{is_right}'
seq_len = bm_batch_size // B
bm_num_betas = body_model.num_betas
J_BODY = len(MANO_JOINTS) - 1 # all joints except root
# print('utils.py, seq_len, T: ', seq_len, T, bm_batch_size, B) # 1, 120, 1, 1
# print('trans, root_orient, body_pose:', trans.shape, root_orient.shape, body_pose.shape)
if T == 1:
raise ValueError
# must expand to use with body model
trans = trans.expand(B, seq_len, 3)
root_orient = root_orient.expand(B, seq_len, 3)
body_pose = body_pose.expand(B, seq_len, J_BODY * 3)
elif T != seq_len:
trans, root_orient, body_pose = zero_pad_tensors(
[trans, root_orient, body_pose], seq_len - T
)
if betas is None:
betas = torch.zeros(B, bm_num_betas, device=trans.device)
betas = betas.reshape((B, 1, bm_num_betas)).expand((B, seq_len, bm_num_betas))
# print('body_pose: ', body_pose.reshape((B * seq_len, -1)).shape)
mano_output = body_model(
hand_pose=body_pose.reshape((B * seq_len, -1)),
betas=betas.reshape((B * seq_len, -1)),
global_orient=root_orient.reshape((B * seq_len, -1)),
transl=trans.reshape((B * seq_len, -1)),
)
joints = mano_output.joints
verts = mano_output.vertices
joints = joints.reshape(B, seq_len, -1, 3)[:, :T]
verts = verts.reshape(B, seq_len, -1, 3)[:, :T]
is_right = is_right.unsqueeze(-1)
joints[:, :, :, 0] = (2*is_right-1)*joints[:, :, :, 0]
verts[:, :, :, 0] = (2*is_right-1)*verts[:, :, :, 0]
return {
"joints": joints,
"vertices": verts,
"l_faces": body_model.faces_tensor[:,[0,2,1]],
"r_faces": body_model.faces_tensor,
"is_right": is_right.squeeze(-1),
'body_pose': body_pose.clone()
}
def zero_pad_tensors(pad_list, pad_size):
"""
Assumes tensors in pad_list are B x T x D and pad temporal dimension
"""
B = pad_list[0].size(0)
new_pad_list = []
for pad_idx, pad_tensor in enumerate(pad_list):
padding = torch.zeros((B, pad_size, pad_tensor.size(2))).to(pad_tensor)
new_pad_list.append(torch.cat([pad_tensor, padding], dim=1))
return new_pad_list