import os import cv2 import numpy as np import plyfile import torch import torch.nn.functional as F from human_body_prior.tools.omni_tools import copy2cpu as c2c from scipy.spatial.transform import Rotation as R from scipy.spatial.transform import Slerp import holden.BVH as BVH from holden.Animation import Animation from holden.Quaternions import Quaternions from typing import List, Tuple, NewType import math Tensor = NewType('Tensor', torch.Tensor) SMPL_JOINTS = { 'Pelvis': 0, 'L_Hip': 1, 'L_Knee': 4, 'L_Ankle': 7, 'L_Foot': 10, 'R_Hip': 2, 'R_Knee': 5, 'R_Ankle': 8, 'R_Foot': 11, 'Spine1': 3, 'Spine2': 6, 'Spine3': 9, 'Neck': 12, 'Head': 15, 'L_Collar': 13, 'L_Shoulder': 16, 'L_Elbow': 18, 'L_Wrist': 20, 'L_Hand': 22, 'R_Collar': 14, 'R_Shoulder': 17, 'R_Elbow': 19, 'R_Wrist': 21, 'R_Hand': 23 } MANO_RIGHT_JOINTS = {'R_Wrist':0, 'rindex0':1, 'rindex1':2, 'rindex2':3, 'rmiddle0':4, 'rmiddle1':5, 'rmiddle2':6, 'rpinky0':7, 'rpinky1':8, 'rpinky2':9, 'rring0':10, 'rring1':11, 'rring2':12, 'rthumb0':13, 'rthumb1':14, 'rthumb2':15} FOOT_IDX = [SMPL_JOINTS['L_Ankle'], SMPL_JOINTS['R_Ankle'], SMPL_JOINTS['L_Foot'], SMPL_JOINTS['R_Foot']] CONTACTS_IDX = [SMPL_JOINTS['L_Ankle'], SMPL_JOINTS['R_Ankle'], SMPL_JOINTS['L_Foot'], SMPL_JOINTS['R_Foot'], SMPL_JOINTS['L_Wrist'], SMPL_JOINTS['R_Wrist'], SMPL_JOINTS['L_Knee'], SMPL_JOINTS['R_Knee']] def align_joints(tensor, smpl_to_bvh=True): """ Args: tensor (T x J x D): The 3D torch tensor we need to process. Returns: The 3D tensor whose joint order is compatible for bvh export. """ order = list(MANO_RIGHT_JOINTS.values()) result = tensor.clone() if isinstance(tensor, torch.Tensor) else tensor.copy() for i in range(len(order)): if smpl_to_bvh: result[:, i] = tensor[:, order[i]] else: result[:, order[i]] = tensor[:, i] return result def ho3d_cam_extrinsics(cam_id): # GPMF case extrinsic_0 = np.array([[-6.409640608401522277e-01, -2.650417602434713693e-01, 7.203595893984829912e-01, -5.675702013852904626e-01], [-7.601847604882365772e-01, 3.490763992765258950e-01, -5.479642300274587541e-01, 3.191066048126701693e-01], [-1.062271275746045074e-01, -8.988317600058360890e-01, -4.252261334538034454e-01, 3.074042501856224519e-01], [0, 0, 0, 1]]) extrinsic_1 = np.array([[-4.228051957326409149e-01, -2.078562115810418665e-01, -8.820609739517225600e-01, 4.637252480950287414e-01], [9.057441941962964815e-01, -6.536944939351106709e-02, -4.187532564239835331e-01, 1.535094236870509776e-01], [2.938062526878559150e-02, -9.759726586299259932e-01, 2.159030997129234852e-01, -1.448621926159473529e-02], [0, 0, 0, 1]]) extrinsic_2 = np.array([[9.846039213996782280e-01, 6.828612271512826681e-02, -1.609102961556882483e-01, 9.897043274267488394e-03], [1.747949086333730750e-01, -3.774480817251544273e-01, 9.093842342584639304e-01, -4.349111883283850455e-01], [1.363040801985337556e-03, -9.235095836246293155e-01, -3.835729279722913110e-01, 2.864983800867586528e-01], [0, 0, 0, 1]]) extrinsic_3 = np.array([[4.290309538248047727e-01, 8.850724776239779490e-02, 8.989432172021233347e-01, -4.754434780295317409e-01], [-9.018737295370956586e-01, 9.767471789067121157e-02, 4.208128152229064223e-01, -2.166588733650218201e-01], [-5.055904104142360661e-02, -9.912749954369395322e-01, 1.217278390117415843e-01, 6.127903398258782025e-02], [0, 0, 0, 1]]) extrinsic_4 = np.array([[-9.650817552456496529e-01, 5.435035576465601509e-02, 2.562484039369455346e-01, -9.846218864128720993e-02], [-2.604576318061128104e-01, -3.033325236743535935e-01, -9.165976227960112022e-01, 3.948193893157375123e-01], [2.791106816171780650e-02, -9.513334951054777111e-01, 3.068966493210200097e-01, -1.383363501223186694e-02], [0, 0, 0, 1]]) R, t = eval(f"extrinsic_{cam_id}")[:3, :3], eval(f"extrinsic_{cam_id}")[:3, 3] return R, t def slerp(quat, trans, key_times, times, mask=True): """ Args: quat: (T x J x 4) trans: (T x 3) """ if mask: quat = c2c(quat[key_times]) trans = c2c(trans[key_times]) else: quat = c2c(quat) trans = c2c(trans) quats = [] for j in range(quat.shape[1]): key_rots = R.from_quat(quat[:, j]) s = Slerp(key_times, key_rots) interp_rots = s(times) quats.append(interp_rots.as_quat()) slerp_quat = np.stack(quats, axis=1) lerp_trans = np.zeros((len(times), 3)) for i in range(3): lerp_trans[:, i] = np.interp(times, key_times, trans[:, i]) return slerp_quat, lerp_trans def compute_orient_angle(matrix, traj): """ Args: matrix (N x T x 3 x 3): The 3D rotation matrix at the root joint traj (N x T x 3): The trajectory to align """ forward = matrix[:, :, :, 2].clone() forward[:, :, 2] = 0 forward = F.normalize(forward, dim=-1) # normalized forward vector (N, T, 3) traj[:, :, 2] = 0 # make sure the trajectory is projected to the plane # first steps is forward diff init_tan = traj[:, 1:2] - traj[:, :1] # middle steps are second order middle_tan = (traj[:, 2:] - traj[:, 0:-2]) / 2 # last step is backward diff final_tan = traj[:, -1:] - traj[:, -2:-1] tangent = torch.cat([init_tan, middle_tan, final_tan], dim=1) tangent = F.normalize(tangent, dim=-1) # normalized tangent vector (N, T, 3) cos = torch.sum(forward * tangent, dim=-1) return cos def compute_trajectory(velocity, up, origin, dt, up_axis='z'): """ Args: velocity: (B, T, 3) up: (B, T) origin: (B, 3) up_axis: x, y, or z Returns: trajectory: (B, T, 3) """ ordermap = { 'x': 0, 'y': 1, 'z': 2, } v_axis = [x for x in ordermap.values() if x != ordermap[up_axis]] origin = origin.unsqueeze(1) # (B, 3) => (B, 1, 3) trajectory = origin.repeat(1, up.shape[1], 1) # (B, 1, 3) => (B, T, 3) for t in range(1, up.shape[1]): trajectory[:, t, v_axis[0]] = trajectory[:, t - 1, v_axis[0]] + velocity[:, t - 1, v_axis[0]] * dt trajectory[:, t, v_axis[1]] = trajectory[:, t - 1, v_axis[1]] + velocity[:, t - 1, v_axis[1]] * dt trajectory[:, :, ordermap[up_axis]] = up return trajectory def build_canonical_frame(forward, up_axis='z'): """ Args: forward: (..., 3) Returns: frame: (..., 3, 3) """ forward[..., 'xyz'.index(up_axis)] = 0 forward = F.normalize(forward, dim=-1) # normalized forward vector up = torch.zeros_like(forward) up[..., 'xyz'.index(up_axis)] = 1 # normalized up vector right = torch.cross(up, forward) frame = torch.stack((right, up, forward), dim=-1) # canonical frame return frame def estimate_linear_velocity(data_seq, dt): ''' Given some batched data sequences of T timesteps in the shape (B, T, ...), estimates the velocity for the middle T-2 steps using a second order central difference scheme. The first and last frames are with forward and backward first-order differences, respectively - h : step size ''' # first steps is forward diff (t+1 - t) / dt init_vel = (data_seq[:, 1:2] - data_seq[:, :1]) / dt # middle steps are second order (t+1 - t-1) / 2dt middle_vel = (data_seq[:, 2:] - data_seq[:, 0:-2]) / (2 * dt) # last step is backward diff (t - t-1) / dt final_vel = (data_seq[:, -1:] - data_seq[:, -2:-1]) / dt vel_seq = torch.cat([init_vel, middle_vel, final_vel], dim=1) return vel_seq def estimate_angular_velocity(rot_seq, dt): ''' Given a batch of sequences of T rotation matrices, estimates angular velocity at T-2 steps. Input sequence should be of shape (B, T, ..., 3, 3) ''' # see https://en.wikipedia.org/wiki/Angular_velocity#Calculation_from_the_orientation_matrix dRdt = estimate_linear_velocity(rot_seq, dt) R = rot_seq RT = R.transpose(-1, -2) # compute skew-symmetric angular velocity tensor w_mat = torch.matmul(dRdt, RT) # pull out angular velocity vector by averaging symmetric entries w_x = (-w_mat[..., 1, 2] + w_mat[..., 2, 1]) / 2.0 w_y = (w_mat[..., 0, 2] - w_mat[..., 2, 0]) / 2.0 w_z = (-w_mat[..., 0, 1] + w_mat[..., 1, 0]) / 2.0 w = torch.stack([w_x, w_y, w_z], axis=-1) return w def normalize(tensor, mean=None, std=None): """ Args: tensor: (B, T, ...) Returns: normalized tensor with 0 mean and 1 standard deviation, std, mean """ if mean is None or std is None: # std, mean = torch.std_mean(tensor, dim=0, unbiased=False, keepdim=True) std, mean = torch.std_mean(tensor, dim=(0, 1), unbiased=False, keepdim=True) std[std == 0.0] = 1.0 return (tensor - mean) / std, mean, std return (tensor - mean) / std def denormalize(tensor, mean, std): """ Args: tensor: B x T x D mean: std: """ return tensor * std + mean def export_bvh_animation(rotations, positions, offsets, parents, output_dir, prefix, joint_names, fps): """ Args: rotations: quaternions of the shape (B, T, J, 4) positions: global translations of the shape (B, T, 3) """ os.makedirs(output_dir, exist_ok=True) for i in range(rotations.shape[0]): rotation = align_joints(rotations[i]) position = positions[i] position = position.unsqueeze(1) anim = Animation(Quaternions(c2c(rotation)), c2c(position), None, offsets=offsets, parents=parents) BVH.save(os.path.join(output_dir, f'{prefix}_{i}.bvh'), anim, names=joint_names, frametime=1 / fps) def export_ply_trajectory(points, color, ply_fname): v = [] for p in points: v += [(p[0], p[1], p[2], color[0], color[1], color[2])] v = np.array(v, dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]) el_v = plyfile.PlyElement.describe(v, 'vertex') e = np.empty(len(points) - 1, dtype=[('vertex1', 'i4'), ('vertex2', 'i4')]) edge_data = np.array([[i, i + 1] for i in range(len(points) - 1)], dtype='i4') e['vertex1'] = edge_data[:, 0] e['vertex2'] = edge_data[:, 1] el_e = plyfile.PlyElement.describe(e, 'edge') plyfile.PlyData([el_v, el_e], text=True).write(ply_fname) def get_keypoints_rectangle(keypoints: np.array, threshold: float) -> Tuple[float, float, float]: """ Compute rectangle enclosing keypoints above the threshold. Args: keypoints (np.array): Keypoint array of shape (N, 3). threshold (float): Confidence visualization threshold. Returns: Tuple[float, float, float]: Rectangle width, height and area. """ # print(keypoints.shape) valid_ind = keypoints[:, -1] > threshold if valid_ind.sum() > 0: valid_keypoints = keypoints[valid_ind][:, :-1] max_x = valid_keypoints[:,0].max() max_y = valid_keypoints[:,1].max() min_x = valid_keypoints[:,0].min() min_y = valid_keypoints[:,1].min() width = max_x - min_x height = max_y - min_y area = width * height return width, height, area else: return 0,0,0 def render_keypoints(img: np.array, keypoints: np.array, pairs: List, colors: List, thickness_circle_ratio: float, thickness_line_ratio_wrt_circle: float, pose_scales: List, threshold: float = 0.1, alpha: float = 1.0) -> np.array: """ Render keypoints on input image. Args: img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. keypoints (np.array): Keypoint array of shape (N, 3). pairs (List): List of keypoint pairs per limb. colors: (List): List of colors per keypoint. thickness_circle_ratio (float): Circle thickness ratio. thickness_line_ratio_wrt_circle (float): Line thickness ratio wrt the circle. pose_scales (List): List of pose scales. threshold (float): Only visualize keypoints with confidence above the threshold. Returns: (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. """ img_orig = img.copy() width, height = img.shape[1], img.shape[2] area = width * height lineType = 8 shift = 0 numberColors = len(colors) thresholdRectangle = 0.1 person_width, person_height, person_area = get_keypoints_rectangle(keypoints, thresholdRectangle) if person_area > 0: ratioAreas = min(1, max(person_width / width, person_height / height)) thicknessRatio = np.maximum(np.round(math.sqrt(area) * thickness_circle_ratio * ratioAreas), 2) thicknessCircle = np.maximum(1, thicknessRatio if ratioAreas > 0.05 else -np.ones_like(thicknessRatio)) thicknessLine = np.maximum(1, np.round(thicknessRatio * thickness_line_ratio_wrt_circle)) radius = thicknessRatio / 2 img = np.ascontiguousarray(img.copy()) for i, pair in enumerate(pairs): index1, index2 = pair if keypoints[index1, -1] > threshold and keypoints[index2, -1] > threshold: thicknessLineScaled = int(round(min(thicknessLine[index1], thicknessLine[index2]) * pose_scales[0])) colorIndex = index2 color = colors[colorIndex % numberColors] keypoint1 = keypoints[index1, :-1].astype(np.int32) keypoint2 = keypoints[index2, :-1].astype(np.int32) cv2.line(img, tuple(keypoint1.tolist()), tuple(keypoint2.tolist()), tuple(color.tolist()), thicknessLineScaled, lineType, shift) for part in range(len(keypoints)): faceIndex = part if keypoints[faceIndex, -1] > threshold: radiusScaled = int(round(radius[faceIndex] * pose_scales[0])) thicknessCircleScaled = int(round(thicknessCircle[faceIndex] * pose_scales[0])) colorIndex = part color = colors[colorIndex % numberColors] center = keypoints[faceIndex, :-1].astype(np.int32) cv2.circle(img, tuple(center.tolist()), radiusScaled, tuple(color.tolist()), thicknessCircleScaled, lineType, shift) return img def render_hand_keypoints(img, right_hand_keypoints, threshold=0.1, use_confidence=False, map_fn=lambda x: np.ones_like(x), alpha=1.0): if use_confidence and map_fn is not None: #thicknessCircleRatioLeft = 1./50 * map_fn(left_hand_keypoints[:, -1]) thicknessCircleRatioRight = 1./50 * map_fn(right_hand_keypoints[:, -1]) else: #thicknessCircleRatioLeft = 1./50 * np.ones(left_hand_keypoints.shape[0]) thicknessCircleRatioRight = 1./50 * np.ones(right_hand_keypoints.shape[0]) thicknessLineRatioWRTCircle = 0.75 pairs = [0,1, 1,2, 2,3, 3,4, 0,5, 5,6, 6,7, 7,8, 0,9, 9,10, 10,11, 11,12, 0,13, 13,14, 14,15, 15,16, 0,17, 17,18, 18,19, 19,20] pairs = np.array(pairs).reshape(-1,2) colors = [100., 100., 100., 100., 0., 0., 150., 0., 0., 200., 0., 0., 255., 0., 0., 100., 100., 0., 150., 150., 0., 200., 200., 0., 255., 255., 0., 0., 100., 50., 0., 150., 75., 0., 200., 100., 0., 255., 125., 0., 50., 100., 0., 75., 150., 0., 100., 200., 0., 125., 255., 100., 0., 100., 150., 0., 150., 200., 0., 200., 255., 0., 255.] colors = np.array(colors).reshape(-1,3) #colors = np.zeros_like(colors) poseScales = [1] #img = render_keypoints(img, left_hand_keypoints, pairs, colors, thicknessCircleRatioLeft, thicknessLineRatioWRTCircle, poseScales, threshold, alpha=alpha) img = render_keypoints(img, right_hand_keypoints, pairs, colors, thicknessCircleRatioRight, thicknessLineRatioWRTCircle, poseScales, threshold, alpha=alpha) #img = render_keypoints(img, right_hand_keypoints, pairs, colors, thickness_circle_ratio, thickness_line_ratio_wrt_circle, pose_scales, 0.1) return img def render_body_keypoints(img: np.array, body_keypoints: np.array) -> np.array: """ Render OpenPose body keypoints on input image. Args: img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. body_keypoints (np.array): Keypoint array of shape (N, 3); 3 <====> (x, y, confidence). Returns: (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. """ thickness_circle_ratio = 1./75. * np.ones(body_keypoints.shape[0]) thickness_line_ratio_wrt_circle = 0.75 pairs = [] pairs = [1,8,1,2,1,5,2,3,3,4,5,6,6,7,8,9,9,10,10,11,8,12,12,13,13,14,1,0,0,15,15,17,0,16,16,18,14,19,19,20,14,21,11,22,22,23,11,24] pairs = np.array(pairs).reshape(-1,2) colors = [255., 0., 85., 255., 0., 0., 255., 85., 0., 255., 170., 0., 255., 255., 0., 170., 255., 0., 85., 255., 0., 0., 255., 0., 255., 0., 0., 0., 255., 85., 0., 255., 170., 0., 255., 255., 0., 170., 255., 0., 85., 255., 0., 0., 255., 255., 0., 170., 170., 0., 255., 255., 0., 255., 85., 0., 255., 0., 0., 255., 0., 0., 255., 0., 0., 255., 0., 255., 255., 0., 255., 255., 0., 255., 255.] colors = np.array(colors).reshape(-1,3) pose_scales = [1] return render_keypoints(img, body_keypoints, pairs, colors, thickness_circle_ratio, thickness_line_ratio_wrt_circle, pose_scales, 0.1) def render_openpose(img: np.array, hand_keypoints: np.array) -> np.array: """ Render keypoints in the OpenPose format on input image. Args: img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. body_keypoints (np.array): Keypoint array of shape (N, 3); 3 <====> (x, y, confidence). Returns: (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. """ #img = render_body_keypoints(img, body_keypoints) img = render_hand_keypoints(img, hand_keypoints) return img