hand-motion-pipeline/third_party/Dyn-HaMR/dyn-hamr/data/tools.py

import os
import json
import functools

import numpy as np

from body_model import OP_NUM_JOINTS
from scipy.interpolate import interp1d
from scipy.spatial.transform import Rotation, Slerp


def read_keypoints(keypoint_fn):
    """
    Only reads body keypoint data of first person.
    """
    empty_kps = np.zeros((OP_NUM_JOINTS, 3), dtype=np.float32)
    if not os.path.isfile(keypoint_fn):
        return empty_kps

    with open(keypoint_fn) as keypoint_file:
        data = json.load(keypoint_file)

    if len(data["people"]) == 0:
        print("WARNING: Found no keypoints in %s! Returning zeros!" % (keypoint_fn))
        return empty_kps

    person_data = data["people"][0]
    body_keypoints = np.array(person_data["pose_keypoints_2d"], dtype=np.float32)
    body_keypoints = body_keypoints.reshape([-1, 3])
    return body_keypoints


def load_keypoints_with_interp(kp_paths, interp=True):
    """
    Load 2D keypoints from paths and interpolate missing frames between tmin and tmax.
    
    Args:
        kp_paths: List of paths to keypoint JSON files
        interp: Whether to interpolate missing frames
    
    Returns:
        joints2d_data: (T, J, 3) array of keypoints with interpolation
    """
    # Load all keypoints
    joints2d_data = np.stack([read_keypoints(p) for p in kp_paths], axis=0).astype(np.float32)
    
    if not interp:
        return joints2d_data
    
    # Find which frames have valid keypoints (file exists and has data)
    vis_mask = np.array([os.path.isfile(p) for p in kp_paths])
    # Also check if keypoints are not all zeros
    has_data = ~np.all(joints2d_data == 0, axis=(1, 2))
    vis_mask = vis_mask & has_data
    vis_idcs = np.where(vis_mask)[0]
    
    if len(vis_idcs) == 0 or len(vis_idcs) == 1:
        # No interpolation needed
        return joints2d_data
    
    # Interpolate only between tmin and tmax (same as MANO interpolation)
    T, J, _ = joints2d_data.shape
    tmin, tmax = min(vis_idcs), max(vis_idcs) + 1
    times = np.arange(tmin, tmax)
    
    # Interpolate each joint's x, y coordinates separately
    for j in range(J):
        # Interpolate x coordinate
        x_interp = interp1d(vis_idcs, joints2d_data[vis_idcs, j, 0], 
                           kind='linear', bounds_error=False)
        # Interpolate y coordinate
        y_interp = interp1d(vis_idcs, joints2d_data[vis_idcs, j, 1], 
                           kind='linear', bounds_error=False)
        
        # Interpolate for frames between tmin and tmax
        for t in times:
            if t not in vis_idcs:
                # Interpolate
                joints2d_data[t, j, 0] = x_interp(t)
                joints2d_data[t, j, 1] = y_interp(t)
                
                # Set confidence as average of neighboring visible frames
                prev_vis = vis_idcs[vis_idcs < t]
                next_vis = vis_idcs[vis_idcs > t]
                if len(prev_vis) > 0 and len(next_vis) > 0:
                    conf_prev = joints2d_data[prev_vis[-1], j, 2]
                    conf_next = joints2d_data[next_vis[0], j, 2]
                    joints2d_data[t, j, 2] = (conf_prev + conf_next) / 2.0 * 0.8  # Slightly lower confidence for interpolated
                elif len(prev_vis) > 0:
                    joints2d_data[t, j, 2] = joints2d_data[prev_vis[-1], j, 2] * 0.8
                elif len(next_vis) > 0:
                    joints2d_data[t, j, 2] = joints2d_data[next_vis[0], j, 2] * 0.8
    
    return joints2d_data


def read_mask_path(path):
    mask_path = None
    if not os.path.isfile(path):
        return mask_path

    with open(path, "r") as f:
        data = json.load(path)

    person_data = data["people"][0]
    if "mask_path" in person_data:
        mask_path = person_data["mask_path"]

    return mask_path


def read_mano_preds(pred_path, tid, num_betas=10):
    """
    reads the betas, body_pose, global orientation and translation of a mano prediction
    exported from phalp outputs
    returns betas (10,), body_pose (23, 3), global_orientation (3,), translation (3,)
    """
    pose = np.zeros((15, 3))
    rot = np.zeros(3)
    trans = np.zeros(3)
    betas = np.zeros(num_betas)
    if not os.path.isfile(pred_path):
        return pose, rot, trans, betas, int(tid)

    with open(pred_path, "r") as f:
        data = json.load(f)

    pose = np.array(data["body_pose"], dtype=np.float32)
    rot = np.array(data["global_orient"], dtype=np.float32)
    trans = np.array(data["cam_trans"], dtype=np.float32)
    betas = np.array(data["betas"], dtype=np.float32)
    is_right = np.array(data["is_right"], dtype=np.float32)

    return pose, rot, trans, betas, is_right


def load_mano_preds(pred_paths, tid, interp=True, num_betas=10):
    vis_mask = np.array([os.path.isfile(x) for x in pred_paths])
    vis_idcs = np.where(vis_mask)[0]

    # load single image mano predictions
    stack_fnc = functools.partial(np.stack, axis=0)
    # (N, 16, 3), (N, 3), (N, 3), (N, 10)
    pose, orient, trans, betas, is_right = map(
        stack_fnc, zip(*[read_mano_preds(p, tid, num_betas=num_betas) for p in pred_paths])
    )

    assert len(np.where(is_right!=int(tid))[0]) == 0
    if not interp:
        return pose, orient, trans, betas, is_right

    # interpolate the occluded tracks
    orient_slerp = Slerp(vis_idcs, Rotation.from_rotvec(orient[vis_idcs]))
    trans_interp = interp1d(vis_idcs, trans[vis_idcs], axis=0)
    betas_interp = interp1d(vis_idcs, betas[vis_idcs], axis=0)

    tmin, tmax = min(vis_idcs), max(vis_idcs) + 1
    times = np.arange(tmin, tmax)
    orient[times] = orient_slerp(times).as_rotvec()
    trans[times] = trans_interp(times)
    betas[times] = betas_interp(times)

    # interpolate for each joint angle
    for i in range(pose.shape[1]):
        pose_slerp = Slerp(vis_idcs, Rotation.from_rotvec(pose[vis_idcs, i]))
        pose[times, i] = pose_slerp(times).as_rotvec()

    return pose, orient, trans, betas, is_right