liyang
ae28d55f81
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
121 lines
4.4 KiB
Python
121 lines
4.4 KiB
Python
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
|
# and proprietary rights in and to this software, related documentation
|
|
# and any modifications thereto. Any use, reproduction, disclosure or
|
|
# distribution of this software and related documentation without an express
|
|
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Projection and transformation matrix helpers.
|
|
#----------------------------------------------------------------------------
|
|
|
|
def projection(x=0.1, n=1.0, f=50.0):
|
|
return np.array([[n/x, 0, 0, 0],
|
|
[ 0, n/x, 0, 0],
|
|
[ 0, 0, -(f+n)/(f-n), -(2*f*n)/(f-n)],
|
|
[ 0, 0, -1, 0]]).astype(np.float32)
|
|
|
|
def translate(x, y, z):
|
|
return np.array([[1, 0, 0, x],
|
|
[0, 1, 0, y],
|
|
[0, 0, 1, z],
|
|
[0, 0, 0, 1]]).astype(np.float32)
|
|
|
|
def rotate_x(a):
|
|
s, c = np.sin(a), np.cos(a)
|
|
return np.array([[1, 0, 0, 0],
|
|
[0, c, -s, 0],
|
|
[0, s, c, 0],
|
|
[0, 0, 0, 1]]).astype(np.float32)
|
|
|
|
def rotate_y(a):
|
|
s, c = np.sin(a), np.cos(a)
|
|
return np.array([[ c, 0, s, 0],
|
|
[ 0, 1, 0, 0],
|
|
[-s, 0, c, 0],
|
|
[ 0, 0, 0, 1]]).astype(np.float32)
|
|
|
|
def random_rotation_translation(t):
|
|
m = np.random.normal(size=[3, 3])
|
|
m[1] = np.cross(m[0], m[2])
|
|
m[2] = np.cross(m[0], m[1])
|
|
m = m / np.linalg.norm(m, axis=1, keepdims=True)
|
|
m = np.pad(m, [[0, 1], [0, 1]], mode='constant')
|
|
m[3, 3] = 1.0
|
|
m[:3, 3] = np.random.uniform(-t, t, size=[3])
|
|
return m
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Bilinear downsample by 2x.
|
|
#----------------------------------------------------------------------------
|
|
|
|
def bilinear_downsample(x):
|
|
w = torch.tensor([[1, 3, 3, 1], [3, 9, 9, 3], [3, 9, 9, 3], [1, 3, 3, 1]], dtype=torch.float32, device=x.device) / 64.0
|
|
w = w.expand(x.shape[-1], 1, 4, 4)
|
|
x = torch.nn.functional.conv2d(x.permute(0, 3, 1, 2), w, padding=1, stride=2, groups=x.shape[-1])
|
|
return x.permute(0, 2, 3, 1)
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Image display function using OpenGL.
|
|
#----------------------------------------------------------------------------
|
|
|
|
_glfw_window = None
|
|
def display_image(image, zoom=None, size=None, title=None): # HWC
|
|
# Import OpenGL and glfw.
|
|
import OpenGL.GL as gl
|
|
import glfw
|
|
|
|
# Zoom image if requested.
|
|
image = np.asarray(image)
|
|
if size is not None:
|
|
assert zoom is None
|
|
zoom = max(1, size // image.shape[0])
|
|
if zoom is not None:
|
|
image = image.repeat(zoom, axis=0).repeat(zoom, axis=1)
|
|
height, width, channels = image.shape
|
|
|
|
# Initialize window.
|
|
if title is None:
|
|
title = 'Debug window'
|
|
global _glfw_window
|
|
if _glfw_window is None:
|
|
glfw.init()
|
|
_glfw_window = glfw.create_window(width, height, title, None, None)
|
|
glfw.make_context_current(_glfw_window)
|
|
glfw.show_window(_glfw_window)
|
|
glfw.swap_interval(0)
|
|
else:
|
|
glfw.make_context_current(_glfw_window)
|
|
glfw.set_window_title(_glfw_window, title)
|
|
glfw.set_window_size(_glfw_window, width, height)
|
|
|
|
# Update window.
|
|
glfw.poll_events()
|
|
gl.glClearColor(0, 0, 0, 1)
|
|
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
|
gl.glWindowPos2f(0, 0)
|
|
gl.glPixelStorei(gl.GL_UNPACK_ALIGNMENT, 1)
|
|
gl_format = {3: gl.GL_RGB, 2: gl.GL_RG, 1: gl.GL_LUMINANCE}[channels]
|
|
gl_dtype = {'uint8': gl.GL_UNSIGNED_BYTE, 'float32': gl.GL_FLOAT}[image.dtype.name]
|
|
gl.glDrawPixels(width, height, gl_format, gl_dtype, image[::-1])
|
|
glfw.swap_buffers(_glfw_window)
|
|
if glfw.window_should_close(_glfw_window):
|
|
return False
|
|
return True
|
|
|
|
#----------------------------------------------------------------------------
|
|
# Image save helper.
|
|
#----------------------------------------------------------------------------
|
|
|
|
def save_image(fn, x):
|
|
import imageio
|
|
x = np.rint(x * 255.0)
|
|
x = np.clip(x, 0, 255).astype(np.uint8)
|
|
imageio.imsave(fn, x)
|
|
|
|
#----------------------------------------------------------------------------
|