liyang
ae28d55f81
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
80 lines
2.5 KiB
Python
80 lines
2.5 KiB
Python
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
|
# and proprietary rights in and to this software, related documentation
|
|
# and any modifications thereto. Any use, reproduction, disclosure or
|
|
# distribution of this software and related documentation without an express
|
|
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
|
|
|
|
|
import os,sys,time,socket
|
|
code_path = os.path.dirname(os.path.realpath(__file__))
|
|
sys.path.append(code_path)
|
|
import open3d as o3d
|
|
import numpy as np
|
|
from PIL import Image
|
|
import cv2,imageio
|
|
import time
|
|
import trimesh
|
|
import pyrender
|
|
from transformations import *
|
|
import numpy as np
|
|
from PIL import Image
|
|
import cv2
|
|
import time
|
|
import argparse,pickle
|
|
from Utils import *
|
|
|
|
|
|
cvcam_in_glcam = np.array([[1,0,0,0],
|
|
[0,-1,0,0],
|
|
[0,0,-1,0],
|
|
[0,0,0,1]])
|
|
|
|
class ModelRendererOffscreen:
|
|
def __init__(self, cam_K, H,W, zfar=100):
|
|
'''
|
|
@window_sizes: H,W
|
|
'''
|
|
self.K = cam_K
|
|
self.scene = pyrender.Scene(ambient_light=[1., 1., 1.],bg_color=[0,0,0])
|
|
self.camera = pyrender.IntrinsicsCamera(fx=cam_K[0,0],fy=cam_K[1,1],cx=cam_K[0,2],cy=cam_K[1,2],znear=0.001,zfar=zfar)
|
|
self.cam_node = self.scene.add(self.camera, pose=np.eye(4), name='cam')
|
|
self.mesh_nodes = []
|
|
|
|
self.H = H
|
|
self.W = W
|
|
self.r = pyrender.OffscreenRenderer(self.W, self.H)
|
|
|
|
|
|
def set_cam_pose(self, cam_pose):
|
|
self.cam_node.matrix = cam_pose
|
|
|
|
def add_mesh(self, mesh):
|
|
mesh = pyrender.Mesh.from_trimesh(mesh, smooth=False)
|
|
mesh_node = self.scene.add(mesh,pose=np.eye(4), name='ob') # Object pose parent is cam
|
|
self.mesh_nodes.append(mesh_node)
|
|
|
|
|
|
def add_point_light(self, intensity=3):
|
|
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=intensity)
|
|
self.scene.add(light, pose=np.eye(4)) # Same as camera position
|
|
|
|
|
|
def clear_mesh_nodes(self):
|
|
for n in self.mesh_nodes:
|
|
self.scene.remove_node(n)
|
|
|
|
|
|
def render(self,mesh=None,ob_in_cvcam=None, get_normal=False):
|
|
if mesh is not None:
|
|
mesh = mesh.copy()
|
|
mesh.apply_transform(cvcam_in_glcam@ob_in_cvcam)
|
|
mesh = pyrender.Mesh.from_trimesh(mesh, smooth=False)
|
|
mesh_node = self.scene.add(mesh, pose=np.eye(4), name='ob') # Object pose parent is cam
|
|
color, depth = self.r.render(self.scene) # depth: float
|
|
if mesh is not None:
|
|
self.scene.remove_node(mesh_node)
|
|
|
|
return color, depth
|