Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
"""HandFlow visualization module: pytorch3d Phong rendering + video I/O.
|
||||
|
||||
- renderer_p3d.py: PhongRenderer — overlay (perspective, OpenCV intrinsics) + orthographic
|
||||
trajectory view (topdown | side), both SoftPhongShader-shaded.
|
||||
- video_io.py: read video frames (cv2) + encode mp4 (imageio/ffmpeg)
|
||||
"""
|
||||
@@ -0,0 +1,308 @@
|
||||
"""pytorch3d Phong renderer for HandFlow demo outputs (HaWoR-style smooth shading).
|
||||
|
||||
Replaces the old cv2 painter's-algorithm renderer (flat triangles -> faceted flicker).
|
||||
Both outputs are Phong-shaded (per-vertex normals + point lights):
|
||||
|
||||
render_overlay : perspective camera (OpenCV intrinsics) -> mesh composited on RGB
|
||||
render_ortho_video : orthographic world/cam view (topdown | side) -> per-frame mesh
|
||||
+ camera trajectory polyline; the mesh and the trajectory are
|
||||
projected through the SAME pytorch3d camera so they stay aligned
|
||||
|
||||
Coordinate system: HandFlow CV convention (X right, Y down, Z forward), meters. Verts are
|
||||
passed to pytorch3d in their native CV space (no axis flip), so right-hand MANO face winding
|
||||
keeps surface normals pointing outward. The ortho look-at is defined directly in CV world.
|
||||
"""
|
||||
|
||||
from typing import List, Sequence
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
from pytorch3d.renderer import (
|
||||
FoVOrthographicCameras,
|
||||
Materials,
|
||||
MeshRasterizer,
|
||||
MeshRenderer,
|
||||
PointLights,
|
||||
RasterizationSettings,
|
||||
SoftPhongShader,
|
||||
TexturesVertex,
|
||||
)
|
||||
from pytorch3d.renderer.camera_conversions import _cameras_from_opencv_projection
|
||||
from pytorch3d.renderer.cameras import look_at_rotation
|
||||
from pytorch3d.structures import Meshes
|
||||
from pytorch3d.structures.meshes import join_meshes_as_scene
|
||||
|
||||
|
||||
def _bgr_to_unit_rgb(color_bgr: Sequence[float]) -> torch.Tensor:
|
||||
"""BGR (uint8 tuple) -> normalized RGB tensor (3,)."""
|
||||
c = np.asarray(color_bgr, dtype=np.float32)
|
||||
if c.max() > 1.0:
|
||||
c = c / 255.0
|
||||
return torch.tensor(np.ascontiguousarray(c[::-1]), dtype=torch.float32) # BGR -> RGB
|
||||
|
||||
|
||||
# Camera marker: a square-base pyramid (pentahedron) drawn as a black hollow wireframe — the
|
||||
# standard camera-symbol used for camera-pose viz (COLMAP / SLAM). Apex = optical center (camera
|
||||
# position); rectangular base in front (+Z) = image plane / field of view.
|
||||
_CAM_PYRAMID_VERTS = np.array([
|
||||
[0.0, 0.0, 0.0], # apex = camera center (focal point)
|
||||
[0.50, 0.38, 0.80], # base corners (forward image plane, ~4:3)
|
||||
[-0.50, 0.38, 0.80],
|
||||
[-0.50, -0.38, 0.80],
|
||||
[0.50, -0.38, 0.80],
|
||||
], dtype=np.float32)
|
||||
_CAM_PYRAMID_EDGES = [(0, 1), (0, 2), (0, 3), (0, 4), # apex -> base (FOV rays)
|
||||
(1, 2), (2, 3), (3, 4), (4, 1)] # base loop (image plane)
|
||||
|
||||
|
||||
|
||||
class PhongRenderer:
|
||||
"""pytorch3d SoftPhongShader renderer for overlay + orthographic views."""
|
||||
|
||||
def __init__(self, device: str | torch.device):
|
||||
self.device = torch.device(device)
|
||||
self._raster_cache: dict[tuple[int, int], MeshRenderer] = {}
|
||||
|
||||
# ── internals ───────────────────────────────────────────────────────────
|
||||
|
||||
def _get_renderer(self, H: int, W: int) -> MeshRenderer:
|
||||
key = (int(H), int(W))
|
||||
if key not in self._raster_cache:
|
||||
# blur_radius=0 keeps each face's coverage hard and full-opacity. A larger blur
|
||||
# erodes the alpha of small faces (e.g. the hand rendered small in the ortho view)
|
||||
# below the composite threshold, making the mesh vanish; and soft alpha at shared
|
||||
# edges lets the background bleed through as triangle "division lines". The binary
|
||||
# mask in _composite then yields a solid, seam-free surface.
|
||||
raster = MeshRasterizer(raster_settings=RasterizationSettings(
|
||||
image_size=(H, W), blur_radius=0.0, faces_per_pixel=1,
|
||||
))
|
||||
self._raster_cache[key] = MeshRenderer(raster, SoftPhongShader(device=self.device))
|
||||
return self._raster_cache[key]
|
||||
|
||||
@staticmethod
|
||||
def _faces(faces: np.ndarray, side: str, device) -> torch.Tensor:
|
||||
f = torch.as_tensor(faces, dtype=torch.int64, device=device)
|
||||
if side == "left": # mirror winding so normals stay outward for the left hand
|
||||
f = f[:, [0, 2, 1]]
|
||||
return f
|
||||
|
||||
def _mesh(self, verts: torch.Tensor, faces_t: torch.Tensor, color_bgr) -> Meshes:
|
||||
rgb = _bgr_to_unit_rgb(color_bgr).to(self.device).view(1, 1, 3).expand(1, verts.shape[0], 3)
|
||||
return Meshes(verts=verts.unsqueeze(0), faces=faces_t.unsqueeze(0),
|
||||
textures=TexturesVertex(verts_features=rgb))
|
||||
|
||||
def _matte_lights(self, location) -> PointLights:
|
||||
"""Diffuse-only point light (ambient + diffuse, no specular) -> matte surface.
|
||||
Colors must be (N_lights, 3) i.e. ((r,g,b),) — a flat (3,) is misread as 3 lights."""
|
||||
return PointLights(
|
||||
device=self.device,
|
||||
ambient_color=((0.45, 0.45, 0.45),),
|
||||
diffuse_color=((0.65, 0.65, 0.65),),
|
||||
specular_color=((0.0, 0.0, 0.0),),
|
||||
location=location,
|
||||
)
|
||||
|
||||
def _render_rgba(self, mesh: Meshes, cameras, lights, H: int, W: int) -> np.ndarray:
|
||||
"""Render -> (H, W, 4) uint8 RGBA."""
|
||||
renderer = self._get_renderer(H, W)
|
||||
materials = Materials(device=self.device, shininess=0.0)
|
||||
out = renderer(mesh, cameras=cameras, lights=lights, materials=materials)[0].detach().cpu().numpy()
|
||||
rgba = np.zeros((H, W, 4), dtype=np.uint8)
|
||||
rgba[..., :3] = np.clip(out[..., :3] * 255.0, 0, 255).astype(np.uint8)
|
||||
rgba[..., 3] = np.clip(out[..., 3] * 255.0, 0, 255).astype(np.uint8)
|
||||
return rgba
|
||||
|
||||
@staticmethod
|
||||
def _composite(background_bgr: np.ndarray, rgba: np.ndarray, opacity: float = 0.8) -> np.ndarray:
|
||||
"""Composite the rendered mesh at a fixed opacity over the background.
|
||||
|
||||
A binary coverage mask (from the alpha channel) is used so the mesh is uniformly
|
||||
opaque across its surface — no per-edge transparency, hence no visible triangle seams.
|
||||
"""
|
||||
bgr = np.ascontiguousarray(rgba[..., :3][:, :, ::-1]) # RGB -> BGR
|
||||
# Low threshold (any real face coverage) so small meshes still show; combined with
|
||||
# blur_radius=0 the surface is solid (no per-edge transparency -> no division lines).
|
||||
a = (rgba[..., 3] > 30).astype(np.float32)[..., None] * opacity
|
||||
out = background_bgr.astype(np.float32) * (1 - a) + bgr.astype(np.float32) * a
|
||||
return np.clip(out, 0, 255).astype(np.uint8)
|
||||
|
||||
# ── overlay: perspective (OpenCV intrinsics) ────────────────────────────
|
||||
|
||||
def render_overlay(
|
||||
self,
|
||||
background_bgr: np.ndarray,
|
||||
verts_cam_m: np.ndarray, # (V,3) camera-space meters (CV convention)
|
||||
faces: np.ndarray, # (F,3)
|
||||
intrinsics: Sequence[float], # [fx, fy, cx, cy]
|
||||
side: str = "right",
|
||||
color_bgr: Sequence[float] = (235, 206, 135), # sky blue #87CEEB (BGR)
|
||||
) -> np.ndarray:
|
||||
"""Phong-shaded MANO mesh alpha-composited onto the RGB background."""
|
||||
H, W = background_bgr.shape[:2]
|
||||
fx, fy, cx, cy = intrinsics
|
||||
K = torch.tensor([[[fx, 0, cx], [0, fy, cy], [0, 0, 1]]], dtype=torch.float32, device=self.device)
|
||||
R = torch.eye(3, device=self.device).unsqueeze(0)
|
||||
T = torch.zeros((1, 3), device=self.device)
|
||||
cameras = _cameras_from_opencv_projection(R, T, K, torch.tensor([[H, W]], device=self.device))
|
||||
|
||||
verts = torch.as_tensor(verts_cam_m, dtype=torch.float32, device=self.device)
|
||||
mesh = self._mesh(verts, self._faces(faces, side, self.device), color_bgr)
|
||||
# Light from the camera, slightly up/right, so the hand reads as 3D (CV: -Y is up).
|
||||
lights = self._matte_lights([[0.10, -0.12, -0.05]])
|
||||
rgba = self._render_rgba(mesh, cameras, lights, H, W)
|
||||
return self._composite(background_bgr, rgba)
|
||||
|
||||
# ── orthographic world / camera view ────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _checkerboard_mesh(center, floor_y, half, n_tiles, color_a_bgr, color_b_bgr, device):
|
||||
"""Checkerboard floor on the world X-Z plane at Y=floor_y (normal points up, -Y in CV)."""
|
||||
cx, _, cz = float(center[0]), 0.0, float(center[2])
|
||||
step = 2 * half / n_tiles
|
||||
verts, feats = [], []
|
||||
faces = []
|
||||
a = _bgr_to_unit_rgb(color_a_bgr).to(device)
|
||||
b = _bgr_to_unit_rgb(color_b_bgr).to(device)
|
||||
for i in range(n_tiles):
|
||||
for j in range(n_tiles):
|
||||
x0, x1 = cx - half + i * step, cx - half + (i + 1) * step
|
||||
z0, z1 = cz - half + j * step, cz - half + (j + 1) * step
|
||||
base = len(verts)
|
||||
verts += [[x0, floor_y, z0], [x1, floor_y, z0], [x1, floor_y, z1], [x0, floor_y, z1]]
|
||||
faces += [[base, base + 1, base + 2], [base, base + 2, base + 3]]
|
||||
c = a if (i + j) % 2 == 0 else b
|
||||
feats += [c, c, c, c]
|
||||
verts_t = torch.tensor(np.asarray(verts, dtype=np.float32), device=device)
|
||||
faces_t = torch.tensor(np.asarray(faces, dtype=np.int64), device=device)
|
||||
feats_t = torch.stack(feats, dim=0)
|
||||
return Meshes(verts=verts_t.unsqueeze(0), faces=faces_t.unsqueeze(0),
|
||||
textures=TexturesVertex(verts_features=feats_t.unsqueeze(0)))
|
||||
|
||||
@staticmethod
|
||||
def _camera_wireframe_px(cam_pos, cam_rot, scale, cameras, S) -> list:
|
||||
"""Camera marker -> 8 edge pixel segments of a square-pyramid (black hollow wireframe).
|
||||
|
||||
Apex at the camera optical center, rectangular base (image plane) in the +Z viewing
|
||||
direction. World verts = rotate by cam_rot, scale, translate to cam_pos; projected through
|
||||
the SAME ortho camera so it stays aligned with the hand/floor/trajectory.
|
||||
"""
|
||||
pts_world = (cam_rot @ (_CAM_PYRAMID_VERTS * scale).T).T + cam_pos # (5,3)
|
||||
ndc = cameras.transform_points(
|
||||
torch.as_tensor(pts_world, dtype=torch.float32, device=cameras.device)
|
||||
).cpu().numpy()[:, :2]
|
||||
# pytorch3d NDC has +X pointing LEFT, so screen px = (0.5 - ndc_x*0.5)*S (and +Y up -> top).
|
||||
px = np.stack([(0.5 - ndc[:, 0] * 0.5) * S, (0.5 - ndc[:, 1] * 0.5) * S], axis=-1).astype(int)
|
||||
return [(tuple(px[a]), tuple(px[b])) for a, b in _CAM_PYRAMID_EDGES]
|
||||
|
||||
def render_ortho_video(
|
||||
self,
|
||||
verts_world_seq: np.ndarray, # (T, V, 3) meters (CV convention)
|
||||
faces: np.ndarray, # (F,3)
|
||||
c2w: np.ndarray, # (T, 4, 4) camera-to-world (identity frames for --fix_camera)
|
||||
view: str = "third_person", # "third_person" (oblique) | "topdown" | "side"
|
||||
side: str = "right",
|
||||
img_size: int = 720,
|
||||
mesh_color_bgr: Sequence[float] = (235, 206, 135), # sky blue #87CEEB (BGR)
|
||||
trail_color_bgr: Sequence[float] = (35, 35, 35),
|
||||
cam_wire_bgr: Sequence[float] = (0, 0, 0), # black hollow tetrahedron wireframe
|
||||
floor_a_bgr: Sequence[float] = (236, 236, 236), # checkerboard light tile
|
||||
floor_b_bgr: Sequence[float] = (208, 208, 208), # checkerboard dark tile
|
||||
bg_color_bgr: Sequence[float] = (250, 250, 250),
|
||||
) -> List[np.ndarray]:
|
||||
"""Orthographic 3D scene video: checkerboard floor (third_person view) + sky-blue hand
|
||||
mesh + camera trajectory polyline + a pyramid marker showing the camera orientation.
|
||||
The SLAM world is gravity-aligned (avg camera-down -> +Y) so the scene renders upright."""
|
||||
S = int(img_size)
|
||||
verts = np.asarray(verts_world_seq, dtype=np.float32)
|
||||
c2w = np.asarray(c2w, dtype=np.float32)
|
||||
n = min(verts.shape[0], c2w.shape[0])
|
||||
verts = verts[:n]
|
||||
camp = c2w[:n, :3, 3]
|
||||
camrot = c2w[:n, :3, :3]
|
||||
|
||||
# Gravity-align the SLAM world: ViPE's world is camera-frame-0 aligned, not gravity aligned,
|
||||
# so for an ego clip the whole scene is rolled about the forward axis. Rotate so the average
|
||||
# camera-down axis (≈ gravity, since the headset is upright on average) -> world +Y. This
|
||||
# unrolls the scene (floor horizontal, camera/hand upright). No-op for --fix_camera (c2w=I).
|
||||
avg_down = camrot[:, :, 1].mean(axis=0)
|
||||
nd = float(np.linalg.norm(avg_down))
|
||||
if nd > 1e-6:
|
||||
avg_down = avg_down / nd
|
||||
tgt = np.array([0.0, 1.0, 0.0], dtype=np.float32)
|
||||
c = float(np.dot(avg_down, tgt))
|
||||
if abs(c - 1.0) > 1e-3:
|
||||
axis = np.cross(avg_down, tgt)
|
||||
an = float(np.linalg.norm(axis))
|
||||
if an > 1e-6:
|
||||
axis = axis / an
|
||||
K = np.array([[0, -axis[2], axis[1]], [axis[2], 0, -axis[0]], [-axis[1], axis[0], 0]], dtype=np.float32)
|
||||
R_align = np.eye(3, dtype=np.float32) + an * K + (1 - c) * (K @ K)
|
||||
else: # anti-parallel: 180° about X
|
||||
R_align = np.diag([1.0, -1.0, -1.0]).astype(np.float32)
|
||||
verts = np.einsum('ij,tvj->tvi', R_align, verts).astype(np.float32)
|
||||
camp = (R_align @ camp.T).T.astype(np.float32)
|
||||
camrot = np.einsum('ij,tjk->tik', R_align, camrot).astype(np.float32)
|
||||
|
||||
all_pts = np.concatenate([verts.reshape(-1, 3), camp.reshape(-1, 3)], axis=0)
|
||||
pmin, pmax = all_pts.min(axis=0), all_pts.max(axis=0)
|
||||
center = (pmin + pmax) / 2.0
|
||||
extent = max(float(np.max((pmax - pmin) / 2.0) * 1.3), 1e-3)
|
||||
floor_y = float(pmax[1]) + 0.02 * extent # CV +Y is down: ground = max Y (just below scene)
|
||||
|
||||
# Look-at in CV world (distance is irrelevant for orthographic projection).
|
||||
if view == "topdown":
|
||||
eye = center + np.array([0.0, -1.0, 0.0], dtype=np.float32); up = np.array([0.0, 0.0, 1.0], dtype=np.float32)
|
||||
elif view == "side":
|
||||
eye = center + np.array([1.0, 0.0, 0.0], dtype=np.float32); up = np.array([0.0, -1.0, 0.0], dtype=np.float32)
|
||||
else: # third_person: over-the-shoulder oblique — behind/above the camera looking forward
|
||||
# at the hand (CV: -Z is behind the camera, -Y is up), like the project-page perspective.
|
||||
eye = center + np.array([-0.45, -0.80, -0.40], dtype=np.float32); up = np.array([0.0, -1.0, 0.0], dtype=np.float32)
|
||||
eye_t = torch.tensor(eye, dtype=torch.float32, device=self.device).view(1, 3)
|
||||
at_t = torch.tensor(center, dtype=torch.float32, device=self.device).view(1, 3)
|
||||
up_t = torch.tensor(up, dtype=torch.float32, device=self.device).view(1, 3)
|
||||
R = look_at_rotation(eye_t, at=at_t, up=up_t, device=self.device)
|
||||
# pytorch3d look_at_rotation returns R (cam<-world uses R^T); T must place the camera
|
||||
# at `eye`: x_cam = R^T (x - eye), so T = -(R^T @ eye). (Using -(R @ eye) mis-aims it.)
|
||||
T_cam = -(R.mT @ eye_t.unsqueeze(-1)).squeeze(-1)
|
||||
cameras = FoVOrthographicCameras(
|
||||
R=R, T=T_cam, min_x=-extent, max_x=extent, min_y=-extent, max_y=extent,
|
||||
znear=-extent * 50.0, zfar=extent * 50.0, scale_xyz=((1.0, 1.0, 1.0),), device=self.device,
|
||||
)
|
||||
lights = self._matte_lights([eye.tolist()])
|
||||
|
||||
# Static checkerboard floor (only for the oblique third-person scene).
|
||||
floor_mesh = None
|
||||
if view == "third_person":
|
||||
floor_mesh = self._checkerboard_mesh(center, floor_y, extent * 1.4, 10, floor_a_bgr, floor_b_bgr, self.device)
|
||||
|
||||
# Camera trajectory -> 2D pixels via the SAME camera (transform_points returns NDC).
|
||||
camp_ndc = cameras.transform_points(
|
||||
torch.as_tensor(camp, dtype=torch.float32, device=self.device)
|
||||
).cpu().numpy()[:, :2]
|
||||
traj_px = np.stack([(0.5 - camp_ndc[:, 0] * 0.5) * S, (0.5 - camp_ndc[:, 1] * 0.5) * S], axis=-1)
|
||||
|
||||
faces_t = self._faces(faces, side, self.device)
|
||||
cam_scale = extent * 0.08
|
||||
bg = np.asarray(bg_color_bgr, dtype=np.uint8)
|
||||
wire = tuple(int(c) for c in cam_wire_bgr)
|
||||
frames: List[np.ndarray] = []
|
||||
for t in range(n):
|
||||
# Scene = floor + hand only (rendered together for correct occlusion).
|
||||
verts_t = torch.as_tensor(verts[t], dtype=torch.float32, device=self.device)
|
||||
hand_mesh = self._mesh(verts_t, faces_t, mesh_color_bgr)
|
||||
scene = join_meshes_as_scene([floor_mesh, hand_mesh]) if floor_mesh is not None else hand_mesh
|
||||
rgba = self._render_rgba(scene, cameras, lights, S, S)
|
||||
canvas = self._composite(np.full((S, S, 3), bg, dtype=np.uint8), rgba, opacity=1.0)
|
||||
|
||||
# Camera trajectory history polyline + current camera as a hollow tetrahedron wireframe
|
||||
# (drawn on top, always visible, apex = camera forward direction).
|
||||
for i in range(1, t + 1):
|
||||
cv2.line(canvas, tuple(traj_px[i - 1].astype(int)), tuple(traj_px[i].astype(int)),
|
||||
trail_color_bgr, 2, cv2.LINE_AA)
|
||||
for p1, p2 in self._camera_wireframe_px(camp[t], camrot[t], cam_scale, cameras, S):
|
||||
cv2.line(canvas, p1, p2, wire, 2, cv2.LINE_AA)
|
||||
frames.append(canvas)
|
||||
|
||||
return frames
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Video I/O utilities: read frames (cv2.VideoCapture) + encode mp4 (imageio/ffmpeg).
|
||||
|
||||
Shared by scripts/demo.py.
|
||||
"""
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
def read_video_frames(path: str) -> Tuple[List[np.ndarray], float, Tuple[int, int]]:
|
||||
"""Read all video frames (BGR uint8).
|
||||
|
||||
Returns:
|
||||
frames_bgr: list of (H, W, 3) BGR uint8
|
||||
fps: frame rate (30 if unreadable)
|
||||
(H, W): frame size
|
||||
"""
|
||||
cap = cv2.VideoCapture(path)
|
||||
if not cap.isOpened():
|
||||
raise FileNotFoundError(f"Cannot open video: {path}")
|
||||
|
||||
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
frames: List[np.ndarray] = []
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
frames.append(frame)
|
||||
cap.release()
|
||||
|
||||
if not frames:
|
||||
raise RuntimeError(f"Video has no valid frames: {path}")
|
||||
|
||||
H, W = frames[0].shape[:2]
|
||||
return frames, float(fps), (H, W)
|
||||
|
||||
|
||||
def write_video_ffmpeg(
|
||||
frames_bgr: List[np.ndarray], path: str, fps: float, quality: int = 8
|
||||
) -> None:
|
||||
"""Encode a BGR frame sequence as mp4 (imageio + libx264).
|
||||
|
||||
Args:
|
||||
frames_bgr: list of (H, W, 3) BGR uint8
|
||||
path: output mp4 path
|
||||
fps: frame rate
|
||||
quality: imageio quality (0-10, higher is better)
|
||||
"""
|
||||
import imageio.v2 as imageio
|
||||
|
||||
writer = imageio.get_writer(path, fps=fps, codec="libx264", quality=quality,
|
||||
macro_block_size=1)
|
||||
try:
|
||||
for frame in frames_bgr:
|
||||
# BGR -> RGB
|
||||
writer.append_data(frame[:, :, ::-1])
|
||||
finally:
|
||||
writer.close()
|
||||
Reference in New Issue
Block a user