hand-motion-pipeline/third_party/Dyn-HaMR/dyn-hamr/vis/viewer.py

import os
import imageio
import numpy as np

import time
import torch
import trimesh

os.environ["PYOPENGL_PLATFORM"] = "egl"

import pyrender
from pyrender.constants import RenderFlags
from pyrender.light import DirectionalLight
from pyrender.node import Node

from geometry.camera import make_rotation, make_translation
from .tools import (
    read_image,
    checkerboard_geometry,
    camera_marker_geometry,
    transform_pyrender,
)


def init_viewer(
    img_size, intrins, vis_scale=1.0, bg_paths=None, fps=30,
):
    img_size = int(vis_scale * img_size[0]), int(vis_scale * img_size[1])
    intrins = vis_scale * intrins

    platform = os.environ.get("PYOPENGL_PLATFORM", "pyglet")
    if platform == "pyglet":
        vis = AnimationViewer(img_size, intrins=intrins, fps=fps)
        print("VIS", vis)
        return vis

    if platform != "egl":
        raise NotImplementedError

    vis = OffscreenAnimation(img_size, intrins=intrins, fps=fps)
    if bg_paths is not None:
        bg_imgs = [
            read_image(p, scale=vis_scale).astype(np.float32) / 255 for p in bg_paths
        ]
        vis.set_bg_seq(bg_imgs)
    print("VIS", vis)
    return vis


class AnimationBase(object):
    """
    Render an animation
    IMPORTANT: pyrender uses coordinates with Y UP (i.e. xyz = right up back),
    we must keep this in mind because the human prior frame has Z UP (i.e.
    xyz = right forward up)
    """

    def __init__(self, img_size, intrins=None):
        self.scene = pyrender.Scene(
            ambient_light=[0.3, 0.3, 0.3], bg_color=[1.0, 1.0, 1.0, 0.0]
        )
        self.scene.bg_color = np.array([1.0, 1.0, 1.0, 0.0])

        self.viewport_size = img_size
        self.camera = make_pyrender_camera(img_size, intrins)

        # set the ground with Z up
        self.ground_pose = np.eye(4)
        ground = pyrender.Mesh.from_trimesh(
            make_checkerboard(color0=[0.9, 0.95, 1.0], color1=[0.7, 0.8, 0.85], up="y", alpha=1.0), smooth=False
        )
        self.ground_node = self.scene.add(ground, name="ground", pose=self.ground_pose)
        self.cam_node = self.scene.add(self.camera, name="camera")

        self.cam_marker_node = None
        self.cam_marker_poses = []

        self.light_nodes = []
        self.light_poses = []

        self.bg_seq = []

        # pyrender nodes, reused between frames
        self.anim_nodes = []
        # list of meshes per frame of animation
        self.anim_meshes = []
        # list of camera poses per frame of animation
        self.anim_cameras = None
        # current timestep of animation
        self.anim_idx = 0

    def acquire_lock(self):
        pass

    def release_lock(self):
        pass

    def close(self):
        pass

    def delete(self):
        pass

    def clear_meshes(self):
        self.anim_meshes = []
        self.anim_idx = 0

    @property
    def anim_len(self):
        return len(self.anim_meshes)

    def add_lighting(self, color=np.ones(3), intensity=1.0):
        self.light_poses = get_light_poses()
        self.light_poses.append(np.eye(4))
        cam_pose = self.scene.get_pose(self.cam_node)
        for i, pose in enumerate(self.light_poses):
            matrix = cam_pose @ pose
            node = Node(
                name=f"light-{i:02d}",
                light=DirectionalLight(color=color, intensity=intensity),
                matrix=matrix,
            )
            if self.scene.has_node(node):
                continue
            self.scene.add_node(node)
            self.light_nodes.append(node)

    def add_mesh_frame(self, meshes, debug=False):
        """
        :param meshes (list) trimesh meshes for this frame timestep
        """
        meshes = [pyrender.Mesh.from_trimesh(m) for m in meshes]
        num_nodes = len(meshes)
        self.anim_meshes.append(meshes)

        if num_nodes > len(self.anim_nodes):
            # need to add a node in the pyrender scene
            self.acquire_lock()
            for i in range(len(self.anim_nodes), num_nodes):
                self.anim_nodes.append(self.scene.add(meshes[i], name=f"mesh_{i:03d}"))
            self.release_lock()

    def add_static_meshes(self, meshes, smooth=True):
        """
        add all meshes to a single frame (timestep)
        """
        meshes = [pyrender.Mesh.from_trimesh(m, smooth=smooth) for m in meshes]
        if len(self.anim_meshes) < 1:
            self.anim_meshes.append([])
        self.anim_meshes[0].extend(meshes)
        num_meshes = len(self.anim_meshes[0])
        num_nodes = len(self.anim_nodes)
        if num_meshes > num_nodes:
            # need to add a node in the pyrender scene
            self.acquire_lock()
            self.anim_nodes.extend(
                [
                    self.scene.add(self.anim_meshes[0][i])
                    for i in range(num_nodes, num_meshes)
                ]
            )
        self.release_lock()

    def set_camera_seq(self, poses):
        """
        :param poses (*, 4, 4)
        """
        poses = transform_pyrender(poses)
        poses = np.asarray(poses)
        assert poses.ndim >= 2 and poses.shape[-2:] == (4, 4)
        if poses.ndim < 3:
            poses = poses[None]
        print(f"Adding camera sequence length {len(poses)}")
        self.anim_cameras = poses

    def add_camera_markers(self, poses):
        """
        Add a single camera marker node that we move around when we animate
        """
        if os.environ.get("DYNHAMR_HIDE_CAMERA_MARKERS") == "1":
            return
        print("ADDING CAMERA MARKERS")
        if self.cam_marker_node is None:
            cam_marker = make_camera_marker(up="y")
            self.cam_marker_node = self.scene.add(
                pyrender.Mesh.from_trimesh(cam_marker, smooth=False)
            )
        poses = transform_pyrender(poses)
        poses = np.asarray(poses)
        self.cam_marker_poses = poses

    def add_camera_markers_static(self, poses):
        """
        Add all camera markers as separate mesh nodes to render in one pass
        """
        print("ADDING STATIC CAMERA MARKERS")
        poses = transform_pyrender(poses)
        poses = np.asarray(poses)
        cam_meshes = [make_camera_marker(up="y", transform=pose) for pose in poses]
        self.add_static_meshes(cam_meshes, smooth=False)

    def update_camera(self, cam_pose):
        self.acquire_lock()
        self.scene.set_pose(self.cam_node, pose=cam_pose)
        self.release_lock()

    def set_ground(self, pose):
        pose = transform_pyrender(pose)
        print("Setting ground pose to be", pose)
        self.ground_pose = np.asarray(pose)
        self.scene.set_pose(self.ground_node, pose=self.ground_pose)

    def set_bg_seq(self, bg_imgs):
        pass

    def set_mesh_visibility(self, vis):
        self.acquire_lock()
        for node in self.anim_nodes:
            node.mesh.is_visible = vis
        self.release_lock()

    def check_mesh_visibility(self):
        for node in self.anim_nodes:
            if node.mesh.is_visible:
                return True

        if self.ground_node.mesh.is_visible:
            return True

        if self.cam_marker_node is not None and self.cam_marker_node.mesh.is_visible:
            return True

        return False

    def update_frame(self):
        t = self.anim_idx % self.anim_len

        # update camera
        if self.anim_cameras is not None:
            cam_t = min(t, len(self.anim_cameras) - 1)
            cam_pose = self.anim_cameras[cam_t]
            self.update_camera(cam_pose)
            if self.cam_marker_node is not None and t < len(self.cam_marker_poses):
                self.acquire_lock()
                self.scene.set_pose(self.cam_marker_node, self.cam_marker_poses[t])
                self.release_lock()

        # update meshes
        meshes = self.anim_meshes[t]
        self.acquire_lock()
        for i, node in enumerate(self.anim_nodes):
            if i < len(meshes):
                node.mesh = meshes[i]
                node.mesh.is_visible = True
            else:
                node.mesh.is_visible = False
        self.release_lock()


class OffscreenAnimation(AnimationBase):
    def __init__(self, img_size, intrins=None, fps=30, ext="mp4"):
        super().__init__(img_size, intrins=intrins)
        self.add_lighting(0.9)

        self.fps = fps
        self.ext = ext
        self.viewer = pyrender.OffscreenRenderer(*self.viewport_size)

        self.bg_seq = []
        self.keypoints_seq = None  # Store GT 2D keypoints for overlay
        self.pred_keypoints_seq = None  # Store predicted 2D keypoints

    def set_bg_seq(self, bg_imgs):
        assert isinstance(bg_imgs, list)
        print(f"setting length {len(bg_imgs)} bg sequence")
        self.bg_seq = bg_imgs

    def set_keypoints_seq(self, keypoints_seq):
        """
        Set GT 2D keypoints sequence for overlay visualization
        Args:
            keypoints_seq: List of (J, 3) arrays with [x, y, confidence] for each frame
        """
        self.keypoints_seq = keypoints_seq
        print(f"setting length {len(keypoints_seq)} GT keypoints sequence")
    
    def set_pred_keypoints_seq(self, pred_keypoints_seq):
        """
        Set predicted 2D keypoints sequence
        Args:
            pred_keypoints_seq: List of (J, 2) arrays with [x, y] for each frame
        """
        self.pred_keypoints_seq = pred_keypoints_seq
        print(f"setting length {len(pred_keypoints_seq)} predicted keypoints sequence")

    def delete(self):
        self.viewer.delete()

    def close(self):
        self.delete()

    def draw_keypoints_on_image(self, img, keypoints, radius=4, color=(0, 255, 0), thickness=-1, conf_threshold=0.3):
        """
        Draw 2D keypoints on an image
        Args:
            img: numpy array (H, W, 3)
            keypoints: (J, 3) array with [x, y, confidence]
            radius: circle radius for keypoints
            color: BGR color tuple
            thickness: -1 for filled circles
            conf_threshold: minimum confidence to draw keypoint
        Returns:
            img with keypoints drawn
        """
        import cv2
        img = img.copy()
        for j in range(keypoints.shape[0]):
            x, y, conf = keypoints[j]
            if conf > conf_threshold:
                # Draw circle at keypoint location
                cv2.circle(img, (int(x), int(y)), radius, color, thickness)
        return img

    def render(
        self, render_bg=True, render_ground=True, render_cam=True, fac=1.0, render_keypoints=False, **kwargs
    ):
        flags = RenderFlags.RGBA | RenderFlags.SHADOWS_DIRECTIONAL
        self.ground_node.mesh.is_visible = render_ground and (not render_bg)
        if self.cam_marker_node is not None:
            self.cam_marker_node.mesh.is_visible = render_cam

        rgba = np.zeros((*self.viewport_size[::-1], 4))
        if self.check_mesh_visibility():  # if any meshes are visible
            rgba, _ = self.viewer.render(self.scene, flags=flags)

        img = rgba
        if render_bg and len(self.bg_seq) > 0:
            t = min(self.anim_idx, len(self.bg_seq) - 1)
            bg = self.bg_seq[t]
            rgba = rgba.astype(np.float32) / 255
            alpha = fac * rgba[..., 3:]
            img = alpha * rgba[..., :3] + (1 - alpha) * bg
            img = (255 * img).astype(np.uint8)

        # Overlay 2D keypoints if available and requested
        if render_keypoints and render_bg:
            # Draw GT keypoints in GREEN (larger)
            if self.keypoints_seq is not None:
                t = min(self.anim_idx, len(self.keypoints_seq) - 1)
                keypoints = self.keypoints_seq[t]
                if keypoints is not None and len(keypoints) > 0:
                    img = self.draw_keypoints_on_image(img, keypoints, color=(0, 255, 0), radius=5)
            
            # Draw predicted keypoints in RED (smaller)
            if self.pred_keypoints_seq is not None:
                t = min(self.anim_idx, len(self.pred_keypoints_seq) - 1)
                pred_kp = self.pred_keypoints_seq[t]
                if pred_kp is not None and len(pred_kp) > 0:
                    # Add confidence=1.0 if only (J, 2)
                    if pred_kp.ndim == 2 and pred_kp.shape[1] == 2:
                        pred_kp = np.concatenate([pred_kp, np.ones((len(pred_kp), 1))], axis=1)
                    img = self.draw_keypoints_on_image(img, pred_kp, color=(0, 0, 255), radius=3, conf_threshold=0.0)

        return img

    def render_mesh_layers(self, render_ground=True, render_cam=True, **kwargs):
        # render each mesh in its own layer
        flags = RenderFlags.RGBA | RenderFlags.SHADOWS_DIRECTIONAL
        render_cam = render_cam and self.cam_marker_node is not None

        # current timestep
        t = self.anim_idx % self.anim_len
        meshes = self.anim_meshes[t]
        layers = []

        # toggle ground and camera
        self.ground_node.mesh.is_visible = False
        if self.cam_marker_node is not None:
            self.cam_marker_node.mesh.is_visible = False

        # render animation meshes for current timestep
        for i in range(len(meshes)):
            for j, node in enumerate(self.anim_nodes):
                if i == j:
                    node.mesh.is_visible = True
                else:
                    node.mesh.is_visible = False
            rgba, _ = self.viewer.render(self.scene, flags=flags)
            layers.append(rgba)

        if not render_ground and not render_cam:
            return layers

        # render the ground and camera
        for node in self.anim_nodes:
            node.mesh.is_visible = False

        if render_cam:
            self.cam_marker_node.mesh.is_visible = render_cam
            rgba, _ = self.viewer.render(self.scene, flags=flags)
            layers.append(rgba)
            self.cam_marker_node.mesh.is_visible = False

        if render_ground:
            self.ground_node.mesh.is_visible = render_ground
            rgba, _ = self.viewer.render(self.scene, flags=flags)
            layers.append(rgba)

        return layers

    def render_frames_layers(self):
        frames_layers = []
        for t in range(self.anim_len):
            self.anim_idx = t
            self.update_frame()
            layers = self.render_mesh_layers()
            frames_layers.append(layers)
        return frames_layers

    def animate(self, save_name, save_frames=False, render_layers=False, **kwargs):
        if render_layers:
            frames_layers = self.render_frames_layers()
            os.makedirs(save_name, exist_ok=True)
            for t, layers in enumerate(frames_layers):
                for l, layer in enumerate(layers):
                    path = f"{save_name}/{t:06d}_{l:03d}.png"
                    imageio.imwrite(path, layer)
            print(f"saved frame layers to {save_name}")
            return save_name

        frames = self.render_frames(**kwargs)
        if len(frames) > 1:
            if save_frames:
                os.makedirs(save_name, exist_ok=True)
                for i, frame in enumerate(frames):
                    path = f"{save_name}/{i:06d}.png"
                    imageio.imwrite(path, frame)
                return save_name
            save_path = f"{save_name}.{self.ext}"
            os.makedirs(save_name, exist_ok=True)
            for idx, i in enumerate(frames):
                imageio.imwrite(f"{save_name}/" + f'{str(idx).zfill(6)}.jpg', i[:, :, :3])
            imageio.mimwrite(save_path, frames, fps=self.fps)
            print("wrote video to", save_path)
            return save_path

        save_path = f"{save_name}.png"
        imageio.imwrite(save_path, frames[0])
        print("wrote image to", save_path)
        return save_path

    def render_frames(self, **kwargs):
        print("ANIMATION LENGTH", self.anim_len)
        frames = []
        for t in range(self.anim_len):
            self.anim_idx = t
            self.update_frame()
            img = self.render(**kwargs)
            frames.append(img)
        return frames

    def render_layers(self, save_dir, composite=False, fac=0.4):
        os.makedirs(save_dir, exist_ok=True)

        # render all frames
        frames = self.render_frames(render_bg=False, render_ground=False)

        # render background
        self.set_mesh_visibility(False)
        bg_img = self.render(render_bg=False, render_ground=True) / 255

        # save frames
        for t, frame in enumerate(frames):
            imageio.imwrite(f"{save_dir}/{t:06d}.png", frame)
        imageio.imwrite(f"{save_dir}/background.png", bg_img)

        if composite:
            rgba = composite_layers(frames, bg_img, fac=fac)
            imageio.imwrite(f"{save_dir}/composite.png", (255 * rgba).astype(np.uint8))


def composite_layers(frames, bg_img, fac=0.4):
    # composite front to back
    H, W = frames[0].shape[:2]
    comp = np.zeros((H, W, 3))
    vis = np.ones((H, W, 1))
    for frame in frames:
        frame = frame / 255
        alpha = fac * frame[..., 3:]
        comp += vis * alpha * frame[..., :3]
        vis *= 1 - alpha

    comp += vis * bg_img[..., 3:] * bg_img[..., :3]
    vis *= 1 - bg_img[..., 3:]
    return comp + vis
    rgba = np.concatenate([comp, 1 - vis], axis=-1)
    return rgba


class AnimationViewer(AnimationBase):
    def __init__(self, img_size, intrins=None, fps=30, num_repeats=1):
        super().__init__(img_size, intrins=intrins)

        self.fps = fps
        self.ext = "gif"

        def pause_play_callback(_, viewer):
            viewer.is_paused = not viewer.is_paused

        def step_callback(_, viewer, step):
            viewer.anim_idx = (viewer.anim_idx + step) % viewer.anim_len

        def stop_callback(_, viewer):
            viewer.do_animate = False

        def loop_callback(_, viewer):
            viewer.num_repeats *= -1

        registered_keys = {
            "p": (pause_play_callback, [self]),
            ".": (step_callback, [self, 1]),
            ",": (step_callback, [self, -1]),
            "l": (loop_callback, [self]),
            "s": (stop_callback, [self]),
        }

        self.viewer = pyrender.Viewer(
            self.scene,
            run_in_thread=True,
            viewport_size=self.viewport_size,
            use_raymond_lighting=True,
            registered_keys=registered_keys,
        )

        self.num_repeats = num_repeats
        self.do_animate = False
        self.is_paused = False

    def acquire_lock(self):
        self.viewer.render_lock.acquire()

    def release_lock(self):
        self.viewer.render_lock.release()

    def close(self):
        self.viewer.close_external()

    def animate(self, save_name=None, **kwargs):
        print("===================")
        print(f"PLAYING ANIMATION {self.num_repeats} TIMES")
        print("VIEWER CONTROLS:")
        print("p\tpause/play")
        print("l\tloop/unloop")
        print(".\tstep forward")
        print(",\tstep backward")
        print("s\tstop")
        print("===================")

        self.anim_idx = 0
        self.do_animate = True
        frame_dur = 1 / self.fps
        if save_name is not None:
            self.viewer.viewer_flags["record"] = True

        # set up initial frame
        self.update_frame()

        while self.do_animate:
            if self.is_paused:
                self.update_frame()
                continue

            if (
                self.num_repeats > 0
                and self.anim_idx >= self.num_repeats * self.anim_len
            ):
                break

            if save_name is not None and self.anim_idx >= self.anim_len:
                self.viewer.viewer_flags["record"] = False

            start_time = time.time()
            self.update_frame()
            render_time = time.time() - start_time

            self.anim_idx += 1
            sleep_time = frame_dur - render_time
            if sleep_time > 0:
                time.sleep(sleep_time)

        if save_name is not None:
            print(f"SAVING RECORDING TO {save_name}")
            self.viewer.save_gif(f"{save_name}.{self.ext}")


def make_checkerboard(
    length=25.0,
    color0=[0.8, 0.9, 0.9],
    color1=[0.6, 0.7, 0.7],
    tile_width=0.5,
    alpha=1.0,
    up="y",
):
    v, f, _, fc = checkerboard_geometry(length, color0, color1, tile_width, alpha, up)
    return trimesh.Trimesh(v, f, face_colors=fc, process=False)


def make_pyrender_camera(img_size, intrins=None):
    if intrins is not None:
        print("USING INTRINSICS CAMERA", intrins)
        fx, fy, cx, cy = intrins
        return pyrender.IntrinsicsCamera(fx, fy, cx, cy)

    W, H = img_size
    focal = 0.5 * (H + W)
    yfov = 2 * np.arctan(0.5 * H / focal)
    print("USING PERSPECTIVE CAMERA", H, W, focal)
    return pyrender.PerspectiveCamera(yfov=yfov, aspectRatio=W / H)


def make_camera_marker(radius=0.1, height=0.2, up="y", transform=None):
    """
    :param radius (default 0.1) radius of pyramid base, diagonal of image plane
    :param height (default 0.2) height of pyramid, focal length
    :param up (default y) camera up vector
    :param transform (default None) (4, 4) cam to world transform
    """
    verts, faces, face_colors = camera_marker_geometry(radius, height, up)
    if transform is not None:
        assert transform.shape == (4, 4)
        verts = (
            np.einsum("ij,nj->ni", transform[:3, :3], verts) + transform[None, :3, 3]
        )
    return trimesh.Trimesh(verts, faces, face_colors=face_colors, process=False)


def get_light_poses(n_lights=5, elevation=np.pi / 3, dist=12):
    # get lights in a circle around origin at elevation
    thetas = elevation * np.ones(n_lights)
    phis = 2 * np.pi * np.arange(n_lights) / n_lights
    poses = []
    trans = make_translation(torch.tensor([0, 0, dist]))
    for phi, theta in zip(phis, thetas):
        rot = make_rotation(rx=-theta, ry=phi, order="xyz")
        poses.append((rot @ trans).numpy())
    return poses


def get_raymond_light_poses(up="z"):
    thetas = np.pi * np.ones(3) / 6
    phis = 2 * np.pi * np.arange(3) / 3

    poses = []

    for phi, theta in zip(phis, thetas):
        xp = np.sin(theta) * np.cos(phi)
        yp = np.sin(theta) * np.sin(phi)
        zp = np.cos(theta)

        if up == "y":
            z = np.array([xp, zp, -yp])
        else:
            z = np.array([xp, yp, zp])
        z = z / np.linalg.norm(z)
        x = np.array([-z[1], z[0], 0.0])
        if np.linalg.norm(x) == 0:
            x = np.array([1.0, 0.0, 0.0])
        x = x / np.linalg.norm(x)
        y = np.cross(z, x)

        matrix = np.eye(4)
        matrix[:3, :3] = np.c_[x, y, z]
        poses.append(matrix)
    return poses