Initial snapshot: hand motion pipeline (Dyn-HaMR + dex-retargeting + SPIDER)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

model/feature_extractors/condition_builder.py

@@ -0,0 +1,67 @@
+"""
+Condition Builder (with cmask mechanism).
+
+Concatenates image token + skeleton token into the condition stream.
+cmask: learnable mask token; both training and inference use HaMeR confidence as a continuous mask.
+
+Inputs:
+  - image_tokens: (B, T, d_model) — from Image Refiner + Compression
+  - skeleton_tokens: (B, T, d_model) — from Skeleton Encoder
+  - hamer_confidence: (B, T) — HaMeR detection confidence
+
+Output: condition (B, 2*T, d_model)
+"""
+
+import torch
+import torch.nn as nn
+
+
+class ConditionBuilder(nn.Module):
+    def __init__(self, d_model: int, mask_ratio: float = 0.2):
+        super().__init__()
+        self.d_model = d_model
+        self.mask_ratio = mask_ratio
+
+        # Learnable mask token — initialized with truncated normal
+        self.c_mask = nn.Parameter(torch.empty(1, 1, d_model))
+        nn.init.trunc_normal_(self.c_mask, std=0.02)
+
+    def forward(
+        self,
+        image_tokens: torch.Tensor,
+        skeleton_tokens: torch.Tensor,
+        hamer_confidence: torch.Tensor,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Args:
+            image_tokens: (B, T, d_model)
+            skeleton_tokens: (B, T, d_model)
+            hamer_confidence: (B, T) — used as m_t during inference
+
+        Returns:
+            condition: (B, 2*T, d_model)
+            m: (B, T) — actual mask values used (for logging)
+        """
+        B, T, D = image_tokens.shape
+
+        # concat: (B, T, 2, d_model) -> (B, 2*T, d_model)
+        tokens = torch.stack([image_tokens, skeleton_tokens], dim=2)  # (B, T, 2, D)
+        tokens = tokens.reshape(B, 2 * T, D)
+
+        # Use HaMeR confidence as a continuous mask (consistent between training and inference)
+        m = hamer_confidence.float()  # (B, T)
+
+        # Optional: randomly dropout some frames during training as regularization
+        if self.training and self.mask_ratio > 0:
+            keep = (torch.rand(B, T, device=tokens.device) >= self.mask_ratio).float()
+            m = m * keep
+
+        # Expand m to match 2*T tokens (the two tokens per frame share the same m)
+        # m: (B, T) -> (B, 2*T)
+        m_expanded = m.unsqueeze(-1).expand(-1, -1, 2).reshape(B, 2 * T)  # (B, 2*T)
+        m_expanded = m_expanded.unsqueeze(-1)  # (B, 2*T, 1)
+
+        # cmask: c'_t = m_t * c_t + (1 - m_t) * c_mask
+        condition = m_expanded * tokens + (1 - m_expanded) * self.c_mask
+
+        return condition, m