Update to 2026-09-17 pipeline snapshot; add weights, L20 assets and recording via Git LFS
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
liyang
@@ -0,0 +1,70 @@
|
||||
"""Use observed skin-surface depth to correct camera-space hand translation.
|
||||
|
||||
This is a robust translation fit, not depth-derived skeleton ground truth.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import argparse
|
||||
import numpy as np
|
||||
import cv2
|
||||
from scipy.ndimage import gaussian_filter1d
|
||||
from scipy.spatial.transform import Rotation
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1];BASE=ROOT/'output/20260915_171525'
|
||||
SRC=ROOT/'docs/20260915_171525'
|
||||
parser=argparse.ArgumentParser();parser.add_argument('--left',action='store_true');args=parser.parse_args()
|
||||
folder='handflow_left' if args.left else 'handflow'
|
||||
suffix='_left' if args.left else ''
|
||||
h=np.load(BASE/folder/'handflow_results.npz');j=dict(np.load(BASE/folder/'human_joints.npz'))
|
||||
meta=json.loads((SRC/'intrinsics.json').read_text());camera=np.load(BASE/'rgbd_camera.npz')
|
||||
K=np.array([meta[k] for k in ['fx','fy','cx','cy']]);N=len(h['pose'])
|
||||
c2w=camera['c2w'];old_camera=h['c2w']
|
||||
wrist_cam=np.einsum('tji,tj->ti',old_camera[:,:3,:3],j['wrist_world']-old_camera[:,:3,3])
|
||||
shifts=[];records=[];valid=[];cap=cv2.VideoCapture(str(SRC/'color.mp4'))
|
||||
for t,verts in enumerate(h['verts_cam']):
|
||||
ok,color=cap.read();assert ok
|
||||
dep=cv2.imread(str(SRC/'depth'/f'{t:06d}.png'),-1).astype(float)*meta['depth_scale_m']
|
||||
ycc=cv2.cvtColor(color,cv2.COLOR_BGR2YCrCb)
|
||||
skin=cv2.inRange(ycc,np.array([0,133,77]),np.array([255,173,127]))>0
|
||||
uv=np.rint(verts[:,:2]/verts[:,2,None]*K[:2]+K[2:]).astype(int)
|
||||
keep=(verts[:,2]>.05)&(uv[:,0]>=2)&(uv[:,0]<color.shape[1]-2)&(uv[:,1]>=2)&(uv[:,1]<color.shape[0]-2)
|
||||
# Keep nearest projected vertex per 4x4 pixel bin, reducing hidden-surface bias.
|
||||
closest={}
|
||||
for index in np.flatnonzero(keep):
|
||||
u,v=uv[index];key=(u//4,v//4)
|
||||
if key not in closest or verts[index,2]<verts[closest[key],2]:closest[key]=index
|
||||
errors=[]
|
||||
for index in closest.values():
|
||||
u,v=uv[index]
|
||||
if not skin[v,u]:continue
|
||||
patch=dep[v-1:v+2,u-1:u+2];values=patch[(patch>.1)&(patch<.85)]
|
||||
if len(values)<5 or np.ptp(values)>.04:continue
|
||||
errors.append(float(np.median(values)-verts[index,2]))
|
||||
dz=float(np.median(errors)) if errors else 0.
|
||||
mad=float(np.median(np.abs(np.asarray(errors)-dz))) if errors else 1.
|
||||
accepted=bool(len(errors)>=25 and mad<.025 and abs(dz)<.25 and h['pred_valid'][t])
|
||||
shifts.append(dz);valid.append(accepted)
|
||||
records.append(dict(frame=t,samples=len(errors),median_depth_residual_m=dz,mad_m=mad,accepted=accepted))
|
||||
cap.release();valid=np.asarray(valid);idx=np.flatnonzero(valid)
|
||||
assert len(idx)>=N*.5, f'Insufficient depth support: {len(idx)}/{N}'
|
||||
filled=np.interp(np.arange(N),idx,np.asarray(shifts)[idx]);smooth=gaussian_filter1d(filled,2)
|
||||
delta=wrist_cam/wrist_cam[:,2,None]*smooth[:,None]
|
||||
corrected=wrist_cam+delta
|
||||
world=np.einsum('tij,tj->ti',c2w[:,:3,:3],corrected)+c2w[:,:3,3]
|
||||
root_world=Rotation.from_matrix(c2w[:,:3,:3]@Rotation.from_rotvec(h['pose'][:,:3]).as_matrix()).as_rotvec()
|
||||
j.update(wrist_world=world,root_orient=root_world,time=camera['time'],
|
||||
before_depth_wrist_camera=wrist_cam,depth_translation_camera=delta,
|
||||
depth_valid=valid,depth_correction_interpolated=~valid,
|
||||
detection_valid=j['detection_valid']&valid&camera['valid'],
|
||||
source=str((BASE/folder/'handflow_results.npz').resolve()),
|
||||
metric_scale_provenance='D405 depth_scale from recording metadata; skin-surface translation fit, model size unchanged; RGBD static-background odometry world. Not calibrated motion ground truth.')
|
||||
np.savez_compressed(BASE/f'human_joints_rgbd{suffix}.npz',**j)
|
||||
np.savez_compressed(BASE/f'depth_hand_fit{suffix}.npz',raw_depth_shift=shifts,applied_depth_shift=smooth,
|
||||
accepted=valid,wrist_camera=corrected,wrist_world=world,c2w=c2w,
|
||||
corrected_verts_camera=h['verts_cam']+delta[:,None,:])
|
||||
report=dict(frames=N,depth_supported_frames=int(valid.sum()),rejected_depth_frames=np.flatnonzero(~valid).tolist(),
|
||||
median_abs_raw_depth_residual_m=float(np.median(np.abs(np.asarray(shifts)[valid]))),
|
||||
median_abs_remaining_sample_residual_m=float(np.median(np.abs(np.asarray(shifts)[valid]-smooth[valid]))),
|
||||
applied_depth_shift_range_m=[float(smooth.min()),float(smooth.max())],frames_detail=records,
|
||||
boundary='Translation-only fit to selected visible skin depth; self-occlusion and skin-mask errors remain. No finger-joint depth truth. Rejected depth corrections interpolated and marked invalid.')
|
||||
(BASE/f'depth_fit_validation{suffix}.json').write_text(json.dumps(report,indent=2));print(json.dumps({k:v for k,v in report.items() if k!='frames_detail'},indent=2))
|
||||
Reference in New Issue
Block a user