liyang
ae28d55f81
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
40 lines
3.3 KiB
Python
40 lines
3.3 KiB
Python
"""Camera overlay and fixed oblique 3D comparison; identical view for both conditions."""
|
|
import sys,json,subprocess
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'third_party/FoundationPose'))
|
|
import cv2,numpy as np,torch,trimesh,nvdiffrast.torch as dr
|
|
from Utils import nvdiffrast_render,make_mesh_tensors
|
|
from scipy.spatial.transform import Rotation
|
|
out=ROOT/'output/depth_ablation_red_20260916';a=np.load(out/'foreground_left_depth_comparison.npz');N=len(a['verts_before']);K=a['K'];P=a['object_pose'];mesh=trimesh.load(out/'red_box_closed.ply');ctx=dr.RasterizeCudaContext();eye=torch.eye(4,device='cuda')[None]
|
|
center=np.median(np.einsum('tij,j->ti',P[:,:3,:3],mesh.bounds.mean(0))+P[:,:3,3],axis=0);R=Rotation.from_euler('y',55,degrees=True).as_matrix();virtualK=np.array([[700.,0,424],[0,700,240],[0,0,1.]])
|
|
def render(v,f,color,k):
|
|
m=trimesh.Trimesh(vertices=v,faces=f,process=False);m.visual.vertex_colors=np.tile([*color,255],(len(v),1));mt=make_mesh_tensors(m)
|
|
with torch.inference_mode():c,d,_=nvdiffrast_render(K=k,H=480,W=848,ob_in_cams=eye,glctx=ctx,mesh_tensors=mt,use_light=True)
|
|
return (c[0].cpu().numpy()[:,:,::-1]*255).astype('uint8'),d[0].cpu().numpy()
|
|
def scene(b,hand,box,k):
|
|
colors=[];depth=[]
|
|
for v,f,c in [(box,mesh.faces,[60,185,95]),(hand,a['faces'],[80,145,235])]:
|
|
cc,d=render(v,f,c,k);colors.append(cc);depth.append(d)
|
|
masks=[d>0 for d in depth];front=masks[1]&((~masks[0])|(depth[1]<depth[0]));img=b.copy()
|
|
for mask,col in [(masks[0]&~front,colors[0]),(front,colors[1])]:img[mask]=(img[mask]*.25+col[mask]*.75).astype('uint8')
|
|
return img
|
|
cap=cv2.VideoCapture(str(ROOT/'docs/20260916_104026/color.mp4'));video=out/'depth_before_after.mp4';w=subprocess.Popen(['ffmpeg','-y','-v','error','-f','rawvideo','-pix_fmt','bgr24','-s','1696x960','-r','30','-i','-','-an','-c:v','libx264','-preset','fast','-crf','20','-pix_fmt','yuv420p','-movflags','+faststart',str(video)],stdin=subprocess.PIPE)
|
|
for t in range(N):
|
|
ok,b=cap.read();assert ok;box=mesh.vertices@P[t,:3,:3].T+P[t,:3,3];panels=[]
|
|
for view in ['camera','oblique']:
|
|
for variant in ['before','after']:
|
|
hand=a['verts_'+variant][t];vv=box
|
|
if view=='oblique':hand=(hand-center)@R.T+[0,0,.65];vv=(box-center)@R.T+[0,0,.65]
|
|
bg=b if view=='camera' else np.full_like(b,35);pic=scene(bg,hand,vv,K if view=='camera' else virtualK)
|
|
name='RGB only' if variant=='before' else 'RGB + depth translation'
|
|
cv2.rectangle(pic,(0,0),(848,52),(0,0,0),-1);cv2.putText(pic,f'{name} | {view} | {t/30:.2f}s',(12,23),0,.62,(240,240,240),1)
|
|
status='paired evaluation' if a['paired'][t] else 'not in paired metrics'
|
|
cv2.putText(pic,status,(12,44),0,.5,(0,220,255),1);panels.append(pic)
|
|
panel=np.vstack([np.hstack(panels[:2]),np.hstack(panels[2:])]);w.stdin.write(panel.tobytes())
|
|
if t in [30,90,150,210,270]:cv2.imwrite(str(out/f'comparison_{t:04d}.jpg'),panel)
|
|
if t%60==0:print('render',t,flush=True)
|
|
w.stdin.close();assert w.wait()==0;cap.release();v=cv2.VideoCapture(str(video));n=0
|
|
while v.read()[0]:n+=1
|
|
assert n==N
|
|
(out/'video_validation.json').write_text(json.dumps({'decoded_frames':n,'fps':30,'width':1696,'height':960,'fixed_oblique_yaw_deg':55,'scope':'foreground left hand; depth-only translation, no pose/shape/smoothing changes'},indent=2));print('Video passed',n,flush=True)
|