liyang
ae28d55f81
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
40 lines
3.6 KiB
Python
40 lines
3.6 KiB
Python
"""Render, audit and package the red-box tracking result without rerunning inference."""
|
|
import sys,json,subprocess
|
|
from pathlib import Path
|
|
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'third_party/FoundationPose'))
|
|
import cv2,numpy as np,torch,trimesh,nvdiffrast.torch as dr
|
|
from scipy.spatial.transform import Rotation
|
|
from Utils import make_mesh_tensors,nvdiffrast_render
|
|
out=ROOT/'output/foundationpose_red_20260916';src=ROOT/'docs/20260916_104026'
|
|
a=np.load(out/'poses.npz');P=a['T_camera_from_object'];metrics=json.loads((out/'frame_metrics.json').read_text());N=len(P)
|
|
mesh=trimesh.load(ROOT/'docs/上半.stl');mt=make_mesh_tensors(mesh);ctx=dr.RasterizeCudaContext();cap=cv2.VideoCapture(str(src/'color.mp4'))
|
|
writer=subprocess.Popen(['ffmpeg','-y','-v','error','-f','rawvideo','-pix_fmt','bgr24','-s','1696x480','-r','30','-i','-','-an','-c:v','libx264','-preset','fast','-crf','20','-pix_fmt','yuv420p','-movflags','+faststart',str(out/'comparison.mp4')],stdin=subprocess.PIPE)
|
|
finite=np.isfinite(P).all((1,2));centers=P[:,:3,:3]@mesh.bounds.mean(0)+P[:,:3,3];steps=[];angles=[]
|
|
for i in range(N):
|
|
ok,b=cap.read();assert ok;pic=b.copy()
|
|
if finite[i]:
|
|
with torch.inference_mode():_,d,_=nvdiffrast_render(K=a['K'],H=480,W=848,ob_in_cams=torch.as_tensor(P[i:i+1],device='cuda',dtype=torch.float32),glctx=ctx,mesh_tensors=mt)
|
|
m=d[0].cpu().numpy()>0
|
|
pic[m]=(pic[m]*.65+np.array([0,220,0])*.35).astype('uint8')
|
|
contours,_=cv2.findContours(m.astype('uint8'),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE);cv2.drawContours(pic,contours,-1,(0,255,0),1)
|
|
if i and finite[i-1]:
|
|
steps.append([i,float(np.linalg.norm(centers[i]-centers[i-1]))]);angles.append([i,float(np.rad2deg(Rotation.from_matrix(P[i-1,:3,:3].T@P[i,:3,:3]).magnitude()))])
|
|
label=f'{i:04d} {i/30:.2f}s '+metrics[i]['status']
|
|
cv2.rectangle(pic,(0,0),(848,45),(0,0,0),-1);cv2.putText(pic,label,(12,29),0,.65,(0,255,255),2)
|
|
cv2.putText(b,'Original RGB',(12,29),0,.7,(0,255,255),2)
|
|
writer.stdin.write(np.hstack([b,pic]).tobytes())
|
|
writer.stdin.close();assert writer.wait()==0;cap.release()
|
|
# Independent checks: homogeneous matrices, SO(3), time synchronization, video decoding.
|
|
assert np.isfinite(P[finite]).all();assert np.allclose(P[finite,3,:],[0,0,0,1],atol=1e-5)
|
|
r=P[finite,:3,:3];ortho=float(np.max(np.abs(r.transpose(0,2,1)@r-np.eye(3))));det=float(np.max(np.abs(np.linalg.det(r)-1)));assert max(ortho,det)<1e-3
|
|
v=cv2.VideoCapture(str(out/'comparison.mp4'));decoded=0
|
|
while v.read()[0]:decoded+=1
|
|
v.release();assert decoded==N
|
|
meta=json.loads((src/'intrinsics.json').read_text());deptherr=[x['depth_median_error_m'] for x in metrics if x.get('depth_median_error_m') is not None]
|
|
valid=a['accepted'];ranges=[]
|
|
for i in np.flatnonzero(~valid):
|
|
if ranges and i==ranges[-1][1]+1:ranges[-1][1]=int(i)
|
|
else:ranges.append([int(i),int(i)])
|
|
report={'video_frames_decoded':decoded,'pose_matrices':int(finite.sum()),'quality_accepted':int(valid.sum()),'invalid_frame_ranges_inclusive':ranges,'rotation_orthogonality_max_error':ortho,'rotation_determinant_max_error':det,'color_depth_timestamp_difference_max_ms':float(np.max(np.abs(np.array(meta['timestamps_ms'])-meta['depth_timestamps_ms']))),'depth_residual_median_mm':float(np.median(deptherr)*1000),'largest_center_steps_m':sorted(steps,key=lambda x:x[1],reverse=True)[:8],'largest_rotation_steps_deg':sorted(angles,key=lambda x:x[1],reverse=True)[:8],'warning':'Residual measures consistency with input depth, not ground-truth pose error. Heuristic quality flags cannot resolve object symmetries or prove contact correctness.'}
|
|
(out/'validation.json').write_text(json.dumps(report,indent=2));print(json.dumps(report,indent=2))
|