liyang
ae28d55f81
Source: RGB-D -> Dyn-HaMR -> L20 retargeting -> FoundationPose -> reference repair -> SPIDER, documented in docs/PIPELINE_LATEST.md and docs/SETUP_AND_WEIGHTS.md. Adds FoundationPose and nvdiffrast upstream snapshots, requirements/pipeline_venv.txt and the FoundationPose weight manifest/downloader. Assets (Git LFS): weights/ (WiLoR detector, HandFlow denoiser, UniDepth-L), FoundationPose checkpoints, HaMeR checkpoint, Dyn-HaMR HMP model and BMC constraints, L20 URDF/meshes, the 20260915_171525 D405 recording and the two box CADs. MANO models are not redistributed (third_party/hamer/_DATA/data/mano/README.txt). Environments, caches and run outputs excluded. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
87 lines
3.6 KiB
Python
87 lines
3.6 KiB
Python
"""Split / reassemble files larger than the hosting platform's 50 MB per-request limit.
|
|
|
|
The Gitea server rejects any single upload above 50 MB (HTTP 413), so weights above that size are stored in
|
|
the repository as 48 MB chunks <file>.part-000, .part-001, ... (each chunk is a Git LFS object) together with
|
|
configs/large_files.json (original path, size, sha256, chunk count).
|
|
|
|
python3 scripts/large_files.py assemble # after clone: rebuild every original file, verify sha256
|
|
python3 scripts/large_files.py split <path>... # maintainer: chunk new large files and update the manifest
|
|
python3 scripts/large_files.py check # verify assembled files against the manifest
|
|
|
|
Chunks are left in place after assembly; delete them with `assemble --clean` if you need the space.
|
|
"""
|
|
import argparse, hashlib, json, sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MANIFEST = ROOT / 'configs/large_files.json'
|
|
CHUNK = 48 * 1024 * 1024
|
|
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with path.open('rb') as f:
|
|
for block in iter(lambda: f.read(1 << 24), b''):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load():
|
|
return json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {'chunk_bytes': CHUNK, 'files': []}
|
|
|
|
|
|
def split(paths):
|
|
m = load(); known = {e['path'] for e in m['files']}
|
|
for p in paths:
|
|
src = (ROOT / p).resolve(); rel = src.relative_to(ROOT).as_posix()
|
|
if rel in known:
|
|
print('already in manifest:', rel); continue
|
|
n = 0
|
|
with src.open('rb') as f:
|
|
while True:
|
|
block = f.read(CHUNK)
|
|
if not block: break
|
|
(src.parent / f'{src.name}.part-{n:03d}').write_bytes(block); n += 1
|
|
m['files'].append({'path': rel, 'bytes': src.stat().st_size, 'sha256': sha256(src), 'parts': n})
|
|
print(f'split {rel}: {n} parts'); src.unlink()
|
|
MANIFEST.write_text(json.dumps(m, indent=2) + '\n')
|
|
|
|
|
|
def assemble(clean=False):
|
|
m = load(); bad = 0
|
|
for e in m['files']:
|
|
dst = ROOT / e['path']; parts = [dst.parent / f'{dst.name}.part-{i:03d}' for i in range(e['parts'])]
|
|
if dst.exists() and dst.stat().st_size == e['bytes'] and sha256(dst) == e['sha256']:
|
|
print('ok ', e['path'])
|
|
else:
|
|
missing = [p.name for p in parts if not p.exists() or p.stat().st_size < 200]
|
|
if missing:
|
|
print('MISSING chunks (run `git lfs pull` first):', e['path'], missing[:3], '...'); bad += 1; continue
|
|
with dst.open('wb') as out:
|
|
for p in parts: out.write(p.read_bytes())
|
|
if sha256(dst) != e['sha256']:
|
|
print('SHA256 MISMATCH', e['path']); bad += 1; continue
|
|
print('assembled', e['path'], f"({e['bytes'] / 1e6:.0f} MB, {e['parts']} parts)")
|
|
if clean:
|
|
for p in parts: p.unlink(missing_ok=True)
|
|
if bad: sys.exit(f'{bad} file(s) could not be assembled')
|
|
|
|
|
|
def check():
|
|
for e in load()['files']:
|
|
dst = ROOT / e['path']
|
|
ok = dst.exists() and dst.stat().st_size == e['bytes'] and sha256(dst) == e['sha256']
|
|
print('ok ' if ok else 'BAD ', e['path'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest='cmd', required=True)
|
|
s = sub.add_parser('split'); s.add_argument('paths', nargs='+')
|
|
a = sub.add_parser('assemble'); a.add_argument('--clean', action='store_true')
|
|
sub.add_parser('check')
|
|
args = ap.parse_args()
|
|
if args.cmd == 'split': split(args.paths)
|
|
elif args.cmd == 'assemble': assemble(args.clean)
|
|
else: check()
|