Files
hand-motion-pipeline/scripts/external_resources.py
T

99 lines
3.9 KiB
Python

#!/usr/bin/env python3
"""List upstream model sources or check locally acquired files; never download models."""
import argparse
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
LFS_HEADER = b'version https://git-lfs.github.com/spec/v1'
def load_resources(root):
data = json.loads((root / 'configs/external_resources.json').read_text())
for group in data['resources']:
for entry in group['files']:
relative = Path(entry['path'])
if relative.is_absolute() or '..' in relative.parts:
raise ValueError(f'Unsafe resource path: {relative}')
return data['resources']
def select_resources(root, scope='rgbd', group_ids=None):
groups = load_resources(root)
ids = set(group_ids or [])
unknown = ids - {g['id'] for g in groups}
if unknown:
raise ValueError(f'Unknown resource groups: {sorted(unknown)}')
return [g for g in groups if (g['id'] in ids if ids else scope == 'all' or g['scope'] == 'rgbd')]
def file_status(path, entry, verify_sha256=False):
if not path.is_file():
return 'missing'
if path.stat().st_size == 0:
return 'empty'
with path.open('rb') as stream:
if stream.read(len(LFS_HEADER)) == LFS_HEADER:
return 'lfs_pointer'
if 'bytes' in entry and path.stat().st_size != entry['bytes']:
return 'size_mismatch'
if verify_sha256 and entry.get('sha256'):
digest = hashlib.sha256()
with path.open('rb') as stream:
for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
digest.update(block)
if digest.hexdigest() != entry['sha256']:
return 'sha256_mismatch'
return 'present'
def check_resources(root, scope='rgbd', group_ids=None, verify_sha256=False):
rows = []
for group in select_resources(root, scope, group_ids):
for entry in group['files']:
rows.append(dict(group=group['id'], path=entry['path'],
status=file_status(root / entry['path'], entry, verify_sha256),
source_url=group['source_url']))
return rows
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=['list', 'check'])
parser.add_argument('--scope', choices=['rgbd', 'all'], default='rgbd')
parser.add_argument('--group', action='append', help='Select a group; repeat to select several.')
parser.add_argument('--verify-sha256', action='store_true', help='Compare with the reference snapshot, including large weights.')
parser.add_argument('--json', action='store_true')
args = parser.parse_args()
try:
if args.command == 'list':
groups = select_resources(ROOT, args.scope, args.group)
if args.json:
print(json.dumps(groups, indent=2))
else:
for group in groups:
print(f"[{group['id']}] {group['name']}\n {group['source_url']}")
if group.get('download_url'):
print(f" Download: {group['download_url']}")
print(f" {group['instructions']}")
for entry in group['files']:
print(f" -> {entry['path']}")
return 0
rows = check_resources(ROOT, args.scope, args.group, args.verify_sha256)
if args.json:
print(json.dumps(rows, indent=2))
else:
for row in rows:
print(f"{row['status']:16} {row['path']}")
if row['status'] != 'present':
print(f" Obtain from: {row['source_url']}")
print('Scope: local files only; model loading and full-pipeline execution require separate checks.')
return int(any(row['status'] != 'present' for row in rows))
except (OSError, ValueError, KeyError) as exc:
parser.exit(2, f'{exc}\n')
if __name__ == '__main__':
raise SystemExit(main())