99 lines
3.9 KiB
Python
99 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""List upstream model sources or check locally acquired files; never download models."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
LFS_HEADER = b'version https://git-lfs.github.com/spec/v1'
|
|
|
|
|
|
def load_resources(root):
|
|
data = json.loads((root / 'configs/external_resources.json').read_text())
|
|
for group in data['resources']:
|
|
for entry in group['files']:
|
|
relative = Path(entry['path'])
|
|
if relative.is_absolute() or '..' in relative.parts:
|
|
raise ValueError(f'Unsafe resource path: {relative}')
|
|
return data['resources']
|
|
|
|
|
|
def select_resources(root, scope='rgbd', group_ids=None):
|
|
groups = load_resources(root)
|
|
ids = set(group_ids or [])
|
|
unknown = ids - {g['id'] for g in groups}
|
|
if unknown:
|
|
raise ValueError(f'Unknown resource groups: {sorted(unknown)}')
|
|
return [g for g in groups if (g['id'] in ids if ids else scope == 'all' or g['scope'] == 'rgbd')]
|
|
|
|
|
|
def file_status(path, entry, verify_sha256=False):
|
|
if not path.is_file():
|
|
return 'missing'
|
|
if path.stat().st_size == 0:
|
|
return 'empty'
|
|
with path.open('rb') as stream:
|
|
if stream.read(len(LFS_HEADER)) == LFS_HEADER:
|
|
return 'lfs_pointer'
|
|
if 'bytes' in entry and path.stat().st_size != entry['bytes']:
|
|
return 'size_mismatch'
|
|
if verify_sha256 and entry.get('sha256'):
|
|
digest = hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
|
|
digest.update(block)
|
|
if digest.hexdigest() != entry['sha256']:
|
|
return 'sha256_mismatch'
|
|
return 'present'
|
|
|
|
|
|
def check_resources(root, scope='rgbd', group_ids=None, verify_sha256=False):
|
|
rows = []
|
|
for group in select_resources(root, scope, group_ids):
|
|
for entry in group['files']:
|
|
rows.append(dict(group=group['id'], path=entry['path'],
|
|
status=file_status(root / entry['path'], entry, verify_sha256),
|
|
source_url=group['source_url']))
|
|
return rows
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command', choices=['list', 'check'])
|
|
parser.add_argument('--scope', choices=['rgbd', 'all'], default='rgbd')
|
|
parser.add_argument('--group', action='append', help='Select a group; repeat to select several.')
|
|
parser.add_argument('--verify-sha256', action='store_true', help='Compare with the reference snapshot, including large weights.')
|
|
parser.add_argument('--json', action='store_true')
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == 'list':
|
|
groups = select_resources(ROOT, args.scope, args.group)
|
|
if args.json:
|
|
print(json.dumps(groups, indent=2))
|
|
else:
|
|
for group in groups:
|
|
print(f"[{group['id']}] {group['name']}\n {group['source_url']}")
|
|
if group.get('download_url'):
|
|
print(f" Download: {group['download_url']}")
|
|
print(f" {group['instructions']}")
|
|
for entry in group['files']:
|
|
print(f" -> {entry['path']}")
|
|
return 0
|
|
rows = check_resources(ROOT, args.scope, args.group, args.verify_sha256)
|
|
if args.json:
|
|
print(json.dumps(rows, indent=2))
|
|
else:
|
|
for row in rows:
|
|
print(f"{row['status']:16} {row['path']}")
|
|
if row['status'] != 'present':
|
|
print(f" Obtain from: {row['source_url']}")
|
|
print('Scope: local files only; model loading and full-pipeline execution require separate checks.')
|
|
return int(any(row['status'] != 'present' for row in rows))
|
|
except (OSError, ValueError, KeyError) as exc:
|
|
parser.exit(2, f'{exc}\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|