"""Split / reassemble files larger than the hosting platform's 50 MB per-request limit. The Gitea server rejects any single upload above 50 MB (HTTP 413), so weights above that size are stored in the repository as 48 MB chunks .part-000, .part-001, ... (each chunk is a Git LFS object) together with configs/large_files.json (original path, size, sha256, chunk count). python3 scripts/large_files.py assemble # after clone: rebuild every original file, verify sha256 python3 scripts/large_files.py split ... # maintainer: chunk new large files and update the manifest python3 scripts/large_files.py check # verify assembled files against the manifest Chunks are left in place after assembly; delete them with `assemble --clean` if you need the space. """ import argparse, hashlib, json, sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / 'configs/large_files.json' CHUNK = 48 * 1024 * 1024 def sha256(path): h = hashlib.sha256() with path.open('rb') as f: for block in iter(lambda: f.read(1 << 24), b''): h.update(block) return h.hexdigest() def load(): return json.loads(MANIFEST.read_text()) if MANIFEST.exists() else {'chunk_bytes': CHUNK, 'files': []} def split(paths): m = load(); known = {e['path'] for e in m['files']} for p in paths: src = (ROOT / p).resolve(); rel = src.relative_to(ROOT).as_posix() if rel in known: print('already in manifest:', rel); continue n = 0 with src.open('rb') as f: while True: block = f.read(CHUNK) if not block: break (src.parent / f'{src.name}.part-{n:03d}').write_bytes(block); n += 1 m['files'].append({'path': rel, 'bytes': src.stat().st_size, 'sha256': sha256(src), 'parts': n}) print(f'split {rel}: {n} parts'); src.unlink() MANIFEST.write_text(json.dumps(m, indent=2) + '\n') def assemble(clean=False): m = load(); bad = 0 for e in m['files']: dst = ROOT / e['path']; parts = [dst.parent / f'{dst.name}.part-{i:03d}' for i in range(e['parts'])] if dst.exists() and dst.stat().st_size == e['bytes'] and sha256(dst) == e['sha256']: print('ok ', e['path']) else: missing = [p.name for p in parts if not p.exists() or p.stat().st_size < 200] if missing: print('MISSING chunks (run `git lfs pull` first):', e['path'], missing[:3], '...'); bad += 1; continue with dst.open('wb') as out: for p in parts: out.write(p.read_bytes()) if sha256(dst) != e['sha256']: print('SHA256 MISMATCH', e['path']); bad += 1; continue print('assembled', e['path'], f"({e['bytes'] / 1e6:.0f} MB, {e['parts']} parts)") if clean: for p in parts: p.unlink(missing_ok=True) if bad: sys.exit(f'{bad} file(s) could not be assembled') def check(): for e in load()['files']: dst = ROOT / e['path'] ok = dst.exists() and dst.stat().st_size == e['bytes'] and sha256(dst) == e['sha256'] print('ok ' if ok else 'BAD ', e['path']) if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest='cmd', required=True) s = sub.add_parser('split'); s.add_argument('paths', nargs='+') a = sub.add_parser('assemble'); a.add_argument('--clean', action='store_true') sub.add_parser('check') args = ap.parse_args() if args.cmd == 'split': split(args.paths) elif args.cmd == 'assemble': assemble(args.clean) else: check()