Files
community/scripts/migrate-gitea-org.py
T

464 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""按组织复制 Git 仓库及 LFS;默认只读预览,不修改业务数据库。"""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import getpass
import json
import os
from pathlib import Path
import re
import sys
import tempfile
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode, urlsplit, urlunsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
class MigrationError(Exception):
"""仅包含可安全记录的错误码,不携带响应正文或凭据。"""
def __init__(self, code: str):
super().__init__(code)
self.code = code
def timestamp() -> str:
return datetime.now(timezone.utc).isoformat()
def name_segment(value: str) -> str:
if (not isinstance(value, str) or not value or value in (".", "..")
or len(value) > 255 or any(char in value for char in "/\\?#%")
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)):
raise MigrationError("INVALID_NAME")
return quote(value, safe="")
def base_url(value: str, allow_http: bool) -> str:
try:
parsed = urlsplit(value)
port = parsed.port
if (parsed.scheme not in ("https", "http") or not parsed.hostname
or (parsed.scheme == "http" and not allow_http)
or parsed.username is not None or parsed.password is not None
or "?" in value or "#" in value or "\\" in value
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
or (port is not None and not 1 <= port <= 65535)
or any(part in (".", "..") for part in parsed.path.split("/"))
or "%" in parsed.path):
raise ValueError()
host = parsed.hostname.lower()
if ":" in host:
host = "[" + host + "]"
if port is not None and port != {"http": 80, "https": 443}[parsed.scheme]:
host += ":" + str(port)
return urlunsplit((parsed.scheme, host, parsed.path.rstrip("/"), "", ""))
except ValueError:
raise MigrationError("INVALID_BASE_URL") from None
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
# 避免重定向转发认证头或再次发送迁移请求。
return None
class ApiClient:
def __init__(self, base: str, token: str, timeout: int = 30):
self.base = base
self.token = token
self.timeout = timeout
self.opener = build_opener(NoRedirect())
self.username = ""
def request(self, method: str, path: str, body: dict | None = None,
timeout: int | None = None, missing: bool = False) -> tuple[int, object]:
request = Request(self.base + "/api/v1" + path, method=method,
data=None if body is None else json.dumps(body).encode("utf-8"),
headers={"Authorization": "token " + self.token,
"Accept": "application/json",
"Content-Type": "application/json"})
try:
with self.opener.open(request, timeout=timeout or self.timeout) as response:
status = response.status
data = response.read(16 * 1024 * 1024 + 1)
if len(data) > 16 * 1024 * 1024:
raise MigrationError("RESPONSE_TOO_LARGE")
return status, json.loads(data) if data else None
except HTTPError as error:
status = error.code
error.close()
if missing and status == 404:
return status, None
# Gitea/反向代理的错误正文可能包含源地址或认证信息,不输出正文。
raise MigrationError("HTTP_" + str(status)) from None
except (URLError, TimeoutError, OSError):
raise MigrationError("NETWORK_ERROR") from None
except (ValueError, UnicodeError):
raise MigrationError("INVALID_JSON_RESPONSE") from None
def get(self, path: str, missing: bool = False):
status, data = self.request("GET", path, missing=missing)
if status == 404 and missing:
return None
if status != 200:
raise MigrationError("UNEXPECTED_GET_STATUS")
return data
def pages(self, path: str) -> list[dict]:
result = []
seen = set()
for page in range(1, 10001):
items = self.get(path + "?" + urlencode({"page": page, "limit": 50}))
if not isinstance(items, list):
raise MigrationError("INVALID_LIST_RESPONSE")
if not items:
return result
for item in items:
if not isinstance(item, dict):
raise MigrationError("INVALID_LIST_ITEM")
# 多个轻量标签可以指向同一个提交(相同 id),分页应按名称去重。
key = item.get("name", item.get("id"))
if not isinstance(key, (str, int)) or key in seen:
raise MigrationError("DUPLICATE_PAGINATION_ITEM")
seen.add(key)
result.append(item)
# 服务端可能将 limit 限制为更小值,不能按 len(items) < 50 提前终止。
raise MigrationError("PAGINATION_LIMIT")
def repo(self, org: str, name: str, missing: bool = False):
return self.get("/repos/" + name_segment(org) + "/" + name_segment(name), missing)
def clone_url(self, org: str, name: str) -> str:
return self.base + "/" + name_segment(org) + "/" + name_segment(name) + ".git"
def identity(info: dict, org: str, name: str) -> int:
if (not isinstance(info, dict) or type(info.get("id")) is not int or info["id"] <= 0
or not isinstance(info.get("owner"), dict)
or not isinstance(info["owner"].get("login"), str)
or info["owner"]["login"].casefold() != org.casefold()
or info.get("name") != name):
raise MigrationError("REPOSITORY_IDENTITY_MISMATCH")
return info["id"]
def preflight(client: ApiClient, org: str, target: bool) -> None:
user = client.get("/user")
if not isinstance(user, dict) or not isinstance(user.get("login"), str):
raise MigrationError("INVALID_USER_RESPONSE")
client.username = user["login"]
name_segment(client.username)
client.get("/orgs/" + name_segment(org))
# 源组织所有者能列出全部私有仓库;迁移 API 也要求目标组织所有者。
if user.get("is_admin") is not True:
owner = False
for team in client.pages("/orgs/" + name_segment(org) + "/teams"):
if team.get("permission") == "owner" and type(team.get("id")) is int:
status, member = client.request("GET", "/teams/" + str(team["id"])
+ "/members/" + name_segment(client.username), missing=True)
if (status == 200 and isinstance(member, dict)
and member.get("login") == client.username):
owner = True
break
if not owner:
raise MigrationError("ORGANIZATION_OWNER_REQUIRED")
settings = client.get("/settings/repository")
if (not isinstance(settings, dict) or settings.get("lfs_disabled") is not False
or settings.get("http_git_disabled") is not False
or (target and settings.get("migrations_disabled") is not False)):
raise MigrationError("GIT_LFS_OR_MIGRATION_DISABLED")
def snapshot(client: ApiClient, org: str, name: str, expected_id: int,
raw_references: bool = False) -> dict:
info = client.repo(org, name)
if identity(info, org, name) != expected_id:
raise MigrationError("REPOSITORY_ID_CHANGED")
if type(info.get("empty")) is not bool or not isinstance(info.get("default_branch"), str):
raise MigrationError("INVALID_REPOSITORY_RESPONSE")
result = {"empty": info["empty"], "default_branch": info["default_branch"],
"branches": {}, "tags": {}}
path = "/repos/" + name_segment(org) + "/" + name_segment(name)
if info["empty"] or raw_references:
# 历史导入中断后,数据库可能标记为空,但磁盘已存在 Git 引用。
for prefix, unit in (("heads", "branches"), ("tags", "tags")):
refs = client.get(path + "/git/refs/" + prefix, missing=True)
if refs is None:
continue
if not isinstance(refs, list):
raise MigrationError("INVALID_GIT_REFERENCE")
for ref in refs:
ref_path = ref.get("ref", "")
sha = ref.get("object", {}).get("sha")
expected_prefix = "refs/" + prefix + "/"
if (not isinstance(ref_path, str) or not ref_path.startswith(expected_prefix)
or not ref_path[len(expected_prefix):]
or not isinstance(sha, str)
or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)):
raise MigrationError("INVALID_GIT_REFERENCE")
ref_name = ref_path[len(expected_prefix):]
if ref_name in result[unit]:
raise MigrationError("INVALID_GIT_REFERENCE")
result[unit][ref_name] = sha
if result["branches"] or result["tags"] or raw_references:
result["empty"] = not (result["branches"] or result["tags"])
result["reference_format"] = "git-object-v1"
if not result["empty"] and result["default_branch"] not in result["branches"]:
raise MigrationError("DEFAULT_BRANCH_MISSING")
return result
for unit in ("branches", "tags"):
for ref in client.pages(path + "/" + unit):
commit = ref.get("commit")
sha = (commit.get("id") or commit.get("sha")) if isinstance(commit, dict) else None
ref_name = ref.get("name")
if (not isinstance(ref_name, str) or not ref_name
or not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)
or ref_name in result[unit]):
raise MigrationError("INVALID_GIT_REFERENCE")
result[unit][ref_name] = sha
if result["default_branch"] not in result["branches"]:
raise MigrationError("DEFAULT_BRANCH_MISSING")
return result
def atomic_json(path: Path, value: dict) -> None:
descriptor, temporary = tempfile.mkstemp(prefix=".migration-", dir=path.parent)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
Path(temporary).unlink(missing_ok=True)
class StateStore:
def __init__(self, directory: Path):
self.directory = directory
self.path = directory / "state.json"
@contextmanager
def locked(self):
if self.directory.is_symlink():
raise MigrationError("INVALID_STATE_DIRECTORY")
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
self.directory.chmod(0o700)
with (self.directory / ".lock").open("a") as stream:
os.fchmod(stream.fileno(), 0o600)
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise MigrationError("STATE_DIRECTORY_LOCKED") from None
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def load(self, binding: dict, inventory: dict) -> dict:
if self.path.exists():
state = json.loads(self.path.read_text(encoding="utf-8"))
if state.get("version") != 1 or state.get("binding") != binding:
raise MigrationError("STATE_ENVIRONMENT_MISMATCH")
if state.get("inventory") != inventory:
raise MigrationError("SOURCE_INVENTORY_CHANGED")
for entry in state["repositories"].values():
if entry["status"] == "MIGRATING":
entry.update(status="REVIEW", error="INTERRUPTED_MIGRATION")
else:
state = {"version": 1, "created_at": timestamp(), "binding": binding,
"inventory": inventory,
"repositories": {name: {"source_id": repo_id, "status": "PLANNED"}
for name, repo_id in inventory.items()}}
self.save(state)
return state
def save(self, state: dict) -> None:
state["updated_at"] = timestamp()
atomic_json(self.path, state)
mappings = []
for name, entry in state["repositories"].items():
if entry["status"] != "VERIFIED":
continue
code = name[2:] if re.fullmatch(r"p_(?:[a-z0-9]{12}|[a-z0-9]{32})", name) else None
mappings.append({"repo_name": name, "project_code": code,
"source_repo_id": entry["source_id"],
"target_repo_id": entry["receipt_id"],
"default_branch": entry["snapshot"]["default_branch"],
"lfs_requested": True,
"lfs_objects_download_verified": entry.get("lfs_objects_download_verified", False),
"completion_evidence": entry.get("completion_evidence", "migration_http_201")})
atomic_json(self.directory / "mapping.json",
{"binding": state["binding"], "repositories": mappings})
def verify(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
name: str, entry: dict) -> None:
info = target.repo(target_org, name)
if identity(info, target_org, name) != entry["receipt_id"]:
raise MigrationError("TARGET_RECEIPT_ID_MISMATCH")
if info.get("private") is not True or info.get("mirror") is not False:
raise MigrationError("TARGET_PRIVACY_OR_MIRROR_MISMATCH")
raw_references = entry["snapshot"].get("reference_format") == "git-object-v1"
if snapshot(target, target_org, name, entry["receipt_id"], raw_references) != entry["snapshot"]:
raise MigrationError("TARGET_GIT_REFERENCES_MISMATCH")
if snapshot(source, source_org, name, entry["source_id"], raw_references) != entry["snapshot"]:
raise MigrationError("SOURCE_CHANGED_DURING_MIGRATION")
def migrate(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
store: StateStore, execute: bool, selected: list[str], limit: int | None,
migration_timeout: int) -> int:
preflight(source, source_org, target=False)
preflight(target, target_org, target=True)
inventory = {}
for info in source.pages("/orgs/" + name_segment(source_org) + "/repos"):
name = info.get("name")
name_segment(name)
repo_id = identity(info, source_org, name)
if name in inventory:
raise MigrationError("DUPLICATE_REPOSITORY_NAME")
inventory[name] = repo_id
inventory = dict(sorted(inventory.items()))
if any(name not in inventory for name in selected):
raise MigrationError("SELECTED_REPOSITORY_NOT_IN_SOURCE_ORG")
binding = {"source_url": source.base, "source_org": source_org,
"target_url": target.base, "target_org": target_org,
"contents": "private-git-and-lfs-v1"}
state = store.load(binding, inventory)
print(f"源组织={source_org} 仓库数={len(inventory)} 目标组织={target_org} 模式={'正式执行' if execute else '只读预览'}")
processed = 0
for name, entry in state["repositories"].items():
if selected and name not in selected:
continue
if entry.get("receipt_id") is not None:
try:
verify(source, target, source_org, target_org, name, entry)
entry.update(status="VERIFIED", error=None)
print(f"已核对 {name}:完成收据、私有属性、默认分支、分支及标签提交一致")
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
print(f"待核对 {name}:{error.code};已保存完成收据,仅重试只读核验")
store.save(state)
continue
if entry["status"] != "PLANNED":
print(f"跳过 {name}:{entry['status']};没有可核对的完成收据,不自动重发")
continue
if limit is not None and processed >= limit:
continue
processed += 1
if target.repo(target_org, name, missing=True) is not None:
entry.update(status="CONFLICT", error="TARGET_ALREADY_EXISTS")
store.save(state)
print(f"同名冲突 {name}:保留生产仓库,不接管、不覆盖")
continue
frozen = snapshot(source, source_org, name, entry["source_id"])
if not execute:
print(f"待迁移 {name}:分支={len(frozen['branches'])} 标签={len(frozen['tags'])} LFS=请求迁移")
continue
# 在发送 POST 前强制落盘。超时、进程退出后不能仅凭仓库存在认定迁移完成。
entry.update(status="MIGRATING", snapshot=frozen, error=None)
store.save(state)
print(f"开始迁移 {name}:{source_org} -> {target_org}", flush=True)
body = {"clone_addr": source.clone_url(source_org, name),
"repo_owner": target_org, "repo_name": name, "service": "git",
"auth_username": source.username, "auth_password": source.token,
"private": True, "mirror": False, "lfs": True,
**{unit: False for unit in ("issues", "labels", "milestones",
"pull_requests", "releases", "wiki")}}
try:
status, receipt = target.request("POST", "/repos/migrate", body,
timeout=migration_timeout)
if status != 201:
raise MigrationError("MIGRATION_COMPLETION_NOT_CONFIRMED")
entry["receipt_id"] = identity(receipt, target_org, name)
entry["status"] = "RECEIVED"
store.save(state)
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
store.save(state)
print(f"迁移未确认 {name}:{error.code};本次停止新增仓库,请检查生产迁移日志")
break
try:
verify(source, target, source_org, target_org, name, entry)
entry.update(status="VERIFIED", error=None)
print(f"迁移完成 {name}:Git 引用已核对,LFS 已请求迁移,实体下载仍需抽查")
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
print(f"待核对 {name}:{error.code};保留已创建仓库和完成收据")
store.save(state)
entries = [entry for name, entry in state["repositories"].items()
if not selected or name in selected]
counts = {status: sum(entry["status"] == status for entry in entries)
for status in ("PLANNED", "VERIFIED", "CONFLICT", "REVIEW")}
print("处理结果 " + " ".join(f"{key}={value}" for key, value in counts.items()))
print("状态及映射目录:" + str(store.directory))
return 2 if counts["CONFLICT"] or counts["REVIEW"] else 0
def read_token(variable: str, prompt: str) -> str:
token = os.environ.get(variable, "")
if not token and sys.stdin.isatty():
token = getpass.getpass(prompt)
if not token or any(char.isspace() or ord(char) < 32 for char in token):
raise MigrationError("MISSING_OR_INVALID_" + variable)
return token
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-url", required=True, help="测试 Gitea 基础地址,不含 /api/v1")
parser.add_argument("--target-url", required=True, help="生产 Gitea 基础地址,不含 /api/v1")
parser.add_argument("--source-org", default="vibemake")
parser.add_argument("--target-org", default="vibemake")
parser.add_argument("--state-dir", type=Path, default=Path("logs/gitea-org-migration"))
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--dry-run", action="store_true", help="只读预览,默认模式")
mode.add_argument("--execute", action="store_true", help="正式创建目标仓库")
parser.add_argument("--repo", action="append", default=[], help="只处理指定仓库,可重复指定")
parser.add_argument("--limit", type=int, help="限制本次尚未处理的仓库数")
parser.add_argument("--migration-timeout", type=int, default=1800, help="单仓库迁移请求超时秒数")
parser.add_argument("--allow-http", action="store_true", help="允许在可信网络通过明文 HTTP 发送凭据")
args = parser.parse_args()
try:
source_url = base_url(args.source_url, args.allow_http)
target_url = base_url(args.target_url, args.allow_http)
if source_url == target_url:
raise MigrationError("SOURCE_AND_TARGET_MUST_DIFFER")
for value in (args.source_org, args.target_org, *args.repo):
name_segment(value)
if (args.limit is not None and args.limit < 1) or args.migration_timeout < 1:
raise MigrationError("INVALID_LIMIT_OR_TIMEOUT")
source_token = read_token("GITEA_MIGRATION_SOURCE_TOKEN", "测试 Gitea Token(输入不回显):")
target_token = read_token("GITEA_MIGRATION_TARGET_TOKEN", "生产 Gitea Token(输入不回显):")
with StateStore(args.state_dir).locked():
return migrate(ApiClient(source_url, source_token), ApiClient(target_url, target_token),
args.source_org, args.target_org, StateStore(args.state_dir),
args.execute, args.repo, args.limit, args.migration_timeout)
except MigrationError as error:
print("操作停止:" + error.code, file=sys.stderr)
return 1
except KeyboardInterrupt:
print("操作已中断。已发出的迁移可能仍在生产执行;保留状态目录后再核对。", file=sys.stderr)
return 130
except Exception as error:
# 不输出异常正文,避免底层网络库或无效输入携带敏感值。
print("操作停止:" + type(error).__name__ + ";检查状态文件和运行环境。", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())