464 lines
23 KiB
Python
464 lines
23 KiB
Python
#!/usr/bin/env python3
|
||
"""按组织复制 Git 仓库及 LFS;默认只读预览,不修改业务数据库。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
from contextlib import contextmanager
|
||
from datetime import datetime, timezone
|
||
import fcntl
|
||
import getpass
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import sys
|
||
import tempfile
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.parse import quote, urlencode, urlsplit, urlunsplit
|
||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||
|
||
|
||
class MigrationError(Exception):
|
||
"""仅包含可安全记录的错误码,不携带响应正文或凭据。"""
|
||
|
||
def __init__(self, code: str):
|
||
super().__init__(code)
|
||
self.code = code
|
||
|
||
|
||
def timestamp() -> str:
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
|
||
def name_segment(value: str) -> str:
|
||
if (not isinstance(value, str) or not value or value in (".", "..")
|
||
or len(value) > 255 or any(char in value for char in "/\\?#%")
|
||
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)):
|
||
raise MigrationError("INVALID_NAME")
|
||
return quote(value, safe="")
|
||
|
||
|
||
def base_url(value: str, allow_http: bool) -> str:
|
||
try:
|
||
parsed = urlsplit(value)
|
||
port = parsed.port
|
||
if (parsed.scheme not in ("https", "http") or not parsed.hostname
|
||
or (parsed.scheme == "http" and not allow_http)
|
||
or parsed.username is not None or parsed.password is not None
|
||
or "?" in value or "#" in value or "\\" in value
|
||
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
|
||
or (port is not None and not 1 <= port <= 65535)
|
||
or any(part in (".", "..") for part in parsed.path.split("/"))
|
||
or "%" in parsed.path):
|
||
raise ValueError()
|
||
host = parsed.hostname.lower()
|
||
if ":" in host:
|
||
host = "[" + host + "]"
|
||
if port is not None and port != {"http": 80, "https": 443}[parsed.scheme]:
|
||
host += ":" + str(port)
|
||
return urlunsplit((parsed.scheme, host, parsed.path.rstrip("/"), "", ""))
|
||
except ValueError:
|
||
raise MigrationError("INVALID_BASE_URL") from None
|
||
|
||
|
||
class NoRedirect(HTTPRedirectHandler):
|
||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||
# 避免重定向转发认证头或再次发送迁移请求。
|
||
return None
|
||
|
||
|
||
class ApiClient:
|
||
def __init__(self, base: str, token: str, timeout: int = 30):
|
||
self.base = base
|
||
self.token = token
|
||
self.timeout = timeout
|
||
self.opener = build_opener(NoRedirect())
|
||
self.username = ""
|
||
|
||
def request(self, method: str, path: str, body: dict | None = None,
|
||
timeout: int | None = None, missing: bool = False) -> tuple[int, object]:
|
||
request = Request(self.base + "/api/v1" + path, method=method,
|
||
data=None if body is None else json.dumps(body).encode("utf-8"),
|
||
headers={"Authorization": "token " + self.token,
|
||
"Accept": "application/json",
|
||
"Content-Type": "application/json"})
|
||
try:
|
||
with self.opener.open(request, timeout=timeout or self.timeout) as response:
|
||
status = response.status
|
||
data = response.read(16 * 1024 * 1024 + 1)
|
||
if len(data) > 16 * 1024 * 1024:
|
||
raise MigrationError("RESPONSE_TOO_LARGE")
|
||
return status, json.loads(data) if data else None
|
||
except HTTPError as error:
|
||
status = error.code
|
||
error.close()
|
||
if missing and status == 404:
|
||
return status, None
|
||
# Gitea/反向代理的错误正文可能包含源地址或认证信息,不输出正文。
|
||
raise MigrationError("HTTP_" + str(status)) from None
|
||
except (URLError, TimeoutError, OSError):
|
||
raise MigrationError("NETWORK_ERROR") from None
|
||
except (ValueError, UnicodeError):
|
||
raise MigrationError("INVALID_JSON_RESPONSE") from None
|
||
|
||
def get(self, path: str, missing: bool = False):
|
||
status, data = self.request("GET", path, missing=missing)
|
||
if status == 404 and missing:
|
||
return None
|
||
if status != 200:
|
||
raise MigrationError("UNEXPECTED_GET_STATUS")
|
||
return data
|
||
|
||
def pages(self, path: str) -> list[dict]:
|
||
result = []
|
||
seen = set()
|
||
for page in range(1, 10001):
|
||
items = self.get(path + "?" + urlencode({"page": page, "limit": 50}))
|
||
if not isinstance(items, list):
|
||
raise MigrationError("INVALID_LIST_RESPONSE")
|
||
if not items:
|
||
return result
|
||
for item in items:
|
||
if not isinstance(item, dict):
|
||
raise MigrationError("INVALID_LIST_ITEM")
|
||
# 多个轻量标签可以指向同一个提交(相同 id),分页应按名称去重。
|
||
key = item.get("name", item.get("id"))
|
||
if not isinstance(key, (str, int)) or key in seen:
|
||
raise MigrationError("DUPLICATE_PAGINATION_ITEM")
|
||
seen.add(key)
|
||
result.append(item)
|
||
# 服务端可能将 limit 限制为更小值,不能按 len(items) < 50 提前终止。
|
||
raise MigrationError("PAGINATION_LIMIT")
|
||
|
||
def repo(self, org: str, name: str, missing: bool = False):
|
||
return self.get("/repos/" + name_segment(org) + "/" + name_segment(name), missing)
|
||
|
||
def clone_url(self, org: str, name: str) -> str:
|
||
return self.base + "/" + name_segment(org) + "/" + name_segment(name) + ".git"
|
||
|
||
|
||
def identity(info: dict, org: str, name: str) -> int:
|
||
if (not isinstance(info, dict) or type(info.get("id")) is not int or info["id"] <= 0
|
||
or not isinstance(info.get("owner"), dict)
|
||
or not isinstance(info["owner"].get("login"), str)
|
||
or info["owner"]["login"].casefold() != org.casefold()
|
||
or info.get("name") != name):
|
||
raise MigrationError("REPOSITORY_IDENTITY_MISMATCH")
|
||
return info["id"]
|
||
|
||
|
||
def preflight(client: ApiClient, org: str, target: bool) -> None:
|
||
user = client.get("/user")
|
||
if not isinstance(user, dict) or not isinstance(user.get("login"), str):
|
||
raise MigrationError("INVALID_USER_RESPONSE")
|
||
client.username = user["login"]
|
||
name_segment(client.username)
|
||
client.get("/orgs/" + name_segment(org))
|
||
# 源组织所有者能列出全部私有仓库;迁移 API 也要求目标组织所有者。
|
||
if user.get("is_admin") is not True:
|
||
owner = False
|
||
for team in client.pages("/orgs/" + name_segment(org) + "/teams"):
|
||
if team.get("permission") == "owner" and type(team.get("id")) is int:
|
||
status, member = client.request("GET", "/teams/" + str(team["id"])
|
||
+ "/members/" + name_segment(client.username), missing=True)
|
||
if (status == 200 and isinstance(member, dict)
|
||
and member.get("login") == client.username):
|
||
owner = True
|
||
break
|
||
if not owner:
|
||
raise MigrationError("ORGANIZATION_OWNER_REQUIRED")
|
||
settings = client.get("/settings/repository")
|
||
if (not isinstance(settings, dict) or settings.get("lfs_disabled") is not False
|
||
or settings.get("http_git_disabled") is not False
|
||
or (target and settings.get("migrations_disabled") is not False)):
|
||
raise MigrationError("GIT_LFS_OR_MIGRATION_DISABLED")
|
||
|
||
|
||
def snapshot(client: ApiClient, org: str, name: str, expected_id: int,
|
||
raw_references: bool = False) -> dict:
|
||
info = client.repo(org, name)
|
||
if identity(info, org, name) != expected_id:
|
||
raise MigrationError("REPOSITORY_ID_CHANGED")
|
||
if type(info.get("empty")) is not bool or not isinstance(info.get("default_branch"), str):
|
||
raise MigrationError("INVALID_REPOSITORY_RESPONSE")
|
||
result = {"empty": info["empty"], "default_branch": info["default_branch"],
|
||
"branches": {}, "tags": {}}
|
||
path = "/repos/" + name_segment(org) + "/" + name_segment(name)
|
||
if info["empty"] or raw_references:
|
||
# 历史导入中断后,数据库可能标记为空,但磁盘已存在 Git 引用。
|
||
for prefix, unit in (("heads", "branches"), ("tags", "tags")):
|
||
refs = client.get(path + "/git/refs/" + prefix, missing=True)
|
||
if refs is None:
|
||
continue
|
||
if not isinstance(refs, list):
|
||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||
for ref in refs:
|
||
ref_path = ref.get("ref", "")
|
||
sha = ref.get("object", {}).get("sha")
|
||
expected_prefix = "refs/" + prefix + "/"
|
||
if (not isinstance(ref_path, str) or not ref_path.startswith(expected_prefix)
|
||
or not ref_path[len(expected_prefix):]
|
||
or not isinstance(sha, str)
|
||
or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)):
|
||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||
ref_name = ref_path[len(expected_prefix):]
|
||
if ref_name in result[unit]:
|
||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||
result[unit][ref_name] = sha
|
||
if result["branches"] or result["tags"] or raw_references:
|
||
result["empty"] = not (result["branches"] or result["tags"])
|
||
result["reference_format"] = "git-object-v1"
|
||
if not result["empty"] and result["default_branch"] not in result["branches"]:
|
||
raise MigrationError("DEFAULT_BRANCH_MISSING")
|
||
return result
|
||
for unit in ("branches", "tags"):
|
||
for ref in client.pages(path + "/" + unit):
|
||
commit = ref.get("commit")
|
||
sha = (commit.get("id") or commit.get("sha")) if isinstance(commit, dict) else None
|
||
ref_name = ref.get("name")
|
||
if (not isinstance(ref_name, str) or not ref_name
|
||
or not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)
|
||
or ref_name in result[unit]):
|
||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||
result[unit][ref_name] = sha
|
||
if result["default_branch"] not in result["branches"]:
|
||
raise MigrationError("DEFAULT_BRANCH_MISSING")
|
||
return result
|
||
|
||
|
||
def atomic_json(path: Path, value: dict) -> None:
|
||
descriptor, temporary = tempfile.mkstemp(prefix=".migration-", dir=path.parent)
|
||
try:
|
||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||
stream.write("\n")
|
||
stream.flush()
|
||
os.fsync(stream.fileno())
|
||
os.replace(temporary, path)
|
||
directory = os.open(path.parent, os.O_RDONLY)
|
||
try:
|
||
os.fsync(directory)
|
||
finally:
|
||
os.close(directory)
|
||
finally:
|
||
Path(temporary).unlink(missing_ok=True)
|
||
|
||
|
||
class StateStore:
|
||
def __init__(self, directory: Path):
|
||
self.directory = directory
|
||
self.path = directory / "state.json"
|
||
|
||
@contextmanager
|
||
def locked(self):
|
||
if self.directory.is_symlink():
|
||
raise MigrationError("INVALID_STATE_DIRECTORY")
|
||
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
self.directory.chmod(0o700)
|
||
with (self.directory / ".lock").open("a") as stream:
|
||
os.fchmod(stream.fileno(), 0o600)
|
||
try:
|
||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
raise MigrationError("STATE_DIRECTORY_LOCKED") from None
|
||
try:
|
||
yield
|
||
finally:
|
||
fcntl.flock(stream, fcntl.LOCK_UN)
|
||
|
||
def load(self, binding: dict, inventory: dict) -> dict:
|
||
if self.path.exists():
|
||
state = json.loads(self.path.read_text(encoding="utf-8"))
|
||
if state.get("version") != 1 or state.get("binding") != binding:
|
||
raise MigrationError("STATE_ENVIRONMENT_MISMATCH")
|
||
if state.get("inventory") != inventory:
|
||
raise MigrationError("SOURCE_INVENTORY_CHANGED")
|
||
for entry in state["repositories"].values():
|
||
if entry["status"] == "MIGRATING":
|
||
entry.update(status="REVIEW", error="INTERRUPTED_MIGRATION")
|
||
else:
|
||
state = {"version": 1, "created_at": timestamp(), "binding": binding,
|
||
"inventory": inventory,
|
||
"repositories": {name: {"source_id": repo_id, "status": "PLANNED"}
|
||
for name, repo_id in inventory.items()}}
|
||
self.save(state)
|
||
return state
|
||
|
||
def save(self, state: dict) -> None:
|
||
state["updated_at"] = timestamp()
|
||
atomic_json(self.path, state)
|
||
mappings = []
|
||
for name, entry in state["repositories"].items():
|
||
if entry["status"] != "VERIFIED":
|
||
continue
|
||
code = name[2:] if re.fullmatch(r"p_(?:[a-z0-9]{12}|[a-z0-9]{32})", name) else None
|
||
mappings.append({"repo_name": name, "project_code": code,
|
||
"source_repo_id": entry["source_id"],
|
||
"target_repo_id": entry["receipt_id"],
|
||
"default_branch": entry["snapshot"]["default_branch"],
|
||
"lfs_requested": True,
|
||
"lfs_objects_download_verified": entry.get("lfs_objects_download_verified", False),
|
||
"completion_evidence": entry.get("completion_evidence", "migration_http_201")})
|
||
atomic_json(self.directory / "mapping.json",
|
||
{"binding": state["binding"], "repositories": mappings})
|
||
|
||
|
||
def verify(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
|
||
name: str, entry: dict) -> None:
|
||
info = target.repo(target_org, name)
|
||
if identity(info, target_org, name) != entry["receipt_id"]:
|
||
raise MigrationError("TARGET_RECEIPT_ID_MISMATCH")
|
||
if info.get("private") is not True or info.get("mirror") is not False:
|
||
raise MigrationError("TARGET_PRIVACY_OR_MIRROR_MISMATCH")
|
||
raw_references = entry["snapshot"].get("reference_format") == "git-object-v1"
|
||
if snapshot(target, target_org, name, entry["receipt_id"], raw_references) != entry["snapshot"]:
|
||
raise MigrationError("TARGET_GIT_REFERENCES_MISMATCH")
|
||
if snapshot(source, source_org, name, entry["source_id"], raw_references) != entry["snapshot"]:
|
||
raise MigrationError("SOURCE_CHANGED_DURING_MIGRATION")
|
||
|
||
|
||
def migrate(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
|
||
store: StateStore, execute: bool, selected: list[str], limit: int | None,
|
||
migration_timeout: int) -> int:
|
||
preflight(source, source_org, target=False)
|
||
preflight(target, target_org, target=True)
|
||
inventory = {}
|
||
for info in source.pages("/orgs/" + name_segment(source_org) + "/repos"):
|
||
name = info.get("name")
|
||
name_segment(name)
|
||
repo_id = identity(info, source_org, name)
|
||
if name in inventory:
|
||
raise MigrationError("DUPLICATE_REPOSITORY_NAME")
|
||
inventory[name] = repo_id
|
||
inventory = dict(sorted(inventory.items()))
|
||
if any(name not in inventory for name in selected):
|
||
raise MigrationError("SELECTED_REPOSITORY_NOT_IN_SOURCE_ORG")
|
||
binding = {"source_url": source.base, "source_org": source_org,
|
||
"target_url": target.base, "target_org": target_org,
|
||
"contents": "private-git-and-lfs-v1"}
|
||
state = store.load(binding, inventory)
|
||
print(f"源组织={source_org} 仓库数={len(inventory)} 目标组织={target_org} 模式={'正式执行' if execute else '只读预览'}")
|
||
processed = 0
|
||
for name, entry in state["repositories"].items():
|
||
if selected and name not in selected:
|
||
continue
|
||
if entry.get("receipt_id") is not None:
|
||
try:
|
||
verify(source, target, source_org, target_org, name, entry)
|
||
entry.update(status="VERIFIED", error=None)
|
||
print(f"已核对 {name}:完成收据、私有属性、默认分支、分支及标签提交一致")
|
||
except MigrationError as error:
|
||
entry.update(status="REVIEW", error=error.code)
|
||
print(f"待核对 {name}:{error.code};已保存完成收据,仅重试只读核验")
|
||
store.save(state)
|
||
continue
|
||
if entry["status"] != "PLANNED":
|
||
print(f"跳过 {name}:{entry['status']};没有可核对的完成收据,不自动重发")
|
||
continue
|
||
if limit is not None and processed >= limit:
|
||
continue
|
||
processed += 1
|
||
if target.repo(target_org, name, missing=True) is not None:
|
||
entry.update(status="CONFLICT", error="TARGET_ALREADY_EXISTS")
|
||
store.save(state)
|
||
print(f"同名冲突 {name}:保留生产仓库,不接管、不覆盖")
|
||
continue
|
||
frozen = snapshot(source, source_org, name, entry["source_id"])
|
||
if not execute:
|
||
print(f"待迁移 {name}:分支={len(frozen['branches'])} 标签={len(frozen['tags'])} LFS=请求迁移")
|
||
continue
|
||
# 在发送 POST 前强制落盘。超时、进程退出后不能仅凭仓库存在认定迁移完成。
|
||
entry.update(status="MIGRATING", snapshot=frozen, error=None)
|
||
store.save(state)
|
||
print(f"开始迁移 {name}:{source_org} -> {target_org}", flush=True)
|
||
body = {"clone_addr": source.clone_url(source_org, name),
|
||
"repo_owner": target_org, "repo_name": name, "service": "git",
|
||
"auth_username": source.username, "auth_password": source.token,
|
||
"private": True, "mirror": False, "lfs": True,
|
||
**{unit: False for unit in ("issues", "labels", "milestones",
|
||
"pull_requests", "releases", "wiki")}}
|
||
try:
|
||
status, receipt = target.request("POST", "/repos/migrate", body,
|
||
timeout=migration_timeout)
|
||
if status != 201:
|
||
raise MigrationError("MIGRATION_COMPLETION_NOT_CONFIRMED")
|
||
entry["receipt_id"] = identity(receipt, target_org, name)
|
||
entry["status"] = "RECEIVED"
|
||
store.save(state)
|
||
except MigrationError as error:
|
||
entry.update(status="REVIEW", error=error.code)
|
||
store.save(state)
|
||
print(f"迁移未确认 {name}:{error.code};本次停止新增仓库,请检查生产迁移日志")
|
||
break
|
||
try:
|
||
verify(source, target, source_org, target_org, name, entry)
|
||
entry.update(status="VERIFIED", error=None)
|
||
print(f"迁移完成 {name}:Git 引用已核对,LFS 已请求迁移,实体下载仍需抽查")
|
||
except MigrationError as error:
|
||
entry.update(status="REVIEW", error=error.code)
|
||
print(f"待核对 {name}:{error.code};保留已创建仓库和完成收据")
|
||
store.save(state)
|
||
entries = [entry for name, entry in state["repositories"].items()
|
||
if not selected or name in selected]
|
||
counts = {status: sum(entry["status"] == status for entry in entries)
|
||
for status in ("PLANNED", "VERIFIED", "CONFLICT", "REVIEW")}
|
||
print("处理结果 " + " ".join(f"{key}={value}" for key, value in counts.items()))
|
||
print("状态及映射目录:" + str(store.directory))
|
||
return 2 if counts["CONFLICT"] or counts["REVIEW"] else 0
|
||
|
||
|
||
def read_token(variable: str, prompt: str) -> str:
|
||
token = os.environ.get(variable, "")
|
||
if not token and sys.stdin.isatty():
|
||
token = getpass.getpass(prompt)
|
||
if not token or any(char.isspace() or ord(char) < 32 for char in token):
|
||
raise MigrationError("MISSING_OR_INVALID_" + variable)
|
||
return token
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--source-url", required=True, help="测试 Gitea 基础地址,不含 /api/v1")
|
||
parser.add_argument("--target-url", required=True, help="生产 Gitea 基础地址,不含 /api/v1")
|
||
parser.add_argument("--source-org", default="community")
|
||
parser.add_argument("--target-org", default="vibemake")
|
||
parser.add_argument("--state-dir", type=Path, default=Path("logs/gitea-org-migration"))
|
||
mode = parser.add_mutually_exclusive_group()
|
||
mode.add_argument("--dry-run", action="store_true", help="只读预览,默认模式")
|
||
mode.add_argument("--execute", action="store_true", help="正式创建目标仓库")
|
||
parser.add_argument("--repo", action="append", default=[], help="只处理指定仓库,可重复指定")
|
||
parser.add_argument("--limit", type=int, help="限制本次尚未处理的仓库数")
|
||
parser.add_argument("--migration-timeout", type=int, default=1800, help="单仓库迁移请求超时秒数")
|
||
parser.add_argument("--allow-http", action="store_true", help="允许在可信网络通过明文 HTTP 发送凭据")
|
||
args = parser.parse_args()
|
||
try:
|
||
source_url = base_url(args.source_url, args.allow_http)
|
||
target_url = base_url(args.target_url, args.allow_http)
|
||
if source_url == target_url:
|
||
raise MigrationError("SOURCE_AND_TARGET_MUST_DIFFER")
|
||
for value in (args.source_org, args.target_org, *args.repo):
|
||
name_segment(value)
|
||
if (args.limit is not None and args.limit < 1) or args.migration_timeout < 1:
|
||
raise MigrationError("INVALID_LIMIT_OR_TIMEOUT")
|
||
source_token = read_token("GITEA_MIGRATION_SOURCE_TOKEN", "测试 Gitea Token(输入不回显):")
|
||
target_token = read_token("GITEA_MIGRATION_TARGET_TOKEN", "生产 Gitea Token(输入不回显):")
|
||
with StateStore(args.state_dir).locked():
|
||
return migrate(ApiClient(source_url, source_token), ApiClient(target_url, target_token),
|
||
args.source_org, args.target_org, StateStore(args.state_dir),
|
||
args.execute, args.repo, args.limit, args.migration_timeout)
|
||
except MigrationError as error:
|
||
print("操作停止:" + error.code, file=sys.stderr)
|
||
return 1
|
||
except KeyboardInterrupt:
|
||
print("操作已中断。已发出的迁移可能仍在生产执行;保留状态目录后再核对。", file=sys.stderr)
|
||
return 130
|
||
except Exception as error:
|
||
# 不输出异常正文,避免底层网络库或无效输入携带敏感值。
|
||
print("操作停止:" + type(error).__name__ + ";检查状态文件和运行环境。", file=sys.stderr)
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|