复制Gitea仓库到生产环境
This commit is contained in:
@@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python3
|
||||
"""按组织复制 Git 仓库及 LFS;默认只读预览,不修改业务数据库。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote, urlencode, urlsplit, urlunsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
|
||||
class MigrationError(Exception):
|
||||
"""仅包含可安全记录的错误码,不携带响应正文或凭据。"""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def timestamp() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def name_segment(value: str) -> str:
|
||||
if (not isinstance(value, str) or not value or value in (".", "..")
|
||||
or len(value) > 255 or any(char in value for char in "/\\?#%")
|
||||
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)):
|
||||
raise MigrationError("INVALID_NAME")
|
||||
return quote(value, safe="")
|
||||
|
||||
|
||||
def base_url(value: str, allow_http: bool) -> str:
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
port = parsed.port
|
||||
if (parsed.scheme not in ("https", "http") or not parsed.hostname
|
||||
or (parsed.scheme == "http" and not allow_http)
|
||||
or parsed.username is not None or parsed.password is not None
|
||||
or "?" in value or "#" in value or "\\" in value
|
||||
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
|
||||
or (port is not None and not 1 <= port <= 65535)
|
||||
or any(part in (".", "..") for part in parsed.path.split("/"))
|
||||
or "%" in parsed.path):
|
||||
raise ValueError()
|
||||
host = parsed.hostname.lower()
|
||||
if ":" in host:
|
||||
host = "[" + host + "]"
|
||||
if port is not None and port != {"http": 80, "https": 443}[parsed.scheme]:
|
||||
host += ":" + str(port)
|
||||
return urlunsplit((parsed.scheme, host, parsed.path.rstrip("/"), "", ""))
|
||||
except ValueError:
|
||||
raise MigrationError("INVALID_BASE_URL") from None
|
||||
|
||||
|
||||
class NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
# 避免重定向转发认证头或再次发送迁移请求。
|
||||
return None
|
||||
|
||||
|
||||
class ApiClient:
|
||||
def __init__(self, base: str, token: str, timeout: int = 30):
|
||||
self.base = base
|
||||
self.token = token
|
||||
self.timeout = timeout
|
||||
self.opener = build_opener(NoRedirect())
|
||||
self.username = ""
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None,
|
||||
timeout: int | None = None, missing: bool = False) -> tuple[int, object]:
|
||||
request = Request(self.base + "/api/v1" + path, method=method,
|
||||
data=None if body is None else json.dumps(body).encode("utf-8"),
|
||||
headers={"Authorization": "token " + self.token,
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json"})
|
||||
try:
|
||||
with self.opener.open(request, timeout=timeout or self.timeout) as response:
|
||||
status = response.status
|
||||
data = response.read(16 * 1024 * 1024 + 1)
|
||||
if len(data) > 16 * 1024 * 1024:
|
||||
raise MigrationError("RESPONSE_TOO_LARGE")
|
||||
return status, json.loads(data) if data else None
|
||||
except HTTPError as error:
|
||||
status = error.code
|
||||
error.close()
|
||||
if missing and status == 404:
|
||||
return status, None
|
||||
# Gitea/反向代理的错误正文可能包含源地址或认证信息,不输出正文。
|
||||
raise MigrationError("HTTP_" + str(status)) from None
|
||||
except (URLError, TimeoutError, OSError):
|
||||
raise MigrationError("NETWORK_ERROR") from None
|
||||
except (ValueError, UnicodeError):
|
||||
raise MigrationError("INVALID_JSON_RESPONSE") from None
|
||||
|
||||
def get(self, path: str, missing: bool = False):
|
||||
status, data = self.request("GET", path, missing=missing)
|
||||
if status == 404 and missing:
|
||||
return None
|
||||
if status != 200:
|
||||
raise MigrationError("UNEXPECTED_GET_STATUS")
|
||||
return data
|
||||
|
||||
def pages(self, path: str) -> list[dict]:
|
||||
result = []
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
items = self.get(path + "?" + urlencode({"page": page, "limit": 50}))
|
||||
if not isinstance(items, list):
|
||||
raise MigrationError("INVALID_LIST_RESPONSE")
|
||||
if not items:
|
||||
return result
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
raise MigrationError("INVALID_LIST_ITEM")
|
||||
# 多个轻量标签可以指向同一个提交(相同 id),分页应按名称去重。
|
||||
key = item.get("name", item.get("id"))
|
||||
if not isinstance(key, (str, int)) or key in seen:
|
||||
raise MigrationError("DUPLICATE_PAGINATION_ITEM")
|
||||
seen.add(key)
|
||||
result.append(item)
|
||||
# 服务端可能将 limit 限制为更小值,不能按 len(items) < 50 提前终止。
|
||||
raise MigrationError("PAGINATION_LIMIT")
|
||||
|
||||
def repo(self, org: str, name: str, missing: bool = False):
|
||||
return self.get("/repos/" + name_segment(org) + "/" + name_segment(name), missing)
|
||||
|
||||
def clone_url(self, org: str, name: str) -> str:
|
||||
return self.base + "/" + name_segment(org) + "/" + name_segment(name) + ".git"
|
||||
|
||||
|
||||
def identity(info: dict, org: str, name: str) -> int:
|
||||
if (not isinstance(info, dict) or type(info.get("id")) is not int or info["id"] <= 0
|
||||
or not isinstance(info.get("owner"), dict)
|
||||
or not isinstance(info["owner"].get("login"), str)
|
||||
or info["owner"]["login"].casefold() != org.casefold()
|
||||
or info.get("name") != name):
|
||||
raise MigrationError("REPOSITORY_IDENTITY_MISMATCH")
|
||||
return info["id"]
|
||||
|
||||
|
||||
def preflight(client: ApiClient, org: str, target: bool) -> None:
|
||||
user = client.get("/user")
|
||||
if not isinstance(user, dict) or not isinstance(user.get("login"), str):
|
||||
raise MigrationError("INVALID_USER_RESPONSE")
|
||||
client.username = user["login"]
|
||||
name_segment(client.username)
|
||||
client.get("/orgs/" + name_segment(org))
|
||||
# 源组织所有者能列出全部私有仓库;迁移 API 也要求目标组织所有者。
|
||||
if user.get("is_admin") is not True:
|
||||
owner = False
|
||||
for team in client.pages("/orgs/" + name_segment(org) + "/teams"):
|
||||
if team.get("permission") == "owner" and type(team.get("id")) is int:
|
||||
status, member = client.request("GET", "/teams/" + str(team["id"])
|
||||
+ "/members/" + name_segment(client.username), missing=True)
|
||||
if (status == 200 and isinstance(member, dict)
|
||||
and member.get("login") == client.username):
|
||||
owner = True
|
||||
break
|
||||
if not owner:
|
||||
raise MigrationError("ORGANIZATION_OWNER_REQUIRED")
|
||||
settings = client.get("/settings/repository")
|
||||
if (not isinstance(settings, dict) or settings.get("lfs_disabled") is not False
|
||||
or settings.get("http_git_disabled") is not False
|
||||
or (target and settings.get("migrations_disabled") is not False)):
|
||||
raise MigrationError("GIT_LFS_OR_MIGRATION_DISABLED")
|
||||
|
||||
|
||||
def snapshot(client: ApiClient, org: str, name: str, expected_id: int,
|
||||
raw_references: bool = False) -> dict:
|
||||
info = client.repo(org, name)
|
||||
if identity(info, org, name) != expected_id:
|
||||
raise MigrationError("REPOSITORY_ID_CHANGED")
|
||||
if type(info.get("empty")) is not bool or not isinstance(info.get("default_branch"), str):
|
||||
raise MigrationError("INVALID_REPOSITORY_RESPONSE")
|
||||
result = {"empty": info["empty"], "default_branch": info["default_branch"],
|
||||
"branches": {}, "tags": {}}
|
||||
path = "/repos/" + name_segment(org) + "/" + name_segment(name)
|
||||
if info["empty"] or raw_references:
|
||||
# 历史导入中断后,数据库可能标记为空,但磁盘已存在 Git 引用。
|
||||
for prefix, unit in (("heads", "branches"), ("tags", "tags")):
|
||||
refs = client.get(path + "/git/refs/" + prefix, missing=True)
|
||||
if refs is None:
|
||||
continue
|
||||
if not isinstance(refs, list):
|
||||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||||
for ref in refs:
|
||||
ref_path = ref.get("ref", "")
|
||||
sha = ref.get("object", {}).get("sha")
|
||||
expected_prefix = "refs/" + prefix + "/"
|
||||
if (not isinstance(ref_path, str) or not ref_path.startswith(expected_prefix)
|
||||
or not ref_path[len(expected_prefix):]
|
||||
or not isinstance(sha, str)
|
||||
or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)):
|
||||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||||
ref_name = ref_path[len(expected_prefix):]
|
||||
if ref_name in result[unit]:
|
||||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||||
result[unit][ref_name] = sha
|
||||
if result["branches"] or result["tags"] or raw_references:
|
||||
result["empty"] = not (result["branches"] or result["tags"])
|
||||
result["reference_format"] = "git-object-v1"
|
||||
if not result["empty"] and result["default_branch"] not in result["branches"]:
|
||||
raise MigrationError("DEFAULT_BRANCH_MISSING")
|
||||
return result
|
||||
for unit in ("branches", "tags"):
|
||||
for ref in client.pages(path + "/" + unit):
|
||||
commit = ref.get("commit")
|
||||
sha = (commit.get("id") or commit.get("sha")) if isinstance(commit, dict) else None
|
||||
ref_name = ref.get("name")
|
||||
if (not isinstance(ref_name, str) or not ref_name
|
||||
or not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)
|
||||
or ref_name in result[unit]):
|
||||
raise MigrationError("INVALID_GIT_REFERENCE")
|
||||
result[unit][ref_name] = sha
|
||||
if result["default_branch"] not in result["branches"]:
|
||||
raise MigrationError("DEFAULT_BRANCH_MISSING")
|
||||
return result
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict) -> None:
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".migration-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
class StateStore:
|
||||
def __init__(self, directory: Path):
|
||||
self.directory = directory
|
||||
self.path = directory / "state.json"
|
||||
|
||||
@contextmanager
|
||||
def locked(self):
|
||||
if self.directory.is_symlink():
|
||||
raise MigrationError("INVALID_STATE_DIRECTORY")
|
||||
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.directory.chmod(0o700)
|
||||
with (self.directory / ".lock").open("a") as stream:
|
||||
os.fchmod(stream.fileno(), 0o600)
|
||||
try:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise MigrationError("STATE_DIRECTORY_LOCKED") from None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(stream, fcntl.LOCK_UN)
|
||||
|
||||
def load(self, binding: dict, inventory: dict) -> dict:
|
||||
if self.path.exists():
|
||||
state = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
if state.get("version") != 1 or state.get("binding") != binding:
|
||||
raise MigrationError("STATE_ENVIRONMENT_MISMATCH")
|
||||
if state.get("inventory") != inventory:
|
||||
raise MigrationError("SOURCE_INVENTORY_CHANGED")
|
||||
for entry in state["repositories"].values():
|
||||
if entry["status"] == "MIGRATING":
|
||||
entry.update(status="REVIEW", error="INTERRUPTED_MIGRATION")
|
||||
else:
|
||||
state = {"version": 1, "created_at": timestamp(), "binding": binding,
|
||||
"inventory": inventory,
|
||||
"repositories": {name: {"source_id": repo_id, "status": "PLANNED"}
|
||||
for name, repo_id in inventory.items()}}
|
||||
self.save(state)
|
||||
return state
|
||||
|
||||
def save(self, state: dict) -> None:
|
||||
state["updated_at"] = timestamp()
|
||||
atomic_json(self.path, state)
|
||||
mappings = []
|
||||
for name, entry in state["repositories"].items():
|
||||
if entry["status"] != "VERIFIED":
|
||||
continue
|
||||
code = name[2:] if re.fullmatch(r"p_(?:[a-z0-9]{12}|[a-z0-9]{32})", name) else None
|
||||
mappings.append({"repo_name": name, "project_code": code,
|
||||
"source_repo_id": entry["source_id"],
|
||||
"target_repo_id": entry["receipt_id"],
|
||||
"default_branch": entry["snapshot"]["default_branch"],
|
||||
"lfs_requested": True,
|
||||
"lfs_objects_download_verified": entry.get("lfs_objects_download_verified", False),
|
||||
"completion_evidence": entry.get("completion_evidence", "migration_http_201")})
|
||||
atomic_json(self.directory / "mapping.json",
|
||||
{"binding": state["binding"], "repositories": mappings})
|
||||
|
||||
|
||||
def verify(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
|
||||
name: str, entry: dict) -> None:
|
||||
info = target.repo(target_org, name)
|
||||
if identity(info, target_org, name) != entry["receipt_id"]:
|
||||
raise MigrationError("TARGET_RECEIPT_ID_MISMATCH")
|
||||
if info.get("private") is not True or info.get("mirror") is not False:
|
||||
raise MigrationError("TARGET_PRIVACY_OR_MIRROR_MISMATCH")
|
||||
raw_references = entry["snapshot"].get("reference_format") == "git-object-v1"
|
||||
if snapshot(target, target_org, name, entry["receipt_id"], raw_references) != entry["snapshot"]:
|
||||
raise MigrationError("TARGET_GIT_REFERENCES_MISMATCH")
|
||||
if snapshot(source, source_org, name, entry["source_id"], raw_references) != entry["snapshot"]:
|
||||
raise MigrationError("SOURCE_CHANGED_DURING_MIGRATION")
|
||||
|
||||
|
||||
def migrate(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
|
||||
store: StateStore, execute: bool, selected: list[str], limit: int | None,
|
||||
migration_timeout: int) -> int:
|
||||
preflight(source, source_org, target=False)
|
||||
preflight(target, target_org, target=True)
|
||||
inventory = {}
|
||||
for info in source.pages("/orgs/" + name_segment(source_org) + "/repos"):
|
||||
name = info.get("name")
|
||||
name_segment(name)
|
||||
repo_id = identity(info, source_org, name)
|
||||
if name in inventory:
|
||||
raise MigrationError("DUPLICATE_REPOSITORY_NAME")
|
||||
inventory[name] = repo_id
|
||||
inventory = dict(sorted(inventory.items()))
|
||||
if any(name not in inventory for name in selected):
|
||||
raise MigrationError("SELECTED_REPOSITORY_NOT_IN_SOURCE_ORG")
|
||||
binding = {"source_url": source.base, "source_org": source_org,
|
||||
"target_url": target.base, "target_org": target_org,
|
||||
"contents": "private-git-and-lfs-v1"}
|
||||
state = store.load(binding, inventory)
|
||||
print(f"源组织={source_org} 仓库数={len(inventory)} 目标组织={target_org} 模式={'正式执行' if execute else '只读预览'}")
|
||||
processed = 0
|
||||
for name, entry in state["repositories"].items():
|
||||
if selected and name not in selected:
|
||||
continue
|
||||
if entry.get("receipt_id") is not None:
|
||||
try:
|
||||
verify(source, target, source_org, target_org, name, entry)
|
||||
entry.update(status="VERIFIED", error=None)
|
||||
print(f"已核对 {name}:完成收据、私有属性、默认分支、分支及标签提交一致")
|
||||
except MigrationError as error:
|
||||
entry.update(status="REVIEW", error=error.code)
|
||||
print(f"待核对 {name}:{error.code};已保存完成收据,仅重试只读核验")
|
||||
store.save(state)
|
||||
continue
|
||||
if entry["status"] != "PLANNED":
|
||||
print(f"跳过 {name}:{entry['status']};没有可核对的完成收据,不自动重发")
|
||||
continue
|
||||
if limit is not None and processed >= limit:
|
||||
continue
|
||||
processed += 1
|
||||
if target.repo(target_org, name, missing=True) is not None:
|
||||
entry.update(status="CONFLICT", error="TARGET_ALREADY_EXISTS")
|
||||
store.save(state)
|
||||
print(f"同名冲突 {name}:保留生产仓库,不接管、不覆盖")
|
||||
continue
|
||||
frozen = snapshot(source, source_org, name, entry["source_id"])
|
||||
if not execute:
|
||||
print(f"待迁移 {name}:分支={len(frozen['branches'])} 标签={len(frozen['tags'])} LFS=请求迁移")
|
||||
continue
|
||||
# 在发送 POST 前强制落盘。超时、进程退出后不能仅凭仓库存在认定迁移完成。
|
||||
entry.update(status="MIGRATING", snapshot=frozen, error=None)
|
||||
store.save(state)
|
||||
print(f"开始迁移 {name}:{source_org} -> {target_org}", flush=True)
|
||||
body = {"clone_addr": source.clone_url(source_org, name),
|
||||
"repo_owner": target_org, "repo_name": name, "service": "git",
|
||||
"auth_username": source.username, "auth_password": source.token,
|
||||
"private": True, "mirror": False, "lfs": True,
|
||||
**{unit: False for unit in ("issues", "labels", "milestones",
|
||||
"pull_requests", "releases", "wiki")}}
|
||||
try:
|
||||
status, receipt = target.request("POST", "/repos/migrate", body,
|
||||
timeout=migration_timeout)
|
||||
if status != 201:
|
||||
raise MigrationError("MIGRATION_COMPLETION_NOT_CONFIRMED")
|
||||
entry["receipt_id"] = identity(receipt, target_org, name)
|
||||
entry["status"] = "RECEIVED"
|
||||
store.save(state)
|
||||
except MigrationError as error:
|
||||
entry.update(status="REVIEW", error=error.code)
|
||||
store.save(state)
|
||||
print(f"迁移未确认 {name}:{error.code};本次停止新增仓库,请检查生产迁移日志")
|
||||
break
|
||||
try:
|
||||
verify(source, target, source_org, target_org, name, entry)
|
||||
entry.update(status="VERIFIED", error=None)
|
||||
print(f"迁移完成 {name}:Git 引用已核对,LFS 已请求迁移,实体下载仍需抽查")
|
||||
except MigrationError as error:
|
||||
entry.update(status="REVIEW", error=error.code)
|
||||
print(f"待核对 {name}:{error.code};保留已创建仓库和完成收据")
|
||||
store.save(state)
|
||||
entries = [entry for name, entry in state["repositories"].items()
|
||||
if not selected or name in selected]
|
||||
counts = {status: sum(entry["status"] == status for entry in entries)
|
||||
for status in ("PLANNED", "VERIFIED", "CONFLICT", "REVIEW")}
|
||||
print("处理结果 " + " ".join(f"{key}={value}" for key, value in counts.items()))
|
||||
print("状态及映射目录:" + str(store.directory))
|
||||
return 2 if counts["CONFLICT"] or counts["REVIEW"] else 0
|
||||
|
||||
|
||||
def read_token(variable: str, prompt: str) -> str:
|
||||
token = os.environ.get(variable, "")
|
||||
if not token and sys.stdin.isatty():
|
||||
token = getpass.getpass(prompt)
|
||||
if not token or any(char.isspace() or ord(char) < 32 for char in token):
|
||||
raise MigrationError("MISSING_OR_INVALID_" + variable)
|
||||
return token
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-url", required=True, help="测试 Gitea 基础地址,不含 /api/v1")
|
||||
parser.add_argument("--target-url", required=True, help="生产 Gitea 基础地址,不含 /api/v1")
|
||||
parser.add_argument("--source-org", default="community")
|
||||
parser.add_argument("--target-org", default="vibemake")
|
||||
parser.add_argument("--state-dir", type=Path, default=Path("logs/gitea-org-migration"))
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--dry-run", action="store_true", help="只读预览,默认模式")
|
||||
mode.add_argument("--execute", action="store_true", help="正式创建目标仓库")
|
||||
parser.add_argument("--repo", action="append", default=[], help="只处理指定仓库,可重复指定")
|
||||
parser.add_argument("--limit", type=int, help="限制本次尚未处理的仓库数")
|
||||
parser.add_argument("--migration-timeout", type=int, default=1800, help="单仓库迁移请求超时秒数")
|
||||
parser.add_argument("--allow-http", action="store_true", help="允许在可信网络通过明文 HTTP 发送凭据")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
source_url = base_url(args.source_url, args.allow_http)
|
||||
target_url = base_url(args.target_url, args.allow_http)
|
||||
if source_url == target_url:
|
||||
raise MigrationError("SOURCE_AND_TARGET_MUST_DIFFER")
|
||||
for value in (args.source_org, args.target_org, *args.repo):
|
||||
name_segment(value)
|
||||
if (args.limit is not None and args.limit < 1) or args.migration_timeout < 1:
|
||||
raise MigrationError("INVALID_LIMIT_OR_TIMEOUT")
|
||||
source_token = read_token("GITEA_MIGRATION_SOURCE_TOKEN", "测试 Gitea Token(输入不回显):")
|
||||
target_token = read_token("GITEA_MIGRATION_TARGET_TOKEN", "生产 Gitea Token(输入不回显):")
|
||||
with StateStore(args.state_dir).locked():
|
||||
return migrate(ApiClient(source_url, source_token), ApiClient(target_url, target_token),
|
||||
args.source_org, args.target_org, StateStore(args.state_dir),
|
||||
args.execute, args.repo, args.limit, args.migration_timeout)
|
||||
except MigrationError as error:
|
||||
print("操作停止:" + error.code, file=sys.stderr)
|
||||
return 1
|
||||
except KeyboardInterrupt:
|
||||
print("操作已中断。已发出的迁移可能仍在生产执行;保留状态目录后再核对。", file=sys.stderr)
|
||||
return 130
|
||||
except Exception as error:
|
||||
# 不输出异常正文,避免底层网络库或无效输入携带敏感值。
|
||||
print("操作停止:" + type(error).__name__ + ";检查状态文件和运行环境。", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,363 @@
|
||||
import copy
|
||||
from contextlib import contextmanager, redirect_stdout
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "migrate-gitea-org.py"
|
||||
spec = importlib.util.spec_from_file_location("gitea_migration", SCRIPT)
|
||||
migration = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(migration)
|
||||
|
||||
SOURCE_TOKEN = "source-test-secret-never-log"
|
||||
TARGET_TOKEN = "target-test-secret-never-log"
|
||||
SHA = "1" * 40
|
||||
OTHER_SHA = "2" * 40
|
||||
FIRST = "p_aaaaaaaaaaaa"
|
||||
SECOND = "p_bbbbbbbbbbbb"
|
||||
|
||||
|
||||
def repository(name, repo_id, owner="community", empty=False):
|
||||
return {"id": repo_id, "name": name, "owner": {"login": owner},
|
||||
"private": True, "mirror": False, "empty": empty, "default_branch": "main",
|
||||
"branches": {} if empty else {"main": SHA, "dev": OTHER_SHA},
|
||||
"tags": {} if empty else {"v1": SHA, "v1-alias": SHA}}
|
||||
|
||||
|
||||
class FakeApi(migration.ApiClient):
|
||||
"""模拟服务端限制分页大小、私有组织及同步仓库迁移,不访问真实服务。"""
|
||||
|
||||
def __init__(self, source=None):
|
||||
self.source = source
|
||||
self.org = "vibemake" if source else "community"
|
||||
super().__init__("https://target.example" if source else "https://source.example",
|
||||
TARGET_TOKEN if source else SOURCE_TOKEN)
|
||||
self.repositories = {} if source else {
|
||||
FIRST: repository(FIRST, 101), SECOND: repository(SECOND, 102)}
|
||||
self.requests = []
|
||||
self.posts = []
|
||||
self.owner = True
|
||||
self.lfs_enabled = True
|
||||
self.post_status = 201
|
||||
self.post_error = None
|
||||
self.interrupt_post = False
|
||||
self.fail_read_after_post = False
|
||||
self.change_target_tag = False
|
||||
self.change_source_after_post = False
|
||||
|
||||
def request(self, method, path, body=None, timeout=None, missing=False):
|
||||
self.requests.append((method, path))
|
||||
parsed = urlsplit(path)
|
||||
route = [unquote(part) for part in parsed.path.strip("/").split("/")]
|
||||
page = int(parse_qs(parsed.query).get("page", ["1"])[0])
|
||||
if method == "POST":
|
||||
if path != "/repos/migrate":
|
||||
raise AssertionError("不允许其他写接口")
|
||||
self.posts.append(copy.deepcopy(body))
|
||||
name = body["repo_name"]
|
||||
info = copy.deepcopy(self.source.repositories[name])
|
||||
info.update(id=1000 + len(self.posts), owner={"login": body["repo_owner"]},
|
||||
private=True, mirror=False)
|
||||
self.repositories[name] = info
|
||||
if self.change_target_tag:
|
||||
info["tags"]["v1"] = OTHER_SHA
|
||||
if self.change_source_after_post:
|
||||
self.source.repositories[name]["branches"]["main"] = OTHER_SHA
|
||||
if self.interrupt_post:
|
||||
raise KeyboardInterrupt()
|
||||
if self.post_error:
|
||||
raise migration.MigrationError(self.post_error)
|
||||
return self.post_status, copy.deepcopy(info)
|
||||
if path == "/user":
|
||||
return 200, {"login": "service", "is_admin": False}
|
||||
if path == "/orgs/" + self.org:
|
||||
return 200, {"name": self.org}
|
||||
if path == "/settings/repository":
|
||||
return 200, {"lfs_disabled": not self.lfs_enabled,
|
||||
"http_git_disabled": False, "migrations_disabled": False}
|
||||
if route == ["orgs", self.org, "teams"]:
|
||||
teams = [{"id": 1, "name": "Owners", "permission": "owner"}]
|
||||
return 200, teams[page - 1:page]
|
||||
if path == "/teams/1/members/service":
|
||||
return (200, {"login": "service"}) if self.owner else (404, None)
|
||||
if route == ["orgs", self.org, "repos"]:
|
||||
# 服务端上限为一条,客户端传 limit=50 也必须继续分页。
|
||||
return 200, copy.deepcopy(list(self.repositories.values())[page - 1:page])
|
||||
if len(route) >= 3 and route[:2] == ["repos", self.org]:
|
||||
info = self.repositories.get(route[2])
|
||||
if info is None:
|
||||
if missing:
|
||||
return 404, None
|
||||
raise migration.MigrationError("HTTP_404")
|
||||
if self.fail_read_after_post and self.posts:
|
||||
raise migration.MigrationError("NETWORK_ERROR")
|
||||
if len(route) == 3:
|
||||
return 200, copy.deepcopy(info)
|
||||
if len(route) == 5 and route[3] == "git" and route[4].startswith("refs"):
|
||||
raise AssertionError("Git 引用路由应包含 heads 或 tags")
|
||||
if len(route) == 6 and route[3:5] == ["git", "refs"]:
|
||||
unit = {"heads": "branches", "tags": "tags"}[route[5]]
|
||||
refs = [{"ref": "refs/" + route[5] + "/" + name, "object": {"sha": sha}}
|
||||
for name, sha in info[unit].items()]
|
||||
return (200, refs) if refs else (404, None)
|
||||
if len(route) == 4 and route[3] in ("branches", "tags"):
|
||||
unit = route[3]
|
||||
refs = [{"name": name, "commit": {"id" if unit == "branches" else "sha": sha},
|
||||
**({"id": sha} if unit == "tags" else {})}
|
||||
for name, sha in info[unit].items()]
|
||||
return 200, refs[page - 1:page]
|
||||
raise AssertionError("未预期的模拟 API 路由")
|
||||
|
||||
|
||||
class MigrationTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.store = migration.StateStore(Path(self.temporary.name) / "state")
|
||||
self.source = FakeApi()
|
||||
self.target = FakeApi(self.source)
|
||||
|
||||
def run_migration(self, execute=False, limit=None, selected=None):
|
||||
output = io.StringIO()
|
||||
with self.store.locked(), redirect_stdout(output):
|
||||
code = migration.migrate(self.source, self.target, "community", "vibemake",
|
||||
self.store, execute, selected or [], limit, 1800)
|
||||
self.assertNotIn(SOURCE_TOKEN, output.getvalue())
|
||||
self.assertNotIn(TARGET_TOKEN, output.getvalue())
|
||||
for path in self.store.directory.iterdir():
|
||||
if path.is_file():
|
||||
text = path.read_text(encoding="utf-8")
|
||||
self.assertNotIn(SOURCE_TOKEN, text)
|
||||
self.assertNotIn(TARGET_TOKEN, text)
|
||||
return code
|
||||
|
||||
def state(self):
|
||||
return json.loads(self.store.path.read_text())
|
||||
|
||||
def mappings(self):
|
||||
return json.loads((self.store.directory / "mapping.json").read_text())["repositories"]
|
||||
|
||||
def test_preview_paginates_all_private_repositories_and_never_writes_remote(self):
|
||||
self.assertEqual(self.run_migration(), 0)
|
||||
self.assertEqual(len(self.state()["inventory"]), 2)
|
||||
self.assertEqual(self.target.posts, [])
|
||||
self.assertEqual(self.mappings(), [])
|
||||
self.assertTrue(all(method == "GET" for method, _ in self.source.requests))
|
||||
self.assertTrue(all(method == "GET" for method, _ in self.target.requests))
|
||||
self.assertIn(("GET", "/orgs/community/repos?page=3&limit=50"), self.source.requests)
|
||||
|
||||
def test_stale_empty_metadata_still_copies_and_verifies_real_git_references(self):
|
||||
self.source.repositories[FIRST]["empty"] = True
|
||||
self.assertEqual(self.run_migration(True, limit=1), 0)
|
||||
entry = self.state()["repositories"][FIRST]
|
||||
self.assertFalse(entry["snapshot"]["empty"])
|
||||
self.assertEqual(entry["snapshot"]["reference_format"], "git-object-v1")
|
||||
self.assertEqual(len(entry["snapshot"]["branches"]), 2)
|
||||
self.assertEqual(len(entry["snapshot"]["tags"]), 2)
|
||||
|
||||
def test_copies_git_lfs_to_new_owner_and_saves_mapping_without_credentials(self):
|
||||
self.assertEqual(self.run_migration(True, limit=1), 0)
|
||||
self.assertEqual(len(self.target.posts), 1)
|
||||
body = self.target.posts[0]
|
||||
self.assertEqual(body["repo_owner"], "vibemake")
|
||||
self.assertEqual(body["repo_name"], FIRST)
|
||||
self.assertEqual(body["clone_addr"], "https://source.example/community/" + FIRST + ".git")
|
||||
self.assertEqual(body["auth_username"], "service")
|
||||
self.assertEqual(body["auth_password"], SOURCE_TOKEN)
|
||||
self.assertTrue(body["lfs"])
|
||||
self.assertTrue(body["private"])
|
||||
self.assertFalse(body["mirror"])
|
||||
for option in ("issues", "labels", "milestones", "pull_requests", "releases", "wiki"):
|
||||
self.assertFalse(body[option])
|
||||
self.assertEqual(self.source.repositories[FIRST]["owner"]["login"], "community")
|
||||
mapping = self.mappings()[0]
|
||||
self.assertEqual(mapping["source_repo_id"], 101)
|
||||
self.assertEqual(mapping["target_repo_id"], 1001)
|
||||
self.assertEqual(mapping["project_code"], "a" * 12)
|
||||
self.assertFalse(mapping["lfs_objects_download_verified"])
|
||||
self.assertEqual(len(self.state()["repositories"][FIRST]["snapshot"]["tags"]), 2)
|
||||
|
||||
def test_resume_does_not_copy_completed_repository_twice(self):
|
||||
self.run_migration(True, limit=1)
|
||||
self.assertEqual(self.run_migration(True), 0)
|
||||
self.assertEqual(len(self.target.posts), 2)
|
||||
self.assertEqual(len(self.mappings()), 2)
|
||||
self.assertEqual(self.run_migration(True), 0)
|
||||
self.assertEqual(len(self.target.posts), 2)
|
||||
|
||||
def test_existing_even_empty_target_is_never_overwritten_or_adopted(self):
|
||||
self.target.repositories[FIRST] = repository(FIRST, 999, "vibemake", empty=True)
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(self.target.posts, [])
|
||||
self.assertEqual(self.target.repositories[FIRST]["id"], 999)
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["status"], "CONFLICT")
|
||||
|
||||
def test_timeout_after_server_created_repository_never_retries_post(self):
|
||||
self.target.post_error = "HTTP_504"
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["status"], "REVIEW")
|
||||
self.target.post_error = None
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(len(self.target.posts), 1)
|
||||
self.assertEqual(self.mappings(), [])
|
||||
|
||||
def test_interrupt_before_receipt_remains_review_on_resume(self):
|
||||
self.target.interrupt_post = True
|
||||
with self.assertRaises(KeyboardInterrupt):
|
||||
self.run_migration(True, selected=[FIRST])
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["status"], "MIGRATING")
|
||||
self.target.interrupt_post = False
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(len(self.target.posts), 1)
|
||||
|
||||
def test_saved_receipt_allows_only_read_verification_after_network_failure(self):
|
||||
self.target.fail_read_after_post = True
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["receipt_id"], 1001)
|
||||
self.target.fail_read_after_post = False
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
|
||||
self.assertEqual(len(self.target.posts), 1)
|
||||
self.assertEqual(len(self.mappings()), 1)
|
||||
|
||||
def test_tag_mismatch_is_not_reported_as_success(self):
|
||||
self.target.change_target_tag = True
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["error"], "TARGET_GIT_REFERENCES_MISMATCH")
|
||||
self.assertEqual(self.mappings(), [])
|
||||
|
||||
def test_source_change_during_migration_requires_review(self):
|
||||
self.target.change_source_after_post = True
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertEqual(self.state()["repositories"][FIRST]["error"], "SOURCE_CHANGED_DURING_MIGRATION")
|
||||
|
||||
def test_unexpected_success_status_does_not_count_as_completion(self):
|
||||
self.target.post_status = 200
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
||||
self.assertNotIn("receipt_id", self.state()["repositories"][FIRST])
|
||||
|
||||
def test_other_organization_cannot_enter_source_inventory(self):
|
||||
self.source.repositories[FIRST]["owner"]["login"] = "other-org"
|
||||
with self.assertRaisesRegex(migration.MigrationError, "REPOSITORY_IDENTITY_MISMATCH"):
|
||||
self.run_migration(True)
|
||||
self.assertEqual(self.target.posts, [])
|
||||
|
||||
def test_owner_and_lfs_checks_happen_before_any_write(self):
|
||||
self.source.owner = False
|
||||
with self.assertRaisesRegex(migration.MigrationError, "ORGANIZATION_OWNER_REQUIRED"):
|
||||
self.run_migration(True)
|
||||
self.source.owner = True
|
||||
self.target.lfs_enabled = False
|
||||
with self.assertRaisesRegex(migration.MigrationError, "GIT_LFS_OR_MIGRATION_DISABLED"):
|
||||
self.run_migration(True)
|
||||
self.assertEqual(self.target.posts, [])
|
||||
|
||||
def test_state_cannot_be_reused_for_another_environment_or_changed_inventory(self):
|
||||
self.run_migration()
|
||||
self.target.base = "https://another-target.example"
|
||||
with self.assertRaisesRegex(migration.MigrationError, "STATE_ENVIRONMENT_MISMATCH"):
|
||||
self.run_migration(True)
|
||||
self.target.base = "https://target.example"
|
||||
self.source.repositories["p_cccccccccccc"] = repository("p_cccccccccccc", 103)
|
||||
with self.assertRaisesRegex(migration.MigrationError, "SOURCE_INVENTORY_CHANGED"):
|
||||
self.run_migration(True)
|
||||
self.assertEqual(self.target.posts, [])
|
||||
|
||||
def test_empty_repository_is_preserved(self):
|
||||
self.source.repositories[FIRST] = repository(FIRST, 101, empty=True)
|
||||
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
|
||||
self.assertTrue(self.target.repositories[FIRST]["empty"])
|
||||
self.assertEqual(len(self.mappings()), 1)
|
||||
|
||||
def test_state_permissions_and_lock_prevent_parallel_writes(self):
|
||||
self.run_migration()
|
||||
self.assertEqual(stat.S_IMODE(self.store.directory.stat().st_mode), 0o700)
|
||||
self.assertEqual(stat.S_IMODE(self.store.path.stat().st_mode), 0o600)
|
||||
with self.store.locked(), self.assertRaisesRegex(migration.MigrationError, "STATE_DIRECTORY_LOCKED"):
|
||||
with migration.StateStore(self.store.directory).locked():
|
||||
self.fail("不应获得第二个锁")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def http_fixture():
|
||||
calls = []
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
calls.append(self.path)
|
||||
self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
if self.path.endswith("/redirect"):
|
||||
self.send_response(302)
|
||||
self.send_header("Location", "/credential-sink")
|
||||
else:
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
self.wfile.write((SOURCE_TOKEN + TARGET_TOKEN).encode())
|
||||
|
||||
def do_GET(self):
|
||||
calls.append(self.path)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b"{}")
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield "http://127.0.0.1:" + str(server.server_port), calls
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
class TransportTest(unittest.TestCase):
|
||||
def test_post_redirect_does_not_forward_credentials_or_repeat_write(self):
|
||||
with http_fixture() as (base, calls):
|
||||
client = migration.ApiClient(base, TARGET_TOKEN)
|
||||
with self.assertRaises(migration.MigrationError) as caught:
|
||||
client.request("POST", "/redirect", {"auth_password": SOURCE_TOKEN})
|
||||
self.assertEqual(str(caught.exception), "HTTP_302")
|
||||
self.assertEqual(calls, ["/api/v1/redirect"])
|
||||
|
||||
def test_http_error_body_with_credentials_is_not_exposed(self):
|
||||
with http_fixture() as (base, _):
|
||||
client = migration.ApiClient(base, TARGET_TOKEN)
|
||||
with self.assertRaises(migration.MigrationError) as caught:
|
||||
client.request("POST", "/repos/migrate", {"auth_password": SOURCE_TOKEN})
|
||||
self.assertEqual(str(caught.exception), "HTTP_500")
|
||||
self.assertNotIn(SOURCE_TOKEN, str(caught.exception))
|
||||
self.assertNotIn(TARGET_TOKEN, str(caught.exception))
|
||||
|
||||
def test_invalid_base_url_does_not_echo_embedded_credentials_in_cli(self):
|
||||
result = subprocess.run([sys.executable, str(SCRIPT),
|
||||
"--source-url", "https://user:" + SOURCE_TOKEN + "@source.example",
|
||||
"--target-url", "https://target.example"],
|
||||
capture_output=True, text=True, check=False)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn(SOURCE_TOKEN, result.stdout + result.stderr)
|
||||
self.assertIn("INVALID_BASE_URL", result.stderr)
|
||||
|
||||
def test_base_url_keeps_subpath_and_rejects_insecure_or_credential_urls(self):
|
||||
self.assertEqual(migration.base_url("https://GITEA.example:443/gitea/", False),
|
||||
"https://gitea.example/gitea")
|
||||
for value in ("http://gitea.example", "https://gitea.example?token=secret",
|
||||
"https://gitea.example/#token", "https://gitea.example/a/../b"):
|
||||
with self.subTest(value=value), self.assertRaises(migration.MigrationError):
|
||||
migration.base_url(value, False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user