复制Gitea仓库到生产环境

This commit is contained in:
lxp
2026-10-20 10:49:23 +08:00
parent f8f2e95dbe
commit ee5c665d6d
3 changed files with 865 additions and 0 deletions
+463
View File
@@ -0,0 +1,463 @@
#!/usr/bin/env python3
"""按组织复制 Git 仓库及 LFS;默认只读预览,不修改业务数据库。"""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import getpass
import json
import os
from pathlib import Path
import re
import sys
import tempfile
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode, urlsplit, urlunsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
class MigrationError(Exception):
"""仅包含可安全记录的错误码,不携带响应正文或凭据。"""
def __init__(self, code: str):
super().__init__(code)
self.code = code
def timestamp() -> str:
return datetime.now(timezone.utc).isoformat()
def name_segment(value: str) -> str:
if (not isinstance(value, str) or not value or value in (".", "..")
or len(value) > 255 or any(char in value for char in "/\\?#%")
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)):
raise MigrationError("INVALID_NAME")
return quote(value, safe="")
def base_url(value: str, allow_http: bool) -> str:
try:
parsed = urlsplit(value)
port = parsed.port
if (parsed.scheme not in ("https", "http") or not parsed.hostname
or (parsed.scheme == "http" and not allow_http)
or parsed.username is not None or parsed.password is not None
or "?" in value or "#" in value or "\\" in value
or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
or (port is not None and not 1 <= port <= 65535)
or any(part in (".", "..") for part in parsed.path.split("/"))
or "%" in parsed.path):
raise ValueError()
host = parsed.hostname.lower()
if ":" in host:
host = "[" + host + "]"
if port is not None and port != {"http": 80, "https": 443}[parsed.scheme]:
host += ":" + str(port)
return urlunsplit((parsed.scheme, host, parsed.path.rstrip("/"), "", ""))
except ValueError:
raise MigrationError("INVALID_BASE_URL") from None
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
# 避免重定向转发认证头或再次发送迁移请求。
return None
class ApiClient:
def __init__(self, base: str, token: str, timeout: int = 30):
self.base = base
self.token = token
self.timeout = timeout
self.opener = build_opener(NoRedirect())
self.username = ""
def request(self, method: str, path: str, body: dict | None = None,
timeout: int | None = None, missing: bool = False) -> tuple[int, object]:
request = Request(self.base + "/api/v1" + path, method=method,
data=None if body is None else json.dumps(body).encode("utf-8"),
headers={"Authorization": "token " + self.token,
"Accept": "application/json",
"Content-Type": "application/json"})
try:
with self.opener.open(request, timeout=timeout or self.timeout) as response:
status = response.status
data = response.read(16 * 1024 * 1024 + 1)
if len(data) > 16 * 1024 * 1024:
raise MigrationError("RESPONSE_TOO_LARGE")
return status, json.loads(data) if data else None
except HTTPError as error:
status = error.code
error.close()
if missing and status == 404:
return status, None
# Gitea/反向代理的错误正文可能包含源地址或认证信息,不输出正文。
raise MigrationError("HTTP_" + str(status)) from None
except (URLError, TimeoutError, OSError):
raise MigrationError("NETWORK_ERROR") from None
except (ValueError, UnicodeError):
raise MigrationError("INVALID_JSON_RESPONSE") from None
def get(self, path: str, missing: bool = False):
status, data = self.request("GET", path, missing=missing)
if status == 404 and missing:
return None
if status != 200:
raise MigrationError("UNEXPECTED_GET_STATUS")
return data
def pages(self, path: str) -> list[dict]:
result = []
seen = set()
for page in range(1, 10001):
items = self.get(path + "?" + urlencode({"page": page, "limit": 50}))
if not isinstance(items, list):
raise MigrationError("INVALID_LIST_RESPONSE")
if not items:
return result
for item in items:
if not isinstance(item, dict):
raise MigrationError("INVALID_LIST_ITEM")
# 多个轻量标签可以指向同一个提交(相同 id),分页应按名称去重。
key = item.get("name", item.get("id"))
if not isinstance(key, (str, int)) or key in seen:
raise MigrationError("DUPLICATE_PAGINATION_ITEM")
seen.add(key)
result.append(item)
# 服务端可能将 limit 限制为更小值,不能按 len(items) < 50 提前终止。
raise MigrationError("PAGINATION_LIMIT")
def repo(self, org: str, name: str, missing: bool = False):
return self.get("/repos/" + name_segment(org) + "/" + name_segment(name), missing)
def clone_url(self, org: str, name: str) -> str:
return self.base + "/" + name_segment(org) + "/" + name_segment(name) + ".git"
def identity(info: dict, org: str, name: str) -> int:
if (not isinstance(info, dict) or type(info.get("id")) is not int or info["id"] <= 0
or not isinstance(info.get("owner"), dict)
or not isinstance(info["owner"].get("login"), str)
or info["owner"]["login"].casefold() != org.casefold()
or info.get("name") != name):
raise MigrationError("REPOSITORY_IDENTITY_MISMATCH")
return info["id"]
def preflight(client: ApiClient, org: str, target: bool) -> None:
user = client.get("/user")
if not isinstance(user, dict) or not isinstance(user.get("login"), str):
raise MigrationError("INVALID_USER_RESPONSE")
client.username = user["login"]
name_segment(client.username)
client.get("/orgs/" + name_segment(org))
# 源组织所有者能列出全部私有仓库;迁移 API 也要求目标组织所有者。
if user.get("is_admin") is not True:
owner = False
for team in client.pages("/orgs/" + name_segment(org) + "/teams"):
if team.get("permission") == "owner" and type(team.get("id")) is int:
status, member = client.request("GET", "/teams/" + str(team["id"])
+ "/members/" + name_segment(client.username), missing=True)
if (status == 200 and isinstance(member, dict)
and member.get("login") == client.username):
owner = True
break
if not owner:
raise MigrationError("ORGANIZATION_OWNER_REQUIRED")
settings = client.get("/settings/repository")
if (not isinstance(settings, dict) or settings.get("lfs_disabled") is not False
or settings.get("http_git_disabled") is not False
or (target and settings.get("migrations_disabled") is not False)):
raise MigrationError("GIT_LFS_OR_MIGRATION_DISABLED")
def snapshot(client: ApiClient, org: str, name: str, expected_id: int,
raw_references: bool = False) -> dict:
info = client.repo(org, name)
if identity(info, org, name) != expected_id:
raise MigrationError("REPOSITORY_ID_CHANGED")
if type(info.get("empty")) is not bool or not isinstance(info.get("default_branch"), str):
raise MigrationError("INVALID_REPOSITORY_RESPONSE")
result = {"empty": info["empty"], "default_branch": info["default_branch"],
"branches": {}, "tags": {}}
path = "/repos/" + name_segment(org) + "/" + name_segment(name)
if info["empty"] or raw_references:
# 历史导入中断后,数据库可能标记为空,但磁盘已存在 Git 引用。
for prefix, unit in (("heads", "branches"), ("tags", "tags")):
refs = client.get(path + "/git/refs/" + prefix, missing=True)
if refs is None:
continue
if not isinstance(refs, list):
raise MigrationError("INVALID_GIT_REFERENCE")
for ref in refs:
ref_path = ref.get("ref", "")
sha = ref.get("object", {}).get("sha")
expected_prefix = "refs/" + prefix + "/"
if (not isinstance(ref_path, str) or not ref_path.startswith(expected_prefix)
or not ref_path[len(expected_prefix):]
or not isinstance(sha, str)
or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)):
raise MigrationError("INVALID_GIT_REFERENCE")
ref_name = ref_path[len(expected_prefix):]
if ref_name in result[unit]:
raise MigrationError("INVALID_GIT_REFERENCE")
result[unit][ref_name] = sha
if result["branches"] or result["tags"] or raw_references:
result["empty"] = not (result["branches"] or result["tags"])
result["reference_format"] = "git-object-v1"
if not result["empty"] and result["default_branch"] not in result["branches"]:
raise MigrationError("DEFAULT_BRANCH_MISSING")
return result
for unit in ("branches", "tags"):
for ref in client.pages(path + "/" + unit):
commit = ref.get("commit")
sha = (commit.get("id") or commit.get("sha")) if isinstance(commit, dict) else None
ref_name = ref.get("name")
if (not isinstance(ref_name, str) or not ref_name
or not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)
or ref_name in result[unit]):
raise MigrationError("INVALID_GIT_REFERENCE")
result[unit][ref_name] = sha
if result["default_branch"] not in result["branches"]:
raise MigrationError("DEFAULT_BRANCH_MISSING")
return result
def atomic_json(path: Path, value: dict) -> None:
descriptor, temporary = tempfile.mkstemp(prefix=".migration-", dir=path.parent)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
Path(temporary).unlink(missing_ok=True)
class StateStore:
def __init__(self, directory: Path):
self.directory = directory
self.path = directory / "state.json"
@contextmanager
def locked(self):
if self.directory.is_symlink():
raise MigrationError("INVALID_STATE_DIRECTORY")
self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
self.directory.chmod(0o700)
with (self.directory / ".lock").open("a") as stream:
os.fchmod(stream.fileno(), 0o600)
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise MigrationError("STATE_DIRECTORY_LOCKED") from None
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def load(self, binding: dict, inventory: dict) -> dict:
if self.path.exists():
state = json.loads(self.path.read_text(encoding="utf-8"))
if state.get("version") != 1 or state.get("binding") != binding:
raise MigrationError("STATE_ENVIRONMENT_MISMATCH")
if state.get("inventory") != inventory:
raise MigrationError("SOURCE_INVENTORY_CHANGED")
for entry in state["repositories"].values():
if entry["status"] == "MIGRATING":
entry.update(status="REVIEW", error="INTERRUPTED_MIGRATION")
else:
state = {"version": 1, "created_at": timestamp(), "binding": binding,
"inventory": inventory,
"repositories": {name: {"source_id": repo_id, "status": "PLANNED"}
for name, repo_id in inventory.items()}}
self.save(state)
return state
def save(self, state: dict) -> None:
state["updated_at"] = timestamp()
atomic_json(self.path, state)
mappings = []
for name, entry in state["repositories"].items():
if entry["status"] != "VERIFIED":
continue
code = name[2:] if re.fullmatch(r"p_(?:[a-z0-9]{12}|[a-z0-9]{32})", name) else None
mappings.append({"repo_name": name, "project_code": code,
"source_repo_id": entry["source_id"],
"target_repo_id": entry["receipt_id"],
"default_branch": entry["snapshot"]["default_branch"],
"lfs_requested": True,
"lfs_objects_download_verified": entry.get("lfs_objects_download_verified", False),
"completion_evidence": entry.get("completion_evidence", "migration_http_201")})
atomic_json(self.directory / "mapping.json",
{"binding": state["binding"], "repositories": mappings})
def verify(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
name: str, entry: dict) -> None:
info = target.repo(target_org, name)
if identity(info, target_org, name) != entry["receipt_id"]:
raise MigrationError("TARGET_RECEIPT_ID_MISMATCH")
if info.get("private") is not True or info.get("mirror") is not False:
raise MigrationError("TARGET_PRIVACY_OR_MIRROR_MISMATCH")
raw_references = entry["snapshot"].get("reference_format") == "git-object-v1"
if snapshot(target, target_org, name, entry["receipt_id"], raw_references) != entry["snapshot"]:
raise MigrationError("TARGET_GIT_REFERENCES_MISMATCH")
if snapshot(source, source_org, name, entry["source_id"], raw_references) != entry["snapshot"]:
raise MigrationError("SOURCE_CHANGED_DURING_MIGRATION")
def migrate(source: ApiClient, target: ApiClient, source_org: str, target_org: str,
store: StateStore, execute: bool, selected: list[str], limit: int | None,
migration_timeout: int) -> int:
preflight(source, source_org, target=False)
preflight(target, target_org, target=True)
inventory = {}
for info in source.pages("/orgs/" + name_segment(source_org) + "/repos"):
name = info.get("name")
name_segment(name)
repo_id = identity(info, source_org, name)
if name in inventory:
raise MigrationError("DUPLICATE_REPOSITORY_NAME")
inventory[name] = repo_id
inventory = dict(sorted(inventory.items()))
if any(name not in inventory for name in selected):
raise MigrationError("SELECTED_REPOSITORY_NOT_IN_SOURCE_ORG")
binding = {"source_url": source.base, "source_org": source_org,
"target_url": target.base, "target_org": target_org,
"contents": "private-git-and-lfs-v1"}
state = store.load(binding, inventory)
print(f"源组织={source_org} 仓库数={len(inventory)} 目标组织={target_org} 模式={'正式执行' if execute else '只读预览'}")
processed = 0
for name, entry in state["repositories"].items():
if selected and name not in selected:
continue
if entry.get("receipt_id") is not None:
try:
verify(source, target, source_org, target_org, name, entry)
entry.update(status="VERIFIED", error=None)
print(f"已核对 {name}:完成收据、私有属性、默认分支、分支及标签提交一致")
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
print(f"待核对 {name}:{error.code};已保存完成收据,仅重试只读核验")
store.save(state)
continue
if entry["status"] != "PLANNED":
print(f"跳过 {name}:{entry['status']};没有可核对的完成收据,不自动重发")
continue
if limit is not None and processed >= limit:
continue
processed += 1
if target.repo(target_org, name, missing=True) is not None:
entry.update(status="CONFLICT", error="TARGET_ALREADY_EXISTS")
store.save(state)
print(f"同名冲突 {name}:保留生产仓库,不接管、不覆盖")
continue
frozen = snapshot(source, source_org, name, entry["source_id"])
if not execute:
print(f"待迁移 {name}:分支={len(frozen['branches'])} 标签={len(frozen['tags'])} LFS=请求迁移")
continue
# 在发送 POST 前强制落盘。超时、进程退出后不能仅凭仓库存在认定迁移完成。
entry.update(status="MIGRATING", snapshot=frozen, error=None)
store.save(state)
print(f"开始迁移 {name}:{source_org} -> {target_org}", flush=True)
body = {"clone_addr": source.clone_url(source_org, name),
"repo_owner": target_org, "repo_name": name, "service": "git",
"auth_username": source.username, "auth_password": source.token,
"private": True, "mirror": False, "lfs": True,
**{unit: False for unit in ("issues", "labels", "milestones",
"pull_requests", "releases", "wiki")}}
try:
status, receipt = target.request("POST", "/repos/migrate", body,
timeout=migration_timeout)
if status != 201:
raise MigrationError("MIGRATION_COMPLETION_NOT_CONFIRMED")
entry["receipt_id"] = identity(receipt, target_org, name)
entry["status"] = "RECEIVED"
store.save(state)
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
store.save(state)
print(f"迁移未确认 {name}:{error.code};本次停止新增仓库,请检查生产迁移日志")
break
try:
verify(source, target, source_org, target_org, name, entry)
entry.update(status="VERIFIED", error=None)
print(f"迁移完成 {name}:Git 引用已核对,LFS 已请求迁移,实体下载仍需抽查")
except MigrationError as error:
entry.update(status="REVIEW", error=error.code)
print(f"待核对 {name}:{error.code};保留已创建仓库和完成收据")
store.save(state)
entries = [entry for name, entry in state["repositories"].items()
if not selected or name in selected]
counts = {status: sum(entry["status"] == status for entry in entries)
for status in ("PLANNED", "VERIFIED", "CONFLICT", "REVIEW")}
print("处理结果 " + " ".join(f"{key}={value}" for key, value in counts.items()))
print("状态及映射目录:" + str(store.directory))
return 2 if counts["CONFLICT"] or counts["REVIEW"] else 0
def read_token(variable: str, prompt: str) -> str:
token = os.environ.get(variable, "")
if not token and sys.stdin.isatty():
token = getpass.getpass(prompt)
if not token or any(char.isspace() or ord(char) < 32 for char in token):
raise MigrationError("MISSING_OR_INVALID_" + variable)
return token
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-url", required=True, help="测试 Gitea 基础地址,不含 /api/v1")
parser.add_argument("--target-url", required=True, help="生产 Gitea 基础地址,不含 /api/v1")
parser.add_argument("--source-org", default="community")
parser.add_argument("--target-org", default="vibemake")
parser.add_argument("--state-dir", type=Path, default=Path("logs/gitea-org-migration"))
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--dry-run", action="store_true", help="只读预览,默认模式")
mode.add_argument("--execute", action="store_true", help="正式创建目标仓库")
parser.add_argument("--repo", action="append", default=[], help="只处理指定仓库,可重复指定")
parser.add_argument("--limit", type=int, help="限制本次尚未处理的仓库数")
parser.add_argument("--migration-timeout", type=int, default=1800, help="单仓库迁移请求超时秒数")
parser.add_argument("--allow-http", action="store_true", help="允许在可信网络通过明文 HTTP 发送凭据")
args = parser.parse_args()
try:
source_url = base_url(args.source_url, args.allow_http)
target_url = base_url(args.target_url, args.allow_http)
if source_url == target_url:
raise MigrationError("SOURCE_AND_TARGET_MUST_DIFFER")
for value in (args.source_org, args.target_org, *args.repo):
name_segment(value)
if (args.limit is not None and args.limit < 1) or args.migration_timeout < 1:
raise MigrationError("INVALID_LIMIT_OR_TIMEOUT")
source_token = read_token("GITEA_MIGRATION_SOURCE_TOKEN", "测试 Gitea Token(输入不回显):")
target_token = read_token("GITEA_MIGRATION_TARGET_TOKEN", "生产 Gitea Token(输入不回显):")
with StateStore(args.state_dir).locked():
return migrate(ApiClient(source_url, source_token), ApiClient(target_url, target_token),
args.source_org, args.target_org, StateStore(args.state_dir),
args.execute, args.repo, args.limit, args.migration_timeout)
except MigrationError as error:
print("操作停止:" + error.code, file=sys.stderr)
return 1
except KeyboardInterrupt:
print("操作已中断。已发出的迁移可能仍在生产执行;保留状态目录后再核对。", file=sys.stderr)
return 130
except Exception as error:
# 不输出异常正文,避免底层网络库或无效输入携带敏感值。
print("操作停止:" + type(error).__name__ + ";检查状态文件和运行环境。", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
+363
View File
@@ -0,0 +1,363 @@
import copy
from contextlib import contextmanager, redirect_stdout
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import importlib.util
import io
import json
from pathlib import Path
import stat
import subprocess
import sys
import tempfile
import threading
import unittest
from urllib.parse import parse_qs, unquote, urlsplit
SCRIPT = Path(__file__).parents[1] / "migrate-gitea-org.py"
spec = importlib.util.spec_from_file_location("gitea_migration", SCRIPT)
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
SOURCE_TOKEN = "source-test-secret-never-log"
TARGET_TOKEN = "target-test-secret-never-log"
SHA = "1" * 40
OTHER_SHA = "2" * 40
FIRST = "p_aaaaaaaaaaaa"
SECOND = "p_bbbbbbbbbbbb"
def repository(name, repo_id, owner="community", empty=False):
return {"id": repo_id, "name": name, "owner": {"login": owner},
"private": True, "mirror": False, "empty": empty, "default_branch": "main",
"branches": {} if empty else {"main": SHA, "dev": OTHER_SHA},
"tags": {} if empty else {"v1": SHA, "v1-alias": SHA}}
class FakeApi(migration.ApiClient):
"""模拟服务端限制分页大小、私有组织及同步仓库迁移,不访问真实服务。"""
def __init__(self, source=None):
self.source = source
self.org = "vibemake" if source else "community"
super().__init__("https://target.example" if source else "https://source.example",
TARGET_TOKEN if source else SOURCE_TOKEN)
self.repositories = {} if source else {
FIRST: repository(FIRST, 101), SECOND: repository(SECOND, 102)}
self.requests = []
self.posts = []
self.owner = True
self.lfs_enabled = True
self.post_status = 201
self.post_error = None
self.interrupt_post = False
self.fail_read_after_post = False
self.change_target_tag = False
self.change_source_after_post = False
def request(self, method, path, body=None, timeout=None, missing=False):
self.requests.append((method, path))
parsed = urlsplit(path)
route = [unquote(part) for part in parsed.path.strip("/").split("/")]
page = int(parse_qs(parsed.query).get("page", ["1"])[0])
if method == "POST":
if path != "/repos/migrate":
raise AssertionError("不允许其他写接口")
self.posts.append(copy.deepcopy(body))
name = body["repo_name"]
info = copy.deepcopy(self.source.repositories[name])
info.update(id=1000 + len(self.posts), owner={"login": body["repo_owner"]},
private=True, mirror=False)
self.repositories[name] = info
if self.change_target_tag:
info["tags"]["v1"] = OTHER_SHA
if self.change_source_after_post:
self.source.repositories[name]["branches"]["main"] = OTHER_SHA
if self.interrupt_post:
raise KeyboardInterrupt()
if self.post_error:
raise migration.MigrationError(self.post_error)
return self.post_status, copy.deepcopy(info)
if path == "/user":
return 200, {"login": "service", "is_admin": False}
if path == "/orgs/" + self.org:
return 200, {"name": self.org}
if path == "/settings/repository":
return 200, {"lfs_disabled": not self.lfs_enabled,
"http_git_disabled": False, "migrations_disabled": False}
if route == ["orgs", self.org, "teams"]:
teams = [{"id": 1, "name": "Owners", "permission": "owner"}]
return 200, teams[page - 1:page]
if path == "/teams/1/members/service":
return (200, {"login": "service"}) if self.owner else (404, None)
if route == ["orgs", self.org, "repos"]:
# 服务端上限为一条,客户端传 limit=50 也必须继续分页。
return 200, copy.deepcopy(list(self.repositories.values())[page - 1:page])
if len(route) >= 3 and route[:2] == ["repos", self.org]:
info = self.repositories.get(route[2])
if info is None:
if missing:
return 404, None
raise migration.MigrationError("HTTP_404")
if self.fail_read_after_post and self.posts:
raise migration.MigrationError("NETWORK_ERROR")
if len(route) == 3:
return 200, copy.deepcopy(info)
if len(route) == 5 and route[3] == "git" and route[4].startswith("refs"):
raise AssertionError("Git 引用路由应包含 heads 或 tags")
if len(route) == 6 and route[3:5] == ["git", "refs"]:
unit = {"heads": "branches", "tags": "tags"}[route[5]]
refs = [{"ref": "refs/" + route[5] + "/" + name, "object": {"sha": sha}}
for name, sha in info[unit].items()]
return (200, refs) if refs else (404, None)
if len(route) == 4 and route[3] in ("branches", "tags"):
unit = route[3]
refs = [{"name": name, "commit": {"id" if unit == "branches" else "sha": sha},
**({"id": sha} if unit == "tags" else {})}
for name, sha in info[unit].items()]
return 200, refs[page - 1:page]
raise AssertionError("未预期的模拟 API 路由")
class MigrationTest(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.store = migration.StateStore(Path(self.temporary.name) / "state")
self.source = FakeApi()
self.target = FakeApi(self.source)
def run_migration(self, execute=False, limit=None, selected=None):
output = io.StringIO()
with self.store.locked(), redirect_stdout(output):
code = migration.migrate(self.source, self.target, "community", "vibemake",
self.store, execute, selected or [], limit, 1800)
self.assertNotIn(SOURCE_TOKEN, output.getvalue())
self.assertNotIn(TARGET_TOKEN, output.getvalue())
for path in self.store.directory.iterdir():
if path.is_file():
text = path.read_text(encoding="utf-8")
self.assertNotIn(SOURCE_TOKEN, text)
self.assertNotIn(TARGET_TOKEN, text)
return code
def state(self):
return json.loads(self.store.path.read_text())
def mappings(self):
return json.loads((self.store.directory / "mapping.json").read_text())["repositories"]
def test_preview_paginates_all_private_repositories_and_never_writes_remote(self):
self.assertEqual(self.run_migration(), 0)
self.assertEqual(len(self.state()["inventory"]), 2)
self.assertEqual(self.target.posts, [])
self.assertEqual(self.mappings(), [])
self.assertTrue(all(method == "GET" for method, _ in self.source.requests))
self.assertTrue(all(method == "GET" for method, _ in self.target.requests))
self.assertIn(("GET", "/orgs/community/repos?page=3&limit=50"), self.source.requests)
def test_stale_empty_metadata_still_copies_and_verifies_real_git_references(self):
self.source.repositories[FIRST]["empty"] = True
self.assertEqual(self.run_migration(True, limit=1), 0)
entry = self.state()["repositories"][FIRST]
self.assertFalse(entry["snapshot"]["empty"])
self.assertEqual(entry["snapshot"]["reference_format"], "git-object-v1")
self.assertEqual(len(entry["snapshot"]["branches"]), 2)
self.assertEqual(len(entry["snapshot"]["tags"]), 2)
def test_copies_git_lfs_to_new_owner_and_saves_mapping_without_credentials(self):
self.assertEqual(self.run_migration(True, limit=1), 0)
self.assertEqual(len(self.target.posts), 1)
body = self.target.posts[0]
self.assertEqual(body["repo_owner"], "vibemake")
self.assertEqual(body["repo_name"], FIRST)
self.assertEqual(body["clone_addr"], "https://source.example/community/" + FIRST + ".git")
self.assertEqual(body["auth_username"], "service")
self.assertEqual(body["auth_password"], SOURCE_TOKEN)
self.assertTrue(body["lfs"])
self.assertTrue(body["private"])
self.assertFalse(body["mirror"])
for option in ("issues", "labels", "milestones", "pull_requests", "releases", "wiki"):
self.assertFalse(body[option])
self.assertEqual(self.source.repositories[FIRST]["owner"]["login"], "community")
mapping = self.mappings()[0]
self.assertEqual(mapping["source_repo_id"], 101)
self.assertEqual(mapping["target_repo_id"], 1001)
self.assertEqual(mapping["project_code"], "a" * 12)
self.assertFalse(mapping["lfs_objects_download_verified"])
self.assertEqual(len(self.state()["repositories"][FIRST]["snapshot"]["tags"]), 2)
def test_resume_does_not_copy_completed_repository_twice(self):
self.run_migration(True, limit=1)
self.assertEqual(self.run_migration(True), 0)
self.assertEqual(len(self.target.posts), 2)
self.assertEqual(len(self.mappings()), 2)
self.assertEqual(self.run_migration(True), 0)
self.assertEqual(len(self.target.posts), 2)
def test_existing_even_empty_target_is_never_overwritten_or_adopted(self):
self.target.repositories[FIRST] = repository(FIRST, 999, "vibemake", empty=True)
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.target.posts, [])
self.assertEqual(self.target.repositories[FIRST]["id"], 999)
self.assertEqual(self.state()["repositories"][FIRST]["status"], "CONFLICT")
def test_timeout_after_server_created_repository_never_retries_post(self):
self.target.post_error = "HTTP_504"
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["status"], "REVIEW")
self.target.post_error = None
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(len(self.target.posts), 1)
self.assertEqual(self.mappings(), [])
def test_interrupt_before_receipt_remains_review_on_resume(self):
self.target.interrupt_post = True
with self.assertRaises(KeyboardInterrupt):
self.run_migration(True, selected=[FIRST])
self.assertEqual(self.state()["repositories"][FIRST]["status"], "MIGRATING")
self.target.interrupt_post = False
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(len(self.target.posts), 1)
def test_saved_receipt_allows_only_read_verification_after_network_failure(self):
self.target.fail_read_after_post = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["receipt_id"], 1001)
self.target.fail_read_after_post = False
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
self.assertEqual(len(self.target.posts), 1)
self.assertEqual(len(self.mappings()), 1)
def test_tag_mismatch_is_not_reported_as_success(self):
self.target.change_target_tag = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["error"], "TARGET_GIT_REFERENCES_MISMATCH")
self.assertEqual(self.mappings(), [])
def test_source_change_during_migration_requires_review(self):
self.target.change_source_after_post = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["error"], "SOURCE_CHANGED_DURING_MIGRATION")
def test_unexpected_success_status_does_not_count_as_completion(self):
self.target.post_status = 200
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertNotIn("receipt_id", self.state()["repositories"][FIRST])
def test_other_organization_cannot_enter_source_inventory(self):
self.source.repositories[FIRST]["owner"]["login"] = "other-org"
with self.assertRaisesRegex(migration.MigrationError, "REPOSITORY_IDENTITY_MISMATCH"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_owner_and_lfs_checks_happen_before_any_write(self):
self.source.owner = False
with self.assertRaisesRegex(migration.MigrationError, "ORGANIZATION_OWNER_REQUIRED"):
self.run_migration(True)
self.source.owner = True
self.target.lfs_enabled = False
with self.assertRaisesRegex(migration.MigrationError, "GIT_LFS_OR_MIGRATION_DISABLED"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_state_cannot_be_reused_for_another_environment_or_changed_inventory(self):
self.run_migration()
self.target.base = "https://another-target.example"
with self.assertRaisesRegex(migration.MigrationError, "STATE_ENVIRONMENT_MISMATCH"):
self.run_migration(True)
self.target.base = "https://target.example"
self.source.repositories["p_cccccccccccc"] = repository("p_cccccccccccc", 103)
with self.assertRaisesRegex(migration.MigrationError, "SOURCE_INVENTORY_CHANGED"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_empty_repository_is_preserved(self):
self.source.repositories[FIRST] = repository(FIRST, 101, empty=True)
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
self.assertTrue(self.target.repositories[FIRST]["empty"])
self.assertEqual(len(self.mappings()), 1)
def test_state_permissions_and_lock_prevent_parallel_writes(self):
self.run_migration()
self.assertEqual(stat.S_IMODE(self.store.directory.stat().st_mode), 0o700)
self.assertEqual(stat.S_IMODE(self.store.path.stat().st_mode), 0o600)
with self.store.locked(), self.assertRaisesRegex(migration.MigrationError, "STATE_DIRECTORY_LOCKED"):
with migration.StateStore(self.store.directory).locked():
self.fail("不应获得第二个锁")
@contextmanager
def http_fixture():
calls = []
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
calls.append(self.path)
self.rfile.read(int(self.headers.get("Content-Length", "0")))
if self.path.endswith("/redirect"):
self.send_response(302)
self.send_header("Location", "/credential-sink")
else:
self.send_response(500)
self.end_headers()
self.wfile.write((SOURCE_TOKEN + TARGET_TOKEN).encode())
def do_GET(self):
calls.append(self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(b"{}")
def log_message(self, *args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
thread.start()
try:
yield "http://127.0.0.1:" + str(server.server_port), calls
finally:
server.shutdown()
server.server_close()
thread.join()
class TransportTest(unittest.TestCase):
def test_post_redirect_does_not_forward_credentials_or_repeat_write(self):
with http_fixture() as (base, calls):
client = migration.ApiClient(base, TARGET_TOKEN)
with self.assertRaises(migration.MigrationError) as caught:
client.request("POST", "/redirect", {"auth_password": SOURCE_TOKEN})
self.assertEqual(str(caught.exception), "HTTP_302")
self.assertEqual(calls, ["/api/v1/redirect"])
def test_http_error_body_with_credentials_is_not_exposed(self):
with http_fixture() as (base, _):
client = migration.ApiClient(base, TARGET_TOKEN)
with self.assertRaises(migration.MigrationError) as caught:
client.request("POST", "/repos/migrate", {"auth_password": SOURCE_TOKEN})
self.assertEqual(str(caught.exception), "HTTP_500")
self.assertNotIn(SOURCE_TOKEN, str(caught.exception))
self.assertNotIn(TARGET_TOKEN, str(caught.exception))
def test_invalid_base_url_does_not_echo_embedded_credentials_in_cli(self):
result = subprocess.run([sys.executable, str(SCRIPT),
"--source-url", "https://user:" + SOURCE_TOKEN + "@source.example",
"--target-url", "https://target.example"],
capture_output=True, text=True, check=False)
self.assertNotEqual(result.returncode, 0)
self.assertNotIn(SOURCE_TOKEN, result.stdout + result.stderr)
self.assertIn("INVALID_BASE_URL", result.stderr)
def test_base_url_keeps_subpath_and_rejects_insecure_or_credential_urls(self):
self.assertEqual(migration.base_url("https://GITEA.example:443/gitea/", False),
"https://gitea.example/gitea")
for value in ("http://gitea.example", "https://gitea.example?token=secret",
"https://gitea.example/#token", "https://gitea.example/a/../b"):
with self.subTest(value=value), self.assertRaises(migration.MigrationError):
migration.base_url(value, False)
if __name__ == "__main__":
unittest.main()