From ee5c665d6d3863409893df29ae875e5c4848d03f Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Tue, 20 Oct 2026 10:49:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=8D=E5=88=B6Gitea=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E5=88=B0=E7=94=9F=E4=BA=A7=E7=8E=AF=E5=A2=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 39 ++ scripts/migrate-gitea-org.py | 463 ++++++++++++++++++++++ scripts/tests/test_gitea_org_migration.py | 363 +++++++++++++++++ 3 files changed, 865 insertions(+) create mode 100644 scripts/migrate-gitea-org.py create mode 100644 scripts/tests/test_gitea_org_migration.py diff --git a/README.md b/README.md index ab863d5..ab4997a 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,45 @@ python3 scripts/test-gitea.py 该测试使用临时 H2 数据库,在配置的组织创建临时项目仓库,验证私有仓库、多文件提交、中文路径、二进制上传、固定发布版本及快照 Fork,结束后仅删除本次测试创建的仓库。Gitea 26.4.0 内置的 1.26.4 支持批量文件变更;以服务器 /swagger.v1.json 为对接依据。 +## 测试 Gitea 组织仓库复制到生产 + +`scripts/migrate-gitea-org.py` 使用 Python 3.9+ 标准库,将测试 `community` 下的 Git 仓库复制到生产 `vibemake`(组织可通过参数修改)。生产 Gitea 应先初始化,并创建目标组织。目标为同名私有普通仓库,保留默认分支、历史、分支和标签,并请求迁移 LFS;不复制 Issue、PR、Release、Wiki、用户、组织成员、权限、Webhook 或 Actions 执行记录,也不导入 Gitea 整库、不复制实例配置。 + +两端账号需是对应组织的所有者或站点管理员。测试 Token 需有 `read:repository`、`read:organization`、`read:user`、`read:misc`,生产 Token 需有 `write:repository`、`write:organization`、`read:user`、`read:misc`;仓库访问范围应包含私有仓库。迁移期间暂停源组织写入、导入任务以及目标后端写入,Gitea 本身保持运行。生产 Gitea 容器必须能通过 HTTP Git 和 LFS 访问测试地址,两端开启 HTTP Git/LFS,生产开启仓库迁移。 + +在项目根目录执行(替换地址): + +```bash +# 两次提示分别输入测试、生产 Token,输入不回显,不会写入命令历史。 +# 默认只读预览;不自动读取现有 .env,避免把两个环境的凭据混用。 +python3 scripts/migrate-gitea-org.py \ + --source-url https://测试Gitea域名 \ + --target-url https://生产Gitea域名 \ + --dry-run + +# 先复制一个尚未处理的仓库。 +python3 scripts/migrate-gitea-org.py \ + --source-url https://测试Gitea域名 \ + --target-url https://生产Gitea域名 \ + --execute --limit 1 + +# 抽查通过后,用相同状态目录续跑其余仓库。 +python3 scripts/migrate-gitea-org.py \ + --source-url https://测试Gitea域名 \ + --target-url https://生产Gitea域名 \ + --execute +``` + +可重复指定 `--repo p_项目编码` 限定仓库;`--source-org` 默认 `community`,`--target-org` 默认 `vibemake`。非交互执行可通过 `GITEA_MIGRATION_SOURCE_TOKEN`、`GITEA_MIGRATION_TARGET_TOKEN` 注入凭据,不接受 Token 命令行参数。默认只允许 HTTPS;仅在可信网络确需明文连接时指定 `--allow-http`。API 请求不跟随重定向,不将凭据放入 URL、状态文件或日志。迁移请求会将源账号和 Token 放在发往生产 Gitea 的请求体中,生产 Gitea 用它们读取源私有 Git/LFS。单仓库请求超时默认 1800 秒,可用 `--migration-timeout` 调整;生产反向代理的超时也应相应配置。 + +`--state-dir` 默认 `logs/gitea-org-migration`,同一批次只允许运行一个进程;目录权限 0700,文件权限 0600。目录绑定源/目标地址、组织和初始仓库清单;组织仓库发生增删或同名重建时停止,不自动改变迁移范围。`state.json` 保存进度、源分支/标签快照及成功迁移返回的目标仓库标识;`mapping.json` 仅包含已取得 HTTP 201 完成收据并核对仓库身份、私有属性、默认分支、分支和标签提交的记录。LFS 由 Gitea 迁移流程处理,脚本不逐对象下载核验;先试迁一个带 LFS 的仓库,用 `git lfs pull`/实际下载检查实体后再批量执行。子模块只保留 Git 引用,不递归复制其他仓库。 + +生产同名仓库标记 `CONFLICT`,不接管、不覆盖。超时、服务端错误、返回状态不明确或迁移中断时标记 `REVIEW`,停止本次新增仓库;Gitea 可能仍在后台执行,不能仅凭仓库存在认定 LFS 已完成。保留状态目录并核对生产日志及仓库;没有成功收据的记录不会自动重发。有成功收据、仅核验失败的记录,在续跑时只读核对。不要删除状态文件来盲目重试,也不要自动删除生产仓库。退出码 0 表示本次无冲突/待核对项(预览或 `--limit` 仍可能有 `PLANNED`),2 表示存在 `CONFLICT`/`REVIEW`,1 表示预检或运行环境错误。 + +历史导入中断后,源数据库的 `empty` 标记可能与磁盘内容不一致;脚本会检查实际 Git 引用,存在引用时按 Git 对象 SHA 核对分支和标签。LFS 还需核查迁移日志:Gitea 可能忽略源端缺失的 LFS 对象后仍返回 HTTP 201。出现 `Ignoring missing upstream LFS object` 时,应按对象 SHA-256、大小建立缺失清单,从可信原始上游或备份恢复,并验证生产端实体可下载;不能把成功响应当作缺失文件已补齐。 + +仓库复制后,生产社区后端设置自己的 `GITEA_BASE_URL`、`GITEA_TOKEN` 和 `GITEA_ORGANIZATION=vibemake`。此工具不连接业务数据库;已复制到生产的 `project` 记录还需按 `mapping.json` 核对项目编码及原仓库标识,再在生产业务库回写 `gitea_owner` 和新的 `gitea_repo_id`,保持 `gitea_repo_name`、发布提交 SHA 等值,并确认已发布 SHA 可读取。`gitea_repo_id` 有唯一索引,回写需处理旧标识与新标识的冲突;备份后在维护事务中执行,不改测试业务库。原 GitHub 导入批次的收据不能直接当作新生产仓库收据复用。OSS 封面、URDF、业务用户及项目记录的复制需单独处理。 + ## 项目封面与 OSS 直传 项目 API 的 `coverUrl` 接收和返回完整 HTTPS 地址,平台封面在数据库中保存 `/project-covers/...` 环境相对路径,外部图片保留完整 URL。无封面时返回 null;创建时可省略,修改时省略或 null 保留原值,传空字符串清除。Fork 继承同一资源引用,替换、清除或删除项目不自动删除 OSS 图片。URDF 同样保存 `/urdf/...` 环境相对路径,上传和 OSS 操作仍使用含环境根目录的真实 key。人工迁移和回退见 [数据库说明](docs/database.md#资源路径历史数据迁移)。 diff --git a/scripts/migrate-gitea-org.py b/scripts/migrate-gitea-org.py new file mode 100644 index 0000000..f3fbd8c --- /dev/null +++ b/scripts/migrate-gitea-org.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +"""按组织复制 Git 仓库及 LFS;默认只读预览,不修改业务数据库。""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import fcntl +import getpass +import json +import os +from pathlib import Path +import re +import sys +import tempfile +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode, urlsplit, urlunsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + + +class MigrationError(Exception): + """仅包含可安全记录的错误码,不携带响应正文或凭据。""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def timestamp() -> str: + return datetime.now(timezone.utc).isoformat() + + +def name_segment(value: str) -> str: + if (not isinstance(value, str) or not value or value in (".", "..") + or len(value) > 255 or any(char in value for char in "/\\?#%") + or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)): + raise MigrationError("INVALID_NAME") + return quote(value, safe="") + + +def base_url(value: str, allow_http: bool) -> str: + try: + parsed = urlsplit(value) + port = parsed.port + if (parsed.scheme not in ("https", "http") or not parsed.hostname + or (parsed.scheme == "http" and not allow_http) + or parsed.username is not None or parsed.password is not None + or "?" in value or "#" in value or "\\" in value + or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value) + or (port is not None and not 1 <= port <= 65535) + or any(part in (".", "..") for part in parsed.path.split("/")) + or "%" in parsed.path): + raise ValueError() + host = parsed.hostname.lower() + if ":" in host: + host = "[" + host + "]" + if port is not None and port != {"http": 80, "https": 443}[parsed.scheme]: + host += ":" + str(port) + return urlunsplit((parsed.scheme, host, parsed.path.rstrip("/"), "", "")) + except ValueError: + raise MigrationError("INVALID_BASE_URL") from None + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # 避免重定向转发认证头或再次发送迁移请求。 + return None + + +class ApiClient: + def __init__(self, base: str, token: str, timeout: int = 30): + self.base = base + self.token = token + self.timeout = timeout + self.opener = build_opener(NoRedirect()) + self.username = "" + + def request(self, method: str, path: str, body: dict | None = None, + timeout: int | None = None, missing: bool = False) -> tuple[int, object]: + request = Request(self.base + "/api/v1" + path, method=method, + data=None if body is None else json.dumps(body).encode("utf-8"), + headers={"Authorization": "token " + self.token, + "Accept": "application/json", + "Content-Type": "application/json"}) + try: + with self.opener.open(request, timeout=timeout or self.timeout) as response: + status = response.status + data = response.read(16 * 1024 * 1024 + 1) + if len(data) > 16 * 1024 * 1024: + raise MigrationError("RESPONSE_TOO_LARGE") + return status, json.loads(data) if data else None + except HTTPError as error: + status = error.code + error.close() + if missing and status == 404: + return status, None + # Gitea/反向代理的错误正文可能包含源地址或认证信息,不输出正文。 + raise MigrationError("HTTP_" + str(status)) from None + except (URLError, TimeoutError, OSError): + raise MigrationError("NETWORK_ERROR") from None + except (ValueError, UnicodeError): + raise MigrationError("INVALID_JSON_RESPONSE") from None + + def get(self, path: str, missing: bool = False): + status, data = self.request("GET", path, missing=missing) + if status == 404 and missing: + return None + if status != 200: + raise MigrationError("UNEXPECTED_GET_STATUS") + return data + + def pages(self, path: str) -> list[dict]: + result = [] + seen = set() + for page in range(1, 10001): + items = self.get(path + "?" + urlencode({"page": page, "limit": 50})) + if not isinstance(items, list): + raise MigrationError("INVALID_LIST_RESPONSE") + if not items: + return result + for item in items: + if not isinstance(item, dict): + raise MigrationError("INVALID_LIST_ITEM") + # 多个轻量标签可以指向同一个提交(相同 id),分页应按名称去重。 + key = item.get("name", item.get("id")) + if not isinstance(key, (str, int)) or key in seen: + raise MigrationError("DUPLICATE_PAGINATION_ITEM") + seen.add(key) + result.append(item) + # 服务端可能将 limit 限制为更小值,不能按 len(items) < 50 提前终止。 + raise MigrationError("PAGINATION_LIMIT") + + def repo(self, org: str, name: str, missing: bool = False): + return self.get("/repos/" + name_segment(org) + "/" + name_segment(name), missing) + + def clone_url(self, org: str, name: str) -> str: + return self.base + "/" + name_segment(org) + "/" + name_segment(name) + ".git" + + +def identity(info: dict, org: str, name: str) -> int: + if (not isinstance(info, dict) or type(info.get("id")) is not int or info["id"] <= 0 + or not isinstance(info.get("owner"), dict) + or not isinstance(info["owner"].get("login"), str) + or info["owner"]["login"].casefold() != org.casefold() + or info.get("name") != name): + raise MigrationError("REPOSITORY_IDENTITY_MISMATCH") + return info["id"] + + +def preflight(client: ApiClient, org: str, target: bool) -> None: + user = client.get("/user") + if not isinstance(user, dict) or not isinstance(user.get("login"), str): + raise MigrationError("INVALID_USER_RESPONSE") + client.username = user["login"] + name_segment(client.username) + client.get("/orgs/" + name_segment(org)) + # 源组织所有者能列出全部私有仓库;迁移 API 也要求目标组织所有者。 + if user.get("is_admin") is not True: + owner = False + for team in client.pages("/orgs/" + name_segment(org) + "/teams"): + if team.get("permission") == "owner" and type(team.get("id")) is int: + status, member = client.request("GET", "/teams/" + str(team["id"]) + + "/members/" + name_segment(client.username), missing=True) + if (status == 200 and isinstance(member, dict) + and member.get("login") == client.username): + owner = True + break + if not owner: + raise MigrationError("ORGANIZATION_OWNER_REQUIRED") + settings = client.get("/settings/repository") + if (not isinstance(settings, dict) or settings.get("lfs_disabled") is not False + or settings.get("http_git_disabled") is not False + or (target and settings.get("migrations_disabled") is not False)): + raise MigrationError("GIT_LFS_OR_MIGRATION_DISABLED") + + +def snapshot(client: ApiClient, org: str, name: str, expected_id: int, + raw_references: bool = False) -> dict: + info = client.repo(org, name) + if identity(info, org, name) != expected_id: + raise MigrationError("REPOSITORY_ID_CHANGED") + if type(info.get("empty")) is not bool or not isinstance(info.get("default_branch"), str): + raise MigrationError("INVALID_REPOSITORY_RESPONSE") + result = {"empty": info["empty"], "default_branch": info["default_branch"], + "branches": {}, "tags": {}} + path = "/repos/" + name_segment(org) + "/" + name_segment(name) + if info["empty"] or raw_references: + # 历史导入中断后,数据库可能标记为空,但磁盘已存在 Git 引用。 + for prefix, unit in (("heads", "branches"), ("tags", "tags")): + refs = client.get(path + "/git/refs/" + prefix, missing=True) + if refs is None: + continue + if not isinstance(refs, list): + raise MigrationError("INVALID_GIT_REFERENCE") + for ref in refs: + ref_path = ref.get("ref", "") + sha = ref.get("object", {}).get("sha") + expected_prefix = "refs/" + prefix + "/" + if (not isinstance(ref_path, str) or not ref_path.startswith(expected_prefix) + or not ref_path[len(expected_prefix):] + or not isinstance(sha, str) + or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)): + raise MigrationError("INVALID_GIT_REFERENCE") + ref_name = ref_path[len(expected_prefix):] + if ref_name in result[unit]: + raise MigrationError("INVALID_GIT_REFERENCE") + result[unit][ref_name] = sha + if result["branches"] or result["tags"] or raw_references: + result["empty"] = not (result["branches"] or result["tags"]) + result["reference_format"] = "git-object-v1" + if not result["empty"] and result["default_branch"] not in result["branches"]: + raise MigrationError("DEFAULT_BRANCH_MISSING") + return result + for unit in ("branches", "tags"): + for ref in client.pages(path + "/" + unit): + commit = ref.get("commit") + sha = (commit.get("id") or commit.get("sha")) if isinstance(commit, dict) else None + ref_name = ref.get("name") + if (not isinstance(ref_name, str) or not ref_name + or not isinstance(sha, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha) + or ref_name in result[unit]): + raise MigrationError("INVALID_GIT_REFERENCE") + result[unit][ref_name] = sha + if result["default_branch"] not in result["branches"]: + raise MigrationError("DEFAULT_BRANCH_MISSING") + return result + + +def atomic_json(path: Path, value: dict) -> None: + descriptor, temporary = tempfile.mkstemp(prefix=".migration-", dir=path.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(value, stream, ensure_ascii=False, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + Path(temporary).unlink(missing_ok=True) + + +class StateStore: + def __init__(self, directory: Path): + self.directory = directory + self.path = directory / "state.json" + + @contextmanager + def locked(self): + if self.directory.is_symlink(): + raise MigrationError("INVALID_STATE_DIRECTORY") + self.directory.mkdir(parents=True, exist_ok=True, mode=0o700) + self.directory.chmod(0o700) + with (self.directory / ".lock").open("a") as stream: + os.fchmod(stream.fileno(), 0o600) + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise MigrationError("STATE_DIRECTORY_LOCKED") from None + try: + yield + finally: + fcntl.flock(stream, fcntl.LOCK_UN) + + def load(self, binding: dict, inventory: dict) -> dict: + if self.path.exists(): + state = json.loads(self.path.read_text(encoding="utf-8")) + if state.get("version") != 1 or state.get("binding") != binding: + raise MigrationError("STATE_ENVIRONMENT_MISMATCH") + if state.get("inventory") != inventory: + raise MigrationError("SOURCE_INVENTORY_CHANGED") + for entry in state["repositories"].values(): + if entry["status"] == "MIGRATING": + entry.update(status="REVIEW", error="INTERRUPTED_MIGRATION") + else: + state = {"version": 1, "created_at": timestamp(), "binding": binding, + "inventory": inventory, + "repositories": {name: {"source_id": repo_id, "status": "PLANNED"} + for name, repo_id in inventory.items()}} + self.save(state) + return state + + def save(self, state: dict) -> None: + state["updated_at"] = timestamp() + atomic_json(self.path, state) + mappings = [] + for name, entry in state["repositories"].items(): + if entry["status"] != "VERIFIED": + continue + code = name[2:] if re.fullmatch(r"p_(?:[a-z0-9]{12}|[a-z0-9]{32})", name) else None + mappings.append({"repo_name": name, "project_code": code, + "source_repo_id": entry["source_id"], + "target_repo_id": entry["receipt_id"], + "default_branch": entry["snapshot"]["default_branch"], + "lfs_requested": True, + "lfs_objects_download_verified": entry.get("lfs_objects_download_verified", False), + "completion_evidence": entry.get("completion_evidence", "migration_http_201")}) + atomic_json(self.directory / "mapping.json", + {"binding": state["binding"], "repositories": mappings}) + + +def verify(source: ApiClient, target: ApiClient, source_org: str, target_org: str, + name: str, entry: dict) -> None: + info = target.repo(target_org, name) + if identity(info, target_org, name) != entry["receipt_id"]: + raise MigrationError("TARGET_RECEIPT_ID_MISMATCH") + if info.get("private") is not True or info.get("mirror") is not False: + raise MigrationError("TARGET_PRIVACY_OR_MIRROR_MISMATCH") + raw_references = entry["snapshot"].get("reference_format") == "git-object-v1" + if snapshot(target, target_org, name, entry["receipt_id"], raw_references) != entry["snapshot"]: + raise MigrationError("TARGET_GIT_REFERENCES_MISMATCH") + if snapshot(source, source_org, name, entry["source_id"], raw_references) != entry["snapshot"]: + raise MigrationError("SOURCE_CHANGED_DURING_MIGRATION") + + +def migrate(source: ApiClient, target: ApiClient, source_org: str, target_org: str, + store: StateStore, execute: bool, selected: list[str], limit: int | None, + migration_timeout: int) -> int: + preflight(source, source_org, target=False) + preflight(target, target_org, target=True) + inventory = {} + for info in source.pages("/orgs/" + name_segment(source_org) + "/repos"): + name = info.get("name") + name_segment(name) + repo_id = identity(info, source_org, name) + if name in inventory: + raise MigrationError("DUPLICATE_REPOSITORY_NAME") + inventory[name] = repo_id + inventory = dict(sorted(inventory.items())) + if any(name not in inventory for name in selected): + raise MigrationError("SELECTED_REPOSITORY_NOT_IN_SOURCE_ORG") + binding = {"source_url": source.base, "source_org": source_org, + "target_url": target.base, "target_org": target_org, + "contents": "private-git-and-lfs-v1"} + state = store.load(binding, inventory) + print(f"源组织={source_org} 仓库数={len(inventory)} 目标组织={target_org} 模式={'正式执行' if execute else '只读预览'}") + processed = 0 + for name, entry in state["repositories"].items(): + if selected and name not in selected: + continue + if entry.get("receipt_id") is not None: + try: + verify(source, target, source_org, target_org, name, entry) + entry.update(status="VERIFIED", error=None) + print(f"已核对 {name}:完成收据、私有属性、默认分支、分支及标签提交一致") + except MigrationError as error: + entry.update(status="REVIEW", error=error.code) + print(f"待核对 {name}:{error.code};已保存完成收据,仅重试只读核验") + store.save(state) + continue + if entry["status"] != "PLANNED": + print(f"跳过 {name}:{entry['status']};没有可核对的完成收据,不自动重发") + continue + if limit is not None and processed >= limit: + continue + processed += 1 + if target.repo(target_org, name, missing=True) is not None: + entry.update(status="CONFLICT", error="TARGET_ALREADY_EXISTS") + store.save(state) + print(f"同名冲突 {name}:保留生产仓库,不接管、不覆盖") + continue + frozen = snapshot(source, source_org, name, entry["source_id"]) + if not execute: + print(f"待迁移 {name}:分支={len(frozen['branches'])} 标签={len(frozen['tags'])} LFS=请求迁移") + continue + # 在发送 POST 前强制落盘。超时、进程退出后不能仅凭仓库存在认定迁移完成。 + entry.update(status="MIGRATING", snapshot=frozen, error=None) + store.save(state) + print(f"开始迁移 {name}:{source_org} -> {target_org}", flush=True) + body = {"clone_addr": source.clone_url(source_org, name), + "repo_owner": target_org, "repo_name": name, "service": "git", + "auth_username": source.username, "auth_password": source.token, + "private": True, "mirror": False, "lfs": True, + **{unit: False for unit in ("issues", "labels", "milestones", + "pull_requests", "releases", "wiki")}} + try: + status, receipt = target.request("POST", "/repos/migrate", body, + timeout=migration_timeout) + if status != 201: + raise MigrationError("MIGRATION_COMPLETION_NOT_CONFIRMED") + entry["receipt_id"] = identity(receipt, target_org, name) + entry["status"] = "RECEIVED" + store.save(state) + except MigrationError as error: + entry.update(status="REVIEW", error=error.code) + store.save(state) + print(f"迁移未确认 {name}:{error.code};本次停止新增仓库,请检查生产迁移日志") + break + try: + verify(source, target, source_org, target_org, name, entry) + entry.update(status="VERIFIED", error=None) + print(f"迁移完成 {name}:Git 引用已核对,LFS 已请求迁移,实体下载仍需抽查") + except MigrationError as error: + entry.update(status="REVIEW", error=error.code) + print(f"待核对 {name}:{error.code};保留已创建仓库和完成收据") + store.save(state) + entries = [entry for name, entry in state["repositories"].items() + if not selected or name in selected] + counts = {status: sum(entry["status"] == status for entry in entries) + for status in ("PLANNED", "VERIFIED", "CONFLICT", "REVIEW")} + print("处理结果 " + " ".join(f"{key}={value}" for key, value in counts.items())) + print("状态及映射目录:" + str(store.directory)) + return 2 if counts["CONFLICT"] or counts["REVIEW"] else 0 + + +def read_token(variable: str, prompt: str) -> str: + token = os.environ.get(variable, "") + if not token and sys.stdin.isatty(): + token = getpass.getpass(prompt) + if not token or any(char.isspace() or ord(char) < 32 for char in token): + raise MigrationError("MISSING_OR_INVALID_" + variable) + return token + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-url", required=True, help="测试 Gitea 基础地址,不含 /api/v1") + parser.add_argument("--target-url", required=True, help="生产 Gitea 基础地址,不含 /api/v1") + parser.add_argument("--source-org", default="community") + parser.add_argument("--target-org", default="vibemake") + parser.add_argument("--state-dir", type=Path, default=Path("logs/gitea-org-migration")) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", help="只读预览,默认模式") + mode.add_argument("--execute", action="store_true", help="正式创建目标仓库") + parser.add_argument("--repo", action="append", default=[], help="只处理指定仓库,可重复指定") + parser.add_argument("--limit", type=int, help="限制本次尚未处理的仓库数") + parser.add_argument("--migration-timeout", type=int, default=1800, help="单仓库迁移请求超时秒数") + parser.add_argument("--allow-http", action="store_true", help="允许在可信网络通过明文 HTTP 发送凭据") + args = parser.parse_args() + try: + source_url = base_url(args.source_url, args.allow_http) + target_url = base_url(args.target_url, args.allow_http) + if source_url == target_url: + raise MigrationError("SOURCE_AND_TARGET_MUST_DIFFER") + for value in (args.source_org, args.target_org, *args.repo): + name_segment(value) + if (args.limit is not None and args.limit < 1) or args.migration_timeout < 1: + raise MigrationError("INVALID_LIMIT_OR_TIMEOUT") + source_token = read_token("GITEA_MIGRATION_SOURCE_TOKEN", "测试 Gitea Token(输入不回显):") + target_token = read_token("GITEA_MIGRATION_TARGET_TOKEN", "生产 Gitea Token(输入不回显):") + with StateStore(args.state_dir).locked(): + return migrate(ApiClient(source_url, source_token), ApiClient(target_url, target_token), + args.source_org, args.target_org, StateStore(args.state_dir), + args.execute, args.repo, args.limit, args.migration_timeout) + except MigrationError as error: + print("操作停止:" + error.code, file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("操作已中断。已发出的迁移可能仍在生产执行;保留状态目录后再核对。", file=sys.stderr) + return 130 + except Exception as error: + # 不输出异常正文,避免底层网络库或无效输入携带敏感值。 + print("操作停止:" + type(error).__name__ + ";检查状态文件和运行环境。", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_gitea_org_migration.py b/scripts/tests/test_gitea_org_migration.py new file mode 100644 index 0000000..9eb2152 --- /dev/null +++ b/scripts/tests/test_gitea_org_migration.py @@ -0,0 +1,363 @@ +import copy +from contextlib import contextmanager, redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import io +import json +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import threading +import unittest +from urllib.parse import parse_qs, unquote, urlsplit + + +SCRIPT = Path(__file__).parents[1] / "migrate-gitea-org.py" +spec = importlib.util.spec_from_file_location("gitea_migration", SCRIPT) +migration = importlib.util.module_from_spec(spec) +spec.loader.exec_module(migration) + +SOURCE_TOKEN = "source-test-secret-never-log" +TARGET_TOKEN = "target-test-secret-never-log" +SHA = "1" * 40 +OTHER_SHA = "2" * 40 +FIRST = "p_aaaaaaaaaaaa" +SECOND = "p_bbbbbbbbbbbb" + + +def repository(name, repo_id, owner="community", empty=False): + return {"id": repo_id, "name": name, "owner": {"login": owner}, + "private": True, "mirror": False, "empty": empty, "default_branch": "main", + "branches": {} if empty else {"main": SHA, "dev": OTHER_SHA}, + "tags": {} if empty else {"v1": SHA, "v1-alias": SHA}} + + +class FakeApi(migration.ApiClient): + """模拟服务端限制分页大小、私有组织及同步仓库迁移,不访问真实服务。""" + + def __init__(self, source=None): + self.source = source + self.org = "vibemake" if source else "community" + super().__init__("https://target.example" if source else "https://source.example", + TARGET_TOKEN if source else SOURCE_TOKEN) + self.repositories = {} if source else { + FIRST: repository(FIRST, 101), SECOND: repository(SECOND, 102)} + self.requests = [] + self.posts = [] + self.owner = True + self.lfs_enabled = True + self.post_status = 201 + self.post_error = None + self.interrupt_post = False + self.fail_read_after_post = False + self.change_target_tag = False + self.change_source_after_post = False + + def request(self, method, path, body=None, timeout=None, missing=False): + self.requests.append((method, path)) + parsed = urlsplit(path) + route = [unquote(part) for part in parsed.path.strip("/").split("/")] + page = int(parse_qs(parsed.query).get("page", ["1"])[0]) + if method == "POST": + if path != "/repos/migrate": + raise AssertionError("不允许其他写接口") + self.posts.append(copy.deepcopy(body)) + name = body["repo_name"] + info = copy.deepcopy(self.source.repositories[name]) + info.update(id=1000 + len(self.posts), owner={"login": body["repo_owner"]}, + private=True, mirror=False) + self.repositories[name] = info + if self.change_target_tag: + info["tags"]["v1"] = OTHER_SHA + if self.change_source_after_post: + self.source.repositories[name]["branches"]["main"] = OTHER_SHA + if self.interrupt_post: + raise KeyboardInterrupt() + if self.post_error: + raise migration.MigrationError(self.post_error) + return self.post_status, copy.deepcopy(info) + if path == "/user": + return 200, {"login": "service", "is_admin": False} + if path == "/orgs/" + self.org: + return 200, {"name": self.org} + if path == "/settings/repository": + return 200, {"lfs_disabled": not self.lfs_enabled, + "http_git_disabled": False, "migrations_disabled": False} + if route == ["orgs", self.org, "teams"]: + teams = [{"id": 1, "name": "Owners", "permission": "owner"}] + return 200, teams[page - 1:page] + if path == "/teams/1/members/service": + return (200, {"login": "service"}) if self.owner else (404, None) + if route == ["orgs", self.org, "repos"]: + # 服务端上限为一条,客户端传 limit=50 也必须继续分页。 + return 200, copy.deepcopy(list(self.repositories.values())[page - 1:page]) + if len(route) >= 3 and route[:2] == ["repos", self.org]: + info = self.repositories.get(route[2]) + if info is None: + if missing: + return 404, None + raise migration.MigrationError("HTTP_404") + if self.fail_read_after_post and self.posts: + raise migration.MigrationError("NETWORK_ERROR") + if len(route) == 3: + return 200, copy.deepcopy(info) + if len(route) == 5 and route[3] == "git" and route[4].startswith("refs"): + raise AssertionError("Git 引用路由应包含 heads 或 tags") + if len(route) == 6 and route[3:5] == ["git", "refs"]: + unit = {"heads": "branches", "tags": "tags"}[route[5]] + refs = [{"ref": "refs/" + route[5] + "/" + name, "object": {"sha": sha}} + for name, sha in info[unit].items()] + return (200, refs) if refs else (404, None) + if len(route) == 4 and route[3] in ("branches", "tags"): + unit = route[3] + refs = [{"name": name, "commit": {"id" if unit == "branches" else "sha": sha}, + **({"id": sha} if unit == "tags" else {})} + for name, sha in info[unit].items()] + return 200, refs[page - 1:page] + raise AssertionError("未预期的模拟 API 路由") + + +class MigrationTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.store = migration.StateStore(Path(self.temporary.name) / "state") + self.source = FakeApi() + self.target = FakeApi(self.source) + + def run_migration(self, execute=False, limit=None, selected=None): + output = io.StringIO() + with self.store.locked(), redirect_stdout(output): + code = migration.migrate(self.source, self.target, "community", "vibemake", + self.store, execute, selected or [], limit, 1800) + self.assertNotIn(SOURCE_TOKEN, output.getvalue()) + self.assertNotIn(TARGET_TOKEN, output.getvalue()) + for path in self.store.directory.iterdir(): + if path.is_file(): + text = path.read_text(encoding="utf-8") + self.assertNotIn(SOURCE_TOKEN, text) + self.assertNotIn(TARGET_TOKEN, text) + return code + + def state(self): + return json.loads(self.store.path.read_text()) + + def mappings(self): + return json.loads((self.store.directory / "mapping.json").read_text())["repositories"] + + def test_preview_paginates_all_private_repositories_and_never_writes_remote(self): + self.assertEqual(self.run_migration(), 0) + self.assertEqual(len(self.state()["inventory"]), 2) + self.assertEqual(self.target.posts, []) + self.assertEqual(self.mappings(), []) + self.assertTrue(all(method == "GET" for method, _ in self.source.requests)) + self.assertTrue(all(method == "GET" for method, _ in self.target.requests)) + self.assertIn(("GET", "/orgs/community/repos?page=3&limit=50"), self.source.requests) + + def test_stale_empty_metadata_still_copies_and_verifies_real_git_references(self): + self.source.repositories[FIRST]["empty"] = True + self.assertEqual(self.run_migration(True, limit=1), 0) + entry = self.state()["repositories"][FIRST] + self.assertFalse(entry["snapshot"]["empty"]) + self.assertEqual(entry["snapshot"]["reference_format"], "git-object-v1") + self.assertEqual(len(entry["snapshot"]["branches"]), 2) + self.assertEqual(len(entry["snapshot"]["tags"]), 2) + + def test_copies_git_lfs_to_new_owner_and_saves_mapping_without_credentials(self): + self.assertEqual(self.run_migration(True, limit=1), 0) + self.assertEqual(len(self.target.posts), 1) + body = self.target.posts[0] + self.assertEqual(body["repo_owner"], "vibemake") + self.assertEqual(body["repo_name"], FIRST) + self.assertEqual(body["clone_addr"], "https://source.example/community/" + FIRST + ".git") + self.assertEqual(body["auth_username"], "service") + self.assertEqual(body["auth_password"], SOURCE_TOKEN) + self.assertTrue(body["lfs"]) + self.assertTrue(body["private"]) + self.assertFalse(body["mirror"]) + for option in ("issues", "labels", "milestones", "pull_requests", "releases", "wiki"): + self.assertFalse(body[option]) + self.assertEqual(self.source.repositories[FIRST]["owner"]["login"], "community") + mapping = self.mappings()[0] + self.assertEqual(mapping["source_repo_id"], 101) + self.assertEqual(mapping["target_repo_id"], 1001) + self.assertEqual(mapping["project_code"], "a" * 12) + self.assertFalse(mapping["lfs_objects_download_verified"]) + self.assertEqual(len(self.state()["repositories"][FIRST]["snapshot"]["tags"]), 2) + + def test_resume_does_not_copy_completed_repository_twice(self): + self.run_migration(True, limit=1) + self.assertEqual(self.run_migration(True), 0) + self.assertEqual(len(self.target.posts), 2) + self.assertEqual(len(self.mappings()), 2) + self.assertEqual(self.run_migration(True), 0) + self.assertEqual(len(self.target.posts), 2) + + def test_existing_even_empty_target_is_never_overwritten_or_adopted(self): + self.target.repositories[FIRST] = repository(FIRST, 999, "vibemake", empty=True) + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(self.target.posts, []) + self.assertEqual(self.target.repositories[FIRST]["id"], 999) + self.assertEqual(self.state()["repositories"][FIRST]["status"], "CONFLICT") + + def test_timeout_after_server_created_repository_never_retries_post(self): + self.target.post_error = "HTTP_504" + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(self.state()["repositories"][FIRST]["status"], "REVIEW") + self.target.post_error = None + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(len(self.target.posts), 1) + self.assertEqual(self.mappings(), []) + + def test_interrupt_before_receipt_remains_review_on_resume(self): + self.target.interrupt_post = True + with self.assertRaises(KeyboardInterrupt): + self.run_migration(True, selected=[FIRST]) + self.assertEqual(self.state()["repositories"][FIRST]["status"], "MIGRATING") + self.target.interrupt_post = False + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(len(self.target.posts), 1) + + def test_saved_receipt_allows_only_read_verification_after_network_failure(self): + self.target.fail_read_after_post = True + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(self.state()["repositories"][FIRST]["receipt_id"], 1001) + self.target.fail_read_after_post = False + self.assertEqual(self.run_migration(True, selected=[FIRST]), 0) + self.assertEqual(len(self.target.posts), 1) + self.assertEqual(len(self.mappings()), 1) + + def test_tag_mismatch_is_not_reported_as_success(self): + self.target.change_target_tag = True + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(self.state()["repositories"][FIRST]["error"], "TARGET_GIT_REFERENCES_MISMATCH") + self.assertEqual(self.mappings(), []) + + def test_source_change_during_migration_requires_review(self): + self.target.change_source_after_post = True + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertEqual(self.state()["repositories"][FIRST]["error"], "SOURCE_CHANGED_DURING_MIGRATION") + + def test_unexpected_success_status_does_not_count_as_completion(self): + self.target.post_status = 200 + self.assertEqual(self.run_migration(True, selected=[FIRST]), 2) + self.assertNotIn("receipt_id", self.state()["repositories"][FIRST]) + + def test_other_organization_cannot_enter_source_inventory(self): + self.source.repositories[FIRST]["owner"]["login"] = "other-org" + with self.assertRaisesRegex(migration.MigrationError, "REPOSITORY_IDENTITY_MISMATCH"): + self.run_migration(True) + self.assertEqual(self.target.posts, []) + + def test_owner_and_lfs_checks_happen_before_any_write(self): + self.source.owner = False + with self.assertRaisesRegex(migration.MigrationError, "ORGANIZATION_OWNER_REQUIRED"): + self.run_migration(True) + self.source.owner = True + self.target.lfs_enabled = False + with self.assertRaisesRegex(migration.MigrationError, "GIT_LFS_OR_MIGRATION_DISABLED"): + self.run_migration(True) + self.assertEqual(self.target.posts, []) + + def test_state_cannot_be_reused_for_another_environment_or_changed_inventory(self): + self.run_migration() + self.target.base = "https://another-target.example" + with self.assertRaisesRegex(migration.MigrationError, "STATE_ENVIRONMENT_MISMATCH"): + self.run_migration(True) + self.target.base = "https://target.example" + self.source.repositories["p_cccccccccccc"] = repository("p_cccccccccccc", 103) + with self.assertRaisesRegex(migration.MigrationError, "SOURCE_INVENTORY_CHANGED"): + self.run_migration(True) + self.assertEqual(self.target.posts, []) + + def test_empty_repository_is_preserved(self): + self.source.repositories[FIRST] = repository(FIRST, 101, empty=True) + self.assertEqual(self.run_migration(True, selected=[FIRST]), 0) + self.assertTrue(self.target.repositories[FIRST]["empty"]) + self.assertEqual(len(self.mappings()), 1) + + def test_state_permissions_and_lock_prevent_parallel_writes(self): + self.run_migration() + self.assertEqual(stat.S_IMODE(self.store.directory.stat().st_mode), 0o700) + self.assertEqual(stat.S_IMODE(self.store.path.stat().st_mode), 0o600) + with self.store.locked(), self.assertRaisesRegex(migration.MigrationError, "STATE_DIRECTORY_LOCKED"): + with migration.StateStore(self.store.directory).locked(): + self.fail("不应获得第二个锁") + + +@contextmanager +def http_fixture(): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + calls.append(self.path) + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path.endswith("/redirect"): + self.send_response(302) + self.send_header("Location", "/credential-sink") + else: + self.send_response(500) + self.end_headers() + self.wfile.write((SOURCE_TOKEN + TARGET_TOKEN).encode()) + + def do_GET(self): + calls.append(self.path) + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True) + thread.start() + try: + yield "http://127.0.0.1:" + str(server.server_port), calls + finally: + server.shutdown() + server.server_close() + thread.join() + + +class TransportTest(unittest.TestCase): + def test_post_redirect_does_not_forward_credentials_or_repeat_write(self): + with http_fixture() as (base, calls): + client = migration.ApiClient(base, TARGET_TOKEN) + with self.assertRaises(migration.MigrationError) as caught: + client.request("POST", "/redirect", {"auth_password": SOURCE_TOKEN}) + self.assertEqual(str(caught.exception), "HTTP_302") + self.assertEqual(calls, ["/api/v1/redirect"]) + + def test_http_error_body_with_credentials_is_not_exposed(self): + with http_fixture() as (base, _): + client = migration.ApiClient(base, TARGET_TOKEN) + with self.assertRaises(migration.MigrationError) as caught: + client.request("POST", "/repos/migrate", {"auth_password": SOURCE_TOKEN}) + self.assertEqual(str(caught.exception), "HTTP_500") + self.assertNotIn(SOURCE_TOKEN, str(caught.exception)) + self.assertNotIn(TARGET_TOKEN, str(caught.exception)) + + def test_invalid_base_url_does_not_echo_embedded_credentials_in_cli(self): + result = subprocess.run([sys.executable, str(SCRIPT), + "--source-url", "https://user:" + SOURCE_TOKEN + "@source.example", + "--target-url", "https://target.example"], + capture_output=True, text=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn(SOURCE_TOKEN, result.stdout + result.stderr) + self.assertIn("INVALID_BASE_URL", result.stderr) + + def test_base_url_keeps_subpath_and_rejects_insecure_or_credential_urls(self): + self.assertEqual(migration.base_url("https://GITEA.example:443/gitea/", False), + "https://gitea.example/gitea") + for value in ("http://gitea.example", "https://gitea.example?token=secret", + "https://gitea.example/#token", "https://gitea.example/a/../b"): + with self.subTest(value=value), self.assertRaises(migration.MigrationError): + migration.base_url(value, False) + + +if __name__ == "__main__": + unittest.main()