Files
community/scripts/tests/test_gitea_org_migration.py
T

364 lines
17 KiB
Python

import copy
from contextlib import contextmanager, redirect_stdout
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import importlib.util
import io
import json
from pathlib import Path
import stat
import subprocess
import sys
import tempfile
import threading
import unittest
from urllib.parse import parse_qs, unquote, urlsplit
SCRIPT = Path(__file__).parents[1] / "migrate-gitea-org.py"
spec = importlib.util.spec_from_file_location("gitea_migration", SCRIPT)
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)
SOURCE_TOKEN = "source-test-secret-never-log"
TARGET_TOKEN = "target-test-secret-never-log"
SHA = "1" * 40
OTHER_SHA = "2" * 40
FIRST = "p_aaaaaaaaaaaa"
SECOND = "p_bbbbbbbbbbbb"
def repository(name, repo_id, owner="community", empty=False):
return {"id": repo_id, "name": name, "owner": {"login": owner},
"private": True, "mirror": False, "empty": empty, "default_branch": "main",
"branches": {} if empty else {"main": SHA, "dev": OTHER_SHA},
"tags": {} if empty else {"v1": SHA, "v1-alias": SHA}}
class FakeApi(migration.ApiClient):
"""模拟服务端限制分页大小、私有组织及同步仓库迁移,不访问真实服务。"""
def __init__(self, source=None):
self.source = source
self.org = "vibemake" if source else "community"
super().__init__("https://target.example" if source else "https://source.example",
TARGET_TOKEN if source else SOURCE_TOKEN)
self.repositories = {} if source else {
FIRST: repository(FIRST, 101), SECOND: repository(SECOND, 102)}
self.requests = []
self.posts = []
self.owner = True
self.lfs_enabled = True
self.post_status = 201
self.post_error = None
self.interrupt_post = False
self.fail_read_after_post = False
self.change_target_tag = False
self.change_source_after_post = False
def request(self, method, path, body=None, timeout=None, missing=False):
self.requests.append((method, path))
parsed = urlsplit(path)
route = [unquote(part) for part in parsed.path.strip("/").split("/")]
page = int(parse_qs(parsed.query).get("page", ["1"])[0])
if method == "POST":
if path != "/repos/migrate":
raise AssertionError("不允许其他写接口")
self.posts.append(copy.deepcopy(body))
name = body["repo_name"]
info = copy.deepcopy(self.source.repositories[name])
info.update(id=1000 + len(self.posts), owner={"login": body["repo_owner"]},
private=True, mirror=False)
self.repositories[name] = info
if self.change_target_tag:
info["tags"]["v1"] = OTHER_SHA
if self.change_source_after_post:
self.source.repositories[name]["branches"]["main"] = OTHER_SHA
if self.interrupt_post:
raise KeyboardInterrupt()
if self.post_error:
raise migration.MigrationError(self.post_error)
return self.post_status, copy.deepcopy(info)
if path == "/user":
return 200, {"login": "service", "is_admin": False}
if path == "/orgs/" + self.org:
return 200, {"name": self.org}
if path == "/settings/repository":
return 200, {"lfs_disabled": not self.lfs_enabled,
"http_git_disabled": False, "migrations_disabled": False}
if route == ["orgs", self.org, "teams"]:
teams = [{"id": 1, "name": "Owners", "permission": "owner"}]
return 200, teams[page - 1:page]
if path == "/teams/1/members/service":
return (200, {"login": "service"}) if self.owner else (404, None)
if route == ["orgs", self.org, "repos"]:
# 服务端上限为一条,客户端传 limit=50 也必须继续分页。
return 200, copy.deepcopy(list(self.repositories.values())[page - 1:page])
if len(route) >= 3 and route[:2] == ["repos", self.org]:
info = self.repositories.get(route[2])
if info is None:
if missing:
return 404, None
raise migration.MigrationError("HTTP_404")
if self.fail_read_after_post and self.posts:
raise migration.MigrationError("NETWORK_ERROR")
if len(route) == 3:
return 200, copy.deepcopy(info)
if len(route) == 5 and route[3] == "git" and route[4].startswith("refs"):
raise AssertionError("Git 引用路由应包含 heads 或 tags")
if len(route) == 6 and route[3:5] == ["git", "refs"]:
unit = {"heads": "branches", "tags": "tags"}[route[5]]
refs = [{"ref": "refs/" + route[5] + "/" + name, "object": {"sha": sha}}
for name, sha in info[unit].items()]
return (200, refs) if refs else (404, None)
if len(route) == 4 and route[3] in ("branches", "tags"):
unit = route[3]
refs = [{"name": name, "commit": {"id" if unit == "branches" else "sha": sha},
**({"id": sha} if unit == "tags" else {})}
for name, sha in info[unit].items()]
return 200, refs[page - 1:page]
raise AssertionError("未预期的模拟 API 路由")
class MigrationTest(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.store = migration.StateStore(Path(self.temporary.name) / "state")
self.source = FakeApi()
self.target = FakeApi(self.source)
def run_migration(self, execute=False, limit=None, selected=None):
output = io.StringIO()
with self.store.locked(), redirect_stdout(output):
code = migration.migrate(self.source, self.target, "community", "vibemake",
self.store, execute, selected or [], limit, 1800)
self.assertNotIn(SOURCE_TOKEN, output.getvalue())
self.assertNotIn(TARGET_TOKEN, output.getvalue())
for path in self.store.directory.iterdir():
if path.is_file():
text = path.read_text(encoding="utf-8")
self.assertNotIn(SOURCE_TOKEN, text)
self.assertNotIn(TARGET_TOKEN, text)
return code
def state(self):
return json.loads(self.store.path.read_text())
def mappings(self):
return json.loads((self.store.directory / "mapping.json").read_text())["repositories"]
def test_preview_paginates_all_private_repositories_and_never_writes_remote(self):
self.assertEqual(self.run_migration(), 0)
self.assertEqual(len(self.state()["inventory"]), 2)
self.assertEqual(self.target.posts, [])
self.assertEqual(self.mappings(), [])
self.assertTrue(all(method == "GET" for method, _ in self.source.requests))
self.assertTrue(all(method == "GET" for method, _ in self.target.requests))
self.assertIn(("GET", "/orgs/community/repos?page=3&limit=50"), self.source.requests)
def test_stale_empty_metadata_still_copies_and_verifies_real_git_references(self):
self.source.repositories[FIRST]["empty"] = True
self.assertEqual(self.run_migration(True, limit=1), 0)
entry = self.state()["repositories"][FIRST]
self.assertFalse(entry["snapshot"]["empty"])
self.assertEqual(entry["snapshot"]["reference_format"], "git-object-v1")
self.assertEqual(len(entry["snapshot"]["branches"]), 2)
self.assertEqual(len(entry["snapshot"]["tags"]), 2)
def test_copies_git_lfs_to_new_owner_and_saves_mapping_without_credentials(self):
self.assertEqual(self.run_migration(True, limit=1), 0)
self.assertEqual(len(self.target.posts), 1)
body = self.target.posts[0]
self.assertEqual(body["repo_owner"], "vibemake")
self.assertEqual(body["repo_name"], FIRST)
self.assertEqual(body["clone_addr"], "https://source.example/community/" + FIRST + ".git")
self.assertEqual(body["auth_username"], "service")
self.assertEqual(body["auth_password"], SOURCE_TOKEN)
self.assertTrue(body["lfs"])
self.assertTrue(body["private"])
self.assertFalse(body["mirror"])
for option in ("issues", "labels", "milestones", "pull_requests", "releases", "wiki"):
self.assertFalse(body[option])
self.assertEqual(self.source.repositories[FIRST]["owner"]["login"], "community")
mapping = self.mappings()[0]
self.assertEqual(mapping["source_repo_id"], 101)
self.assertEqual(mapping["target_repo_id"], 1001)
self.assertEqual(mapping["project_code"], "a" * 12)
self.assertFalse(mapping["lfs_objects_download_verified"])
self.assertEqual(len(self.state()["repositories"][FIRST]["snapshot"]["tags"]), 2)
def test_resume_does_not_copy_completed_repository_twice(self):
self.run_migration(True, limit=1)
self.assertEqual(self.run_migration(True), 0)
self.assertEqual(len(self.target.posts), 2)
self.assertEqual(len(self.mappings()), 2)
self.assertEqual(self.run_migration(True), 0)
self.assertEqual(len(self.target.posts), 2)
def test_existing_even_empty_target_is_never_overwritten_or_adopted(self):
self.target.repositories[FIRST] = repository(FIRST, 999, "vibemake", empty=True)
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.target.posts, [])
self.assertEqual(self.target.repositories[FIRST]["id"], 999)
self.assertEqual(self.state()["repositories"][FIRST]["status"], "CONFLICT")
def test_timeout_after_server_created_repository_never_retries_post(self):
self.target.post_error = "HTTP_504"
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["status"], "REVIEW")
self.target.post_error = None
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(len(self.target.posts), 1)
self.assertEqual(self.mappings(), [])
def test_interrupt_before_receipt_remains_review_on_resume(self):
self.target.interrupt_post = True
with self.assertRaises(KeyboardInterrupt):
self.run_migration(True, selected=[FIRST])
self.assertEqual(self.state()["repositories"][FIRST]["status"], "MIGRATING")
self.target.interrupt_post = False
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(len(self.target.posts), 1)
def test_saved_receipt_allows_only_read_verification_after_network_failure(self):
self.target.fail_read_after_post = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["receipt_id"], 1001)
self.target.fail_read_after_post = False
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
self.assertEqual(len(self.target.posts), 1)
self.assertEqual(len(self.mappings()), 1)
def test_tag_mismatch_is_not_reported_as_success(self):
self.target.change_target_tag = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["error"], "TARGET_GIT_REFERENCES_MISMATCH")
self.assertEqual(self.mappings(), [])
def test_source_change_during_migration_requires_review(self):
self.target.change_source_after_post = True
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertEqual(self.state()["repositories"][FIRST]["error"], "SOURCE_CHANGED_DURING_MIGRATION")
def test_unexpected_success_status_does_not_count_as_completion(self):
self.target.post_status = 200
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
self.assertNotIn("receipt_id", self.state()["repositories"][FIRST])
def test_other_organization_cannot_enter_source_inventory(self):
self.source.repositories[FIRST]["owner"]["login"] = "other-org"
with self.assertRaisesRegex(migration.MigrationError, "REPOSITORY_IDENTITY_MISMATCH"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_owner_and_lfs_checks_happen_before_any_write(self):
self.source.owner = False
with self.assertRaisesRegex(migration.MigrationError, "ORGANIZATION_OWNER_REQUIRED"):
self.run_migration(True)
self.source.owner = True
self.target.lfs_enabled = False
with self.assertRaisesRegex(migration.MigrationError, "GIT_LFS_OR_MIGRATION_DISABLED"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_state_cannot_be_reused_for_another_environment_or_changed_inventory(self):
self.run_migration()
self.target.base = "https://another-target.example"
with self.assertRaisesRegex(migration.MigrationError, "STATE_ENVIRONMENT_MISMATCH"):
self.run_migration(True)
self.target.base = "https://target.example"
self.source.repositories["p_cccccccccccc"] = repository("p_cccccccccccc", 103)
with self.assertRaisesRegex(migration.MigrationError, "SOURCE_INVENTORY_CHANGED"):
self.run_migration(True)
self.assertEqual(self.target.posts, [])
def test_empty_repository_is_preserved(self):
self.source.repositories[FIRST] = repository(FIRST, 101, empty=True)
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
self.assertTrue(self.target.repositories[FIRST]["empty"])
self.assertEqual(len(self.mappings()), 1)
def test_state_permissions_and_lock_prevent_parallel_writes(self):
self.run_migration()
self.assertEqual(stat.S_IMODE(self.store.directory.stat().st_mode), 0o700)
self.assertEqual(stat.S_IMODE(self.store.path.stat().st_mode), 0o600)
with self.store.locked(), self.assertRaisesRegex(migration.MigrationError, "STATE_DIRECTORY_LOCKED"):
with migration.StateStore(self.store.directory).locked():
self.fail("不应获得第二个锁")
@contextmanager
def http_fixture():
calls = []
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
calls.append(self.path)
self.rfile.read(int(self.headers.get("Content-Length", "0")))
if self.path.endswith("/redirect"):
self.send_response(302)
self.send_header("Location", "/credential-sink")
else:
self.send_response(500)
self.end_headers()
self.wfile.write((SOURCE_TOKEN + TARGET_TOKEN).encode())
def do_GET(self):
calls.append(self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(b"{}")
def log_message(self, *args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
thread.start()
try:
yield "http://127.0.0.1:" + str(server.server_port), calls
finally:
server.shutdown()
server.server_close()
thread.join()
class TransportTest(unittest.TestCase):
def test_post_redirect_does_not_forward_credentials_or_repeat_write(self):
with http_fixture() as (base, calls):
client = migration.ApiClient(base, TARGET_TOKEN)
with self.assertRaises(migration.MigrationError) as caught:
client.request("POST", "/redirect", {"auth_password": SOURCE_TOKEN})
self.assertEqual(str(caught.exception), "HTTP_302")
self.assertEqual(calls, ["/api/v1/redirect"])
def test_http_error_body_with_credentials_is_not_exposed(self):
with http_fixture() as (base, _):
client = migration.ApiClient(base, TARGET_TOKEN)
with self.assertRaises(migration.MigrationError) as caught:
client.request("POST", "/repos/migrate", {"auth_password": SOURCE_TOKEN})
self.assertEqual(str(caught.exception), "HTTP_500")
self.assertNotIn(SOURCE_TOKEN, str(caught.exception))
self.assertNotIn(TARGET_TOKEN, str(caught.exception))
def test_invalid_base_url_does_not_echo_embedded_credentials_in_cli(self):
result = subprocess.run([sys.executable, str(SCRIPT),
"--source-url", "https://user:" + SOURCE_TOKEN + "@source.example",
"--target-url", "https://target.example"],
capture_output=True, text=True, check=False)
self.assertNotEqual(result.returncode, 0)
self.assertNotIn(SOURCE_TOKEN, result.stdout + result.stderr)
self.assertIn("INVALID_BASE_URL", result.stderr)
def test_base_url_keeps_subpath_and_rejects_insecure_or_credential_urls(self):
self.assertEqual(migration.base_url("https://GITEA.example:443/gitea/", False),
"https://gitea.example/gitea")
for value in ("http://gitea.example", "https://gitea.example?token=secret",
"https://gitea.example/#token", "https://gitea.example/a/../b"):
with self.subTest(value=value), self.assertRaises(migration.MigrationError):
migration.base_url(value, False)
if __name__ == "__main__":
unittest.main()