364 lines
17 KiB
Python
364 lines
17 KiB
Python
import copy
|
|
from contextlib import contextmanager, redirect_stdout
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from urllib.parse import parse_qs, unquote, urlsplit
|
|
|
|
|
|
SCRIPT = Path(__file__).parents[1] / "migrate-gitea-org.py"
|
|
spec = importlib.util.spec_from_file_location("gitea_migration", SCRIPT)
|
|
migration = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(migration)
|
|
|
|
SOURCE_TOKEN = "source-test-secret-never-log"
|
|
TARGET_TOKEN = "target-test-secret-never-log"
|
|
SHA = "1" * 40
|
|
OTHER_SHA = "2" * 40
|
|
FIRST = "p_aaaaaaaaaaaa"
|
|
SECOND = "p_bbbbbbbbbbbb"
|
|
|
|
|
|
def repository(name, repo_id, owner="vibemake-test", empty=False):
|
|
return {"id": repo_id, "name": name, "owner": {"login": owner},
|
|
"private": True, "mirror": False, "empty": empty, "default_branch": "main",
|
|
"branches": {} if empty else {"main": SHA, "dev": OTHER_SHA},
|
|
"tags": {} if empty else {"v1": SHA, "v1-alias": SHA}}
|
|
|
|
|
|
class FakeApi(migration.ApiClient):
|
|
"""模拟服务端限制分页大小、私有组织及同步仓库迁移,不访问真实服务。"""
|
|
|
|
def __init__(self, source=None):
|
|
self.source = source
|
|
self.org = "vibemake" if source else "vibemake-test"
|
|
super().__init__("https://target.example" if source else "https://source.example",
|
|
TARGET_TOKEN if source else SOURCE_TOKEN)
|
|
self.repositories = {} if source else {
|
|
FIRST: repository(FIRST, 101), SECOND: repository(SECOND, 102)}
|
|
self.requests = []
|
|
self.posts = []
|
|
self.owner = True
|
|
self.lfs_enabled = True
|
|
self.post_status = 201
|
|
self.post_error = None
|
|
self.interrupt_post = False
|
|
self.fail_read_after_post = False
|
|
self.change_target_tag = False
|
|
self.change_source_after_post = False
|
|
|
|
def request(self, method, path, body=None, timeout=None, missing=False):
|
|
self.requests.append((method, path))
|
|
parsed = urlsplit(path)
|
|
route = [unquote(part) for part in parsed.path.strip("/").split("/")]
|
|
page = int(parse_qs(parsed.query).get("page", ["1"])[0])
|
|
if method == "POST":
|
|
if path != "/repos/migrate":
|
|
raise AssertionError("不允许其他写接口")
|
|
self.posts.append(copy.deepcopy(body))
|
|
name = body["repo_name"]
|
|
info = copy.deepcopy(self.source.repositories[name])
|
|
info.update(id=1000 + len(self.posts), owner={"login": body["repo_owner"]},
|
|
private=True, mirror=False)
|
|
self.repositories[name] = info
|
|
if self.change_target_tag:
|
|
info["tags"]["v1"] = OTHER_SHA
|
|
if self.change_source_after_post:
|
|
self.source.repositories[name]["branches"]["main"] = OTHER_SHA
|
|
if self.interrupt_post:
|
|
raise KeyboardInterrupt()
|
|
if self.post_error:
|
|
raise migration.MigrationError(self.post_error)
|
|
return self.post_status, copy.deepcopy(info)
|
|
if path == "/user":
|
|
return 200, {"login": "service", "is_admin": False}
|
|
if path == "/orgs/" + self.org:
|
|
return 200, {"name": self.org}
|
|
if path == "/settings/repository":
|
|
return 200, {"lfs_disabled": not self.lfs_enabled,
|
|
"http_git_disabled": False, "migrations_disabled": False}
|
|
if route == ["orgs", self.org, "teams"]:
|
|
teams = [{"id": 1, "name": "Owners", "permission": "owner"}]
|
|
return 200, teams[page - 1:page]
|
|
if path == "/teams/1/members/service":
|
|
return (200, {"login": "service"}) if self.owner else (404, None)
|
|
if route == ["orgs", self.org, "repos"]:
|
|
# 服务端上限为一条,客户端传 limit=50 也必须继续分页。
|
|
return 200, copy.deepcopy(list(self.repositories.values())[page - 1:page])
|
|
if len(route) >= 3 and route[:2] == ["repos", self.org]:
|
|
info = self.repositories.get(route[2])
|
|
if info is None:
|
|
if missing:
|
|
return 404, None
|
|
raise migration.MigrationError("HTTP_404")
|
|
if self.fail_read_after_post and self.posts:
|
|
raise migration.MigrationError("NETWORK_ERROR")
|
|
if len(route) == 3:
|
|
return 200, copy.deepcopy(info)
|
|
if len(route) == 5 and route[3] == "git" and route[4].startswith("refs"):
|
|
raise AssertionError("Git 引用路由应包含 heads 或 tags")
|
|
if len(route) == 6 and route[3:5] == ["git", "refs"]:
|
|
unit = {"heads": "branches", "tags": "tags"}[route[5]]
|
|
refs = [{"ref": "refs/" + route[5] + "/" + name, "object": {"sha": sha}}
|
|
for name, sha in info[unit].items()]
|
|
return (200, refs) if refs else (404, None)
|
|
if len(route) == 4 and route[3] in ("branches", "tags"):
|
|
unit = route[3]
|
|
refs = [{"name": name, "commit": {"id" if unit == "branches" else "sha": sha},
|
|
**({"id": sha} if unit == "tags" else {})}
|
|
for name, sha in info[unit].items()]
|
|
return 200, refs[page - 1:page]
|
|
raise AssertionError("未预期的模拟 API 路由")
|
|
|
|
|
|
class MigrationTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.store = migration.StateStore(Path(self.temporary.name) / "state")
|
|
self.source = FakeApi()
|
|
self.target = FakeApi(self.source)
|
|
|
|
def run_migration(self, execute=False, limit=None, selected=None):
|
|
output = io.StringIO()
|
|
with self.store.locked(), redirect_stdout(output):
|
|
code = migration.migrate(self.source, self.target, "vibemake-test", "vibemake",
|
|
self.store, execute, selected or [], limit, 1800)
|
|
self.assertNotIn(SOURCE_TOKEN, output.getvalue())
|
|
self.assertNotIn(TARGET_TOKEN, output.getvalue())
|
|
for path in self.store.directory.iterdir():
|
|
if path.is_file():
|
|
text = path.read_text(encoding="utf-8")
|
|
self.assertNotIn(SOURCE_TOKEN, text)
|
|
self.assertNotIn(TARGET_TOKEN, text)
|
|
return code
|
|
|
|
def state(self):
|
|
return json.loads(self.store.path.read_text())
|
|
|
|
def mappings(self):
|
|
return json.loads((self.store.directory / "mapping.json").read_text())["repositories"]
|
|
|
|
def test_preview_paginates_all_private_repositories_and_never_writes_remote(self):
|
|
self.assertEqual(self.run_migration(), 0)
|
|
self.assertEqual(len(self.state()["inventory"]), 2)
|
|
self.assertEqual(self.target.posts, [])
|
|
self.assertEqual(self.mappings(), [])
|
|
self.assertTrue(all(method == "GET" for method, _ in self.source.requests))
|
|
self.assertTrue(all(method == "GET" for method, _ in self.target.requests))
|
|
self.assertIn(("GET", "/orgs/vibemake-test/repos?page=3&limit=50"), self.source.requests)
|
|
|
|
def test_stale_empty_metadata_still_copies_and_verifies_real_git_references(self):
|
|
self.source.repositories[FIRST]["empty"] = True
|
|
self.assertEqual(self.run_migration(True, limit=1), 0)
|
|
entry = self.state()["repositories"][FIRST]
|
|
self.assertFalse(entry["snapshot"]["empty"])
|
|
self.assertEqual(entry["snapshot"]["reference_format"], "git-object-v1")
|
|
self.assertEqual(len(entry["snapshot"]["branches"]), 2)
|
|
self.assertEqual(len(entry["snapshot"]["tags"]), 2)
|
|
|
|
def test_copies_git_lfs_to_new_owner_and_saves_mapping_without_credentials(self):
|
|
self.assertEqual(self.run_migration(True, limit=1), 0)
|
|
self.assertEqual(len(self.target.posts), 1)
|
|
body = self.target.posts[0]
|
|
self.assertEqual(body["repo_owner"], "vibemake")
|
|
self.assertEqual(body["repo_name"], FIRST)
|
|
self.assertEqual(body["clone_addr"], "https://source.example/vibemake-test/" + FIRST + ".git")
|
|
self.assertEqual(body["auth_username"], "service")
|
|
self.assertEqual(body["auth_password"], SOURCE_TOKEN)
|
|
self.assertTrue(body["lfs"])
|
|
self.assertTrue(body["private"])
|
|
self.assertFalse(body["mirror"])
|
|
for option in ("issues", "labels", "milestones", "pull_requests", "releases", "wiki"):
|
|
self.assertFalse(body[option])
|
|
self.assertEqual(self.source.repositories[FIRST]["owner"]["login"], "vibemake-test")
|
|
mapping = self.mappings()[0]
|
|
self.assertEqual(mapping["source_repo_id"], 101)
|
|
self.assertEqual(mapping["target_repo_id"], 1001)
|
|
self.assertEqual(mapping["project_code"], "a" * 12)
|
|
self.assertFalse(mapping["lfs_objects_download_verified"])
|
|
self.assertEqual(len(self.state()["repositories"][FIRST]["snapshot"]["tags"]), 2)
|
|
|
|
def test_resume_does_not_copy_completed_repository_twice(self):
|
|
self.run_migration(True, limit=1)
|
|
self.assertEqual(self.run_migration(True), 0)
|
|
self.assertEqual(len(self.target.posts), 2)
|
|
self.assertEqual(len(self.mappings()), 2)
|
|
self.assertEqual(self.run_migration(True), 0)
|
|
self.assertEqual(len(self.target.posts), 2)
|
|
|
|
def test_existing_even_empty_target_is_never_overwritten_or_adopted(self):
|
|
self.target.repositories[FIRST] = repository(FIRST, 999, "vibemake", empty=True)
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(self.target.posts, [])
|
|
self.assertEqual(self.target.repositories[FIRST]["id"], 999)
|
|
self.assertEqual(self.state()["repositories"][FIRST]["status"], "CONFLICT")
|
|
|
|
def test_timeout_after_server_created_repository_never_retries_post(self):
|
|
self.target.post_error = "HTTP_504"
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(self.state()["repositories"][FIRST]["status"], "REVIEW")
|
|
self.target.post_error = None
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(len(self.target.posts), 1)
|
|
self.assertEqual(self.mappings(), [])
|
|
|
|
def test_interrupt_before_receipt_remains_review_on_resume(self):
|
|
self.target.interrupt_post = True
|
|
with self.assertRaises(KeyboardInterrupt):
|
|
self.run_migration(True, selected=[FIRST])
|
|
self.assertEqual(self.state()["repositories"][FIRST]["status"], "MIGRATING")
|
|
self.target.interrupt_post = False
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(len(self.target.posts), 1)
|
|
|
|
def test_saved_receipt_allows_only_read_verification_after_network_failure(self):
|
|
self.target.fail_read_after_post = True
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(self.state()["repositories"][FIRST]["receipt_id"], 1001)
|
|
self.target.fail_read_after_post = False
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
|
|
self.assertEqual(len(self.target.posts), 1)
|
|
self.assertEqual(len(self.mappings()), 1)
|
|
|
|
def test_tag_mismatch_is_not_reported_as_success(self):
|
|
self.target.change_target_tag = True
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(self.state()["repositories"][FIRST]["error"], "TARGET_GIT_REFERENCES_MISMATCH")
|
|
self.assertEqual(self.mappings(), [])
|
|
|
|
def test_source_change_during_migration_requires_review(self):
|
|
self.target.change_source_after_post = True
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertEqual(self.state()["repositories"][FIRST]["error"], "SOURCE_CHANGED_DURING_MIGRATION")
|
|
|
|
def test_unexpected_success_status_does_not_count_as_completion(self):
|
|
self.target.post_status = 200
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 2)
|
|
self.assertNotIn("receipt_id", self.state()["repositories"][FIRST])
|
|
|
|
def test_other_organization_cannot_enter_source_inventory(self):
|
|
self.source.repositories[FIRST]["owner"]["login"] = "other-org"
|
|
with self.assertRaisesRegex(migration.MigrationError, "REPOSITORY_IDENTITY_MISMATCH"):
|
|
self.run_migration(True)
|
|
self.assertEqual(self.target.posts, [])
|
|
|
|
def test_owner_and_lfs_checks_happen_before_any_write(self):
|
|
self.source.owner = False
|
|
with self.assertRaisesRegex(migration.MigrationError, "ORGANIZATION_OWNER_REQUIRED"):
|
|
self.run_migration(True)
|
|
self.source.owner = True
|
|
self.target.lfs_enabled = False
|
|
with self.assertRaisesRegex(migration.MigrationError, "GIT_LFS_OR_MIGRATION_DISABLED"):
|
|
self.run_migration(True)
|
|
self.assertEqual(self.target.posts, [])
|
|
|
|
def test_state_cannot_be_reused_for_another_environment_or_changed_inventory(self):
|
|
self.run_migration()
|
|
self.target.base = "https://another-target.example"
|
|
with self.assertRaisesRegex(migration.MigrationError, "STATE_ENVIRONMENT_MISMATCH"):
|
|
self.run_migration(True)
|
|
self.target.base = "https://target.example"
|
|
self.source.repositories["p_cccccccccccc"] = repository("p_cccccccccccc", 103)
|
|
with self.assertRaisesRegex(migration.MigrationError, "SOURCE_INVENTORY_CHANGED"):
|
|
self.run_migration(True)
|
|
self.assertEqual(self.target.posts, [])
|
|
|
|
def test_empty_repository_is_preserved(self):
|
|
self.source.repositories[FIRST] = repository(FIRST, 101, empty=True)
|
|
self.assertEqual(self.run_migration(True, selected=[FIRST]), 0)
|
|
self.assertTrue(self.target.repositories[FIRST]["empty"])
|
|
self.assertEqual(len(self.mappings()), 1)
|
|
|
|
def test_state_permissions_and_lock_prevent_parallel_writes(self):
|
|
self.run_migration()
|
|
self.assertEqual(stat.S_IMODE(self.store.directory.stat().st_mode), 0o700)
|
|
self.assertEqual(stat.S_IMODE(self.store.path.stat().st_mode), 0o600)
|
|
with self.store.locked(), self.assertRaisesRegex(migration.MigrationError, "STATE_DIRECTORY_LOCKED"):
|
|
with migration.StateStore(self.store.directory).locked():
|
|
self.fail("不应获得第二个锁")
|
|
|
|
|
|
@contextmanager
|
|
def http_fixture():
|
|
calls = []
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
calls.append(self.path)
|
|
self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
|
if self.path.endswith("/redirect"):
|
|
self.send_response(302)
|
|
self.send_header("Location", "/credential-sink")
|
|
else:
|
|
self.send_response(500)
|
|
self.end_headers()
|
|
self.wfile.write((SOURCE_TOKEN + TARGET_TOKEN).encode())
|
|
|
|
def do_GET(self):
|
|
calls.append(self.path)
|
|
self.send_response(200)
|
|
self.end_headers()
|
|
self.wfile.write(b"{}")
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield "http://127.0.0.1:" + str(server.server_port), calls
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join()
|
|
|
|
|
|
class TransportTest(unittest.TestCase):
|
|
def test_post_redirect_does_not_forward_credentials_or_repeat_write(self):
|
|
with http_fixture() as (base, calls):
|
|
client = migration.ApiClient(base, TARGET_TOKEN)
|
|
with self.assertRaises(migration.MigrationError) as caught:
|
|
client.request("POST", "/redirect", {"auth_password": SOURCE_TOKEN})
|
|
self.assertEqual(str(caught.exception), "HTTP_302")
|
|
self.assertEqual(calls, ["/api/v1/redirect"])
|
|
|
|
def test_http_error_body_with_credentials_is_not_exposed(self):
|
|
with http_fixture() as (base, _):
|
|
client = migration.ApiClient(base, TARGET_TOKEN)
|
|
with self.assertRaises(migration.MigrationError) as caught:
|
|
client.request("POST", "/repos/migrate", {"auth_password": SOURCE_TOKEN})
|
|
self.assertEqual(str(caught.exception), "HTTP_500")
|
|
self.assertNotIn(SOURCE_TOKEN, str(caught.exception))
|
|
self.assertNotIn(TARGET_TOKEN, str(caught.exception))
|
|
|
|
def test_invalid_base_url_does_not_echo_embedded_credentials_in_cli(self):
|
|
result = subprocess.run([sys.executable, str(SCRIPT),
|
|
"--source-url", "https://user:" + SOURCE_TOKEN + "@source.example",
|
|
"--target-url", "https://target.example"],
|
|
capture_output=True, text=True, check=False)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertNotIn(SOURCE_TOKEN, result.stdout + result.stderr)
|
|
self.assertIn("INVALID_BASE_URL", result.stderr)
|
|
|
|
def test_base_url_keeps_subpath_and_rejects_insecure_or_credential_urls(self):
|
|
self.assertEqual(migration.base_url("https://GITEA.example:443/gitea/", False),
|
|
"https://gitea.example/gitea")
|
|
for value in ("http://gitea.example", "https://gitea.example?token=secret",
|
|
"https://gitea.example/#token", "https://gitea.example/a/../b"):
|
|
with self.subTest(value=value), self.assertRaises(migration.MigrationError):
|
|
migration.base_url(value, False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|