From 211f6c96f913827e29d54ab635082d057579e1e5 Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Thu, 15 Oct 2026 16:48:30 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E7=BC=96=E7=A0=81=E7=BC=A9?= =?UTF-8?q?=E7=9F=AD=EF=BC=8Cgithub=E5=BC=80=E6=BA=90=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E7=A4=BE=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 + README.md | 35 +- .../linkerbot/community/api/AiSearchTest.java | 22 +- .../community/api/GiteaLiveTest.java | 3 +- .../linkerbot/community/api/WorkflowTest.java | 21 ++ .../community/common/ProjectCodes.java | 41 ++ .../community/common/ProjectCodesTest.java | 40 ++ community-import/pom.xml | 12 + .../importer/DatabaseConfiguration.java | 27 ++ .../community/importer/ImportApplication.java | 304 +++++++++++++++ .../community/importer/ImportCoordinator.java | 350 ++++++++++++++++++ .../community/importer/ImportFailure.java | 15 + .../community/importer/ImportModels.java | 79 ++++ .../community/importer/SourceChecks.java | 206 +++++++++++ .../community/importer/StateStore.java | 130 +++++++ .../community/importer/WorkbookReader.java | 261 +++++++++++++ .../src/main/resources/application.yml | 28 ++ .../importer/ImportCoordinatorTest.java | 332 +++++++++++++++++ .../community/importer/SourceChecksTest.java | 45 +++ .../importer/WorkbookReaderTest.java | 195 ++++++++++ .../community/service/gitea/GiteaClient.java | 24 ++ .../service/gitea/RestGiteaClient.java | 93 ++++- .../service/impl/ProjectServiceImpl.java | 46 ++- .../service/support/ProjectInputs.java | 38 ++ .../service/ProjectCodeAllocationTest.java | 89 +++++ .../service/gitea/GiteaMigrationTest.java | 138 +++++++ docs/community-api.md | 2 + docs/database.md | 2 + pom.xml | 1 + scripts/import-github-projects.sh | 11 + 30 files changed, 2567 insertions(+), 25 deletions(-) create mode 100644 community-common/src/main/java/com/linkerbot/community/common/ProjectCodes.java create mode 100644 community-common/src/test/java/com/linkerbot/community/common/ProjectCodesTest.java create mode 100644 community-import/pom.xml create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/ImportFailure.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/ImportModels.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/SourceChecks.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/StateStore.java create mode 100644 community-import/src/main/java/com/linkerbot/community/importer/WorkbookReader.java create mode 100644 community-import/src/main/resources/application.yml create mode 100644 community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java create mode 100644 community-import/src/test/java/com/linkerbot/community/importer/SourceChecksTest.java create mode 100644 community-import/src/test/java/com/linkerbot/community/importer/WorkbookReaderTest.java create mode 100644 community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java create mode 100644 community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java create mode 100644 community-service/src/test/java/com/linkerbot/community/service/gitea/GiteaMigrationTest.java create mode 100644 scripts/import-github-projects.sh diff --git a/.env.example b/.env.example index 2729ac6..422867d 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,8 @@ DB_PASSWORD='replace-with-your-database-password' GITEA_BASE_URL=http://localhost:3000 GITEA_ORGANIZATION=community GITEA_TOKEN='replace-with-your-gitea-token' +# 可选,仅供批量导入读取 GitHub 元数据;不能填写 Gitea Token。 +GITHUB_TOKEN='' # 默认 test,部署生产环境时改为 prod;连接信息也应对应生产环境。 SPRING_PROFILES_ACTIVE=test # 与 cadworld 一致,默认日志目录为 logs,可改为部署环境中的绝对路径。 diff --git a/README.md b/README.md index 62d7772..f78de76 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Spring Boot 社区后端,工程分层参考 cadworld。使用 Java 21、Spring | community-service | 业务 Service 接口在 service 根包,实现类在 service.impl;包含 mapper、config、gitea、ai、support | | community-security | 平台登录会话、验证码、身份上下文、认证过滤器和安全配置 | | community-common | 通用响应、分页、异常处理、requestId 和基础工具 | +| community-import | 固定 Excel 的 GitHub 项目运维导入命令,不提供 HTTP 接口 | UserService、ProjectService、FileService、InteractionService、CoverUploadService 采用接口加实现类,Controller、安全模块和服务之间通过接口注入。GiteaClient 与 RestGiteaClient 定义仓库调用边界;项目锁、UTC 时间和分页检查归入 support。安全模块内的 TokenService、CaptchaService 保留具体类,首期沿用本机缓存。 @@ -75,7 +76,7 @@ SPRING_PROFILES_ACTIVE=prod java -Djava.awt.headless=true -jar community-api/tar mvn clean verify ``` -普通测试固定使用 test 配置,覆盖连接为临时 H2 MySQL 模式及模拟 Gitea/AI,验证登录、权限、提交冲突、发布快照、Fork、评论、点赞,以及 AI 筛选、引用校验、SSE、限流、超时和取消,不访问真实数据库、仓库或模型。完整安装验证可执行 `mvn clean install`。 +普通测试固定使用 test 配置,覆盖连接为临时 H2 MySQL 模式及模拟 Gitea/AI,验证登录、权限、提交冲突、发布快照、Fork、评论、点赞,以及 AI 筛选、引用校验、SSE、限流、超时和取消,不访问真实数据库、仓库或模型。未加载外部服务凭据时,执行 `AI_ENABLED=false OSS_ENABLED=false mvn test`;AI 流程测试自行启用模拟模型。完整安装验证可执行 `AI_ENABLED=false OSS_ENABLED=false mvn clean install`。 真实 Gitea 联调: @@ -125,3 +126,35 @@ python3 scripts/test-ai.py 创建或 Fork 前先写入 CREATING 项目,成功后标记 READY。结果不明或数据库回写失败需要人工核对;不提供后台自动恢复、无条件自动重试或跨重启完整幂等。具体恢复方式见接口文档。 + +## 项目短编码与 GitHub 初始化导入 + +普通创建、Fork 和导入的新项目使用 12 位小写项目编码,字符集为 `0123456789abcdefghjkmnpqrstvwxyz`。普通项目用安全随机数生成,项目编码唯一约束冲突最多重试 5 次;其他约束错误不重试。导入编码由 `github-import:v1:` 的 SHA-256 前 60 位确定,避免重跑或仓库更名后重复创建。Gitea 仓库名称为 `p_{projectCode}`。历史 32 位编码、仓库及链接继续使用,数据库无需改表。 + +导入工具仅接受以下固定 Sheet:`1_轮式机器人`、`2_四足机器人`、`3_小人形机器人`、`4_大人形机器人`、`5_机械臂`、`6_其他类型`、`7_开源硬件`。只选 I 列精确为“可访问”、L 列有合法 HTTPS 封面、B 列为具体 GitHub 仓库的行。其他 Sheet 不参与导入;K 列图片公式不会执行。原始 A~H 数据列与表头存在错位,工具按实际位置读取,忽略 H 列的无意义备注 `100`,不将“待查”等信息改写成已确认。 + +封面保留原 URL 和 GIF,不转存 OSS。描述由原中文简介和技术资料组合,不调用 AI。类型沿用现有配置:轮式、四足、人形分别映射对应类型,机械臂、其他和硬件映射 GENERAL,原始分类保留在描述里。 + +脚本加载本项目 `.env`,依赖 Java 21、Maven,以及正式执行时可用的 MySQL 和 Gitea。独立导入模块不启动 Web 服务,也不加载登录、AI 或 OSS 组件。Gitea 技术账号需具备组织所有者及仓库写权限,实例需开启迁移、HTTP Git 和 LFS。`GITHUB_TOKEN` 可选,仅用于 GitHub 元数据 API;API 限流时读取同一公开仓库的 GitHub 网页,交叉核验仓库 ID、规范地址、公开状态和默认分支,缺失或不一致则预检失败。网页和图片请求均不携带凭据。`GITEA_TOKEN` 仅访问配置的 Gitea,不能代替 `GITHUB_TOKEN`;切勿把 Token 放进 URL 或命令参数。 + +```bash +IMPORT_EXCEL='/绝对路径/项目.xlsx' +# 仅验证静态筛选,不访问网络、业务库或 Gitea。 +bash scripts/import-github-projects.sh --excel "$IMPORT_EXCEL" --offline +# 默认 dry-run:在线检查仓库和图片,并生成清单;不写业务库或 Gitea。 +bash scripts/import-github-projects.sh --excel "$IMPORT_EXCEL" --dry-run +# 先试导前三个尚未完成的候选。必须使用已有有效用户的 userCode。 +bash scripts/import-github-projects.sh --excel "$IMPORT_EXCEL" --execute --owner-user-code '<用户编码>' --limit 3 +# 试导检查通过后,用相同状态目录续跑其余项目。 +bash scripts/import-github-projects.sh --excel "$IMPORT_EXCEL" --execute --owner-user-code '<用户编码>' +``` + +`--owner-email` 可替代 `--owner-user-code` 查找已有账号,二者互斥,工具不会创建用户;状态中始终保存查到的用户编码。`--state-dir` 默认 `logs/github-import`,包含原子写入的 `manifest.json` 和 `report.json`;此目录必须保留并备份,不能在迁移进行中删除。清单绑定 Excel 指纹、归属用户和 Gitea 目标,不能复用于另一文件或环境。同一目录有进程锁,`--concurrency` 默认为 2,可选 1~4;同一批次只运行一个进程。`--limit` 限制尚未完成候选的处理数量;已完成项目、迁移结果未知或不具备收据核对条件的 REVIEW 项目不计入。 + +工具先创建 CREATING 项目记录,直接调用 Gitea `/repos/migrate`,不先创建空仓库。迁移使用私有普通仓库、开启 LFS,保留 Git 内容、历史、分支和标签;不迁移工单、PR、Wiki 或 Release 附件。迁移成功收据先落盘,再关联项目并自动发布。独立迁移请求默认超时 1800 秒,可用 `--migration-timeout-seconds` 调整,普通 Gitea 请求超时不变。原默认分支不是 main 时补建 main 并设为默认,保留原分支;同名 main 内容冲突时停止该项目,不覆盖源分支或已冻结版本。 + +进程在 MIGRATING 阶段中断,或迁移超时、结果不明确时,项目进入 REVIEW,禁止自动重发或仅凭仓库存在判定成功。需要核对 Gitea 迁移及 LFS 完成情况、来源和提交后由维护人员处理;不要直接把 REVIEW 改为 READY 或删除仓库再重跑。已确认的迁移收据会用于数据库回写或发布失败后的续跑,避免重复迁移。对于已持久化成功收据、仅在主分支适配时出现不明写入结果的项目,续跑先只读核对仓库身份、来源、私有属性、默认分支和两条分支的冻结 SHA;全部一致才继续关联和发布,否则保持 REVIEW,不重发分支写入。已完成项目不会因后续人工撤回、修改或删除而被批次重新发布。 + +报告逐项包含 Sheet、行号、短编码、阶段、错误码和兼容性提醒。退出码 0 表示本次选定工作完成,1 表示命令、配置或持久化等致命错误,2 表示存在预检未通过、失败或待核对项目。在线仓库或图片失效会减少实际导入量;报告中的候选数仍与静态筛选对账。 + +本次只完成仓库迁移、关联和发布。社区仍有单文件读取限制,LFS 实体读取、子模块及符号链接的浏览/Fork 能力没有扩展;当前发布快照检测到的限制会写入报告。子模块只保留引用,不递归迁移独立仓库,也不下载网盘或其他外链资源。保留所有分支不代表社区提供分支切换界面。 diff --git a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java index 2cdff5d..2469c98 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/AiSearchTest.java @@ -12,6 +12,7 @@ import com.linkerbot.community.bean.entity.Project; import com.linkerbot.community.bean.entity.User; import com.linkerbot.community.common.BusinessException; import com.linkerbot.community.common.Codes; +import com.linkerbot.community.common.ProjectCodes; import com.linkerbot.community.security.TokenService; import com.linkerbot.community.service.ai.AiModelClient; import com.linkerbot.community.service.config.AiProperties; @@ -99,7 +100,7 @@ class AiSearchTest { private Project project(String name, String type, String description) { Project project = new Project(); - project.setProjectCode(Codes.next()); + project.setProjectCode(ProjectCodes.next()); project.setOwnerUserId(user.getId()); project.setName(name); project.setDescription(description); @@ -165,6 +166,25 @@ class AiSearchTest { return events.getLast(); } + @Test void mixedLegacyAndShortCodesSupportFollowupReferences() throws Exception { + Project current = project("短编码机器人", "GENERAL", "支持教学"); + Project legacy = project("旧编码机器人", "GENERAL", "支持教学"); + legacy.setProjectCode(Codes.next()); + legacy.setGiteaRepoName("p_" + legacy.getProjectCode()); + projects.updateById(legacy); + model.handler = (name, input) -> { + if (!name.endsWith("intent")) return rank(input); + assertThat(input.path("previousProjects").size()).isEqualTo(2); + ObjectNode intent = intent(null, List.of()); + intent.withArray("focusProjectCodes").add(legacy.getProjectCode()); + return intent; + }; + List events = finish(start(Map.of("message", "第二个项目支持什么", "previousProjectCodes", + List.of(current.getProjectCode(), legacy.getProjectCode())))); + assertThat(events.getLast().path("data").path("projects").get(0).path("project").path("projectCode").asText()) + .isEqualTo(legacy.getProjectCode()).hasSize(32); + } + @ParameterizedTest @ValueSource(strings = {"GENERAL", "HUMANOID", "QUADRUPED", "WHEELED"}) void filtersConfiguredTypesAndKeepsProjectContract(String type) throws Exception { diff --git a/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java b/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java index bfb7dbf..b3df147 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/GiteaLiveTest.java @@ -70,7 +70,8 @@ class GiteaLiveTest { void removeOnlyRepositoriesCreatedByThisTest() throws Exception { HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); for (String repo : jdbc.queryForList("SELECT gitea_repo_name FROM project", String.class)) { - if (!repo.matches("p_[0-9a-f]{32}")) throw new IllegalStateException("拒绝清理非测试仓库"); + if (!repo.matches("p_([0-9a-f]{32}|[0-9abcdefghjkmnpqrstvwxyz]{12})")) + throw new IllegalStateException("拒绝清理非测试仓库"); HttpRequest request = HttpRequest.newBuilder( URI.create( diff --git a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java index d6b87e4..d37d89c 100644 --- a/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java +++ b/community-api/src/test/java/com/linkerbot/community/api/WorkflowTest.java @@ -171,6 +171,27 @@ class WorkflowTest { 200); } + @Test + void shortProjectsAndLegacyProjectsRemainUsableTogether() throws Exception { + String owner = login("short-owner@example.com"); + String code = create(owner, "旧项目"); + assertThat(code).matches("[0-9abcdefghjkmnpqrstvwxyz]{12}"); + assertThat(jdbc.queryForObject("SELECT project_code FROM project WHERE project_code = ?", String.class, code)).isEqualTo(code); + String legacy = "abcdef0123456789abcdef0123456789"; + jdbc.update("UPDATE project SET project_code = ?, gitea_repo_name = ? WHERE project_code = ?", legacy, "p_" + legacy, code); + String sha = head(owner, legacy); + publish(owner, legacy, sha); + assertThat(perform(get("/api/community/projects/" + legacy), 200).path("data").path("projectCode").asText()).isEqualTo(legacy); + JsonNode copy = perform(body(auth(post("/api/community/projects/" + legacy + "/forks"), owner), + Map.of("name", "新 Fork", "publishedCommitSha", sha)), 200).path("data"); + assertThat(copy.path("projectCode").asText()).matches("[0-9abcdefghjkmnpqrstvwxyz]{12}"); + assertThat(copy.path("sourceProjectCode").asText()).isEqualTo(legacy); + perform(auth(put("/api/community/projects/" + legacy + "/like"), owner), 200); + JsonNode comment = perform(body(auth(post("/api/community/projects/" + legacy + "/comments"), owner), Map.of("content", "兼容测试")), 200).path("data"); + assertThat(comment.path("commentCode").asText()).hasSize(32); + assertThat(copy.path("owner").path("userCode").asText()).hasSize(32); + } + @Test void coverRoundTripsThroughMetadataPublicationAndForkWithoutFileCommit() throws Exception { String owner = login("cover-owner@example.com"); diff --git a/community-common/src/main/java/com/linkerbot/community/common/ProjectCodes.java b/community-common/src/main/java/com/linkerbot/community/common/ProjectCodes.java new file mode 100644 index 0000000..3bcbcbd --- /dev/null +++ b/community-common/src/main/java/com/linkerbot/community/common/ProjectCodes.java @@ -0,0 +1,41 @@ +package com.linkerbot.community.common; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; + +/** 仅用于项目编码;旧的 UUID 编码继续有效。 */ +public final class ProjectCodes { + public static final String ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"; + private static final SecureRandom RANDOM = new SecureRandom(); + + private ProjectCodes() {} + + public static String next() { + StringBuilder code = new StringBuilder(12); + for (int i = 0; i < 12; i++) code.append(ALPHABET.charAt(RANDOM.nextInt(32))); + return code.toString(); + } + + public static String github(long repositoryId) { + if (repositoryId <= 0) throw new IllegalArgumentException("GitHub 仓库 ID 必须为正数"); + try { + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest( + ("github-import:v1:" + repositoryId) + .getBytes(StandardCharsets.UTF_8)); + StringBuilder code = new StringBuilder(12); + for (int i = 0; i < 12; i++) { + int offset = i * 5; + int pair = (hash[offset / 8] & 255) << 8; + if (offset / 8 + 1 < hash.length) pair |= hash[offset / 8 + 1] & 255; + code.append(ALPHABET.charAt((pair >>> (11 - offset % 8)) & 31)); + } + return code.toString(); + } catch (NoSuchAlgorithmException error) { + throw new IllegalStateException("SHA-256 不可用", error); + } + } +} diff --git a/community-common/src/test/java/com/linkerbot/community/common/ProjectCodesTest.java b/community-common/src/test/java/com/linkerbot/community/common/ProjectCodesTest.java new file mode 100644 index 0000000..9e25579 --- /dev/null +++ b/community-common/src/test/java/com/linkerbot/community/common/ProjectCodesTest.java @@ -0,0 +1,40 @@ +package com.linkerbot.community.common; + +import static org.assertj.core.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.HashSet; +import java.util.Set; + +class ProjectCodesTest { + @Test + void randomCodesAndOtherIdentifiersStaySeparate() { + Set codes = new HashSet<>(); + for (int i = 0; i < 10000; i++) { + String code = ProjectCodes.next(); + assertThat(code).matches("[0-9abcdefghjkmnpqrstvwxyz]{12}"); + assertThat(codes.add(code)).isTrue(); + } + assertThat(Codes.next()).matches("[a-f0-9]{32}"); + } + + @Test + void importUsesExactlyFirstSixtySha256Bits() throws Exception { + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest("github-import:v1:12345".getBytes(StandardCharsets.UTF_8)); + BigInteger firstSixty = new BigInteger(1, hash).shiftRight(196); + StringBuilder expected = new StringBuilder(); + for (int i = 11; i >= 0; i--) + expected.append( + ProjectCodes.ALPHABET.charAt(firstSixty.shiftRight(i * 5).intValue() & 31)); + assertThat(ProjectCodes.github(12345)).isEqualTo(expected.toString()); + assertThat(ProjectCodes.github(12345)).isNotEqualTo(ProjectCodes.github(12346)); + assertThatThrownBy(() -> ProjectCodes.github(0)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/community-import/pom.xml b/community-import/pom.xml new file mode 100644 index 0000000..ea089a8 --- /dev/null +++ b/community-import/pom.xml @@ -0,0 +1,12 @@ + + + 4.0.0 + com.linkerbotcommunity0.0.1-SNAPSHOT + community-import + + com.linkerbotcommunity-service${project.version} + com.mysqlmysql-connector-jruntime + com.h2databaseh2test + + org.springframework.bootspring-boot-maven-plugin + diff --git a/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java b/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java new file mode 100644 index 0000000..4ef8d88 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/DatabaseConfiguration.java @@ -0,0 +1,27 @@ +package com.linkerbot.community.importer; + +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.config.MybatisConfiguration; +import com.linkerbot.community.service.gitea.RestGiteaClient; +import com.linkerbot.community.service.impl.ProjectServiceImpl; +import com.linkerbot.community.service.impl.UserServiceImpl; +import com.linkerbot.community.service.support.ProjectLocks; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** 显式加载所需组件,避免扫描在线应用的安全、AI 和 OSS 模块。 */ +@Configuration(proxyBeanMethods = false) +@EnableAutoConfiguration +@MapperScan("com.linkerbot.community.service.mapper") +@Import({ + CommunityProperties.class, + MybatisConfiguration.class, + RestGiteaClient.class, + ProjectServiceImpl.class, + UserServiceImpl.class, + ProjectLocks.class +}) +public class DatabaseConfiguration {} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java new file mode 100644 index 0000000..657e3ef --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportApplication.java @@ -0,0 +1,304 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.community.bean.entity.User; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.common.ProjectCodes; +import com.linkerbot.community.service.ProjectService; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.gitea.*; +import com.linkerbot.community.service.mapper.ProjectMapper; +import com.linkerbot.community.service.mapper.UserMapper; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ConfigurableApplicationContext; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.*; + +public final class ImportApplication { + public record Options( + Path excel, + Path state, + String owner, + String ownerEmail, + boolean execute, + boolean offline, + int limit, + int concurrency, + Duration migrationTimeout) { + public static Options parse(String[] args) { + Map values = new HashMap<>(); + Set flags = new HashSet<>(); + Set booleanOptions = Set.of("--execute", "--dry-run", "--offline"); + Set valuedOptions = + Set.of( + "--excel", + "--state-dir", + "--owner-user-code", + "--owner-email", + "--limit", + "--concurrency", + "--migration-timeout-seconds"); + for (int i = 0; i < args.length; i++) { + String key = args[i]; + if (values.containsKey(key) || flags.contains(key)) + throw new IllegalArgumentException("参数重复:" + key); + if (booleanOptions.contains(key)) flags.add(key); + else if (valuedOptions.contains(key) + && i + 1 < args.length + && !args[i + 1].startsWith("--")) values.put(key, args[++i]); + else throw new IllegalArgumentException("不支持的参数或缺少参数值"); + } + if (!values.containsKey("--excel")) throw new IllegalArgumentException("必须提供 --excel"); + boolean execute = flags.contains("--execute"); + if (execute && (flags.contains("--dry-run") || flags.contains("--offline"))) + throw new IllegalArgumentException("--execute 不能与 --dry-run 或 --offline 同时使用"); + String owner = values.getOrDefault("--owner-user-code", ""); + String ownerEmail = + values.getOrDefault("--owner-email", "").trim().toLowerCase(Locale.ROOT); + if (!owner.isBlank() && !ownerEmail.isBlank()) + throw new IllegalArgumentException("用户编码和邮箱只能提供一个"); + if (execute && owner.isBlank() && ownerEmail.isBlank()) + throw new IllegalArgumentException("正式执行必须提供 --owner-user-code 或 --owner-email"); + int limit = + Integer.parseInt( + values.getOrDefault("--limit", String.valueOf(Integer.MAX_VALUE))); + int concurrency = Integer.parseInt(values.getOrDefault("--concurrency", "2")); + long seconds = + Long.parseLong(values.getOrDefault("--migration-timeout-seconds", "1800")); + if (limit < 1 || concurrency < 1 || concurrency > 4 || seconds < 1) + throw new IllegalArgumentException("数量、并发或超时参数不合法(并发范围 1~4)"); + return new Options( + Path.of(values.get("--excel")), + Path.of(values.getOrDefault("--state-dir", "logs/github-import")), + owner, + ownerEmail, + execute, + flags.contains("--offline"), + limit, + concurrency, + Duration.ofSeconds(seconds)); + } + } + + public static void main(String[] args) { + if (args.length == 0 || Arrays.asList(args).contains("--help")) { + System.out.println( + "用法:--excel <文件> [--state-dir <目录>] [--dry-run | --execute (--owner-user-code" + + " <编码> 或 --owner-email <已有邮箱>)] [--limit N] [--concurrency 2] [--offline]" + + " [--migration-timeout-seconds 1800]"); + return; + } + int status; + try { + status = run(Options.parse(args)); + } catch (Exception error) { + String code = + error instanceof ImportFailure failure + ? failure.code() + : error instanceof BusinessException business + ? business.code() + : error.getClass().getSimpleName(); + String detail = + error instanceof ImportFailure failure + ? ":" + failure.getMessage() + : ";检查参数、配置及运行报告"; + System.err.println("导入命令终止 code=" + code + detail); + status = 1; + } + if (status != 0) System.exit(status); + } + + public static int run(Options options) throws Exception { + WorkbookData workbook = new WorkbookReader().read(options.excel()); + try (StateStore store = new StateStore(options.state(), workbook)) { + System.out.printf( + "静态筛选 candidates=%d excluded=%d fingerprint=%s%n", + workbook.candidates().size(), + workbook.excluded().size(), + workbook.fingerprint()); + if (options.offline()) { + store.report(); + return 0; + } + CommunityProperties config = environmentConfig(); + if (store.manifest.giteaBaseUrl != null + && (!store.manifest.giteaBaseUrl.equals(config.getGitea().getBaseUrl()) + || !store.manifest.giteaOrganization.equals( + config.getGitea().getOwner()))) + throw new ImportFailure("TARGET_CHANGED", "状态目录绑定的 Gitea 目标已变化"); + GiteaClient remote = new RestGiteaClient(config, new ObjectMapper()); + remote.checkMigrationSettings(); + SourceChecks source = new SourceChecks(System.getenv("GITHUB_TOKEN")); + Set ids = new HashSet<>(); + int selected = 0; + List work = new ArrayList<>(); + for (Entry entry : store.manifest.entries) { + if (entry.source != null && !ids.add(entry.source.id())) + throw new ImportFailure("DUPLICATE_SOURCE", "清单中 GitHub 仓库重复"); + if (entry.phase == Phase.MIGRATING) + store.update( + entry, + e -> { + e.phase = Phase.REVIEW; + e.errorCode = "MIGRATION_RESULT_UNKNOWN"; + e.message = "进程中断后的迁移结果未知,禁止自动重发"; + }); + if (entry.phase == Phase.PUBLISHED + || entry.phase == Phase.REVIEW + && !ImportCoordinator.canVerifyNormalization(entry) + || selected >= options.limit()) continue; + selected++; + GitHubRepository found; + try { + found = source.repository(entry.candidate.repositoryUrl()); + if (entry.source != null && found.id() != entry.source.id()) + throw new ImportFailure("SOURCE_CHANGED", "来源仓库身份变化,需人工核对"); + if (entry.source == null && !ids.add(found.id())) + throw new ImportFailure("DUPLICATE_SOURCE", "不同表格行指向同一个仓库"); + source.cover(entry.candidate.coverUrl()); + } catch (IOException | InterruptedException error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + throw error; + } + preflightFailure(store, entry, "SOURCE_NETWORK_ERROR"); + store.report(); + continue; + } catch (ImportFailure error) { + preflightFailure(store, entry, error.code()); + store.report(); + continue; + } + GitHubRepository verified = found; + store.update( + entry, + e -> { + if (e.source == null) e.source = verified; + e.projectCode = ProjectCodes.github(verified.id()); + e.preflightPassed = true; + e.errorCode = null; + e.message = null; + if (e.phase == Phase.PENDING || e.phase == Phase.PREFLIGHT_FAILED) + e.phase = Phase.READY; + }); + work.add(entry); + System.out.printf( + "预检通过 sheet=%s row=%d projectCode=%s%n", + entry.candidate.sheet(), entry.candidate.row(), entry.projectCode); + store.report(); + } + store.report(); + if (!options.execute()) return work.size() == selected ? 0 : 2; + try (ConfigurableApplicationContext context = + new SpringApplicationBuilder(DatabaseConfiguration.class) + .web(WebApplicationType.NONE) + .logStartupInfo(false) + .run()) { + UserMapper users = context.getBean(UserMapper.class); + LambdaQueryWrapper ownerQuery = new LambdaQueryWrapper<>(); + if (!options.owner().isBlank()) ownerQuery.eq(User::getUserCode, options.owner()); + else ownerQuery.eq(User::getEmail, options.ownerEmail()); + User owner = users.selectOne(ownerQuery); + if (owner == null || !"ACTIVE".equals(owner.getStatus())) + throw new ImportFailure("INVALID_OWNER", "归属用户不存在或不可用"); + System.out.println("导入归属已核验 userCode=" + owner.getUserCode()); + if (store.manifest.ownerUserCode != null + && !store.manifest.ownerUserCode.equals(owner.getUserCode())) + throw new ImportFailure("OWNER_CHANGED", "批次归属用户已变化"); + store.manifest.ownerUserCode = owner.getUserCode(); + store.manifest.giteaBaseUrl = config.getGitea().getBaseUrl(); + store.manifest.giteaOrganization = config.getGitea().getOwner(); + store.save(); + ImportCoordinator coordinator = + new ImportCoordinator( + context.getBean(ProjectMapper.class), + context.getBean(ProjectService.class), + context.getBean(GiteaClient.class), + context.getBean(CommunityProperties.class), + store, + owner, + options.migrationTimeout()); + try (ExecutorService executor = + Executors.newFixedThreadPool(options.concurrency())) { + try { + CompletionService jobs = new ExecutorCompletionService<>(executor); + for (Entry entry : work) + jobs.submit( + () -> { + try { + coordinator.run(entry); + store.report(); + } catch (IOException error) { + throw new java.io.UncheckedIOException(error); + } + return null; + }); + for (int completed = 0; completed < work.size(); completed++) + jobs.take().get(); + } finally { + executor.shutdownNow(); + } + } + } finally { + store.report(); + } + return store.manifest.entries.stream() + .anyMatch( + e -> + e.phase == Phase.FAILED + || e.phase == Phase.REVIEW + || e.phase == Phase.PREFLIGHT_FAILED) + ? 2 + : 0; + } + } + + private static void preflightFailure(StateStore store, Entry entry, String code) + throws IOException { + store.update( + entry, + e -> { + boolean reviewing = e.phase == Phase.REVIEW; + e.preflightPassed = false; + e.errorCode = code; + e.message = "在线预检失败,请核对网络或来源后重跑"; + e.phase = + reviewing + || code.equals("SOURCE_CHANGED") + || code.equals("DUPLICATE_SOURCE") + ? Phase.REVIEW + : Phase.PREFLIGHT_FAILED; + }); + System.out.printf( + "预检未通过 sheet=%s row=%d code=%s%n", + entry.candidate.sheet(), entry.candidate.row(), code); + } + + private static CommunityProperties environmentConfig() { + CommunityProperties config = new CommunityProperties(); + String baseUrl = + System.getenv() + .getOrDefault("GITEA_BASE_URL", "http://localhost:3000") + .replaceAll("/+$", ""); + java.net.URI base = java.net.URI.create(baseUrl); + if (base.getUserInfo() != null + || base.getQuery() != null + || base.getFragment() != null + || base.getHost() == null + || !("http".equals(base.getScheme()) || "https".equals(base.getScheme()))) + throw new ImportFailure("INVALID_GITEA_URL", "Gitea 基础地址不能包含凭据、查询或片段"); + config.getGitea().setBaseUrl(baseUrl); + config.getGitea().setOwner(System.getenv().getOrDefault("GITEA_ORGANIZATION", "community")); + config.getGitea().setToken(System.getenv().getOrDefault("GITEA_TOKEN", "")); + return config; + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java new file mode 100644 index 0000000..bda611f --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportCoordinator.java @@ -0,0 +1,350 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.linkerbot.community.bean.entity.Project; +import com.linkerbot.community.bean.entity.User; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.ProjectService; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.gitea.*; +import com.linkerbot.community.service.gitea.GiteaClient.RemoteRepository; +import com.linkerbot.community.service.mapper.ProjectMapper; +import com.linkerbot.community.service.support.ProjectInputs; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.*; +import java.time.temporal.ChronoUnit; +import java.util.*; + +public final class ImportCoordinator { + private static final Logger log = LoggerFactory.getLogger(ImportCoordinator.class); + private final ProjectMapper projects; + private final ProjectService service; + private final GiteaClient gitea; + private final CommunityProperties config; + private final StateStore store; + private final User owner; + private final Duration timeout; + private final String batch; + + public static boolean canVerifyNormalization(Entry entry) { + return entry.phase == Phase.REVIEW + && entry.receipt != null + && entry.targetSha != null + && "NORMALIZING".equals(entry.failedPhase) + && !"SOURCE_CHANGED".equals(entry.errorCode) + && !"DUPLICATE_SOURCE".equals(entry.errorCode); + } + + public ImportCoordinator( + ProjectMapper projects, + ProjectService service, + GiteaClient gitea, + CommunityProperties config, + StateStore store, + User owner, + Duration timeout) { + this.projects = projects; + this.service = service; + this.gitea = gitea; + this.config = config; + this.store = store; + this.owner = owner; + this.timeout = timeout; + this.batch = + store.manifest.fingerprint.substring( + 0, Math.min(12, store.manifest.fingerprint.length())); + } + + public void run(Entry entry) throws IOException { + long started = System.nanoTime(); + Project project = null; + boolean owned = false; + try { + if (!entry.preflightPassed + || entry.phase == Phase.REVIEW && !canVerifyNormalization(entry) + || entry.phase == Phase.PUBLISHED) return; + ProjectInputs.validate(entry.candidate.input(), config); + String repoName = "p_" + entry.projectCode; + project = + projects.selectOne( + new LambdaQueryWrapper() + .eq(Project::getProjectCode, entry.projectCode)); + if (project == null) { + if (entry.receipt != null + || entry.reservedAt != null + && entry.phase != Phase.RESERVING + && !"RESERVING".equals(entry.failedPhase)) + throw new ImportFailure("PROJECT_MISSING", "已有进度记录对应的项目丢失,需人工核对"); + ensureAbsent(repoName); + store.update( + entry, + e -> { + if (e.reservedAt == null) + e.reservedAt = + LocalDateTime.now(Clock.systemUTC()) + .truncatedTo(ChronoUnit.MILLIS) + .toString(); + e.phase = Phase.RESERVING; + }); + project = new Project(); + project.setProjectCode(entry.projectCode); + project.setOwnerUserId(owner.getId()); + project.setName(entry.candidate.name().trim()); + project.setDescription(entry.candidate.description()); + project.setCoverUrl(ProjectInputs.coverUrl(entry.candidate.coverUrl())); + project.setProjectType(entry.candidate.projectType()); + project.setGiteaOwner(config.getGitea().getOwner()); + project.setGiteaRepoName(repoName); + project.setRepoStatus("CREATING"); + project.setPublishStatus("UNPUBLISHED"); + project.setDeleted(false); + project.setCreatedAt(LocalDateTime.parse(entry.reservedAt)); + project.setUpdatedAt(project.getCreatedAt()); + projects.insert(project); + owned = true; + store.phase(entry, Phase.RESERVED); + log.info( + "导入项目记录创建 batch={} projectCode={} sheet={} row={}", + batch, + entry.projectCode, + entry.candidate.sheet(), + entry.candidate.row()); + } else { + if (entry.reservedAt == null + || Boolean.TRUE.equals(project.getDeleted()) + || !Objects.equals(project.getOwnerUserId(), owner.getId()) + || !Objects.equals( + project.getCreatedAt(), LocalDateTime.parse(entry.reservedAt)) + || !Objects.equals(project.getGiteaOwner(), config.getGitea().getOwner()) + || !Objects.equals(project.getGiteaRepoName(), repoName)) + throw new ImportFailure("PROJECT_CONFLICT", "编码已被其他项目占用或项目归属变化,禁止接管"); + owned = true; + if ("PUBLISHED".equals(project.getPublishStatus())) { + validateRemote(entry, gitea.inspect(repoName)); + if (!Objects.equals( + project.getGiteaRepoId(), + entry.receipt == null ? null : entry.receipt.id()) + || !Objects.equals(project.getPublishedCommitSha(), entry.targetSha)) + throw new ImportFailure("PUBLICATION_CONFLICT", "已发布项目与导入收据不一致"); + store.phase(entry, Phase.PUBLISHED); + return; + } + } + + if (entry.phase == Phase.REVIEW) { + RemoteRepository confirmed = gitea.inspect(repoName); + validateRemote(entry, confirmed); + boolean applied = + confirmed.id() == entry.receipt.id() + && "main".equals(confirmed.defaultBranch()); + try { + applied = + applied + && entry.targetSha.equals( + gitea.branch( + confirmed.repository(), + entry.source.defaultBranch())) + && entry.targetSha.equals( + gitea.branch(confirmed.repository(), "main")); + } catch (GiteaFailure missing) { + if (missing.status() != 404) throw missing; + applied = false; + } + if (!applied) { + store.update( + entry, + e -> { + e.errorCode = "NORMALIZATION_NOT_CONFIRMED"; + e.message = "迁移收据已确认,但目标 main、默认分支或 SHA 尚未核对通过;不重发远端写入"; + }); + return; + } + store.phase(entry, Phase.BRANCH_READY); + log.info("分支适配结果已核对 batch={} projectCode={}", batch, entry.projectCode); + } + + if (entry.receipt == null) { + ensureAbsent(repoName); + store.phase(entry, Phase.MIGRATING); + log.info( + "原生迁移开始 batch={} projectCode={} source={}", + batch, + entry.projectCode, + entry.source.url()); + RemoteRepository receipt = + gitea.migrate(entry.source.url() + ".git", repoName, timeout); + // 收据必须先持久化;之后数据库或主分支处理失败可安全续跑。 + store.update( + entry, + e -> { + e.receipt = receipt; + e.phase = Phase.MIGRATED; + }); + log.info("原生迁移完成 batch={} projectCode={}", batch, entry.projectCode); + } + RemoteRepository remote = gitea.inspect(repoName); + validateRemote(entry, remote); + if (remote.id() != entry.receipt.id()) + throw new ImportFailure("REPOSITORY_REPLACED", "仓库标识与迁移收据不一致"); + store.phase(entry, Phase.NORMALIZING); + String sourceSha = gitea.branch(remote.repository(), entry.source.defaultBranch()); + if (entry.targetSha != null && !entry.targetSha.equals(sourceSha)) + throw new ImportFailure("SOURCE_BRANCH_CHANGED", "目标仓库的源分支已变化,禁止自动改写冻结版本"); + if (entry.targetSha == null) store.update(entry, e -> e.targetSha = sourceSha); + String mainSha; + try { + mainSha = gitea.branch(remote.repository(), "main"); + } catch (GiteaFailure missing) { + if (missing.status() != 404) throw missing; + gitea.createBranch(remote.repository(), "main", sourceSha); + mainSha = gitea.branch(remote.repository(), "main"); + } + if (!sourceSha.equals(mainSha)) + throw new ImportFailure("MAIN_BRANCH_CONFLICT", "main 与源默认分支内容不同,禁止覆盖"); + if (!"main".equals(remote.defaultBranch())) + gitea.defaultBranch(remote.repository(), "main"); + store.update( + entry, + e -> { + e.targetSha = sourceSha; + e.phase = Phase.BRANCH_READY; + }); + + project.setGiteaRepoId(remote.id()); + project.setRepoStatus("READY"); + project.setUpdatedAt(LocalDateTime.now(Clock.systemUTC())); + if (projects.updateById(project) != 1) + throw new ImportFailure("DATABASE_UPDATE_FAILED", "项目仓库关联写入失败"); + store.phase(entry, Phase.LINKED); + List warnings; + try { + warnings = + gitea.migrationWarnings( + remote.repository(), sourceSha, config.getMaxFileBytes()); + } catch (RuntimeException warningFailure) { + warnings = List.of("兼容性扫描未完成;请人工检查大文件、LFS、子模块和符号链接"); + } + List checkedWarnings = warnings; + store.update(entry, e -> e.warnings = new ArrayList<>(checkedWarnings)); + store.phase(entry, Phase.PUBLISHING); + service.publish(owner.getId(), entry.projectCode, sourceSha); + store.phase(entry, Phase.PUBLISHED); + log.info( + "导入发布完成 batch={} projectCode={} durationMs={}", + batch, + entry.projectCode, + (System.nanoTime() - started) / 1_000_000); + } catch (IOException persistenceFailure) { + // 状态落盘失败必须终止本次执行,不能接着发出新的远端写入。 + log.error( + "导入状态落盘失败 batch={} projectCode={} stage={} errorType={}", + batch, + entry.projectCode, + entry.phase, + persistenceFailure.getClass().getSimpleName()); + throw persistenceFailure; + } catch (RuntimeException error) { + String failedPhase = + entry.phase == Phase.REVIEW && entry.failedPhase != null + ? entry.failedPhase + : entry.phase.name(); + boolean review = + entry.phase == Phase.MIGRATING + || entry.phase == Phase.REVIEW + || error instanceof GiteaFailure failure && failure.uncertain(); + String code = + error instanceof ImportFailure failure + ? failure.code() + : error instanceof BusinessException business + ? business.code() + : "IMPORT_INTERNAL_ERROR"; + if (code.equals("REMOTE_ALREADY_EXISTS") || code.equals("PROJECT_MISSING")) + review = true; + if (entry.phase == Phase.MIGRATING + && error instanceof GiteaFailure failure + && !failure.uncertain()) { + try { + gitea.inspect("p_" + entry.projectCode); + } catch (GiteaFailure missing) { + if (missing.status() == 404) review = false; + } + } + if (owned && project != null && !"READY".equals(project.getRepoStatus())) { + project.setRepoStatus(review ? "CREATING" : "FAILED"); + project.setUpdatedAt(LocalDateTime.now(Clock.systemUTC())); + try { + projects.updateById(project); + } catch (RuntimeException databaseFailure) { + log.error( + "导入失败状态回写异常 batch={} projectCode={} errorType={}", + batch, + entry.projectCode, + databaseFailure.getClass().getSimpleName()); + } + } + String message = + error instanceof ImportFailure failure + ? failure.getMessage() + : "导入阶段失败,按错误码和服务端日志核对"; + boolean requiresReview = review; + store.update( + entry, + e -> { + e.phase = requiresReview ? Phase.REVIEW : Phase.FAILED; + e.errorCode = code; + e.failedPhase = failedPhase; + e.message = message; + }); + String location = + error.getStackTrace().length == 0 + ? "unknown" + : error.getStackTrace()[0].toString(); + log.error( + "导入异常 batch={} projectCode={} source={} stage={} code={} review={} errorType={}" + + " location={} durationMs={}", + batch, + entry.projectCode, + entry.source.url(), + failedPhase, + code, + review, + error.getClass().getSimpleName(), + location, + (System.nanoTime() - started) / 1_000_000); + } + } + + private void ensureAbsent(String name) { + try { + gitea.inspect(name); + } catch (GiteaFailure missing) { + if (missing.status() == 404) return; + throw missing; + } + throw new ImportFailure("REMOTE_ALREADY_EXISTS", "目标仓库已存在但没有已确认迁移收据,需人工核对"); + } + + private void validateRemote(Entry entry, RemoteRepository remote) { + String original; + try { + original = WorkbookReader.repositoryKey(remote.originalUrl()); + } catch (RuntimeException invalid) { + throw new ImportFailure("SOURCE_CONFLICT", "目标仓库缺少合法来源信息"); + } + if (!remote.owner().equals(config.getGitea().getOwner()) + || !remote.name().equals("p_" + entry.projectCode) + || !remote.privateRepo() + || remote.mirror() + || remote.empty() + || !(original.equals(WorkbookReader.repositoryKey(entry.source.url())) + || original.equals( + WorkbookReader.repositoryKey(entry.candidate.repositoryUrl())))) + throw new ImportFailure("REPOSITORY_CONFLICT", "目标仓库归属、来源、可见性或内容不符合迁移要求"); + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportFailure.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportFailure.java new file mode 100644 index 0000000..61a1525 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportFailure.java @@ -0,0 +1,15 @@ +package com.linkerbot.community.importer; + +/** 仅携带允许输出的诊断信息,不保留远端响应或认证信息。 */ +public final class ImportFailure extends RuntimeException { + private final String code; + + public ImportFailure(String code, String message) { + super(message); + this.code = code; + } + + public String code() { + return code; + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/ImportModels.java b/community-import/src/main/java/com/linkerbot/community/importer/ImportModels.java new file mode 100644 index 0000000..7ea4ea7 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/ImportModels.java @@ -0,0 +1,79 @@ +package com.linkerbot.community.importer; + +import com.linkerbot.community.bean.Contracts.ProjectInput; +import com.linkerbot.community.service.gitea.GiteaClient.RemoteRepository; + +import java.util.ArrayList; +import java.util.List; + +public final class ImportModels { + private ImportModels() {} + + public record Candidate( + String sheet, + int row, + String name, + String repositoryUrl, + String description, + String projectType, + String coverUrl) { + public ProjectInput input() { + return new ProjectInput(name, description, projectType, coverUrl); + } + } + + public record Excluded(String sheet, int row, String name, String reason) {} + + public record WorkbookData( + String fingerprint, List candidates, List excluded) {} + + public record GitHubRepository(long id, String url, String defaultBranch) {} + + public enum Phase { + PENDING, + PREFLIGHT_FAILED, + READY, + RESERVING, + RESERVED, + MIGRATING, + MIGRATED, + NORMALIZING, + BRANCH_READY, + LINKED, + PUBLISHING, + PUBLISHED, + FAILED, + REVIEW + } + + public static final class Entry { + public Candidate candidate; + public GitHubRepository source; + public String projectCode; + public Phase phase = Phase.PENDING; + public boolean preflightPassed; + public String reservedAt; + public RemoteRepository receipt; + public String targetSha; + public String errorCode; + public String failedPhase; + public String message; + public List warnings = new ArrayList<>(); + + public Entry() {} + + public Entry(Candidate candidate) { + this.candidate = candidate; + } + } + + public static final class Manifest { + public int version = 1; + public String fingerprint; + public String ownerUserCode; + public String giteaBaseUrl; + public String giteaOrganization; + public List entries = new ArrayList<>(); + public List excluded = new ArrayList<>(); + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/SourceChecks.java b/community-import/src/main/java/com/linkerbot/community/importer/SourceChecks.java new file mode 100644 index 0000000..2b50ca4 --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/SourceChecks.java @@ -0,0 +1,206 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.InputStream; +import java.net.URI; +import java.net.http.*; +import java.time.Duration; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SourceChecks { + private final HttpClient http = + HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NEVER) + .build(); + private final ObjectMapper json = new ObjectMapper(); + private final String token; + private boolean apiLimited; + + public SourceChecks(String token) { + this.token = token == null ? "" : token; + } + + public GitHubRepository repository(String url) throws Exception { + if (apiLimited) return publicPage(url); + URI uri = URI.create("https://api.github.com/repos/" + WorkbookReader.repositoryKey(url)); + for (int redirects = 0; redirects <= 5; redirects++) { + if (!"https".equalsIgnoreCase(uri.getScheme()) + || !"api.github.com".equalsIgnoreCase(uri.getHost()) + || uri.getUserInfo() != null + || (uri.getPort() != -1 && uri.getPort() != 443)) + throw new ImportFailure("GITHUB_REDIRECT", "GitHub API 跳转地址不符合要求"); + HttpRequest.Builder request = + HttpRequest.newBuilder(uri) + .timeout(Duration.ofSeconds(30)) + .header("Accept", "application/vnd.github+json") + .header("User-Agent", "community-import/1"); + if (!token.isBlank()) request.header("Authorization", "Bearer " + token); + HttpResponse response = + http.send(request.GET().build(), HttpResponse.BodyHandlers.ofString()); + int status = response.statusCode(); + if (status >= 300 && status < 400) { + uri = redirect(uri, response); + continue; + } + if (status == 403 || status == 429) { + apiLimited = true; + return publicPage(url); + } + if (status == 404) throw new ImportFailure("GITHUB_NOT_FOUND", "GitHub 仓库不可访问"); + if (status != 200) throw new ImportFailure("GITHUB_HTTP_" + status, "GitHub 返回异常状态"); + JsonNode data = json.readTree(response.body()); + if (data.path("id").asLong() <= 0 + || data.path("private").asBoolean() + || data.path("default_branch").asText().isBlank() + || data.path("size").asLong(-1) < 0) + throw new ImportFailure("GITHUB_INVALID_REPOSITORY", "仓库元数据无效或不是公开仓库"); + String canonical = + "https://github.com/" + + WorkbookReader.repositoryKey(data.path("html_url").asText()); + return new GitHubRepository( + data.path("id").asLong(), canonical, data.path("default_branch").asText()); + } + throw new ImportFailure("GITHUB_REDIRECT", "GitHub 跳转次数过多"); + } + + private GitHubRepository publicPage(String url) throws Exception { + URI uri = URI.create("https://github.com/" + WorkbookReader.repositoryKey(url)); + for (int redirects = 0; redirects <= 5; redirects++) { + if (!"https".equalsIgnoreCase(uri.getScheme()) + || !"github.com".equalsIgnoreCase(uri.getHost()) + || uri.getUserInfo() != null + || (uri.getPort() != -1 && uri.getPort() != 443)) + throw new ImportFailure("GITHUB_REDIRECT", "GitHub 网页跳转地址不符合要求"); + HttpRequest request = + HttpRequest.newBuilder(uri) + .timeout(Duration.ofSeconds(30)) + .header("Accept", "text/html") + .header("User-Agent", "community-import/1") + .GET() + .build(); + // 网页请求不携带任何 GitHub 或 Gitea 凭据。 + HttpResponse response = + http.send(request, HttpResponse.BodyHandlers.ofString()); + int status = response.statusCode(); + if (status >= 300 && status < 400) { + uri = redirect(uri, response); + continue; + } + if (status == 403 || status == 429) + throw new ImportFailure("GITHUB_ACCESS_OR_RATE_LIMIT", "GitHub API 及公开网页均不可访问"); + if (status != 200) + throw new ImportFailure("GITHUB_PAGE_HTTP_" + status, "GitHub 公开仓库网页不可访问"); + return parsePublicPage(response.body(), uri.toString()); + } + throw new ImportFailure("GITHUB_REDIRECT", "GitHub 网页跳转次数过多"); + } + + static GitHubRepository parsePublicPage(String html, String url) throws Exception { + String key = WorkbookReader.repositoryKey(url); + String id = pageMeta(html, "repository_id"); + if (!id.matches("[1-9][0-9]*") + || !key.equalsIgnoreCase(pageMeta(html, "repository_nwo")) + || !"true".equals(pageMeta(html, "repository_public"))) + throw new ImportFailure("GITHUB_PAGE_METADATA", "GitHub 网页仓库身份或公开状态无法核验"); + Matcher scripts = + Pattern.compile( + "]*data-target=\"react-app.embeddedData\"[^>]*>(.*?)", + Pattern.DOTALL) + .matcher(html); + while (scripts.find()) { + JsonNode repo = + new ObjectMapper() + .readTree(scripts.group(1)) + .path("payload") + .path("codeViewLayoutRoute") + .path("repo"); + if (repo.path("id").asLong() != Long.parseLong(id)) continue; + String name = repo.path("ownerLogin").asText() + "/" + repo.path("name").asText(); + if (!name.equalsIgnoreCase(key) + || !repo.path("public").asBoolean() + || !repo.path("private").isBoolean() + || repo.path("private").asBoolean() + || !repo.path("isEmpty").isBoolean() + || repo.path("isEmpty").asBoolean() + || repo.path("defaultBranch").asText().isBlank()) break; + return new GitHubRepository( + Long.parseLong(id), + "https://github.com/" + name, + repo.path("defaultBranch").asText()); + } + throw new ImportFailure("GITHUB_PAGE_METADATA", "GitHub 网页缺少一致的仓库 ID、公开状态或默认分支"); + } + + private static String pageMeta(String html, String field) { + Matcher meta = + Pattern.compile( + "]*>") + .matcher(html); + return meta.find() ? meta.group(1) : ""; + } + + public void cover(String url) throws Exception { + URI uri = URI.create(url); + for (int redirects = 0; redirects <= 5; redirects++) { + if (!"https".equalsIgnoreCase(uri.getScheme()) + || uri.getHost() == null + || uri.getUserInfo() != null) + throw new ImportFailure("COVER_REDIRECT", "封面跳转地址必须为 HTTPS"); + HttpRequest request = + HttpRequest.newBuilder(uri) + .timeout(Duration.ofSeconds(30)) + .header("User-Agent", "community-import/1") + .header("Range", "bytes=0-511") + .GET() + .build(); + HttpResponse response = + http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream body = response.body()) { + int status = response.statusCode(); + if (status >= 300 && status < 400) { + uri = redirect(uri, response); + continue; + } + if (status != 200 && status != 206) + throw new ImportFailure("COVER_HTTP_" + status, "封面链接无法读取"); + byte[] signature = body.readNBytes(16); + boolean image = + signature.length >= 3 + && ((signature[0] & 255) == 255 && (signature[1] & 255) == 216 + || signature.length >= 8 + && (signature[0] & 255) == 137 + && signature[1] == 'P' + && signature[2] == 'N' + && signature[3] == 'G' + || signature[0] == 'G' + && signature[1] == 'I' + && signature[2] == 'F' + || signature.length >= 12 + && signature[0] == 'R' + && signature[1] == 'I' + && signature[8] == 'W' + && signature[9] == 'E'); + if (!image) + throw new ImportFailure("COVER_NOT_IMAGE", "封面未返回 JPG、PNG、GIF 或 WebP 图片内容"); + return; + } + } + throw new ImportFailure("COVER_REDIRECT", "封面跳转次数过多"); + } + + private URI redirect(URI uri, HttpResponse response) { + return uri.resolve( + response.headers() + .firstValue("Location") + .orElseThrow( + () -> new ImportFailure("INVALID_REDIRECT", "跳转响应缺少 Location"))); + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/StateStore.java b/community-import/src/main/java/com/linkerbot/community/importer/StateStore.java new file mode 100644 index 0000000..287c70d --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/StateStore.java @@ -0,0 +1,130 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.*; +import java.util.*; +import java.util.function.Consumer; + +public final class StateStore implements AutoCloseable { + private final Path directory; + private final ObjectMapper json = new ObjectMapper().findAndRegisterModules(); + private final FileChannel channel; + private final FileLock lock; + public final Manifest manifest; + + public StateStore(Path directory, WorkbookData data) throws IOException { + this.directory = directory; + Files.createDirectories(directory); + channel = + FileChannel.open( + directory.resolve(".lock"), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE); + FileLock acquired; + try { + acquired = channel.tryLock(); + } catch (RuntimeException error) { + channel.close(); + throw error; + } + if (acquired == null) { + channel.close(); + throw new ImportFailure("BATCH_LOCKED", "同一状态目录已有导入进程"); + } + lock = acquired; + try { + Path file = directory.resolve("manifest.json"); + if (Files.exists(file)) { + manifest = json.readValue(file.toFile(), Manifest.class); + if (manifest.version != 1 + || !data.fingerprint().equals(manifest.fingerprint) + || !data.candidates() + .equals(manifest.entries.stream().map(e -> e.candidate).toList()) + || !data.excluded().equals(manifest.excluded)) + throw new ImportFailure("BATCH_CHANGED", "Excel 或清单内容已变化,请使用新的状态目录"); + } else { + manifest = new Manifest(); + manifest.fingerprint = data.fingerprint(); + manifest.excluded.addAll(data.excluded()); + data.candidates().forEach(c -> manifest.entries.add(new Entry(c))); + save(); + } + } catch (IOException | RuntimeException error) { + lock.release(); + channel.close(); + throw error; + } + } + + public synchronized void update(Entry entry, Consumer change) throws IOException { + change.accept(entry); + save(); + } + + public synchronized void phase(Entry entry, Phase phase) throws IOException { + update( + entry, + e -> { + e.phase = phase; + e.errorCode = null; + e.message = null; + e.failedPhase = null; + }); + } + + public synchronized void save() throws IOException { + atomic("manifest.json", manifest); + } + + public synchronized void report() throws IOException { + Map counts = new LinkedHashMap<>(); + for (Phase phase : Phase.values()) + counts.put( + phase.name(), manifest.entries.stream().filter(e -> e.phase == phase).count()); + Map report = new LinkedHashMap<>(); + report.put("fingerprint", manifest.fingerprint); + report.put("candidateCount", manifest.entries.size()); + report.put("excludedCount", manifest.excluded.size()); + report.put("counts", counts); + report.put("entries", manifest.entries); + report.put("excluded", manifest.excluded); + atomic("report.json", report); + } + + private void atomic(String name, Object value) throws IOException { + Path temporary = directory.resolve(name + ".tmp"); + byte[] bytes = json.writerWithDefaultPrettyPrinter().writeValueAsBytes(value); + try (FileChannel file = + FileChannel.open( + temporary, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE)) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) file.write(buffer); + file.force(true); + } + // 不支持原子替换的状态目录必须停止执行,不能降级为可能损坏进度的普通覆盖。 + Files.move( + temporary, + directory.resolve(name), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public void close() throws IOException { + try { + lock.release(); + } finally { + channel.close(); + } + } +} diff --git a/community-import/src/main/java/com/linkerbot/community/importer/WorkbookReader.java b/community-import/src/main/java/com/linkerbot/community/importer/WorkbookReader.java new file mode 100644 index 0000000..fb003da --- /dev/null +++ b/community-import/src/main/java/com/linkerbot/community/importer/WorkbookReader.java @@ -0,0 +1,261 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import com.linkerbot.community.service.support.ProjectInputs; + +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamReader; + +/** 此处只支持经确认的固定工作簿布局;不执行公式或文档中的指令。 */ +public final class WorkbookReader { + public static final Map SHEETS; + + static { + LinkedHashMap types = new LinkedHashMap<>(); + types.put("1_轮式机器人", "WHEELED"); + types.put("2_四足机器人", "QUADRUPED"); + types.put("3_小人形机器人", "HUMANOID"); + types.put("4_大人形机器人", "HUMANOID"); + types.put("5_机械臂", "GENERAL"); + types.put("6_其他类型", "GENERAL"); + types.put("7_开源硬件", "GENERAL"); + SHEETS = Collections.unmodifiableMap(types); + } + + private final XMLInputFactory xml = XMLInputFactory.newFactory(); + + public WorkbookReader() { + xml.setProperty(XMLInputFactory.SUPPORT_DTD, false); + xml.setProperty("javax.xml.stream.isSupportingExternalEntities", false); + } + + private XMLStreamReader open(ZipFile zip, String name) throws Exception { + ZipEntry entry = zip.getEntry(name); + if (entry == null || entry.getSize() > 32L * 1024 * 1024) + throw new IllegalArgumentException("Excel XML 缺失或过大:" + name); + return xml.createXMLStreamReader(zip.getInputStream(entry)); + } + + public WorkbookData read(Path path) throws Exception { + String fingerprint; + try (InputStream input = Files.newInputStream(path)) { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) digest.update(buffer, 0, count); + fingerprint = HexFormat.of().formatHex(digest.digest()); + } + List candidates = new ArrayList<>(); + List excluded = new ArrayList<>(); + try (ZipFile zip = new ZipFile(path.toFile())) { + List strings = sharedStrings(zip); + Map relationships = new HashMap<>(); + XMLStreamReader rel = open(zip, "xl/_rels/workbook.xml.rels"); + while (rel.hasNext()) { + if (rel.next() == XMLStreamConstants.START_ELEMENT + && rel.getLocalName().equals("Relationship")) + relationships.put( + rel.getAttributeValue(null, "Id"), + rel.getAttributeValue(null, "Target")); + } + rel.close(); + Map sheets = new HashMap<>(); + XMLStreamReader book = open(zip, "xl/workbook.xml"); + while (book.hasNext()) { + if (book.next() == XMLStreamConstants.START_ELEMENT + && book.getLocalName().equals("sheet")) { + String name = book.getAttributeValue(null, "name"); + if (!SHEETS.containsKey(name)) continue; + String id = + book.getAttributeValue( + "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "id"); + String target = relationships.get(id); + if (target == null) throw new IllegalArgumentException("Sheet 关联缺失:" + name); + String resolved = + target.startsWith("/") + ? target.substring(1) + : Path.of("xl") + .resolve(target) + .normalize() + .toString() + .replace('\\', '/'); + if (!resolved.startsWith("xl/")) + throw new IllegalArgumentException("非法 Sheet 路径"); + if (sheets.put(name, resolved) != null) + throw new IllegalArgumentException("重复 Sheet:" + name); + } + } + book.close(); + if (!sheets.keySet().equals(SHEETS.keySet())) + throw new IllegalArgumentException("缺少固定的七个 Sheet,停止导入"); + for (String sheet : SHEETS.keySet()) + readSheet(zip, sheets.get(sheet), sheet, strings, candidates, excluded); + } + return new WorkbookData(fingerprint, List.copyOf(candidates), List.copyOf(excluded)); + } + + private List sharedStrings(ZipFile zip) throws Exception { + List strings = new ArrayList<>(); + if (zip.getEntry("xl/sharedStrings.xml") == null) return strings; + XMLStreamReader reader = open(zip, "xl/sharedStrings.xml"); + StringBuilder text = new StringBuilder(); + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT && reader.getLocalName().equals("si")) + text.setLength(0); + else if (event == XMLStreamConstants.START_ELEMENT && reader.getLocalName().equals("t")) + text.append(reader.getElementText()); + else if (event == XMLStreamConstants.END_ELEMENT && reader.getLocalName().equals("si")) + strings.add(text.toString()); + } + reader.close(); + return strings; + } + + private void readSheet( + ZipFile zip, + String path, + String sheet, + List strings, + List candidates, + List excluded) + throws Exception { + XMLStreamReader reader = open(zip, path); + Map cells = new HashMap<>(); + int row = 0; + String column = ""; + String type = ""; + StringBuilder value = new StringBuilder(); + boolean headersValid = false; + while (reader.hasNext()) { + int event = reader.next(); + String tag = + event == XMLStreamConstants.START_ELEMENT + || event == XMLStreamConstants.END_ELEMENT + ? reader.getLocalName() + : ""; + if (event == XMLStreamConstants.START_ELEMENT) { + switch (tag) { + case "row" -> { + row = Integer.parseInt(reader.getAttributeValue(null, "r")); + cells.clear(); + } + case "c" -> { + column = reader.getAttributeValue(null, "r").replaceAll("[0-9]", ""); + type = reader.getAttributeValue(null, "t"); + value.setLength(0); + } + case "v", "t" -> value.append(reader.getElementText()); + default -> {} + } + } else if (event == XMLStreamConstants.END_ELEMENT && tag.equals("c")) { + String text = value.toString(); + if ("s".equals(type) && !text.isEmpty()) text = strings.get(Integer.parseInt(text)); + cells.put(column, text.trim()); + } else if (event == XMLStreamConstants.END_ELEMENT && tag.equals("row")) { + if (row == 2) { + headersValid = + "GitHub链接状态".equals(cells.get("I")) + && "封面图片 URL".equals(cells.get("L")); + if (!headersValid) throw new IllegalArgumentException("Sheet 布局不符合约定:" + sheet); + } else if (row >= 3 && !cells.values().stream().allMatch(String::isBlank)) { + if (!headersValid) + throw new IllegalArgumentException("Sheet 缺少第 2 行表头:" + sheet); + addRow(sheet, row, cells, candidates, excluded); + } + } + } + reader.close(); + if (!headersValid) throw new IllegalArgumentException("Sheet 没有有效表头:" + sheet); + } + + private void addRow( + String sheet, + int row, + Map cells, + List candidates, + List excluded) { + String name = cells.getOrDefault("A", ""); + String reason = null; + if (!"可访问".equals(cells.get("I"))) reason = "LINK_STATUS_NOT_ACCESSIBLE"; + else if (cells.getOrDefault("L", "").isBlank()) reason = "NO_COVER"; + else { + try { + ProjectInputs.coverUrl(cells.get("L")); + repositoryKey(cells.get("B")); + } catch (RuntimeException invalid) { + reason = "INVALID_URL"; + } + } + if (name.isBlank() || name.length() > 100) reason = "INVALID_NAME"; + if (reason != null) { + excluded.add(new Excluded(sheet, row, name, reason)); + return; + } + StringBuilder description = + new StringBuilder("原始分类:") + .append(sheet.substring(2)) + .append("\n简介:") + .append(cells.getOrDefault("C", "")); + for (Map.Entry field : + List.of( + Map.entry("D", "CAD 文件类型"), + Map.entry("E", "类型/尺寸"), + Map.entry("F", "URDF 情况"), + Map.entry("G", "资料验证状态"))) + description + .append('\n') + .append(field.getValue()) + .append(":") + .append(cells.getOrDefault(field.getKey(), "未提供")); + String remark = cells.getOrDefault("H", ""); + if (!remark.isBlank() && !remark.equals("100")) description.append("\n备注:").append(remark); + String source = "\n来源:" + cells.get("B"); + String body = description.toString(); + if (body.length() + source.length() > 2000) + body = body.substring(0, 2000 - source.length() - 1) + "…"; + candidates.add( + new Candidate( + sheet, + row, + name, + cells.get("B"), + body + source, + SHEETS.get(sheet), + cells.get("L"))); + } + + public static String repositoryKey(String url) { + if (url == null) throw new IllegalArgumentException("仓库地址为空"); + URI uri = URI.create(url.trim()); + String path = uri.getPath() == null ? "" : uri.getPath().replaceAll("/$", ""); + if (!"https".equalsIgnoreCase(uri.getScheme()) + || !"github.com".equalsIgnoreCase(uri.getHost()) + || uri.getUserInfo() != null + || (uri.getPort() != -1 && uri.getPort() != 443) + || uri.getQuery() != null + || uri.getFragment() != null + || !path.matches("/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")) + throw new IllegalArgumentException("必须是 GitHub 仓库根地址"); + if (path.endsWith(".git")) path = path.substring(0, path.length() - 4); + String[] parts = path.substring(1).split("/"); + if (parts.length != 2 + || parts[0].equals(".") + || parts[0].equals("..") + || parts[1].equals(".") + || parts[1].equals("..")) throw new IllegalArgumentException("仓库地址不合法"); + return path.substring(1).toLowerCase(Locale.ROOT); + } +} diff --git a/community-import/src/main/resources/application.yml b/community-import/src/main/resources/application.yml new file mode 100644 index 0000000..9f51282 --- /dev/null +++ b/community-import/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + main: + web-application-type: none + banner-mode: off + sql: + init: + mode: never + datasource: + url: ${DB_URL:jdbc:mysql://localhost:3306/community?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC} + username: ${DB_USERNAME:root} + password: ${DB_PASSWORD:} + driver-class-name: com.mysql.cj.jdbc.Driver + hikari: + maximum-pool-size: 4 + connection-timeout: 5000 +mybatis-plus: + configuration: + map-underscore-to-camel-case: true +community: + gitea: + base-url: ${GITEA_BASE_URL:http://localhost:3000} + owner: ${GITEA_ORGANIZATION:community} + token: ${GITEA_TOKEN:} +logging: + level: + root: WARN + com.linkerbot.community: INFO + com.linkerbot.community.service.mapper: WARN diff --git a/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java b/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java new file mode 100644 index 0000000..dc4f97a --- /dev/null +++ b/community-import/src/test/java/com/linkerbot/community/importer/ImportCoordinatorTest.java @@ -0,0 +1,332 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.linkerbot.community.bean.entity.Project; +import com.linkerbot.community.bean.entity.User; +import com.linkerbot.community.common.*; +import com.linkerbot.community.service.ProjectService; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.gitea.*; +import com.linkerbot.community.service.mapper.*; + +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.AdditionalAnswers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.*; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.nio.file.*; +import java.time.*; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.*; + +@SpringBootTest( + classes = {DatabaseConfiguration.class, ImportCoordinatorTest.Fakes.class}, + properties = { + "spring.datasource.url=jdbc:h2:mem:importer;MODE=MySQL;DB_CLOSE_DELAY=-1", + "spring.datasource.username=sa", + "spring.datasource.password=", + "spring.datasource.driver-class-name=org.h2.Driver", + "spring.main.web-application-type=none", + "logging.level.root=WARN" + }) +class ImportCoordinatorTest { + @Configuration(proxyBeanMethods = false) + static class Fakes { + @Bean + @Primary + GiteaClient fake() { + return mock(GiteaClient.class); + } + } + + @Autowired ProjectMapper projects; + @Autowired UserMapper users; + @Autowired ProjectService service; + @Autowired GiteaClient gitea; + @Autowired CommunityProperties config; + @Autowired JdbcTemplate jdbc; + @TempDir Path directory; + private StateStore store; + private Entry entry; + private User owner; + private GiteaClient.RemoteRepository remote; + private AtomicBoolean exists; + private AtomicBoolean main; + private static final String SHA = "a".repeat(40); + + @BeforeEach + void prepare() throws Exception { + Path schema = Path.of("../docs/database.md"); + Matcher tables = + Pattern.compile("CREATE TABLE [\\s\\S]*?;").matcher(Files.readString(schema)); + jdbc.execute("DROP ALL OBJECTS"); + while (tables.find()) + jdbc.execute( + tables.group() + .replaceAll(" COMMENT '[^']*'", "") + .replaceAll("\\) ENGINE=InnoDB[^;]*;", ");")); + owner = new User(); + owner.setUserCode(Codes.next()); + owner.setEmail("owner@example.invalid"); + owner.setNickname("整理账号"); + owner.setStatus("ACTIVE"); + owner.setCreatedAt(LocalDateTime.now()); + owner.setUpdatedAt(owner.getCreatedAt()); + users.insert(owner); + Candidate candidate = + new Candidate( + "1_轮式机器人", + 3, + "迁移项目", + "https://github.com/example/robot", + "中文简介;URDF:待查", + "WHEELED", + "https://images.example.com/a.gif"); + store = + new StateStore( + directory.resolve("state"), + new WorkbookData("fingerprint", List.of(candidate), List.of())); + entry = store.manifest.entries.getFirst(); + entry.source = new GitHubRepository(12345, candidate.repositoryUrl(), "master"); + entry.projectCode = ProjectCodes.github(12345); + entry.preflightPassed = true; + entry.phase = Phase.READY; + store.save(); + remote = + new GiteaClient.RemoteRepository( + 77, + "community", + "p_" + entry.projectCode, + true, + false, + false, + candidate.repositoryUrl() + ".git", + "master"); + reset(gitea); + exists = new AtomicBoolean(); + main = new AtomicBoolean(); + when(gitea.inspect(anyString())) + .thenAnswer( + call -> { + if (!exists.get()) + throw new GiteaFailure(404, "NOT_FOUND", "不存在", false); + return remote; + }); + when(gitea.migrate(anyString(), anyString(), any(Duration.class))) + .thenAnswer( + call -> { + exists.set(true); + return remote; + }); + when(gitea.branch(any(), anyString())) + .thenAnswer( + call -> { + if (call.getArgument(1).equals("main") && !main.get()) + throw new GiteaFailure(404, "NOT_FOUND", "无 main", false); + return SHA; + }); + doAnswer( + call -> { + main.set(true); + return null; + }) + .when(gitea) + .createBranch(any(), eq("main"), eq(SHA)); + when(gitea.head(any())).thenReturn(SHA); + when(gitea.migrationWarnings(any(), anyString(), anyLong())) + .thenReturn(List.of("LFS 社区读取限制")); + } + + @AfterEach + void close() throws Exception { + store.close(); + } + + private ImportCoordinator worker(ProjectMapper mapper, ProjectService publisher) { + return new ImportCoordinator( + mapper, publisher, gitea, config, store, owner, Duration.ofMinutes(30)); + } + + @Test + void migrationBindsShortCodeAdaptsMasterAndPublishesWithoutEmptyRepository() throws Exception { + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + Project project = service.published(entry.projectCode); + assertThat(project.getProjectCode()).isEqualTo(entry.projectCode).hasSize(12); + assertThat(project.getGiteaRepoId()).isEqualTo(77L); + assertThat(project.getPublishedCommitSha()).isEqualTo(SHA); + assertThat(project.getSourceProjectId()).isNull(); + assertThat(project.getCoverUrl()).endsWith("a.gif"); + assertThat(service.view(project, owner.getId()).projectCode()).isEqualTo(entry.projectCode); + verify(gitea, never()).createRepository(anyString(), anyBoolean()); + verify(gitea).createBranch(remote.repository(), "main", SHA); + worker(projects, service).run(entry); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void normalizationErrorResumesOnlyAfterReadOnlyConfirmationOfAppliedState() throws Exception { + doThrow(new GiteaFailure(502, "GITEA_ERROR", "写入后响应失败", true)) + .when(gitea) + .defaultBranch(any(), eq("main")); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.REVIEW); + assertThat(ImportCoordinator.canVerifyNormalization(entry)).isTrue(); + // 默认分支仍未生效时,只读核对,不重发写入。 + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.REVIEW); + assertThat(entry.errorCode).isEqualTo("NORMALIZATION_NOT_CONFIRMED"); + verify(gitea, times(1)).defaultBranch(any(), anyString()); + remote = + new GiteaClient.RemoteRepository( + remote.id(), + remote.owner(), + remote.name(), + true, + false, + false, + remote.originalUrl(), + "main"); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + verify(gitea, times(1)).createBranch(any(), anyString(), anyString()); + verify(gitea, times(1)).defaultBranch(any(), anyString()); + } + + @Test + void unknownMigrationIsNeverAutomaticallyRepeated() throws Exception { + when(gitea.migrate(anyString(), anyString(), any(Duration.class))) + .thenAnswer( + call -> { + exists.set(true); + throw new GiteaFailure(504, "REMOTE_RESULT_UNKNOWN", "超时", true); + }); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.REVIEW); + assertThat( + jdbc.queryForObject( + "SELECT repo_status FROM project WHERE project_code = ?", + String.class, + entry.projectCode)) + .isEqualTo("CREATING"); + worker(projects, service).run(entry); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void databaseReservationFailureCanResumeWhenBothProjectAndRemoteAreAbsent() throws Exception { + ProjectMapper failing = mock(ProjectMapper.class, AdditionalAnswers.delegatesTo(projects)); + doThrow(new IllegalStateException("模拟写入前数据库故障")).when(failing).insert(any(Project.class)); + worker(failing, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.FAILED); + assertThat(entry.failedPhase).isEqualTo("RESERVING"); + String reservedAt = entry.reservedAt; + verify(gitea, never()).migrate(anyString(), anyString(), any(Duration.class)); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + assertThat(entry.reservedAt).isEqualTo(reservedAt); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void persistenceFailureBeforeReservationStopsRemoteWrites() throws Exception { + Files.createDirectory(directory.resolve("state/manifest.json.tmp")); + assertThatThrownBy(() -> worker(projects, service).run(entry)) + .isInstanceOf(java.io.IOException.class); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM project", Integer.class)).isZero(); + verify(gitea, never()).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void confirmedReceiptSurvivesDatabaseWriteFailureAndRestart() throws Exception { + ProjectMapper failing = mock(ProjectMapper.class, AdditionalAnswers.delegatesTo(projects)); + AtomicBoolean fail = new AtomicBoolean(true); + doAnswer( + call -> { + if (fail.getAndSet(false)) throw new IllegalStateException("模拟数据库回写失败"); + return projects.updateById((Project) call.getArgument(0)); + }) + .when(failing) + .updateById(any(Project.class)); + worker(failing, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.FAILED); + assertThat(entry.receipt).isNotNull(); + store.close(); + store = + new StateStore( + directory.resolve("state"), + new WorkbookData("fingerprint", List.of(entry.candidate), List.of())); + entry = store.manifest.entries.getFirst(); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void publicationFailureResumesWithoutMigratingAgain() throws Exception { + ProjectService publisher = mock(ProjectService.class); + when(publisher.publish(owner.getId(), entry.projectCode, SHA)) + .thenThrow(BusinessException.conflict("模拟发布失败")) + .thenAnswer(call -> service.publish(owner.getId(), entry.projectCode, SHA)); + worker(projects, publisher).run(entry); + assertThat(entry.phase).isEqualTo(Phase.FAILED); + worker(projects, publisher).run(entry); + assertThat(entry.phase).isEqualTo(Phase.PUBLISHED); + verify(gitea, times(1)).migrate(anyString(), anyString(), any(Duration.class)); + } + + @Test + void conflictingMainIsPreservedAndNotPublished() throws Exception { + when(gitea.branch(any(), eq("main"))).thenReturn("b".repeat(40)); + worker(projects, service).run(entry); + assertThat(entry.errorCode).isEqualTo("MAIN_BRANCH_CONFLICT"); + assertThat( + jdbc.queryForObject( + "SELECT publish_status FROM project WHERE project_code = ?", + String.class, + entry.projectCode)) + .isEqualTo("UNPUBLISHED"); + verify(gitea, never()).defaultBranch(any(), anyString()); + } + + @Test + void unrelatedProjectAndRemoteRepositoryAreNeverAdopted() throws Exception { + exists.set(true); + worker(projects, service).run(entry); + assertThat(entry.phase).isEqualTo(Phase.REVIEW); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM project", Integer.class)).isZero(); + verify(gitea, never()).migrate(anyString(), anyString(), any(Duration.class)); + exists.set(false); + entry.phase = Phase.READY; + when(gitea.createRepository(anyString(), eq(true))) + .thenReturn(new GiteaClient.Repository(88, "community", "temporary")); + String existing = service.create(owner.getId(), entry.candidate.input()).projectCode(); + jdbc.update( + "UPDATE project SET project_code = ? WHERE project_code = ?", + entry.projectCode, + existing); + String before = + jdbc.queryForObject( + "SELECT repo_status FROM project WHERE project_code = ?", + String.class, + entry.projectCode); + worker(projects, service).run(entry); + assertThat(entry.errorCode).isEqualTo("PROJECT_CONFLICT"); + assertThat( + jdbc.queryForObject( + "SELECT repo_status FROM project WHERE project_code = ?", + String.class, + entry.projectCode)) + .isEqualTo(before); + } +} diff --git a/community-import/src/test/java/com/linkerbot/community/importer/SourceChecksTest.java b/community-import/src/test/java/com/linkerbot/community/importer/SourceChecksTest.java new file mode 100644 index 0000000..c1dc94c --- /dev/null +++ b/community-import/src/test/java/com/linkerbot/community/importer/SourceChecksTest.java @@ -0,0 +1,45 @@ +package com.linkerbot.community.importer; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class SourceChecksTest { + private static final String PAGE = + """ + + + + +"""; + + @Test + void publicPageRequiresMatchingIdentityVisibilityAndBranch() throws Exception { + ImportModels.GitHubRepository repo = + SourceChecks.parsePublicPage(PAGE, "https://github.com/neurobionics/opensourceleg"); + assertEquals(482055059L, repo.id()); + assertEquals("main", repo.defaultBranch()); + for (String changed : + new String[] { + PAGE.replace("content=\"482055059\"", "content=\"42\""), + PAGE.replace("\"public\":true", "\"public\":false"), + PAGE.replace("\"private\":false", "\"private\":true"), + PAGE.replace("\"isEmpty\":false", "\"isEmpty\":true"), + PAGE.replace("\"defaultBranch\":\"main\"", "\"defaultBranch\":\"\""), + PAGE.replace("react-app.embeddedData", "other-data") + }) { + assertThrows( + ImportFailure.class, + () -> + SourceChecks.parsePublicPage( + changed, "https://github.com/neurobionics/opensourceleg")); + } + assertThrows( + ImportFailure.class, + () -> SourceChecks.parsePublicPage(PAGE, "https://github.com/someone/other")); + } +} diff --git a/community-import/src/test/java/com/linkerbot/community/importer/WorkbookReaderTest.java b/community-import/src/test/java/com/linkerbot/community/importer/WorkbookReaderTest.java new file mode 100644 index 0000000..ef9d37f --- /dev/null +++ b/community-import/src/test/java/com/linkerbot/community/importer/WorkbookReaderTest.java @@ -0,0 +1,195 @@ +package com.linkerbot.community.importer; + +import static com.linkerbot.community.importer.ImportModels.*; + +import static org.assertj.core.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import java.util.zip.*; + +class WorkbookReaderTest { + @TempDir Path directory; + + static Path fixture(Path path) throws Exception { + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(path))) { + List names = new ArrayList<>(WorkbookReader.SHEETS.keySet()); + names.add("10_项目封面图鉴"); + names.add("0_汇总总表"); + StringBuilder book = + new StringBuilder( + ""); + StringBuilder rel = new StringBuilder(""); + for (int i = 0; i < names.size(); i++) { + book.append(""); + rel.append(""); + String header = + "" + + cell("I2", "GitHub链接状态") + + cell("L2", "封面图片 URL") + + ""; + String rows = + row( + 3, + "可访问", + "https://images.example.com/a.gif", + "https://github.com/example/repo" + i); + if (i == 0) + rows += + row( + 4, + "可访问(已重定向)", + "https://images.example.com/a.png", + "https://github.com/example/redirect") + + row(5, "可访问", "", "https://github.com/example/no-cover") + + row( + 6, + "可访问", + "https://images.example.com/a.png", + "https://github.com/example") + + row( + 7, + "可访问", + "https://images.example.com/a.png", + "https://github.com/example/repo/tree/main") + + row( + 8, + "可访问", + "http://images.example.com/a.png", + "https://github.com/example/plain") + + row( + 9, + "可访问", + "https://images.example.com/a.png", + "https://gitlab.com/example/repo"); + entry( + zip, + "xl/worksheets/s" + i + ".xml", + "" + header + rows + ""); + } + entry(zip, "xl/workbook.xml", book + ""); + entry(zip, "xl/_rels/workbook.xml.rels", rel + ""); + } + return path; + } + + private static String row(int row, String status, String cover, String url) { + return "" + + cell("A" + row, "测试项目") + + cell("B" + row, url) + + cell("C" + row, "中文简介") + + cell("D" + row, "STEP") + + cell("E" + row, "小型") + + cell("F" + row, "待查") + + cell("G" + row, "待验证") + + cell("H" + row, "100") + + cell("I" + row, status) + + "IMAGE(L" + + row + + ")点开图片URL" + + cell("L" + row, cover) + + ""; + } + + private static String cell(String ref, String value) { + return "" + value + ""; + } + + private static void entry(ZipOutputStream zip, String name, String value) throws Exception { + zip.putNextEntry(new ZipEntry(name)); + zip.write(value.getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + + @Test + void fixedSheetsAndActualColumnsAreUsedWithoutExecutingCoverFormula() throws Exception { + WorkbookData data = new WorkbookReader().read(fixture(directory.resolve("source.xlsx"))); + assertThat(data.candidates()).hasSize(7); + assertThat(data.excluded()).hasSize(6); + assertThat(data.candidates().stream().map(Candidate::sheet)) + .containsExactlyElementsOf(WorkbookReader.SHEETS.keySet()); + Candidate first = data.candidates().getFirst(); + assertThat(first.name()).isEqualTo("测试项目"); + assertThat(first.projectType()).isEqualTo("WHEELED"); + assertThat(first.description()) + .contains("中文简介", "CAD 文件类型:STEP", "URDF 情况:待查", "资料验证状态:待验证") + .doesNotContain("100"); + assertThat(first.coverUrl()).endsWith("a.gif"); + assertThat(new WorkbookReader().read(directory.resolve("source.xlsx"))).isEqualTo(data); + } + + @Test + void stateLockFingerprintAndReceiptsSurviveRestart() throws Exception { + WorkbookData data = new WorkbookReader().read(fixture(directory.resolve("source.xlsx"))); + Path state = directory.resolve("state"); + try (StateStore store = new StateStore(state, data)) { + store.update( + store.manifest.entries.getFirst(), + e -> { + e.projectCode = "111111111111"; + e.phase = Phase.MIGRATING; + }); + assertThatThrownBy(() -> new StateStore(state, data)) + .isInstanceOf(RuntimeException.class); + store.report(); + } + try (StateStore store = new StateStore(state, data)) { + assertThat(store.manifest.entries.getFirst().phase).isEqualTo(Phase.MIGRATING); + assertThat(store.manifest.entries.getFirst().projectCode).isEqualTo("111111111111"); + } + assertThatThrownBy( + () -> + new StateStore( + state, + new WorkbookData( + "other", data.candidates(), data.excluded()))) + .isInstanceOf(ImportFailure.class); + try (StateStore ignored = new StateStore(state, data)) { + assertThat(Files.exists(state.resolve("report.json"))).isTrue(); + } + } + + @Test + void cliIsNonMutatingUnlessExplicitlyExecuting() { + ImportApplication.Options options = + ImportApplication.Options.parse(new String[] {"--excel", "source.xlsx"}); + assertThat(options.execute()).isFalse(); + assertThat(options.concurrency()).isEqualTo(2); + assertThat(options.migrationTimeout().toMinutes()).isEqualTo(30); + assertThatThrownBy( + () -> + ImportApplication.Options.parse( + new String[] {"--excel", "s", "--execute"})) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + ImportApplication.Options.parse( + new String[] { + "--excel", + "s", + "--execute", + "--offline", + "--owner-user-code", + "u" + })) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java b/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java index 366db71..0b11ee7 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java +++ b/community-service/src/main/java/com/linkerbot/community/service/gitea/GiteaClient.java @@ -4,6 +4,7 @@ import com.linkerbot.community.bean.Contracts.*; import com.linkerbot.community.common.PageResponse; import java.util.List; +import java.time.Duration; public interface GiteaClient { record Repository(long id, String owner, String name) {} @@ -12,6 +13,29 @@ public interface GiteaClient { record Change(String operation, String path, String blobSha, byte[] bytes) {} + record RemoteRepository(long id, String owner, String name, boolean privateRepo, + boolean mirror, boolean empty, String originalUrl, String defaultBranch) { + public Repository repository() { return new Repository(id, owner, name); } + } + + default RemoteRepository migrate(String cloneUrl, String name, Duration timeout) { + throw new UnsupportedOperationException("未实现迁移"); + } + + default RemoteRepository inspect(String name) { throw new UnsupportedOperationException("未实现仓库检查"); } + + default String branch(Repository repository, String branch) { throw new UnsupportedOperationException("未实现分支检查"); } + + default void createBranch(Repository repository, String name, String sha) { throw new UnsupportedOperationException("未实现分支创建"); } + + default void defaultBranch(Repository repository, String name) { throw new UnsupportedOperationException("未实现默认分支设置"); } + + default void checkMigrationSettings() { throw new UnsupportedOperationException("未实现迁移能力检查"); } + + default List migrationWarnings(Repository repository, String sha, long maxFileBytes) { + return List.of("社区下载仍受单文件限制,LFS 实体读取及特殊 Git 文件的 Fork 能力未扩展"); + } + Repository createRepository(String name, boolean initialize); String head(Repository repository); diff --git a/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java b/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java index d218869..7a495bc 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java +++ b/community-service/src/main/java/com/linkerbot/community/service/gitea/RestGiteaClient.java @@ -18,6 +18,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.*; +import java.time.Duration; @Component public class RestGiteaClient implements GiteaClient { @@ -51,13 +52,17 @@ public class RestGiteaClient implements GiteaClient { } private HttpResponse request(String method, String path, Object body) { + return request(method, path, body, properties.getGitea().getRequestTimeout()); + } + + private HttpResponse request(String method, String path, Object body, Duration timeout) { if (properties.getGitea().getToken().isBlank()) throw new GiteaFailure(503, "GITEA_NOT_CONFIGURED", "请配置 Gitea Token", false); try { String base = properties.getGitea().getBaseUrl().replaceAll("/+$", ""); HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create(base + "/api/v1" + path)) - .timeout(properties.getGitea().getRequestTimeout()) + .timeout(timeout) .header("Accept", "application/json") .header("Authorization", "token " + properties.getGitea().getToken()); if (body == null) builder.method(method, HttpRequest.BodyPublishers.noBody()); @@ -72,6 +77,8 @@ public class RestGiteaClient implements GiteaClient { int status = response.statusCode(); if (status < 200 || status >= 300) { if (status == 404) throw new GiteaFailure(404, "NOT_FOUND", "仓库文件或版本不存在", false); + if (status == 422 && path.equals("/repos/migrate")) + throw new GiteaFailure(422, "GITEA_MIGRATION_REJECTED", "Gitea 迁移失败,需核对源站访问和服务端迁移日志", false); if (status == 409 || status == 422) throw new GiteaFailure(409, "GITEA_CONFLICT", "仓库内容冲突,请刷新后重试", false); throw new GiteaFailure( @@ -117,11 +124,93 @@ public class RestGiteaClient implements GiteaClient { @Override public String head(Repository repo) { + return branch(repo, "main"); + } + + @Override + public String branch(Repository repo, String branchName) { JsonNode branch = - parse(request("GET", repoPath(repo) + "/branches/main", null).body(), false); + parse(request("GET", repoPath(repo) + "/branches/" + encode(branchName), null).body(), false); return FilePaths.sha(branch.path("commit").path("id").asText()); } + private RemoteRepository remote(JsonNode data) { + if (data.path("id").asLong() <= 0) + throw new GiteaFailure(502, "GITEA_INVALID_RESPONSE", "仓库响应缺少标识", true); + return new RemoteRepository(data.path("id").asLong(), data.path("owner").path("login").asText(), + data.path("name").asText(), data.path("private").asBoolean(), data.path("mirror").asBoolean(), + data.path("empty").asBoolean(), data.path("original_url").asText(), data.path("default_branch").asText()); + } + + @Override + public RemoteRepository migrate(String cloneUrl, String name, Duration timeout) { + Map body = new LinkedHashMap<>(); + body.put("clone_addr", cloneUrl); + body.put("repo_owner", properties.getGitea().getOwner()); + body.put("repo_name", name); + body.put("service", "git"); + body.put("private", true); + body.put("mirror", false); + body.put("lfs", true); + for (String option : List.of("issues", "labels", "milestones", "pull_requests", "releases", "wiki")) + body.put(option, false); + HttpResponse response = request("POST", "/repos/migrate", body, timeout); + if (response.statusCode() != 201) + throw new GiteaFailure(502, "MIGRATION_NOT_CONFIRMED", "迁移没有返回已完成的 201 收据,需人工核对", true); + return remote(parse(response.body(), true)); + } + + @Override + public RemoteRepository inspect(String name) { + return remote(parse(request("GET", "/repos/" + encode(properties.getGitea().getOwner()) + "/" + encode(name), null).body(), false)); + } + + @Override + public void createBranch(Repository repo, String name, String sha) { + request("POST", repoPath(repo) + "/branches", Map.of("new_branch_name", name, "old_ref_name", FilePaths.sha(sha))); + } + + @Override + public void defaultBranch(Repository repo, String name) { + request("PATCH", repoPath(repo), Map.of("default_branch", name)); + } + + @Override + public void checkMigrationSettings() { + JsonNode settings = parse(request("GET", "/settings/repository", null).body(), false); + if (!settings.has("lfs_disabled") || settings.path("lfs_disabled").asBoolean() + || settings.path("migrations_disabled").asBoolean() || settings.path("http_git_disabled").asBoolean()) + throw BusinessException.badRequest("Gitea 必须启用迁移、HTTP Git 和 LFS"); + } + + @Override + public List migrationWarnings(Repository repo, String sha, long maxFileBytes) { + List warnings = new ArrayList<>(); + warnings.add("已请求迁移 LFS;社区 API 的 LFS 实体读取能力未扩展"); + long large = 0; + long links = 0; + long modules = 0; + long seen = 0; + for (int page = 1; page <= 10000; page++) { + JsonNode tree = parse(request("GET", repoPath(repo) + "/git/trees/" + FilePaths.sha(sha) + + "?recursive=true&per_page=1000&page=" + page, null).body(), false); + JsonNode entries = tree.path("tree"); + if (!entries.isArray()) throw new GiteaFailure(502, "GITEA_INVALID_RESPONSE", "Git 文件树响应格式异常", false); + for (JsonNode entry : entries) { + if (entry.path("size").asLong() > maxFileBytes) large++; + if (entry.path("mode").asText().equals("120000")) links++; + if (entry.path("mode").asText().equals("160000")) modules++; + } + seen += entries.size(); + if (seen >= tree.path("total_count").asLong(seen) && !tree.path("truncated").asBoolean()) break; + if (entries.isEmpty() || page == 10000) { warnings.add("文件树扫描不完整,需要人工核对剩余文件"); break; } + } + if (large > 0) warnings.add("发布快照有 " + large + " 个文件超过社区单文件读取限制 " + maxFileBytes + " 字节"); + if (links > 0) warnings.add("发布快照有 " + links + " 个符号链接,社区普通文件读取和 Fork 不支持"); + if (modules > 0) warnings.add("发布快照有 " + modules + " 个子模块引用,未递归迁移且社区 Fork 不支持"); + return List.copyOf(warnings); + } + private JsonNode contents(Repository repo, String sha, String path) { String suffix = path.isEmpty() ? "" : "/" + filePath(path); return parse( diff --git a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java index 3a89f77..e950413 100644 --- a/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java +++ b/community-service/src/main/java/com/linkerbot/community/service/impl/ProjectServiceImpl.java @@ -15,6 +15,8 @@ import com.linkerbot.community.service.config.CommunityProperties; import com.linkerbot.community.service.gitea.*; import com.linkerbot.community.service.mapper.*; import com.linkerbot.community.service.support.ProjectLocks; +import com.linkerbot.community.service.support.ProjectInputs; +import org.springframework.dao.DuplicateKeyException; import lombok.RequiredArgsConstructor; @@ -22,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; -import java.net.URI; import java.util.*; import java.util.function.Function; @@ -46,25 +47,11 @@ public class ProjectServiceImpl implements ProjectService { } private void validateInput(ProjectInput input) { - if (!config.getTypes().containsKey(input.projectType())) - throw BusinessException.badRequest("项目类型不在可选列表中"); + ProjectInputs.validate(input, config); } private String normalizeCoverUrl(String value) { - if (value == null || value.isBlank()) return null; - String normalized = value.trim(); - try { - URI uri = URI.create(normalized); - if (normalized.length() > 2048 - || !"https".equalsIgnoreCase(uri.getScheme()) - || uri.getHost() == null - || uri.getRawUserInfo() != null - || (uri.getPort() != -1 && (uri.getPort() < 1 || uri.getPort() > 65535))) - throw new IllegalArgumentException(); - } catch (IllegalArgumentException error) { - throw BusinessException.badRequest("封面地址必须是最多 2048 字符且不含用户凭据的 HTTPS 地址"); - } - return normalized; + return ProjectInputs.coverUrl(value); } @Override @@ -185,7 +172,7 @@ public class ProjectServiceImpl implements ProjectService { private Project initialize(long userId, ProjectInput input, Project source) { validateInput(input); Project project = new Project(); - project.setProjectCode(Codes.next()); + project.setProjectCode(ProjectCodes.next()); project.setOwnerUserId(userId); project.setName(input.name().trim()); project.setDescription(input.description() == null ? "" : input.description()); @@ -202,8 +189,27 @@ public class ProjectServiceImpl implements ProjectService { project.setSourceProjectId(source.getId()); project.setSourceCommitSha(source.getPublishedCommitSha()); } - projects.insert(project); - return project; + for (int attempt = 1; attempt <= 5; attempt++) { + try { + projects.insert(project); + return project; + } catch (DuplicateKeyException duplicate) { + boolean codeConflict = false; + for (Throwable cause = duplicate; cause != null; cause = cause.getCause()) { + if (cause.getMessage() != null + && cause.getMessage().toLowerCase(Locale.ROOT).contains("uk_project_code")) + codeConflict = true; + } + if (!codeConflict) throw duplicate; + log.warn("项目编码冲突 projectCode={} attempt={}", project.getProjectCode(), attempt); + if (attempt == 5) + throw new BusinessException(503, "PROJECT_CODE_EXHAUSTED", "项目编码分配失败,请稍后重试"); + project.setId(null); + project.setProjectCode(ProjectCodes.next()); + project.setGiteaRepoName("p_" + project.getProjectCode()); + } + } + throw new IllegalStateException("项目编码分配未完成"); } @Override diff --git a/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java b/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java new file mode 100644 index 0000000..bd43cac --- /dev/null +++ b/community-service/src/main/java/com/linkerbot/community/service/support/ProjectInputs.java @@ -0,0 +1,38 @@ +package com.linkerbot.community.service.support; + +import com.linkerbot.community.bean.Contracts.ProjectInput; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.service.config.CommunityProperties; + +import java.net.URI; + +public final class ProjectInputs { + private ProjectInputs() {} + + public static void validate(ProjectInput input, CommunityProperties config) { + if (input.name() == null || input.name().isBlank() || input.name().length() > 100) + throw BusinessException.badRequest("项目名称必须为 1~100 字符"); + if (input.description() != null && input.description().length() > 2000) + throw BusinessException.badRequest("项目描述不能超过 2000 字符"); + if (!config.getTypes().containsKey(input.projectType())) + throw BusinessException.badRequest("项目类型不在可选列表中"); + coverUrl(input.coverUrl()); + } + + public static String coverUrl(String value) { + if (value == null || value.isBlank()) return null; + String normalized = value.trim(); + try { + URI uri = URI.create(normalized); + if (normalized.length() > 2048 + || !"https".equalsIgnoreCase(uri.getScheme()) + || uri.getHost() == null + || uri.getRawUserInfo() != null + || (uri.getPort() != -1 && (uri.getPort() < 1 || uri.getPort() > 65535))) + throw new IllegalArgumentException(); + } catch (IllegalArgumentException error) { + throw BusinessException.badRequest("封面地址必须是最多 2048 字符且不含用户凭据的 HTTPS 地址"); + } + return normalized; + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java new file mode 100644 index 0000000..cfe2b91 --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/ProjectCodeAllocationTest.java @@ -0,0 +1,89 @@ +package com.linkerbot.community.service; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import com.linkerbot.community.bean.Contracts.ProjectInput; +import com.linkerbot.community.bean.entity.Project; +import com.linkerbot.community.bean.entity.User; +import com.linkerbot.community.common.BusinessException; +import com.linkerbot.community.common.ProjectCodes; +import com.linkerbot.community.service.config.CommunityProperties; +import com.linkerbot.community.service.gitea.GiteaClient; +import com.linkerbot.community.service.impl.ProjectServiceImpl; +import com.linkerbot.community.service.mapper.*; +import com.linkerbot.community.service.support.ProjectLocks; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.springframework.dao.DuplicateKeyException; + +class ProjectCodeAllocationTest { + @Test + void retriesOnlyProjectCodeConstraintBeforeAnyRemoteCreation() { + ProjectMapper mapper = mock(ProjectMapper.class); + GiteaClient gitea = mock(GiteaClient.class); + when(mapper.insert(any(Project.class))) + .thenThrow(new DuplicateKeyException("UK_PROJECT_CODE_INDEX")) + .thenAnswer( + call -> { + Project project = call.getArgument(0); + project.setId(1L); + return 1; + }); + when(gitea.createRepository(anyString(), eq(true))) + .thenReturn(new GiteaClient.Repository(1, "community", "p_222222222222")); + try (MockedStatic codes = mockStatic(ProjectCodes.class)) { + codes.when(ProjectCodes::next).thenReturn("111111111111", "222222222222"); + assertThat( + service(mapper, gitea) + .create(1, new ProjectInput("测试", "", "GENERAL", null)) + .projectCode()) + .isEqualTo("222222222222"); + } + verify(mapper, times(2)).insert(any(Project.class)); + verify(gitea).createRepository("p_222222222222", true); + verify(gitea, never()).createRepository("p_111111111111", true); + } + + @Test + void stopsAfterFiveCollisionsAndDoesNotRetryOtherConstraints() { + ProjectMapper mapper = mock(ProjectMapper.class); + GiteaClient gitea = mock(GiteaClient.class); + when(mapper.insert(any(Project.class))) + .thenThrow(new DuplicateKeyException("uk_project_code")); + assertThatThrownBy( + () -> + service(mapper, gitea) + .create(1, new ProjectInput("测试", "", "GENERAL", null))) + .isInstanceOfSatisfying( + BusinessException.class, + e -> assertThat(e.code()).isEqualTo("PROJECT_CODE_EXHAUSTED")); + verify(mapper, times(5)).insert(any(Project.class)); + verifyNoInteractions(gitea); + reset(mapper); + when(mapper.insert(any(Project.class))) + .thenThrow(new DuplicateKeyException("uk_project_repo")); + assertThatThrownBy( + () -> + service(mapper, gitea) + .create(1, new ProjectInput("测试", "", "GENERAL", null))) + .isInstanceOf(DuplicateKeyException.class); + verify(mapper).insert(any(Project.class)); + verifyNoInteractions(gitea); + } + + private ProjectServiceImpl service(ProjectMapper projects, GiteaClient gitea) { + UserService users = mock(UserService.class); + when(users.require(1)).thenReturn(new User()); + return new ProjectServiceImpl( + projects, + mock(LikeMapper.class), + mock(CommentMapper.class), + users, + gitea, + new CommunityProperties(), + new ProjectLocks()); + } +} diff --git a/community-service/src/test/java/com/linkerbot/community/service/gitea/GiteaMigrationTest.java b/community-service/src/test/java/com/linkerbot/community/service/gitea/GiteaMigrationTest.java new file mode 100644 index 0000000..0803a3a --- /dev/null +++ b/community-service/src/test/java/com/linkerbot/community/service/gitea/GiteaMigrationTest.java @@ -0,0 +1,138 @@ +package com.linkerbot.community.service.gitea; + +import static org.assertj.core.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkerbot.community.service.config.CommunityProperties; +import com.sun.net.httpserver.HttpServer; + +import org.junit.jupiter.api.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; + +class GiteaMigrationTest { + @Test + void nativeMigrationUsesConfirmedSchemaAndKeepsLfsAndPrivateEnabled() throws Exception { + ObjectMapper json = new ObjectMapper(); + AtomicReference body = new AtomicReference<>(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/api/v1/repos/migrate", + exchange -> { + body.set(json.readTree(exchange.getRequestBody())); + assertThat(exchange.getRequestHeaders().getFirst("Authorization")) + .isEqualTo("token fixture-token"); + byte[] response = + "{\"id\":77,\"owner\":{\"login\":\"community\"},\"name\":\"p_111111111111\",\"private\":true,\"mirror\":false,\"empty\":false,\"default_branch\":\"master\",\"original_url\":\"https://github.com/example/robot.git\"}" + .getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(201, response.length); + exchange.getResponseBody().write(response); + exchange.close(); + }); + server.start(); + try { + CommunityProperties config = new CommunityProperties(); + config.getGitea().setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + config.getGitea().setToken("fixture-token"); + GiteaClient client = new RestGiteaClient(config, json); + GiteaClient.RemoteRepository result = + client.migrate( + "https://github.com/example/robot.git", + "p_111111111111", + Duration.ofSeconds(2)); + assertThat(result.id()).isEqualTo(77); + assertThat(result.defaultBranch()).isEqualTo("master"); + assertThat(body.get().path("repo_owner").asText()).isEqualTo("community"); + assertThat(body.get().path("repo_name").asText()).isEqualTo("p_111111111111"); + assertThat(body.get().path("service").asText()).isEqualTo("git"); + assertThat(body.get().path("private").asBoolean()).isTrue(); + assertThat(body.get().path("lfs").asBoolean()).isTrue(); + for (String key : + new String[] { + "mirror", + "issues", + "labels", + "milestones", + "pull_requests", + "releases", + "wiki" + }) assertThat(body.get().path(key).asBoolean()).as(key).isFalse(); + } finally { + server.stop(0); + } + } + + @Test + void migrationValidationOrCloneFailureIsNotReportedAsARepositoryNameConflict() + throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/api/v1/repos/migrate", + exchange -> { + byte[] response = + "{\"message\":\"Migration failed: fatal: connection failed\"}" + .getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(422, response.length); + exchange.getResponseBody().write(response); + exchange.close(); + }); + server.start(); + try { + CommunityProperties config = new CommunityProperties(); + config.getGitea().setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + config.getGitea().setToken("fixture-token"); + GiteaClient client = new RestGiteaClient(config, new ObjectMapper()); + assertThatThrownBy( + () -> + client.migrate( + "https://github.com/example/robot.git", + "p_111111111111", + Duration.ofSeconds(2))) + .isInstanceOfSatisfying( + GiteaFailure.class, + error -> { + assertThat(error.status()).isEqualTo(422); + assertThat(error.code()).isEqualTo("GITEA_MIGRATION_REJECTED"); + assertThat(error.uncertain()).isFalse(); + }); + } finally { + server.stop(0); + } + } + + @Test + void migrationTimeoutIsAnUnknownRemoteWriteResult() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/api/v1/repos/migrate", + exchange -> { + try { + Thread.sleep(300); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + exchange.close(); + }); + server.start(); + try { + CommunityProperties config = new CommunityProperties(); + config.getGitea().setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort()); + config.getGitea().setToken("fixture-token"); + assertThatThrownBy( + () -> + new RestGiteaClient(config, new ObjectMapper()) + .migrate( + "https://github.com/example/robot.git", + "p_111111111111", + Duration.ofMillis(50))) + .isInstanceOfSatisfying( + GiteaFailure.class, error -> assertThat(error.uncertain()).isTrue()); + } finally { + server.stop(0); + } + } +} diff --git a/docs/community-api.md b/docs/community-api.md index a29fba7..c0310e8 100644 --- a/docs/community-api.md +++ b/docs/community-api.md @@ -89,6 +89,8 @@ Git SHA 使用完整的 40 或 64 位十六进制字符串,不使用缩写。 | publishedCommitSha | 项目详情 | 社区当前发布版本;Fork 时原样提交此值 | | blobSha | 文件或目录条目 | 文件或目录对应的 Git 对象标识,前端不用它提交文件 | +新建、Fork 和 GitHub 初始化导入的 `projectCode` 为 12 位小写字母和数字,例如 `7m4k9x2p8q6r`。已有 32 位项目编码及链接继续有效。前端将编码作为不透明字符串传递,不截取、不补齐,也不固定要求 32 位;`sourceProjectCode`、AI 的 `previousProjectCodes` 及结果编码同样兼容两种长度,最大长度仍为 32。用户编码、评论编码及 Commit SHA 的规则不变。 + 项目详情不包含工作区当前 main 的 SHA。进入编辑器、提交或发布前,从工作区目录或文件接口获取 `commitSha`。 ## 接口总览 diff --git a/docs/database.md b/docs/database.md index fdb6aa3..9b518ce 100644 --- a/docs/database.md +++ b/docs/database.md @@ -80,6 +80,8 @@ CREATE TABLE project_like ( 内部 ID 仅用于数据库关联,对外使用业务编码。时间均由后端写入 UTC。`CREATING` 项目表示仓库初始化未完成,不代表编辑草稿。Gitea 数据库与业务库分离,不修改 Gitea 内部表。 +新增项目使用 12 位项目编码,历史 32 位编码继续保留。现有 `project_code CHAR(32)` 可容纳两者,不需要修改表结构或历史记录;仓库名称仍为 `p_{projectCode}`。用户编码和评论编码保持 32 位。GitHub 运维导入的来源和进度保存于批次状态目录,GitHub 来源不占用社区内部 Fork 来源字段。 + ## 已有数据库增加项目封面 已使用旧版结构的环境,先在对应业务库执行以下 SQL,再部署新版本应用;新建数据库已包含此字段,不重复执行。应用不会自动迁移数据库。 diff --git a/pom.xml b/pom.xml index a6c34a6..b292674 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ community-service community-security community-api + community-import 21 diff --git a/scripts/import-github-projects.sh b/scripts/import-github-projects.sh new file mode 100644 index 0000000..381a85f --- /dev/null +++ b/scripts/import-github-projects.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +COMMUNITY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -f "$COMMUNITY_ROOT/.env" ]]; then + set -a + source "$COMMUNITY_ROOT/.env" + set +a +fi +cd "$COMMUNITY_ROOT" +mvn -q -pl community-import -am -DskipTests package +exec java -Djava.awt.headless=true -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8 -jar community-import/target/community-import-0.0.1-SNAPSHOT.jar "$@"